AReaL/examples/vlm/clevr_count_70k_sft.yaml at main · inclusionAI/AReaL · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
experiment_name: clevr_count_70k-sft
trial_name: trial0

seed: 1
enable_offload: false
total_train_epochs: 3
tokenizer_path: ${actor.path}

cluster:
  n_nodes: 1
  n_gpus_per_node: 8
  fileroot: /tmp/areal/experiments
  name_resolve:
    type: nfs
    nfs_record_root: /tmp/areal/name_resolve


actor:
  backend: "fsdp:d8p1t1"
  experiment_name: ${experiment_name}
  trial_name: ${trial_name}
  path: Qwen/Qwen2-VL-7B
  init_from_scratch: false
  gradient_checkpointing: true
  dtype: bfloat16
  mb_spec:
    max_tokens_per_mb: 4096
  optimizer:
    type: adam
    lr: 2e-5
    weight_decay: 0.05
    beta1: 0.9
    beta2: 0.95
    eps: 1e-5
    lr_scheduler_type: cosine
    gradient_clipping: 1.0
  scheduling_spec:
    - task_type: worker
      port_count: 2
      gpu: 1
      mem: 32
      cmd: python3 -m areal.infra.rpc.rpc_server
      env_vars: {}

train_dataset:
  batch_size: 128
  shuffle: true
  pin_memory: true
  num_workers: 4
  path: BUAADreamer/clevr_count_70k
  type: sft

valid_dataset:
  batch_size: 128
  pin_memory: true
  num_workers: 4
  path: BUAADreamer/clevr_count_70k
  type: sft

# Utilities
saver:
  experiment_name: ${experiment_name}
  trial_name: ${trial_name}
  fileroot: ${cluster.fileroot}
  freq_epochs: 1
  freq_steps: null
  freq_secs: null

recover:
  mode: disabled
  experiment_name: ${experiment_name}
  trial_name: ${trial_name}
  fileroot: ${cluster.fileroot}
  freq_epochs: 1
  freq_steps: null
  freq_secs: 3600

evaluator:
  experiment_name: ${experiment_name}
  trial_name: ${trial_name}
  fileroot: ${cluster.fileroot}
  freq_epochs: 1
  freq_steps: null
  freq_secs: null

stats_logger:
  experiment_name: ${experiment_name}
  trial_name: ${trial_name}
  fileroot: ${cluster.fileroot}
  wandb:
    mode: disabled