vnt-research-prototype/config_longctx_vnt_stable.yaml at main · AloneMath/vnt-research-prototype · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
seed: 42
exp_name: longctx_vnt_stable
device: auto
save_dir: output/checkpoints

model:
  d_model: 256
  n_heads: 8
  d_ff: 1024
  n_layers: 4
  vocab_size: 5000
  max_len: 128
  memory_slots: 48
  micro_steps: 2
  n_ops: 3
  dropout: 0.1
  stop_threshold: 0.60
  confidence_stop_threshold: 0.72

train:
  total_steps: 1200
  batch_size: 16
  seq_len: 128
  lr: 0.0003
  weight_decay: 0.01
  grad_clip: 1.0
  eval_interval: 200
  log_interval: 50
  warmup_steps: 100
  tau_start: 1.0
  tau_end: 0.5
  label_smoothing: 0.0
  budget_weight: 0.01
  vnt_budget_weight: 0.10
  entropy_weight: 0.01

optim:
  betas: [0.9, 0.95]
  eps: 1.0e-8

data:
  dataset: wikitext
  subset: wikitext-2-raw-v1
  text_file: "./input.txt"
  split_train: train
  split_valid: validation
  split_test: test
  tokenizer: whitespace
  lowercase: true
  min_freq: 2
  max_vocab: 5000

logging:
  use_tensorboard: false

experiment:
  auto_align_params: true
  align_tolerance_ratio: 0.02