Ethanfel
f9d092158a
n4_baseline showed token_norm growing linearly without plateau — classic sign of lr too high relative to parameter count. With only K×1024 params, gradient signal per param is already high-magnitude; high lr causes overshoot rather than convergence. - Default lr: 1e-3 → 2e-4 (matches LoRA working regime) - Default batch_size: 16 → 4 (more diverse gradients, helps norm saturate) - ti_sweep_1.json: add lr_batch group (lr_low_b4, lr_mid_b8, lr_low_b4_prefix, lr_2e3), restructure with clearer groups, annotate n4_baseline as completed with findings Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
85 lines
2.8 KiB
JSON
85 lines
2.8 KiB
JSON
{
|
|
"name": "ti_sweep_1",
|
|
"description": "First TI sweep. n4_baseline (suffix, batch=16, lr=1e-3) completed — loss 1.025→0.963, plateau after step 1500, token_norm grew linearly without saturation (overshoot sign). Now testing: prefix injection, lower LR, smaller batch.",
|
|
"data_dir": "/media/unraid/davinci/Selva/BJ/features",
|
|
"output_root": "/media/unraid/davinci/Selva/BJ/experiment/ti_sweep_1",
|
|
"base": {
|
|
"steps": 3000,
|
|
"batch_size": 16,
|
|
"warmup_steps": 100,
|
|
"save_every": 1000,
|
|
"seed": 42,
|
|
"init_text": "",
|
|
"lr": 1e-3,
|
|
"n_tokens": 4,
|
|
"inject_mode": "suffix"
|
|
},
|
|
"experiments": [
|
|
|
|
{
|
|
"id": "n4_baseline",
|
|
"group": "reference",
|
|
"description": "COMPLETED. batch=16, lr=1e-3, suffix. Reference. Loss plateau ~0.963, token_norm linear growth to 3.2 — LR too high for the parameter count."
|
|
},
|
|
|
|
{
|
|
"id": "n4_prefix",
|
|
"group": "prefix_inject",
|
|
"description": "Same as baseline but prefix injection. Tests whether suffix positions are limiting signal — if prefix loss goes lower or converges faster, suffix was the bottleneck.",
|
|
"inject_mode": "prefix"
|
|
},
|
|
|
|
{
|
|
"id": "lr_low_b4",
|
|
"group": "lr_batch",
|
|
"description": "lr=2e-4, batch=4. Matches LoRA's working regime. Smaller batch = noisier but more diverse gradients; lower LR = smaller steps, token_norm should plateau rather than drift.",
|
|
"lr": 2e-4,
|
|
"batch_size": 4
|
|
},
|
|
{
|
|
"id": "lr_mid_b8",
|
|
"group": "lr_batch",
|
|
"description": "lr=5e-4, batch=8. Middle ground — half the baseline LR and batch. Token norm should grow slower and saturate.",
|
|
"lr": 5e-4,
|
|
"batch_size": 8
|
|
},
|
|
{
|
|
"id": "lr_low_b4_prefix",
|
|
"group": "lr_batch",
|
|
"description": "lr=2e-4, batch=4, prefix. Best LR/batch regime + best injection position combined.",
|
|
"lr": 2e-4,
|
|
"batch_size": 4,
|
|
"inject_mode": "prefix"
|
|
},
|
|
|
|
{
|
|
"id": "n8_prefix",
|
|
"group": "prefix_inject",
|
|
"description": "8 tokens, prefix, baseline LR/batch. More capacity at the better injection position.",
|
|
"n_tokens": 8,
|
|
"inject_mode": "prefix"
|
|
},
|
|
{
|
|
"id": "n4_prefix_warm",
|
|
"group": "prefix_inject",
|
|
"description": "4 tokens, prefix, warm-started from 'mechanical impact sound design'.",
|
|
"inject_mode": "prefix",
|
|
"init_text": "mechanical impact sound design"
|
|
},
|
|
|
|
{
|
|
"id": "n8",
|
|
"group": "suffix_token_count",
|
|
"description": "8 tokens, suffix, baseline LR/batch. Capacity ablation vs n4_baseline.",
|
|
"n_tokens": 8
|
|
},
|
|
{
|
|
"id": "lr_2e3",
|
|
"group": "lr_batch",
|
|
"description": "lr=2e-3, baseline batch. Expected to plateau earlier and higher than baseline — confirms LR is the issue.",
|
|
"lr": 2e-3
|
|
}
|
|
|
|
]
|
|
}
|