Ethanfel
b89167cfae
Diagnosis: learned tokens grew to norm ~3.2 while real CLIP content tokens sit at ~1.0. Model never trained on embeddings that large — activates buzz artifact instead of semantic style shift. Fix: measure mean token norm from content positions (1–20) of dataset CLIP embeddings at startup, clamp learned_tokens per-token after every optimizer step to max 1.5× that reference (50% headroom). Token norm is now logged as current/limit for easy monitoring. ti_sweep_1.json: rebuild around norm_clamp group — n4_clamped (primary diagnostic), prefix_clamped, n8_prefix_clamped, warm_clamped. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
53 lines
1.8 KiB
JSON
53 lines
1.8 KiB
JSON
{
|
|
"name": "ti_sweep_1",
|
|
"description": "First TI sweep. n4_baseline (suffix, batch=16, lr=1e-3) completed — buzz artifact diagnosed as token norm drifting to 3.2x outside CLIP manifold. All new experiments use norm clamping (auto from dataset) + corrected lr/batch.",
|
|
"data_dir": "/media/unraid/davinci/Selva/BJ/features",
|
|
"output_root": "/media/unraid/davinci/Selva/BJ/experiment/ti_sweep_1",
|
|
"base": {
|
|
"steps": 3000,
|
|
"batch_size": 4,
|
|
"warmup_steps": 100,
|
|
"save_every": 1000,
|
|
"seed": 42,
|
|
"init_text": "",
|
|
"lr": 2e-4,
|
|
"n_tokens": 4,
|
|
"inject_mode": "suffix"
|
|
},
|
|
"experiments": [
|
|
|
|
{
|
|
"id": "n4_baseline",
|
|
"group": "reference",
|
|
"description": "COMPLETED (old code, no norm clamp). batch=16, lr=1e-3. Token norm drifted to 3.2 → buzz artifact. Kept for loss curve comparison only."
|
|
},
|
|
|
|
{
|
|
"id": "n4_clamped",
|
|
"group": "norm_clamp",
|
|
"description": "Same as baseline but with norm clamping enabled. Primary diagnostic: does clamping alone fix the buzz? lr=2e-4, batch=4, suffix."
|
|
},
|
|
{
|
|
"id": "n4_prefix_clamped",
|
|
"group": "norm_clamp",
|
|
"description": "Prefix injection + norm clamping. Best of both: high-attention positions, tokens stay on CLIP manifold.",
|
|
"inject_mode": "prefix"
|
|
},
|
|
{
|
|
"id": "n8_prefix_clamped",
|
|
"group": "norm_clamp",
|
|
"description": "8 tokens, prefix, clamped. More capacity without the artifact.",
|
|
"n_tokens": 8,
|
|
"inject_mode": "prefix"
|
|
},
|
|
{
|
|
"id": "n4_prefix_warm_clamped",
|
|
"group": "norm_clamp",
|
|
"description": "4 tokens, prefix, warm init from 'mechanical impact sound design', clamped. Should converge fastest — starts in-manifold, stays in-manifold.",
|
|
"inject_mode": "prefix",
|
|
"init_text": "mechanical impact sound design"
|
|
}
|
|
|
|
]
|
|
}
|