feat: SelVA Skip Experiment node + save partial scalars on skip

- New node: SelVA Skip Experiment — writes skip_current.flag from UI,
  queue in a second workflow tab while scheduler is running
- SkipExperiment now attaches partial loss/grad/spectral data to the
  exception so the scheduler saves all collected scalars in the summary

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-08 13:10:43 +02:00
parent 264dc49d42
commit 58e1985af2
4 changed files with 69 additions and 4 deletions
+10 -3
View File
@@ -478,10 +478,17 @@ class SelvaLoraScheduler:
except SkipExperiment as e:
duration = time.monotonic() - t_start
print(f"[LoRA Scheduler] Experiment '{exp_id}' skipped: {e}", flush=True)
partial = getattr(e, "partial", {})
lh = partial.get("loss_history", [])
smoothed = _smooth_losses(lh) if lh else []
exp_record["results"] = {
"status": "skipped",
"error": str(e),
"duration_seconds": round(duration, 1),
"status": "skipped",
"stopped_at_step": partial.get("stopped_at_step"),
"final_loss": round(smoothed[-1], 6) if smoothed else None,
"loss_history": [round(v, 6) for v in lh],
"grad_norm_history": partial.get("grad_norm_history", []),
"spectral_metrics": {str(k): v for k, v in partial.get("spectral_metrics", {}).items()},
"duration_seconds": round(duration, 1),
}
_write_summary()
pbar_outer.update(1)