Add SxCP Audio Wave + Segments: interactive waveform node (upload/play/click-segments)

New node with a JS widget (web/audio_wave.js): upload an audio clip, play it, and click
the waveform to place segment boundaries (click add / drag move / dblclick note /
shift|right-click delete). Boundaries+notes serialize to a hidden segments_json that
drives Python segmentation (falls back to auto-split / notes syntax). Python node
(nodes/audio_wave_segments.py) loads the file (torchaudio/soundfile/librosa), builds
segments from the boundaries, and outputs waveform_image + audio_summary + AUDIO — same
contract as Audio Prompt Guide, so it feeds chat mode the same way. _attach_notes now
merges (keeps clicked notes). WEB_DIRECTORY re-enabled. JS is a first cut — needs testing
in ComfyUI (console logs on error); the Python node works standalone via segments_json/notes.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

README.md

@@ -101,6 +101,16 @@ seg1: slow dreamy intro          # by segment number (also "1:" or "S1:")
 cinematic, moody grade           # no prefix = global (applies throughout)
 ```
 
+### Interactive: `SxCP Audio Wave + Segments`
+
+Same outputs, but with an in-node waveform widget: **upload** an audio clip, **play** it,
+and **click the waveform** to place segment boundaries — click to add a split, drag to
+move, double-click a segment for its note, shift/right-click to delete. The boundaries +
+notes are saved to a hidden `segments_json` and drive the segmentation (falls back to
+auto-split / the `notes` syntax if none placed). Outputs `waveform_image`, `audio_summary`,
+and `audio`. Needs `torchaudio`/`soundfile`/`librosa` to load the file (torchaudio usually
+ships with torch).
+
 ## Performance / speed
 
 This node runs models through **transformers `.generate()`** — the simplest path, but the

__init__.py

@@ -12,8 +12,13 @@ from .nodes.audio_guide import (
     NODE_CLASS_MAPPINGS as _AUDIO_CLASSES,
     NODE_DISPLAY_NAME_MAPPINGS as _AUDIO_NAMES,
 )
+from .nodes.audio_wave_segments import (
+    NODE_CLASS_MAPPINGS as _WAVE_CLASSES,
+    NODE_DISPLAY_NAME_MAPPINGS as _WAVE_NAMES,
+)
 
-NODE_CLASS_MAPPINGS = {**_JUDGE_CLASSES, **_RECEPTOR_CLASSES, **_AUDIO_CLASSES}
+NODE_CLASS_MAPPINGS = {**_JUDGE_CLASSES, **_RECEPTOR_CLASSES, **_AUDIO_CLASSES, **_WAVE_CLASSES}
-NODE_DISPLAY_NAME_MAPPINGS = {**_JUDGE_NAMES, **_RECEPTOR_NAMES, **_AUDIO_NAMES}
+NODE_DISPLAY_NAME_MAPPINGS = {**_JUDGE_NAMES, **_RECEPTOR_NAMES, **_AUDIO_NAMES, **_WAVE_NAMES}
 
-__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
+WEB_DIRECTORY = "./web"  # serves web/audio_wave.js (the waveform + segment widget)
+__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]

nodes/audio_guide.py

@@ -56,7 +56,8 @@ def _attach_notes(segs, notes):
     seg_map, time_notes, glob = _split_notes(notes)
     for s in segs:
         s0, s1 = s["start_s"], s["start_s"] + s["duration_s"]
-        found = list(seg_map.get(s["segment"], []))
+        existing = (s.get("note") or "").strip()
+        found = ([existing] if existing else []) + list(seg_map.get(s["segment"], []))
         found += [txt for (t0, t1, txt) in time_notes if t0 < s1 and t1 > s0]
         s["note"] = "; ".join(found)
     return glob

nodes/audio_wave_segments.py

@@ -0,0 +1,157 @@
+"""
+Audio Wave Segments node for ComfyUI.
+
+An audio node with its own upload that (via the JS widget in web/audio_wave.js)
+displays the waveform, plays the clip, and lets you click segment boundaries on
+the waveform and give each a note. It outputs the same waveform IMAGE + timing
+`audio_summary` as SxCP Audio Prompt Guide (feed those to the judge in chat mode),
+plus the AUDIO for downstream use.
+
+Works without the JS too: leave `segments_json` as "[]" and it auto-splits, or type
+per-segment notes in `notes` using the seg1:/10s:/global syntax.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+import numpy as np
+import torch
+
+from .audio_guide import (
+    _rms_envelope, _tempo_beats, _snap8, _segments, _render, _summary, _attach_notes,
+)
+
+
+def _load_audio_file(path):
+    """Load an audio file -> (waveform [C, N] float32 tensor, sample_rate).
+    Tries torchaudio, then soundfile, then librosa."""
+    try:
+        import torchaudio
+        wav, sr = torchaudio.load(path)
+        return wav.to(torch.float32), int(sr)
+    except Exception:
+        pass
+    try:
+        import soundfile as sf
+        data, sr = sf.read(path, dtype="float32", always_2d=True)  # [N, C]
+        return torch.from_numpy(data.T.copy()), int(sr)
+    except Exception:
+        pass
+    import librosa
+    y, sr = librosa.load(path, sr=None, mono=False)
+    y = np.atleast_2d(y)
+    return torch.from_numpy(np.ascontiguousarray(y, dtype=np.float32)), int(sr)
+
+
+def _segments_from_boundaries(rms_n, times, duration, fps, starts, beats):
+    """Build segments from user-placed boundary start times (JS click points)."""
+    starts = sorted({0.0} | {round(float(s), 3) for s in starts if 0.0 < float(s) < duration})
+    stages = ["establish", "build", "peak", "settle"]
+    bounds = np.array(starts + [duration])
+    segs = []
+    for i, t0 in enumerate(starts):
+        t1 = starts[i + 1] if i + 1 < len(starts) else duration
+        mask = (times >= t0) & (times < t1)
+        e = float(rms_n[mask].mean()) if mask.any() else 0.0
+        peak_t = float(times[mask][np.argmax(rms_n[mask])]) if mask.any() else t0
+        dur = round(t1 - t0, 2)
+        label = "high" if e > 0.66 else ("medium" if e > 0.33 else "low")
+        stage = stages[i] if i < len(stages) else ("peak" if e > 0.6 else "settle")
+        segs.append({
+            "segment": i + 1, "start_s": round(t0, 2), "duration_s": dur,
+            "frames": _snap8(round(dur * fps)), "energy": label,
+            "energy_val": round(e, 3), "peak_s": round(peak_t, 2),
+            "stage_hint": stage, "note": "",
+            "beats_in": [round(b, 2) for b in beats if t0 <= b < t1],
+        })
+    return segs, bounds
+
+
+def _audio_files():
+    try:
+        import folder_paths
+        d = folder_paths.get_input_directory()
+        try:
+            return sorted(folder_paths.filter_files_content_types(os.listdir(d), ["audio", "video"]))
+        except Exception:
+            exts = (".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aac")
+            return sorted(f for f in os.listdir(d) if f.lower().endswith(exts))
+    except Exception:
+        return []
+
+
+class AudioWaveSegments:
+    CATEGORY = "prompt_calibrator"
+    FUNCTION = "run"
+    RETURN_TYPES = ("IMAGE", "STRING", "AUDIO")
+    RETURN_NAMES = ("waveform_image", "audio_summary", "audio")
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "audio": (_audio_files(), {"audio_upload": True}),
+                "fps": ("INT", {"default": 24, "min": 1, "max": 120}),
+                "max_segments": ("INT", {"default": 6, "min": 3, "max": 12}),
+                "notes": ("STRING", {"default": "", "multiline": True}),
+                # Written by the JS waveform widget: [{"start_s": 0.0, "note": "..."}, ...].
+                # Leave as "[]" to auto-split. Also editable by hand.
+                "segments_json": ("STRING", {"default": "[]"}),
+            },
+        }
+
+    @classmethod
+    def IS_CHANGED(cls, audio, fps, max_segments, notes, segments_json):
+        try:
+            import folder_paths
+            p = folder_paths.get_annotated_filepath(audio)
+            mt = os.path.getmtime(p) if os.path.isfile(p) else ""
+        except Exception:
+            mt = ""
+        return f"{audio}|{fps}|{max_segments}|{notes}|{segments_json}|{mt}"
+
+    def run(self, audio, fps, max_segments, notes, segments_json):
+        try:
+            import folder_paths
+            path = folder_paths.get_annotated_filepath(audio)
+        except Exception:
+            path = audio
+        if not path or not os.path.isfile(path):
+            blank = torch.zeros((1, 64, 512, 3))
+            return (blank, f"[AudioWaveSegments] audio not found: {audio}", None)
+
+        wav, sr = _load_audio_file(path)              # wav: [C, N]
+        y = wav.mean(dim=0).cpu().numpy().astype(np.float32)
+        duration = len(y) / sr if sr else 0.0
+        rms_n, times = _rms_envelope(y, sr)
+        bpm, beats = _tempo_beats(y, sr)
+
+        # User boundaries from the JS widget, else auto-split.
+        starts, seg_notes = [], {}
+        try:
+            data = json.loads(segments_json) if segments_json.strip() else []
+            for i, seg in enumerate(data):
+                starts.append(float(seg.get("start_s", 0.0)))
+                seg_notes[i + 1] = str(seg.get("note", "") or "")
+        except Exception as e:
+            print(f"[AudioWaveSegments] bad segments_json ({e}); auto-splitting.")
+            data = []
+
+        if data:
+            segs, bounds = _segments_from_boundaries(rms_n, times, duration, fps, starts, beats)
+            for s in segs:                            # notes placed on the waveform
+                s["note"] = seg_notes.get(s["segment"], "")
+        else:
+            segs, bounds = _segments(rms_n, times, duration, fps, max_segments, beats)
+
+        global_notes = _attach_notes(segs, notes)     # merge the text-box notes on top
+        image = _render(rms_n, times, duration, beats, bounds, segs)
+        summary = _summary(duration, sr, bpm, beats, segs, global_notes)
+        audio_out = {"waveform": wav.unsqueeze(0), "sample_rate": sr}  # [1, C, N]
+        return (image, summary, audio_out)
+
+
+NODE_CLASS_MAPPINGS = {"AudioWaveSegments": AudioWaveSegments}
+NODE_DISPLAY_NAME_MAPPINGS = {"AudioWaveSegments": "SxCP Audio Wave + Segments"}

requirements.txt

@@ -11,5 +11,8 @@ bitsandbytes
 # optional, for the Audio Prompt Guide node — adds tempo (BPM) + beat times
 # (energy envelope + segments work without it):
 # librosa
+# for the Audio Wave + Segments node to LOAD an uploaded file (one of these; torchaudio
+# usually ships with torch): torchaudio / soundfile / librosa
+# soundfile
 # optional, for faster attention on the RTX 5090:
 # flash-attn

web/audio_wave.js

@@ -0,0 +1,158 @@
+// SxCP Audio Wave + Segments — waveform display + playback + click-to-segment.
+// Writes the segment boundaries/notes into the node's hidden `segments_json` widget.
+// First cut: open the browser console for [audiowave] logs if something misbehaves.
+import { app } from "../../scripts/app.js";
+import { api } from "../../scripts/api.js";
+
+function getW(node, name) { return node.widgets?.find((w) => w.name === name); }
+
+function computePeaks(buf, n) {
+  const data = buf.getChannelData(0);
+  const block = Math.max(1, Math.floor(data.length / n));
+  const peaks = new Float32Array(n);
+  let max = 1e-6;
+  for (let i = 0; i < n; i++) {
+    let m = 0;
+    const start = i * block;
+    for (let j = 0; j < block && start + j < data.length; j++) {
+      const v = Math.abs(data[start + j]);
+      if (v > m) m = v;
+    }
+    peaks[i] = m;
+    if (m > max) max = m;
+  }
+  for (let i = 0; i < n; i++) peaks[i] /= max;
+  return peaks;
+}
+
+function setupWave(node) {
+  const st = { duration: 0, peaks: null, boundaries: [], notes: {}, audio: new Audio(), playing: false };
+  node._wave = st;
+
+  const wrap = document.createElement("div");
+  wrap.style.cssText = "display:flex;flex-direction:column;gap:4px;width:100%;";
+  const bar = document.createElement("div");
+  bar.style.cssText = "display:flex;gap:6px;align-items:center;font-size:10px;color:#bbb;flex-wrap:wrap;";
+  const mk = (t) => { const b = document.createElement("button"); b.textContent = t; b.style.cssText = "font-size:10px;padding:1px 6px;"; return b; };
+  const playBtn = mk("▶ play"), clearBtn = mk("clear"), upBtn = mk("upload");
+  const hint = document.createElement("span");
+  hint.textContent = "click=add split · drag=move · dblclick=note · shift/right-click=delete";
+  bar.append(playBtn, clearBtn, upBtn, hint);
+  const canvas = document.createElement("canvas");
+  canvas.width = 640; canvas.height = 150;
+  canvas.style.cssText = "width:100%;height:150px;background:#141418;border-radius:4px;cursor:crosshair;";
+  wrap.append(bar, canvas);
+  node.addDOMWidget("wave", "wave", wrap, { serialize: false });
+  const ctx = canvas.getContext("2d");
+
+  const t2x = (t) => (st.duration ? (t / st.duration) * canvas.width : 0);
+  const x2t = (x) => (st.duration ? (x / canvas.width) * st.duration : 0);
+  const starts = () => [0, ...st.boundaries.slice().sort((a, b) => a - b)];
+
+  function serialize() {
+    const segs = starts().map((s, i) => ({ start_s: Math.round(s * 100) / 100, note: st.notes[i] || "" }));
+    const w = getW(node, "segments_json");
+    if (w) { w.value = JSON.stringify(segs); w.callback?.(w.value); }
+  }
+
+  function draw() {
+    ctx.fillStyle = "#141418"; ctx.fillRect(0, 0, canvas.width, canvas.height);
+    if (st.peaks) {
+      ctx.fillStyle = "#3c8cdc";
+      const n = st.peaks.length, bw = canvas.width / n;
+      for (let i = 0; i < n; i++) { const h = st.peaks[i] * (canvas.height - 26); ctx.fillRect(i * bw, canvas.height - h, Math.max(1, bw), h); }
+    }
+    ctx.font = "10px monospace";
+    starts().forEach((s, i) => {
+      const x = t2x(s);
+      ctx.strokeStyle = "#eee"; ctx.beginPath(); ctx.moveTo(x, 0); ctx.lineTo(x, canvas.height); ctx.stroke();
+      ctx.fillStyle = "#fff"; ctx.fillText("S" + (i + 1), x + 3, 11);
+      if (st.notes[i]) { ctx.fillStyle = "#ffd27a"; ctx.fillText(st.notes[i].slice(0, 24), x + 3, 23); }
+    });
+    if (st.playing) { const x = t2x(st.audio.currentTime); ctx.strokeStyle = "#ff5a3c"; ctx.beginPath(); ctx.moveTo(x, 0); ctx.lineTo(x, canvas.height); ctx.stroke(); }
+  }
+
+  async function loadFile(name) {
+    if (!name) return;
+    const url = `/view?filename=${encodeURIComponent(name)}&type=input&subfolder=`;
+    try {
+      const ab = await (await fetch(url)).arrayBuffer();
+      const ac = new (window.AudioContext || window.webkitAudioContext)();
+      const buf = await ac.decodeAudioData(ab.slice(0));
+      st.duration = buf.duration; st.peaks = computePeaks(buf, canvas.width);
+      st.audio.src = url; draw();
+    } catch (e) { console.error("[audiowave] could not load/decode", name, e); }
+  }
+
+  const tol = () => x2t(6);
+  const nearBoundary = (t) => { let bi = -1, bd = 1e9; st.boundaries.forEach((b, i) => { const d = Math.abs(b - t); if (d < tol() && d < bd) { bd = d; bi = i; } }); return bi; };
+  let dragIdx = -1;
+
+  canvas.addEventListener("mousedown", (e) => {
+    const r = canvas.getBoundingClientRect();
+    const t = x2t(((e.clientX - r.left) / r.width) * canvas.width);
+    if (e.button === 2 || e.shiftKey) { const bi = nearBoundary(t); if (bi >= 0) { st.boundaries.splice(bi, 1); serialize(); draw(); } e.preventDefault(); return; }
+    const bi = nearBoundary(t);
+    if (bi >= 0) { dragIdx = bi; }
+    else { st.boundaries.push(t); serialize(); draw(); }
+  });
+  canvas.addEventListener("mousemove", (e) => {
+    if (dragIdx < 0) return;
+    const r = canvas.getBoundingClientRect();
+    st.boundaries[dragIdx] = Math.max(0.01, Math.min(st.duration - 0.01, x2t(((e.clientX - r.left) / r.width) * canvas.width)));
+    draw();
+  });
+  window.addEventListener("mouseup", () => { if (dragIdx >= 0) { serialize(); dragIdx = -1; draw(); } });
+  canvas.addEventListener("dblclick", (e) => {
+    const r = canvas.getBoundingClientRect();
+    const t = x2t(((e.clientX - r.left) / r.width) * canvas.width);
+    const ss = starts(); let si = 0; for (let i = 0; i < ss.length; i++) if (t >= ss[i]) si = i;
+    const val = window.prompt(`Note for segment ${si + 1}:`, st.notes[si] || "");
+    if (val !== null) { st.notes[si] = val; serialize(); draw(); }
+  });
+  canvas.addEventListener("contextmenu", (e) => e.preventDefault());
+
+  const loop = () => { if (!st.playing) return; draw(); requestAnimationFrame(loop); };
+  playBtn.onclick = () => {
+    if (st.audio.paused) { st.audio.play().catch((e) => console.error("[audiowave] play", e)); st.playing = true; playBtn.textContent = "⏸ pause"; loop(); }
+    else { st.audio.pause(); st.playing = false; playBtn.textContent = "▶ play"; }
+  };
+  st.audio.onended = () => { st.playing = false; playBtn.textContent = "▶ play"; draw(); };
+  clearBtn.onclick = () => { st.boundaries = []; st.notes = {}; serialize(); draw(); };
+
+  upBtn.onclick = () => {
+    const inp = document.createElement("input"); inp.type = "file"; inp.accept = "audio/*";
+    inp.onchange = async () => {
+      const f = inp.files[0]; if (!f) return;
+      const fd = new FormData(); fd.append("image", f, f.name); fd.append("type", "input");
+      try {
+        const res = await api.fetchApi("/upload/image", { method: "POST", body: fd });
+        const j = await res.json(); const name = j.name || f.name;
+        const w = getW(node, "audio");
+        if (w) { if (!w.options.values.includes(name)) w.options.values.push(name); w.value = name; }
+        await loadFile(name);
+      } catch (e) { console.error("[audiowave] upload failed (drop the file in ComfyUI/input instead)", e); }
+    };
+    inp.click();
+  };
+
+  // react to the audio combo changing + initial load
+  const aw = getW(node, "audio");
+  if (aw) { const cb = aw.callback; aw.callback = function () { const r = cb ? cb.apply(this, arguments) : undefined; loadFile(aw.value); return r; }; if (aw.value) loadFile(aw.value); }
+  // restore boundaries/notes from a reloaded workflow
+  try { const sj = getW(node, "segments_json"); if (sj && sj.value) { const arr = JSON.parse(sj.value); st.boundaries = arr.slice(1).map((s) => s.start_s); arr.forEach((s, i) => { if (s.note) st.notes[i] = s.note; }); } } catch (e) { /* ignore */ }
+  draw();
+}
+
+app.registerExtension({
+  name: "sxcp.audiowave",
+  async beforeRegisterNodeDef(nodeType, nodeData) {
+    if (nodeData?.name !== "AudioWaveSegments") return;
+    const orig = nodeType.prototype.onNodeCreated;
+    nodeType.prototype.onNodeCreated = function () {
+      const r = orig?.apply(this, arguments);
+      try { setupWave(this); } catch (e) { console.error("[audiowave] setup failed", e); }
+      return r;
+    };
+  },
+});