From 57b459b956ada2c9bc178cc2576e1d5f6a50cc65 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 4 Jul 2026 22:31:45 +0200 Subject: [PATCH] Add SxCP Audio Wave + Segments: interactive waveform node (upload/play/click-segments) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New node with a JS widget (web/audio_wave.js): upload an audio clip, play it, and click the waveform to place segment boundaries (click add / drag move / dblclick note / shift|right-click delete). Boundaries+notes serialize to a hidden segments_json that drives Python segmentation (falls back to auto-split / notes syntax). Python node (nodes/audio_wave_segments.py) loads the file (torchaudio/soundfile/librosa), builds segments from the boundaries, and outputs waveform_image + audio_summary + AUDIO — same contract as Audio Prompt Guide, so it feeds chat mode the same way. _attach_notes now merges (keeps clicked notes). WEB_DIRECTORY re-enabled. JS is a first cut — needs testing in ComfyUI (console logs on error); the Python node works standalone via segments_json/notes. Co-Authored-By: Claude Opus 4.8 --- README.md | 10 +++ __init__.py | 11 ++- nodes/audio_guide.py | 3 +- nodes/audio_wave_segments.py | 157 ++++++++++++++++++++++++++++++++++ requirements.txt | 3 + web/audio_wave.js | 158 +++++++++++++++++++++++++++++++++++ 6 files changed, 338 insertions(+), 4 deletions(-) create mode 100644 nodes/audio_wave_segments.py create mode 100644 web/audio_wave.js diff --git a/README.md b/README.md index d818443..9d9aff6 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,16 @@ seg1: slow dreamy intro # by segment number (also "1:" or "S1:") cinematic, moody grade # no prefix = global (applies throughout) ``` +### Interactive: `SxCP Audio Wave + Segments` + +Same outputs, but with an in-node waveform widget: **upload** an audio clip, **play** it, +and **click the waveform** to place segment boundaries — click to add a split, drag to +move, double-click a segment for its note, shift/right-click to delete. The boundaries + +notes are saved to a hidden `segments_json` and drive the segmentation (falls back to +auto-split / the `notes` syntax if none placed). Outputs `waveform_image`, `audio_summary`, +and `audio`. Needs `torchaudio`/`soundfile`/`librosa` to load the file (torchaudio usually +ships with torch). + ## Performance / speed This node runs models through **transformers `.generate()`** — the simplest path, but the diff --git a/__init__.py b/__init__.py index 5ae8e40..e09c11e 100644 --- a/__init__.py +++ b/__init__.py @@ -12,8 +12,13 @@ from .nodes.audio_guide import ( NODE_CLASS_MAPPINGS as _AUDIO_CLASSES, NODE_DISPLAY_NAME_MAPPINGS as _AUDIO_NAMES, ) +from .nodes.audio_wave_segments import ( + NODE_CLASS_MAPPINGS as _WAVE_CLASSES, + NODE_DISPLAY_NAME_MAPPINGS as _WAVE_NAMES, +) -NODE_CLASS_MAPPINGS = {**_JUDGE_CLASSES, **_RECEPTOR_CLASSES, **_AUDIO_CLASSES} -NODE_DISPLAY_NAME_MAPPINGS = {**_JUDGE_NAMES, **_RECEPTOR_NAMES, **_AUDIO_NAMES} +NODE_CLASS_MAPPINGS = {**_JUDGE_CLASSES, **_RECEPTOR_CLASSES, **_AUDIO_CLASSES, **_WAVE_CLASSES} +NODE_DISPLAY_NAME_MAPPINGS = {**_JUDGE_NAMES, **_RECEPTOR_NAMES, **_AUDIO_NAMES, **_WAVE_NAMES} -__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] +WEB_DIRECTORY = "./web" # serves web/audio_wave.js (the waveform + segment widget) +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] diff --git a/nodes/audio_guide.py b/nodes/audio_guide.py index 2f3a1d7..0a9dbf0 100644 --- a/nodes/audio_guide.py +++ b/nodes/audio_guide.py @@ -56,7 +56,8 @@ def _attach_notes(segs, notes): seg_map, time_notes, glob = _split_notes(notes) for s in segs: s0, s1 = s["start_s"], s["start_s"] + s["duration_s"] - found = list(seg_map.get(s["segment"], [])) + existing = (s.get("note") or "").strip() + found = ([existing] if existing else []) + list(seg_map.get(s["segment"], [])) found += [txt for (t0, t1, txt) in time_notes if t0 < s1 and t1 > s0] s["note"] = "; ".join(found) return glob diff --git a/nodes/audio_wave_segments.py b/nodes/audio_wave_segments.py new file mode 100644 index 0000000..b150283 --- /dev/null +++ b/nodes/audio_wave_segments.py @@ -0,0 +1,157 @@ +""" +Audio Wave Segments node for ComfyUI. + +An audio node with its own upload that (via the JS widget in web/audio_wave.js) +displays the waveform, plays the clip, and lets you click segment boundaries on +the waveform and give each a note. It outputs the same waveform IMAGE + timing +`audio_summary` as SxCP Audio Prompt Guide (feed those to the judge in chat mode), +plus the AUDIO for downstream use. + +Works without the JS too: leave `segments_json` as "[]" and it auto-splits, or type +per-segment notes in `notes` using the seg1:/10s:/global syntax. +""" + +from __future__ import annotations + +import json +import os + +import numpy as np +import torch + +from .audio_guide import ( + _rms_envelope, _tempo_beats, _snap8, _segments, _render, _summary, _attach_notes, +) + + +def _load_audio_file(path): + """Load an audio file -> (waveform [C, N] float32 tensor, sample_rate). + Tries torchaudio, then soundfile, then librosa.""" + try: + import torchaudio + wav, sr = torchaudio.load(path) + return wav.to(torch.float32), int(sr) + except Exception: + pass + try: + import soundfile as sf + data, sr = sf.read(path, dtype="float32", always_2d=True) # [N, C] + return torch.from_numpy(data.T.copy()), int(sr) + except Exception: + pass + import librosa + y, sr = librosa.load(path, sr=None, mono=False) + y = np.atleast_2d(y) + return torch.from_numpy(np.ascontiguousarray(y, dtype=np.float32)), int(sr) + + +def _segments_from_boundaries(rms_n, times, duration, fps, starts, beats): + """Build segments from user-placed boundary start times (JS click points).""" + starts = sorted({0.0} | {round(float(s), 3) for s in starts if 0.0 < float(s) < duration}) + stages = ["establish", "build", "peak", "settle"] + bounds = np.array(starts + [duration]) + segs = [] + for i, t0 in enumerate(starts): + t1 = starts[i + 1] if i + 1 < len(starts) else duration + mask = (times >= t0) & (times < t1) + e = float(rms_n[mask].mean()) if mask.any() else 0.0 + peak_t = float(times[mask][np.argmax(rms_n[mask])]) if mask.any() else t0 + dur = round(t1 - t0, 2) + label = "high" if e > 0.66 else ("medium" if e > 0.33 else "low") + stage = stages[i] if i < len(stages) else ("peak" if e > 0.6 else "settle") + segs.append({ + "segment": i + 1, "start_s": round(t0, 2), "duration_s": dur, + "frames": _snap8(round(dur * fps)), "energy": label, + "energy_val": round(e, 3), "peak_s": round(peak_t, 2), + "stage_hint": stage, "note": "", + "beats_in": [round(b, 2) for b in beats if t0 <= b < t1], + }) + return segs, bounds + + +def _audio_files(): + try: + import folder_paths + d = folder_paths.get_input_directory() + try: + return sorted(folder_paths.filter_files_content_types(os.listdir(d), ["audio", "video"])) + except Exception: + exts = (".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aac") + return sorted(f for f in os.listdir(d) if f.lower().endswith(exts)) + except Exception: + return [] + + +class AudioWaveSegments: + CATEGORY = "prompt_calibrator" + FUNCTION = "run" + RETURN_TYPES = ("IMAGE", "STRING", "AUDIO") + RETURN_NAMES = ("waveform_image", "audio_summary", "audio") + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "audio": (_audio_files(), {"audio_upload": True}), + "fps": ("INT", {"default": 24, "min": 1, "max": 120}), + "max_segments": ("INT", {"default": 6, "min": 3, "max": 12}), + "notes": ("STRING", {"default": "", "multiline": True}), + # Written by the JS waveform widget: [{"start_s": 0.0, "note": "..."}, ...]. + # Leave as "[]" to auto-split. Also editable by hand. + "segments_json": ("STRING", {"default": "[]"}), + }, + } + + @classmethod + def IS_CHANGED(cls, audio, fps, max_segments, notes, segments_json): + try: + import folder_paths + p = folder_paths.get_annotated_filepath(audio) + mt = os.path.getmtime(p) if os.path.isfile(p) else "" + except Exception: + mt = "" + return f"{audio}|{fps}|{max_segments}|{notes}|{segments_json}|{mt}" + + def run(self, audio, fps, max_segments, notes, segments_json): + try: + import folder_paths + path = folder_paths.get_annotated_filepath(audio) + except Exception: + path = audio + if not path or not os.path.isfile(path): + blank = torch.zeros((1, 64, 512, 3)) + return (blank, f"[AudioWaveSegments] audio not found: {audio}", None) + + wav, sr = _load_audio_file(path) # wav: [C, N] + y = wav.mean(dim=0).cpu().numpy().astype(np.float32) + duration = len(y) / sr if sr else 0.0 + rms_n, times = _rms_envelope(y, sr) + bpm, beats = _tempo_beats(y, sr) + + # User boundaries from the JS widget, else auto-split. + starts, seg_notes = [], {} + try: + data = json.loads(segments_json) if segments_json.strip() else [] + for i, seg in enumerate(data): + starts.append(float(seg.get("start_s", 0.0))) + seg_notes[i + 1] = str(seg.get("note", "") or "") + except Exception as e: + print(f"[AudioWaveSegments] bad segments_json ({e}); auto-splitting.") + data = [] + + if data: + segs, bounds = _segments_from_boundaries(rms_n, times, duration, fps, starts, beats) + for s in segs: # notes placed on the waveform + s["note"] = seg_notes.get(s["segment"], "") + else: + segs, bounds = _segments(rms_n, times, duration, fps, max_segments, beats) + + global_notes = _attach_notes(segs, notes) # merge the text-box notes on top + image = _render(rms_n, times, duration, beats, bounds, segs) + summary = _summary(duration, sr, bpm, beats, segs, global_notes) + audio_out = {"waveform": wav.unsqueeze(0), "sample_rate": sr} # [1, C, N] + return (image, summary, audio_out) + + +NODE_CLASS_MAPPINGS = {"AudioWaveSegments": AudioWaveSegments} +NODE_DISPLAY_NAME_MAPPINGS = {"AudioWaveSegments": "SxCP Audio Wave + Segments"} diff --git a/requirements.txt b/requirements.txt index e2f2fe7..179a15d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,5 +11,8 @@ bitsandbytes # optional, for the Audio Prompt Guide node — adds tempo (BPM) + beat times # (energy envelope + segments work without it): # librosa +# for the Audio Wave + Segments node to LOAD an uploaded file (one of these; torchaudio +# usually ships with torch): torchaudio / soundfile / librosa +# soundfile # optional, for faster attention on the RTX 5090: # flash-attn diff --git a/web/audio_wave.js b/web/audio_wave.js new file mode 100644 index 0000000..afadee4 --- /dev/null +++ b/web/audio_wave.js @@ -0,0 +1,158 @@ +// SxCP Audio Wave + Segments — waveform display + playback + click-to-segment. +// Writes the segment boundaries/notes into the node's hidden `segments_json` widget. +// First cut: open the browser console for [audiowave] logs if something misbehaves. +import { app } from "../../scripts/app.js"; +import { api } from "../../scripts/api.js"; + +function getW(node, name) { return node.widgets?.find((w) => w.name === name); } + +function computePeaks(buf, n) { + const data = buf.getChannelData(0); + const block = Math.max(1, Math.floor(data.length / n)); + const peaks = new Float32Array(n); + let max = 1e-6; + for (let i = 0; i < n; i++) { + let m = 0; + const start = i * block; + for (let j = 0; j < block && start + j < data.length; j++) { + const v = Math.abs(data[start + j]); + if (v > m) m = v; + } + peaks[i] = m; + if (m > max) max = m; + } + for (let i = 0; i < n; i++) peaks[i] /= max; + return peaks; +} + +function setupWave(node) { + const st = { duration: 0, peaks: null, boundaries: [], notes: {}, audio: new Audio(), playing: false }; + node._wave = st; + + const wrap = document.createElement("div"); + wrap.style.cssText = "display:flex;flex-direction:column;gap:4px;width:100%;"; + const bar = document.createElement("div"); + bar.style.cssText = "display:flex;gap:6px;align-items:center;font-size:10px;color:#bbb;flex-wrap:wrap;"; + const mk = (t) => { const b = document.createElement("button"); b.textContent = t; b.style.cssText = "font-size:10px;padding:1px 6px;"; return b; }; + const playBtn = mk("▶ play"), clearBtn = mk("clear"), upBtn = mk("upload"); + const hint = document.createElement("span"); + hint.textContent = "click=add split · drag=move · dblclick=note · shift/right-click=delete"; + bar.append(playBtn, clearBtn, upBtn, hint); + const canvas = document.createElement("canvas"); + canvas.width = 640; canvas.height = 150; + canvas.style.cssText = "width:100%;height:150px;background:#141418;border-radius:4px;cursor:crosshair;"; + wrap.append(bar, canvas); + node.addDOMWidget("wave", "wave", wrap, { serialize: false }); + const ctx = canvas.getContext("2d"); + + const t2x = (t) => (st.duration ? (t / st.duration) * canvas.width : 0); + const x2t = (x) => (st.duration ? (x / canvas.width) * st.duration : 0); + const starts = () => [0, ...st.boundaries.slice().sort((a, b) => a - b)]; + + function serialize() { + const segs = starts().map((s, i) => ({ start_s: Math.round(s * 100) / 100, note: st.notes[i] || "" })); + const w = getW(node, "segments_json"); + if (w) { w.value = JSON.stringify(segs); w.callback?.(w.value); } + } + + function draw() { + ctx.fillStyle = "#141418"; ctx.fillRect(0, 0, canvas.width, canvas.height); + if (st.peaks) { + ctx.fillStyle = "#3c8cdc"; + const n = st.peaks.length, bw = canvas.width / n; + for (let i = 0; i < n; i++) { const h = st.peaks[i] * (canvas.height - 26); ctx.fillRect(i * bw, canvas.height - h, Math.max(1, bw), h); } + } + ctx.font = "10px monospace"; + starts().forEach((s, i) => { + const x = t2x(s); + ctx.strokeStyle = "#eee"; ctx.beginPath(); ctx.moveTo(x, 0); ctx.lineTo(x, canvas.height); ctx.stroke(); + ctx.fillStyle = "#fff"; ctx.fillText("S" + (i + 1), x + 3, 11); + if (st.notes[i]) { ctx.fillStyle = "#ffd27a"; ctx.fillText(st.notes[i].slice(0, 24), x + 3, 23); } + }); + if (st.playing) { const x = t2x(st.audio.currentTime); ctx.strokeStyle = "#ff5a3c"; ctx.beginPath(); ctx.moveTo(x, 0); ctx.lineTo(x, canvas.height); ctx.stroke(); } + } + + async function loadFile(name) { + if (!name) return; + const url = `/view?filename=${encodeURIComponent(name)}&type=input&subfolder=`; + try { + const ab = await (await fetch(url)).arrayBuffer(); + const ac = new (window.AudioContext || window.webkitAudioContext)(); + const buf = await ac.decodeAudioData(ab.slice(0)); + st.duration = buf.duration; st.peaks = computePeaks(buf, canvas.width); + st.audio.src = url; draw(); + } catch (e) { console.error("[audiowave] could not load/decode", name, e); } + } + + const tol = () => x2t(6); + const nearBoundary = (t) => { let bi = -1, bd = 1e9; st.boundaries.forEach((b, i) => { const d = Math.abs(b - t); if (d < tol() && d < bd) { bd = d; bi = i; } }); return bi; }; + let dragIdx = -1; + + canvas.addEventListener("mousedown", (e) => { + const r = canvas.getBoundingClientRect(); + const t = x2t(((e.clientX - r.left) / r.width) * canvas.width); + if (e.button === 2 || e.shiftKey) { const bi = nearBoundary(t); if (bi >= 0) { st.boundaries.splice(bi, 1); serialize(); draw(); } e.preventDefault(); return; } + const bi = nearBoundary(t); + if (bi >= 0) { dragIdx = bi; } + else { st.boundaries.push(t); serialize(); draw(); } + }); + canvas.addEventListener("mousemove", (e) => { + if (dragIdx < 0) return; + const r = canvas.getBoundingClientRect(); + st.boundaries[dragIdx] = Math.max(0.01, Math.min(st.duration - 0.01, x2t(((e.clientX - r.left) / r.width) * canvas.width))); + draw(); + }); + window.addEventListener("mouseup", () => { if (dragIdx >= 0) { serialize(); dragIdx = -1; draw(); } }); + canvas.addEventListener("dblclick", (e) => { + const r = canvas.getBoundingClientRect(); + const t = x2t(((e.clientX - r.left) / r.width) * canvas.width); + const ss = starts(); let si = 0; for (let i = 0; i < ss.length; i++) if (t >= ss[i]) si = i; + const val = window.prompt(`Note for segment ${si + 1}:`, st.notes[si] || ""); + if (val !== null) { st.notes[si] = val; serialize(); draw(); } + }); + canvas.addEventListener("contextmenu", (e) => e.preventDefault()); + + const loop = () => { if (!st.playing) return; draw(); requestAnimationFrame(loop); }; + playBtn.onclick = () => { + if (st.audio.paused) { st.audio.play().catch((e) => console.error("[audiowave] play", e)); st.playing = true; playBtn.textContent = "⏸ pause"; loop(); } + else { st.audio.pause(); st.playing = false; playBtn.textContent = "▶ play"; } + }; + st.audio.onended = () => { st.playing = false; playBtn.textContent = "▶ play"; draw(); }; + clearBtn.onclick = () => { st.boundaries = []; st.notes = {}; serialize(); draw(); }; + + upBtn.onclick = () => { + const inp = document.createElement("input"); inp.type = "file"; inp.accept = "audio/*"; + inp.onchange = async () => { + const f = inp.files[0]; if (!f) return; + const fd = new FormData(); fd.append("image", f, f.name); fd.append("type", "input"); + try { + const res = await api.fetchApi("/upload/image", { method: "POST", body: fd }); + const j = await res.json(); const name = j.name || f.name; + const w = getW(node, "audio"); + if (w) { if (!w.options.values.includes(name)) w.options.values.push(name); w.value = name; } + await loadFile(name); + } catch (e) { console.error("[audiowave] upload failed (drop the file in ComfyUI/input instead)", e); } + }; + inp.click(); + }; + + // react to the audio combo changing + initial load + const aw = getW(node, "audio"); + if (aw) { const cb = aw.callback; aw.callback = function () { const r = cb ? cb.apply(this, arguments) : undefined; loadFile(aw.value); return r; }; if (aw.value) loadFile(aw.value); } + // restore boundaries/notes from a reloaded workflow + try { const sj = getW(node, "segments_json"); if (sj && sj.value) { const arr = JSON.parse(sj.value); st.boundaries = arr.slice(1).map((s) => s.start_s); arr.forEach((s, i) => { if (s.note) st.notes[i] = s.note; }); } } catch (e) { /* ignore */ } + draw(); +} + +app.registerExtension({ + name: "sxcp.audiowave", + async beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData?.name !== "AudioWaveSegments") return; + const orig = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function () { + const r = orig?.apply(this, arguments); + try { setupWave(this); } catch (e) { console.error("[audiowave] setup failed", e); } + return r; + }; + }, +});