Audio Wave: correct model — 721 groups are hard splits, segments inside; range select; mirrored render; fix reload explosion
Rebuilt around the real model: group_frames (721) are HARD splits; segments = user
splits UNION the group lines, so a segment never crosses a group boundary (last segment
in a group ends exactly on frame 721). segment_select is now a RANGE string ('A-B' /
'N' / '' = all) that crops waveform_image + audio + summary to segments A..B. Render
rewritten: mirrored waveform (uses top+bottom), bold group grid, thin segment lines,
labels along the top and notes along the bottom, selected range shaded. JS: guards
frame values (0 -> default, fixing the 400+-segments-on-reload explosion), writes only
USER splits to segments_json, dblclick=add split / shift-click=remove / click=seek.
group number fixed (boundary rounding). Workflows + README updated.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+29
-22
@@ -131,43 +131,50 @@ def _segments(rms_n, times, duration, fps, max_segments, beats):
|
||||
|
||||
|
||||
def _render(rms_n, times, duration, beats, bounds, segs,
|
||||
fps=None, frames_marker=0, window=None):
|
||||
"""Render the envelope + beats + segment boundaries + time labels to a ComfyUI IMAGE.
|
||||
window=(t0,t1) crops to that time span (for a selected segment). frames_marker draws
|
||||
a green line every N frames (N/fps seconds) — LTX clip-length grid."""
|
||||
fps=None, group_frames=0, frames_marker=0, window=None, sel=None):
|
||||
"""Render a mirrored waveform + the 721-frame group grid (bold) + segment lines (thin),
|
||||
labels along the top and per-segment notes along the bottom. window=(t0,t1) crops to a
|
||||
range; sel=(a,b) shades segments a..b."""
|
||||
W, H = 1024, 256
|
||||
img = Image.new("RGB", (W, H), (18, 18, 22))
|
||||
d = ImageDraw.Draw(img)
|
||||
t0, t1 = window if window else (0.0, duration)
|
||||
span = max(t1 - t0, 1e-6)
|
||||
mid = H // 2
|
||||
|
||||
def X(t):
|
||||
return int(max(0, min(W - 1, (t - t0) / span * (W - 1))))
|
||||
|
||||
pts = [(0, H)] + [(X(times[i]), H - int(rms_n[i] * (H - 30)))
|
||||
for i in range(len(rms_n)) if t0 <= times[i] <= t1] + [(W - 1, H)]
|
||||
if len(pts) > 2:
|
||||
d.polygon(pts, fill=(60, 140, 220))
|
||||
if fps and frames_marker: # frame grid (green)
|
||||
step = frames_marker / fps
|
||||
if sel and not window: # shade the selected segment range
|
||||
chosen = [s for s in segs if sel[0] <= s["segment"] <= sel[1]]
|
||||
if chosen:
|
||||
xa = X(chosen[0]["start_s"])
|
||||
xb = X(chosen[-1]["start_s"] + chosen[-1]["duration_s"])
|
||||
d.rectangle([xa, 0, xb, H], fill=(38, 54, 82))
|
||||
amp = H * 0.44 # mirrored waveform around the centre line
|
||||
for i in range(len(rms_n)):
|
||||
if t0 <= times[i] <= t1:
|
||||
x = X(times[i]); h = int(rms_n[i] * amp)
|
||||
d.line([(x, mid - h), (x, mid + h)], fill=(60, 140, 220))
|
||||
if fps and group_frames: # bold 721-frame group grid
|
||||
gstep = group_frames / fps
|
||||
k = 1
|
||||
while k * step < duration + 1e-6:
|
||||
mt = k * step
|
||||
if t0 <= mt <= t1:
|
||||
d.line([(X(mt), 0), (X(mt), H)], fill=(70, 200, 120), width=1)
|
||||
d.text((X(mt) + 2, H - 13), f"{frames_marker * k}f", fill=(70, 200, 120))
|
||||
while k * gstep < duration + 1e-6:
|
||||
gt = k * gstep
|
||||
if t0 <= gt <= t1:
|
||||
d.line([(X(gt), 0), (X(gt), H)], fill=(235, 235, 242), width=2)
|
||||
k += 1
|
||||
for b in beats: # beat markers (orange)
|
||||
for b in beats: # beat ticks (faint, centre band)
|
||||
if t0 <= b <= t1:
|
||||
d.line([(X(b), 0), (X(b), H)], fill=(230, 110, 60), width=1)
|
||||
for s in segs: # boundaries + label + time + note
|
||||
d.line([(X(b), mid - 4), (X(b), mid + 4)], fill=(150, 90, 60), width=1)
|
||||
for s in segs: # segment line + label top + note bottom
|
||||
if not (t0 <= s["start_s"] <= t1):
|
||||
continue
|
||||
x = X(s["start_s"])
|
||||
d.line([(x, 0), (x, H)], fill=(240, 240, 240), width=1)
|
||||
d.text((x + 4, 4), f"S{s['segment']} {s['energy']} {s['start_s']}s/{s['frames']}f", fill=(255, 255, 255))
|
||||
if s.get("note"): # per-segment note (amber)
|
||||
d.text((x + 4, 18), s["note"][:30], fill=(255, 210, 110))
|
||||
d.line([(x, 0), (x, H)], fill=(120, 190, 150), width=1)
|
||||
d.text((x + 3, 3), f"S{s['segment']} {s['start_s']}s/{s['frames']}f", fill=(240, 240, 240))
|
||||
if s.get("note"):
|
||||
d.text((x + 3, H - 13), s["note"][:28], fill=(255, 210, 110))
|
||||
arr = np.asarray(img, dtype=np.float32) / 255.0
|
||||
return torch.from_numpy(arr)[None, ...] # [1, H, W, 3]
|
||||
|
||||
|
||||
@@ -15,12 +15,13 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from .audio_guide import (
|
||||
_rms_envelope, _tempo_beats, _snap8, _segments, _render, _summary, _attach_notes,
|
||||
_rms_envelope, _tempo_beats, _snap8, _render, _summary, _attach_notes,
|
||||
)
|
||||
|
||||
|
||||
@@ -88,31 +89,52 @@ def _audio_files():
|
||||
return files or [_NO_AUDIO]
|
||||
|
||||
|
||||
def _segments_fixed(rms_n, times, duration, fps, sub_frames, beats):
|
||||
"""Split into fixed chunks of `sub_frames` frames (the LTX clip length). The last
|
||||
chunk gets whatever's left (snapped to 8n+1)."""
|
||||
step = max(sub_frames, 1) / max(fps, 1) # seconds per chunk
|
||||
n = max(1, int(np.ceil(duration / step - 1e-6)))
|
||||
def _parse_range(sel, n):
|
||||
"""'' / '0' / 'all' -> (0,0) = whole clip; 'N' -> (N,N); 'A-B' -> (A,B). 1-based, clamped."""
|
||||
sel = (sel or "").strip()
|
||||
if not sel or sel in ("0", "all"):
|
||||
return (0, 0)
|
||||
m = re.match(r"^(\d+)\s*[-–:]\s*(\d+)$", sel)
|
||||
if m:
|
||||
a, b = int(m.group(1)), int(m.group(2))
|
||||
else:
|
||||
try:
|
||||
a = b = int(sel)
|
||||
except ValueError:
|
||||
return (0, 0)
|
||||
if a > b:
|
||||
a, b = b, a
|
||||
return (max(1, min(a, n)), max(1, min(b, n)))
|
||||
|
||||
|
||||
def _segments_grouped(rms_n, times, duration, fps, group_frames, user_starts, beats):
|
||||
"""Segments = user split points UNION the fixed group boundaries (every group_frames
|
||||
frames). The group lines are hard splits, so a segment never crosses one — the last
|
||||
segment in a group ends exactly on the group boundary (e.g. frame 721)."""
|
||||
gstep = max(group_frames, 1) / max(fps, 1)
|
||||
gbounds, k = [], 1
|
||||
while k * gstep < duration - 1e-6:
|
||||
gbounds.append(round(k * gstep, 3)); k += 1
|
||||
allb = sorted({round(b, 3) for b in list(user_starts) + gbounds if 0 < b < duration - 1e-6})
|
||||
starts = [0.0] + allb
|
||||
stages = ["establish", "build", "peak", "settle"]
|
||||
segs, bounds = [], [0.0]
|
||||
for i in range(n):
|
||||
t0 = i * step
|
||||
t1 = min(duration, (i + 1) * step)
|
||||
for i, t0 in enumerate(starts):
|
||||
t1 = starts[i + 1] if i + 1 < len(starts) else duration
|
||||
bounds.append(round(t1, 3))
|
||||
mask = (times >= t0) & (times < t1)
|
||||
e = float(rms_n[mask].mean()) if mask.any() else 0.0
|
||||
peak_t = float(times[mask][np.argmax(rms_n[mask])]) if mask.any() else t0
|
||||
dur = round(t1 - t0, 2)
|
||||
frames = sub_frames if i < n - 1 else _snap8(round(dur * fps))
|
||||
label = "high" if e > 0.66 else ("medium" if e > 0.33 else "low")
|
||||
stage = stages[i] if i < len(stages) else ("peak" if e > 0.6 else "settle")
|
||||
segs.append({
|
||||
"segment": i + 1, "start_s": round(t0, 2), "duration_s": dur, "frames": frames,
|
||||
"energy": label, "energy_val": round(e, 3), "peak_s": round(peak_t, 2),
|
||||
"stage_hint": stage, "note": "",
|
||||
"beats_in": [round(b, 2) for b in beats if t0 <= b < t1],
|
||||
"segment": i + 1, "group": 1 + sum(1 for gb in gbounds if t0 >= gb - 0.02),
|
||||
"start_s": round(t0, 2), "duration_s": dur, "frames": _snap8(round(dur * fps)),
|
||||
"energy": "high" if e > 0.66 else ("medium" if e > 0.33 else "low"),
|
||||
"energy_val": round(e, 3), "peak_s": round(peak_t, 2),
|
||||
"stage_hint": stages[i] if i < len(stages) else ("peak" if e > 0.6 else "settle"),
|
||||
"note": "", "beats_in": [round(b, 2) for b in beats if t0 <= b < t1],
|
||||
})
|
||||
return segs, np.array(bounds)
|
||||
return segs, np.array(bounds), gbounds
|
||||
|
||||
|
||||
class AudioWaveSegments:
|
||||
@@ -128,29 +150,29 @@ class AudioWaveSegments:
|
||||
# Pick a file from ComfyUI/input, or use the widget's "upload" button (JS).
|
||||
"audio": (_audio_files(),),
|
||||
"fps": ("INT", {"default": 24, "min": 1, "max": 120}),
|
||||
# Fixed LTX clip length. Segments = chunks of this many frames (721@24fps
|
||||
# ~= 30.04s). This is also the waveform grid. Set 0 to use segments_json.
|
||||
"subsegment_frames": ("INT", {"default": 721, "min": 0, "max": 100000}),
|
||||
# 0 = whole clip. N = output ONLY subsegment N (crops image + audio + summary),
|
||||
# so you can generate/skip one beat at a time.
|
||||
"segment_select": ("INT", {"default": 0, "min": 0, "max": 999}),
|
||||
# Hard split every this many frames (721@24fps ~= 30.04s = one LTX clip).
|
||||
# Segments live inside a group and never cross a group line.
|
||||
"group_frames": ("INT", {"default": 721, "min": 1, "max": 100000}),
|
||||
# "" / "0" = whole clip. "N" = only segment N. "A-B" = segments A..B —
|
||||
# crops image + audio + summary to that range (for generating/skipping beats).
|
||||
"segment_select": ("STRING", {"default": ""}),
|
||||
"notes": ("STRING", {"default": "", "multiline": True}),
|
||||
# Machine field: per-chunk notes/boundaries from the JS widget (hidden).
|
||||
# Machine field: the JS widget writes only USER split points here (hidden).
|
||||
"segments_json": ("STRING", {"default": "[]"}),
|
||||
},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, audio, fps, subsegment_frames, segment_select, notes, segments_json):
|
||||
def IS_CHANGED(cls, audio, fps, group_frames, segment_select, notes, segments_json):
|
||||
try:
|
||||
import folder_paths
|
||||
p = folder_paths.get_annotated_filepath(audio)
|
||||
mt = os.path.getmtime(p) if os.path.isfile(p) else ""
|
||||
except Exception:
|
||||
mt = ""
|
||||
return f"{audio}|{fps}|{subsegment_frames}|{segment_select}|{notes}|{segments_json}|{mt}"
|
||||
return f"{audio}|{fps}|{group_frames}|{segment_select}|{notes}|{segments_json}|{mt}"
|
||||
|
||||
def run(self, audio, fps, subsegment_frames, segment_select, notes, segments_json):
|
||||
def run(self, audio, fps, group_frames, segment_select, notes, segments_json):
|
||||
try:
|
||||
import folder_paths
|
||||
path = folder_paths.get_annotated_filepath(audio)
|
||||
@@ -166,44 +188,44 @@ class AudioWaveSegments:
|
||||
rms_n, times = _rms_envelope(y, sr)
|
||||
bpm, beats = _tempo_beats(y, sr)
|
||||
|
||||
if subsegment_frames > 0: # fixed LTX-clip grid
|
||||
segs, bounds = _segments_fixed(rms_n, times, duration, fps, subsegment_frames, beats)
|
||||
else: # manual boundaries from the JS widget
|
||||
starts = []
|
||||
try:
|
||||
for seg in (json.loads(segments_json) if segments_json.strip() else []):
|
||||
starts.append(float(seg.get("start_s", 0.0)))
|
||||
except Exception as e:
|
||||
print(f"[AudioWaveSegments] bad segments_json ({e})")
|
||||
segs, bounds = _segments_from_boundaries(rms_n, times, duration, fps, starts, beats)
|
||||
|
||||
# per-chunk notes from segments_json (JS) then the notes-box (segN: syntax) on top.
|
||||
user_starts, seg_notes = [], {} # user split points from the JS widget
|
||||
try:
|
||||
for i, seg in enumerate(json.loads(segments_json) if segments_json.strip() else []):
|
||||
if i < len(segs) and seg.get("note"):
|
||||
segs[i]["note"] = str(seg["note"])
|
||||
except Exception:
|
||||
pass
|
||||
global_notes = _attach_notes(segs, notes)
|
||||
user_starts.append(float(seg.get("start_s", 0.0)))
|
||||
if seg.get("note"):
|
||||
seg_notes[round(float(seg.get("start_s", 0.0)), 2)] = str(seg["note"])
|
||||
except Exception as e:
|
||||
print(f"[AudioWaveSegments] bad segments_json ({e})")
|
||||
|
||||
window, render_segs = None, segs
|
||||
if group_frames < 1:
|
||||
group_frames = 721
|
||||
segs, bounds, gbounds = _segments_grouped(rms_n, times, duration, fps, group_frames, user_starts, beats)
|
||||
for s in segs: # notes attached by start time
|
||||
if s["start_s"] in seg_notes:
|
||||
s["note"] = seg_notes[s["start_s"]]
|
||||
global_notes = _attach_notes(segs, notes) # notes-box (segN:) merges on top
|
||||
|
||||
a, b = _parse_range(segment_select, len(segs))
|
||||
window, sel = None, None
|
||||
audio_out = {"waveform": wav.unsqueeze(0), "sample_rate": sr}
|
||||
sel = int(segment_select)
|
||||
if 1 <= sel <= len(segs): # crop to one subsegment
|
||||
seg = segs[sel - 1]
|
||||
s0 = seg["start_s"]
|
||||
s1 = min(duration, s0 + seg["duration_s"])
|
||||
a, b = int(s0 * sr), int(s1 * sr)
|
||||
if b > a:
|
||||
audio_out = {"waveform": wav[:, a:b].unsqueeze(0), "sample_rate": sr}
|
||||
window, render_segs = (s0, s1), [seg]
|
||||
summary = (f"SELECTED SUBSEGMENT {sel} of {len(segs)} — generate ONLY this beat.\n"
|
||||
+ _summary(duration, sr, bpm, beats, [seg], global_notes))
|
||||
if a >= 1 and segs: # crop to segment range A..B
|
||||
s0 = segs[a - 1]["start_s"]
|
||||
s1 = min(duration, segs[b - 1]["start_s"] + segs[b - 1]["duration_s"])
|
||||
i0, i1 = int(s0 * sr), int(s1 * sr)
|
||||
if i1 > i0:
|
||||
audio_out = {"waveform": wav[:, i0:i1].unsqueeze(0), "sample_rate": sr}
|
||||
window, sel = (s0, s1), (a, b)
|
||||
total = sum(s["frames"] for s in segs[a - 1:b])
|
||||
g0, g1 = segs[a - 1]["group"], segs[b - 1]["group"]
|
||||
gtxt = f"group {g0}" + ("" if g0 == g1 else f"–{g1}")
|
||||
summary = (f"SELECTED SEGMENTS {a}-{b} of {len(segs)} ({gtxt}), {total} frames total "
|
||||
f"— generate these beats.\n"
|
||||
+ _summary(duration, sr, bpm, beats, segs[a - 1:b], global_notes))
|
||||
else:
|
||||
summary = _summary(duration, sr, bpm, beats, segs, global_notes)
|
||||
|
||||
image = _render(rms_n, times, duration, beats, bounds, render_segs,
|
||||
fps=fps, frames_marker=subsegment_frames, window=window)
|
||||
image = _render(rms_n, times, duration, beats, bounds, segs,
|
||||
fps=fps, group_frames=group_frames, window=window, sel=sel)
|
||||
return (image, summary, audio_out)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user