diff --git a/README.md b/README.md index 10b44b7..5da149f 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,15 @@ The model then gets the audio's timing/energy + your motion notes and writes bea camera moves, and deltas that escalate with the music. `librosa` is optional (BPM/beats); without it you still get the energy envelope + segments. +**Notes can target a segment** (the note overrides that segment's energy default, and is drawn +on the waveform). Syntax in the `notes` box, one per line: + +``` +seg1: slow dreamy intro # by segment number (also "1:" or "S1:") +10s-15s: explosive drop # by time range (also "12.3s:") +cinematic, moody grade # no prefix = global (applies throughout) +``` + ## Performance / speed This node runs models through **transformers `.generate()`** β€” the simplest path, but the diff --git a/nodes/audio_guide.py b/nodes/audio_guide.py index 141f031..2f3a1d7 100644 --- a/nodes/audio_guide.py +++ b/nodes/audio_guide.py @@ -19,11 +19,49 @@ with it you also get tempo (BPM) and beat times. from __future__ import annotations +import re + import numpy as np import torch from PIL import Image, ImageDraw +def _split_notes(notes: str): + """Parse the notes box into per-segment / per-time / global notes. Lines like: + seg2: fast motion (or 2: fast motion / S2: ...) -> segment 2 + 5s: build (or 0-5s: ... / 12.3s: ...) -> by time + cinematic, moody (no prefix) -> global + Returns (seg_map {num: [notes]}, time_notes [(t0,t1,text)], global_notes [str]).""" + seg_map, time_notes, glob = {}, [], [] + for raw in notes.splitlines(): + line = raw.strip() + if not line: + continue + mt = re.match(r"(?i)^(\d+(?:\.\d+)?)\s*s(?:\s*[-–to]+\s*(\d+(?:\.\d+)?)\s*s?)?\s*[:)\-]\s*(.+)$", line) + ms = (re.match(r"(?i)^(?:seg(?:ment)?|s)\s*(\d+)\s*[:)\-]\s*(.+)$", line) + or re.match(r"^(\d+)\s*[:)]\s*(.+)$", line)) + if mt: + t0 = float(mt.group(1)); t1 = float(mt.group(2)) if mt.group(2) else t0 + time_notes.append((t0, t1, mt.group(3).strip())) + elif ms: + seg_map.setdefault(int(ms.group(1)), []).append(ms.group(2).strip()) + else: + glob.append(line) + return seg_map, time_notes, glob + + +def _attach_notes(segs, notes): + """Attach the parsed notes to each segment (by number or overlapping time). + Returns the leftover global notes.""" + seg_map, time_notes, glob = _split_notes(notes) + for s in segs: + s0, s1 = s["start_s"], s["start_s"] + s["duration_s"] + found = list(seg_map.get(s["segment"], [])) + found += [txt for (t0, t1, txt) in time_notes if t0 < s1 and t1 > s0] + s["note"] = "; ".join(found) + return glob + + def _to_mono(audio) -> tuple[np.ndarray, int]: """ComfyUI AUDIO dict -> (mono float32 samples, sample_rate).""" wf = audio["waveform"] @@ -108,12 +146,15 @@ def _render(rms_n, times, duration, beats, bounds, segs): for bd in bounds: # segment boundaries (white) d.line([(X(bd), 0), (X(bd), H)], fill=(240, 240, 240), width=1) for s in segs: - d.text((X(s["start_s"]) + 4, 4), f"S{s['segment']} {s['energy']}", fill=(255, 255, 255)) + x = X(s["start_s"]) + 4 + d.text((x, 4), f"S{s['segment']} {s['energy']}", fill=(255, 255, 255)) + if s.get("note"): # per-segment note (amber) + d.text((x, 18), s["note"][:30], fill=(255, 210, 110)) arr = np.asarray(img, dtype=np.float32) / 255.0 return torch.from_numpy(arr)[None, ...] # [1, H, W, 3] -def _summary(duration, sr, bpm, beats, segs, notes): +def _summary(duration, sr, bpm, beats, segs, global_notes): lines = ["AUDIO GUIDE β€” align the video beats to this audio:", f"- duration: {duration:.2f}s | sample_rate: {sr} Hz"] if bpm: @@ -121,21 +162,23 @@ def _summary(duration, sr, bpm, beats, segs, notes): if beats: preview = ", ".join(f"{b:.2f}" for b in beats[:24]) lines.append(f"- beat times (s): {preview}{' ...' if len(beats) > 24 else ''}") - lines.append("- suggested segments (use these durations/frames and energy):") + lines.append("- suggested segments (use these durations/frames and energy; honor the NOTE):") for s in segs: peak = f", energy peak @ {s['peak_s']}s" if s["energy"] == "high" else "" + note = f" <<< NOTE: {s['note']}" if s.get("note") else "" lines.append( f" seg{s['segment']}: {s['start_s']}–{s['start_s'] + s['duration_s']:.2f}s, " f"{s['duration_s']}s, {s['frames']} frames, energy {s['energy'].upper()} " - f"({s['stage_hint']}){peak}") + f"({s['stage_hint']}){peak}{note}") lines.append("") lines.append("Guidance: higher energy -> faster motion and bigger camera moves; " "lower energy -> slower, settle. Put escalation on rising energy and the " - "release on the final segment. Snap each beat's frames to 8n+1.") - if notes.strip(): + "release on the final segment. Snap each beat's frames to 8n+1. Where a " + "segment has a NOTE, that instruction OVERRIDES the energy default.") + if global_notes: lines.append("") - lines.append("USER NOTES (map these onto the matching segments/times):") - lines.append(notes.strip()) + lines.append("GLOBAL NOTES (apply throughout):") + lines.extend(f" - {n}" for n in global_notes) return "\n".join(lines) @@ -162,8 +205,9 @@ class AudioPromptGuide: rms_n, times = _rms_envelope(y, sr) bpm, beats = _tempo_beats(y, sr) segs, bounds = _segments(rms_n, times, duration, fps, max_segments, beats) + global_notes = _attach_notes(segs, notes) # per-segment / per-time / global image = _render(rms_n, times, duration, beats, bounds, segs) - summary = _summary(duration, sr, bpm, beats, segs, notes) + summary = _summary(duration, sr, bpm, beats, segs, global_notes) return (image, summary)