diff --git a/README.md b/README.md index 2db527f..6544c97 100644 --- a/README.md +++ b/README.md @@ -101,12 +101,13 @@ seg1: slow dreamy intro # by segment number (also "1:" or "S1:") cinematic, moody grade # no prefix = global (applies throughout) ``` -**Ready-made workflow:** `workflow/workflow_ltx_audio_api.json` wires it all up — -`Audio Wave + Segments` + `LoadImage` (subject) + `Text (Prompt)` (the LTX system prompt -from `prompts/ltx_audio_relay_system_prompt.txt`) → the judge in chat mode -(`json_output=true`, model `3.5-9b`). Load it, upload your audio + set your subject image, -and the judge's `analysis` output is the LTX beats JSON. (`Text (Prompt)` is a small STRING -node this pack adds to feed the judge's `system_prompt`/`user_prompt`/`axes` sockets.) +**Ready-made workflow:** load **`workflow/workflow_ltx_audio.json`** (UI format — drag it +onto the canvas) — it wires `Audio Wave + Segments` + `LoadImage` (subject) + `Text (Prompt)` +(the LTX system prompt from `prompts/ltx_audio_relay_system_prompt.txt`) → the judge in chat +mode (`json_output=true`, model `3.5-9b`). Set your subject image + audio, queue, and the +judge's `analysis` output is the LTX beats JSON. (`workflow_ltx_audio_api.json` is the same +graph in API format for `agent_bridge.py`. `Text (Prompt)` is a small STRING node this pack +adds to feed the judge's `system_prompt`/`user_prompt`/`axes` sockets.) ### Interactive: `Audio Wave + Segments` diff --git a/nodes/audio_wave_segments.py b/nodes/audio_wave_segments.py index 19394da..1222947 100644 --- a/nodes/audio_wave_segments.py +++ b/nodes/audio_wave_segments.py @@ -92,7 +92,8 @@ class AudioWaveSegments: def INPUT_TYPES(cls): return { "required": { - "audio": (_audio_files(), {"audio_upload": True}), + # Pick a file from ComfyUI/input, or use the widget's "upload" button (JS). + "audio": (_audio_files(),), "fps": ("INT", {"default": 24, "min": 1, "max": 120}), "max_segments": ("INT", {"default": 6, "min": 3, "max": 12}), "notes": ("STRING", {"default": "", "multiline": True}), diff --git a/workflow/workflow_ltx_audio.json b/workflow/workflow_ltx_audio.json new file mode 100644 index 0000000..0ea6bde --- /dev/null +++ b/workflow/workflow_ltx_audio.json @@ -0,0 +1,258 @@ +{ + "id": "ltx-audio-relay", + "last_node_id": 4, + "last_link_id": 4, + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [ + 40, + 40 + ], + "size": [ + 320, + 380 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 1 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "reference.png", + "image" + ] + }, + { + "id": 3, + "type": "CalibratorText", + "pos": [ + 40, + 460 + ], + "size": [ + 360, + 300 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text", + "type": "STRING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "CalibratorText" + }, + "widgets_values": [ + "You are an expert prompt engineer for the LTX-2.3 video model, generating a\nmulti-segment \"Prompt Relay\" timeline TIMED TO AUDIO. Output ONLY valid JSON — no\nmarkdown, no code fences, no text before or after.\n\nYou are given:\n- IMAGE 1 = the REFERENCE subject/scene. Use it ONLY for persistent visual appearance\n (subject's look, setting, lighting, color palette, style) in global_prompt.\n- IMAGE 2 = the AUDIO ENERGY WAVEFORM (energy envelope with segment boundaries and beat\n markers). Read the overall energy shape and where it peaks.\n- An \"AUDIO GUIDE\" text block: per-segment start, duration_s, frames (already snapped to\n 8n+1), energy level, stage hint, per-segment NOTE, and GLOBAL NOTES.\n\nMatch this schema exactly:\n\n{\n \"fps\": 24,\n \"global_prompt\": \"string\",\n \"negative_prompt\": \"string\",\n \"beats\": [\n {\n \"segment\": 1,\n \"duration_s\": 5,\n \"frames\": 121,\n \"camera\": \"string\",\n \"delta\": \"string\",\n \"sound\": \"string\",\n \"dialogue\": \"string\",\n \"prompt\": \"string\"\n }\n ]\n}\n\nRules:\n- fps: take from the AUDIO GUIDE.\n- global_prompt: one present-tense paragraph with ALL persistent elements taken from\n IMAGE 1 — subject's physical appearance (age, hair, clothing as visual cues), setting,\n lighting, color palette, style — plus the sustained ambient/audio layer. No camera\n moves. Never repeated in beats.\n- negative_prompt: \"shaky, glitchy, low quality, worst quality, deformed, distorted,\n disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, ugly, static\".\n- beats: EXACTLY ONE beat per AUDIO GUIDE segment, in order. Copy that segment's\n \"segment\", \"duration_s\", and \"frames\" verbatim (do not recompute).\n- Beat 1: \"delta\" = the static visible starting state only; \"camera\" = \"static hold\".\n- Every later beat: \"delta\" = ONLY what CHANGES from the previous beat. Never repeat the\n global_prompt or earlier beats.\n- MOTION FROM AUDIO: drive each beat's motion from its energy — higher energy => faster\n motion and bigger camera moves; lower energy => slower, settle. Follow the energy\n envelope across the timeline (establish -> build -> peak -> settle), putting the release\n on the final segment.\n- NOTES OVERRIDE: if a segment has a NOTE, that instruction OVERRIDES the energy default\n for that beat (e.g. NOTE \"slow\" on a HIGH-energy segment => slow, deliberate motion).\n Apply GLOBAL NOTES throughout.\n- \"camera\" starts with a camera verb (push in, pan, arc, handheld drift, hold).\n- \"sound\" = short sound cue; \"dialogue\" = spoken line in quotes or \"\".\n- \"prompt\" = the final assembled beat text in order: camera, then delta action, then any\n dialogue/sound — present tense, concrete on-screen verbs, no emotion labels. Ready to paste.\n" + ] + }, + { + "id": 2, + "type": "AudioWaveSegments", + "pos": [ + 40, + 800 + ], + "size": [ + 420, + 320 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "waveform_image", + "type": "IMAGE", + "links": [ + 2 + ] + }, + { + "name": "audio_summary", + "type": "STRING", + "links": [ + 4 + ] + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + } + ], + "properties": { + "Node name for S&R": "AudioWaveSegments" + }, + "widgets_values": [ + "audio.wav", + 24, + 6, + "", + "[]" + ] + }, + { + "id": 4, + "type": "QwenVLImageJudge", + "pos": [ + 540, + 40 + ], + "size": [ + 420, + 640 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "reference_image", + "type": "IMAGE", + "link": 1 + }, + { + "name": "generated_image", + "type": "IMAGE", + "link": 2, + "shape": 7 + }, + { + "name": "axes", + "type": "STRING", + "link": null, + "shape": 7 + }, + { + "name": "reference_description", + "type": "STRING", + "link": null, + "shape": 7 + }, + { + "name": "system_prompt", + "type": "STRING", + "link": 3, + "shape": 7 + }, + { + "name": "user_prompt", + "type": "STRING", + "link": 4, + "shape": 7 + } + ], + "outputs": [ + { + "name": "overall_score", + "type": "FLOAT", + "links": null + }, + { + "name": "axis_scores_json", + "type": "STRING", + "links": null + }, + { + "name": "analysis", + "type": "STRING", + "links": null + }, + { + "name": "raw", + "type": "STRING", + "links": null + }, + { + "name": "report_path", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "QwenVLImageJudge" + }, + "widgets_values": [ + "chat", + "general", + "(manual — use model_path below)", + "3.5-9b", + "bf16", + 3072, + 0.4, + false, + false, + true, + true, + true, + "/media/p5/Comfyui/output/ltx", + "ltx" + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 4, + 0, + "IMAGE" + ], + [ + 2, + 2, + 0, + 4, + 1, + "IMAGE" + ], + [ + 3, + 3, + 0, + 4, + 4, + "STRING" + ], + [ + 4, + 2, + 1, + 4, + 5, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file