Add loadable UI-format LTX audio workflow (API format didn't load in the canvas)
The canvas Load expects the UI graph format (nodes[]+links[]+positions), not the API format — so the _api.json came up empty. Added workflow_ltx_audio.json in UI format (matched the schema from a real workflow: node dicts with pos/size/inputs/ outputs/widgets_values, link tuples, version 0.4), validated slots/links. Dropped audio_upload from AudioWaveSegments so the widget layout is predictable (the JS upload button stays). README points to the loadable file; _api.json kept for the bridge. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -101,12 +101,13 @@ seg1: slow dreamy intro # by segment number (also "1:" or "S1:")
|
|||||||
cinematic, moody grade # no prefix = global (applies throughout)
|
cinematic, moody grade # no prefix = global (applies throughout)
|
||||||
```
|
```
|
||||||
|
|
||||||
**Ready-made workflow:** `workflow/workflow_ltx_audio_api.json` wires it all up —
|
**Ready-made workflow:** load **`workflow/workflow_ltx_audio.json`** (UI format — drag it
|
||||||
`Audio Wave + Segments` + `LoadImage` (subject) + `Text (Prompt)` (the LTX system prompt
|
onto the canvas) — it wires `Audio Wave + Segments` + `LoadImage` (subject) + `Text (Prompt)`
|
||||||
from `prompts/ltx_audio_relay_system_prompt.txt`) → the judge in chat mode
|
(the LTX system prompt from `prompts/ltx_audio_relay_system_prompt.txt`) → the judge in chat
|
||||||
(`json_output=true`, model `3.5-9b`). Load it, upload your audio + set your subject image,
|
mode (`json_output=true`, model `3.5-9b`). Set your subject image + audio, queue, and the
|
||||||
and the judge's `analysis` output is the LTX beats JSON. (`Text (Prompt)` is a small STRING
|
judge's `analysis` output is the LTX beats JSON. (`workflow_ltx_audio_api.json` is the same
|
||||||
node this pack adds to feed the judge's `system_prompt`/`user_prompt`/`axes` sockets.)
|
graph in API format for `agent_bridge.py`. `Text (Prompt)` is a small STRING node this pack
|
||||||
|
adds to feed the judge's `system_prompt`/`user_prompt`/`axes` sockets.)
|
||||||
|
|
||||||
### Interactive: `Audio Wave + Segments`
|
### Interactive: `Audio Wave + Segments`
|
||||||
|
|
||||||
|
|||||||
@@ -92,7 +92,8 @@ class AudioWaveSegments:
|
|||||||
def INPUT_TYPES(cls):
|
def INPUT_TYPES(cls):
|
||||||
return {
|
return {
|
||||||
"required": {
|
"required": {
|
||||||
"audio": (_audio_files(), {"audio_upload": True}),
|
# Pick a file from ComfyUI/input, or use the widget's "upload" button (JS).
|
||||||
|
"audio": (_audio_files(),),
|
||||||
"fps": ("INT", {"default": 24, "min": 1, "max": 120}),
|
"fps": ("INT", {"default": 24, "min": 1, "max": 120}),
|
||||||
"max_segments": ("INT", {"default": 6, "min": 3, "max": 12}),
|
"max_segments": ("INT", {"default": 6, "min": 3, "max": 12}),
|
||||||
"notes": ("STRING", {"default": "", "multiline": True}),
|
"notes": ("STRING", {"default": "", "multiline": True}),
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
{
|
||||||
|
"id": "ltx-audio-relay",
|
||||||
|
"last_node_id": 4,
|
||||||
|
"last_link_id": 4,
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"type": "LoadImage",
|
||||||
|
"pos": [
|
||||||
|
40,
|
||||||
|
40
|
||||||
|
],
|
||||||
|
"size": [
|
||||||
|
320,
|
||||||
|
380
|
||||||
|
],
|
||||||
|
"flags": {},
|
||||||
|
"order": 0,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "IMAGE",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"links": [
|
||||||
|
1
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "MASK",
|
||||||
|
"type": "MASK",
|
||||||
|
"links": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "LoadImage"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"reference.png",
|
||||||
|
"image"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"type": "CalibratorText",
|
||||||
|
"pos": [
|
||||||
|
40,
|
||||||
|
460
|
||||||
|
],
|
||||||
|
"size": [
|
||||||
|
360,
|
||||||
|
300
|
||||||
|
],
|
||||||
|
"flags": {},
|
||||||
|
"order": 1,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "text",
|
||||||
|
"type": "STRING",
|
||||||
|
"links": [
|
||||||
|
3
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "CalibratorText"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"You are an expert prompt engineer for the LTX-2.3 video model, generating a\nmulti-segment \"Prompt Relay\" timeline TIMED TO AUDIO. Output ONLY valid JSON — no\nmarkdown, no code fences, no text before or after.\n\nYou are given:\n- IMAGE 1 = the REFERENCE subject/scene. Use it ONLY for persistent visual appearance\n (subject's look, setting, lighting, color palette, style) in global_prompt.\n- IMAGE 2 = the AUDIO ENERGY WAVEFORM (energy envelope with segment boundaries and beat\n markers). Read the overall energy shape and where it peaks.\n- An \"AUDIO GUIDE\" text block: per-segment start, duration_s, frames (already snapped to\n 8n+1), energy level, stage hint, per-segment NOTE, and GLOBAL NOTES.\n\nMatch this schema exactly:\n\n{\n \"fps\": 24,\n \"global_prompt\": \"string\",\n \"negative_prompt\": \"string\",\n \"beats\": [\n {\n \"segment\": 1,\n \"duration_s\": 5,\n \"frames\": 121,\n \"camera\": \"string\",\n \"delta\": \"string\",\n \"sound\": \"string\",\n \"dialogue\": \"string\",\n \"prompt\": \"string\"\n }\n ]\n}\n\nRules:\n- fps: take from the AUDIO GUIDE.\n- global_prompt: one present-tense paragraph with ALL persistent elements taken from\n IMAGE 1 — subject's physical appearance (age, hair, clothing as visual cues), setting,\n lighting, color palette, style — plus the sustained ambient/audio layer. No camera\n moves. Never repeated in beats.\n- negative_prompt: \"shaky, glitchy, low quality, worst quality, deformed, distorted,\n disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, ugly, static\".\n- beats: EXACTLY ONE beat per AUDIO GUIDE segment, in order. Copy that segment's\n \"segment\", \"duration_s\", and \"frames\" verbatim (do not recompute).\n- Beat 1: \"delta\" = the static visible starting state only; \"camera\" = \"static hold\".\n- Every later beat: \"delta\" = ONLY what CHANGES from the previous beat. Never repeat the\n global_prompt or earlier beats.\n- MOTION FROM AUDIO: drive each beat's motion from its energy — higher energy => faster\n motion and bigger camera moves; lower energy => slower, settle. Follow the energy\n envelope across the timeline (establish -> build -> peak -> settle), putting the release\n on the final segment.\n- NOTES OVERRIDE: if a segment has a NOTE, that instruction OVERRIDES the energy default\n for that beat (e.g. NOTE \"slow\" on a HIGH-energy segment => slow, deliberate motion).\n Apply GLOBAL NOTES throughout.\n- \"camera\" starts with a camera verb (push in, pan, arc, handheld drift, hold).\n- \"sound\" = short sound cue; \"dialogue\" = spoken line in quotes or \"\".\n- \"prompt\" = the final assembled beat text in order: camera, then delta action, then any\n dialogue/sound — present tense, concrete on-screen verbs, no emotion labels. Ready to paste.\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"type": "AudioWaveSegments",
|
||||||
|
"pos": [
|
||||||
|
40,
|
||||||
|
800
|
||||||
|
],
|
||||||
|
"size": [
|
||||||
|
420,
|
||||||
|
320
|
||||||
|
],
|
||||||
|
"flags": {},
|
||||||
|
"order": 2,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "waveform_image",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"links": [
|
||||||
|
2
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "audio_summary",
|
||||||
|
"type": "STRING",
|
||||||
|
"links": [
|
||||||
|
4
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "audio",
|
||||||
|
"type": "AUDIO",
|
||||||
|
"links": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "AudioWaveSegments"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"audio.wav",
|
||||||
|
24,
|
||||||
|
6,
|
||||||
|
"",
|
||||||
|
"[]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"type": "QwenVLImageJudge",
|
||||||
|
"pos": [
|
||||||
|
540,
|
||||||
|
40
|
||||||
|
],
|
||||||
|
"size": [
|
||||||
|
420,
|
||||||
|
640
|
||||||
|
],
|
||||||
|
"flags": {},
|
||||||
|
"order": 3,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "reference_image",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"link": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "generated_image",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"link": 2,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "axes",
|
||||||
|
"type": "STRING",
|
||||||
|
"link": null,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "reference_description",
|
||||||
|
"type": "STRING",
|
||||||
|
"link": null,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "system_prompt",
|
||||||
|
"type": "STRING",
|
||||||
|
"link": 3,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "user_prompt",
|
||||||
|
"type": "STRING",
|
||||||
|
"link": 4,
|
||||||
|
"shape": 7
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "overall_score",
|
||||||
|
"type": "FLOAT",
|
||||||
|
"links": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "axis_scores_json",
|
||||||
|
"type": "STRING",
|
||||||
|
"links": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "analysis",
|
||||||
|
"type": "STRING",
|
||||||
|
"links": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "raw",
|
||||||
|
"type": "STRING",
|
||||||
|
"links": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "report_path",
|
||||||
|
"type": "STRING",
|
||||||
|
"links": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "QwenVLImageJudge"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"chat",
|
||||||
|
"general",
|
||||||
|
"(manual — use model_path below)",
|
||||||
|
"3.5-9b",
|
||||||
|
"bf16",
|
||||||
|
3072,
|
||||||
|
0.4,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
"/media/p5/Comfyui/output/ltx",
|
||||||
|
"ltx"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"links": [
|
||||||
|
[
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
4,
|
||||||
|
0,
|
||||||
|
"IMAGE"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
2,
|
||||||
|
2,
|
||||||
|
0,
|
||||||
|
4,
|
||||||
|
1,
|
||||||
|
"IMAGE"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
3,
|
||||||
|
3,
|
||||||
|
0,
|
||||||
|
4,
|
||||||
|
4,
|
||||||
|
"STRING"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
4,
|
||||||
|
2,
|
||||||
|
1,
|
||||||
|
4,
|
||||||
|
5,
|
||||||
|
"STRING"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"groups": [],
|
||||||
|
"config": {},
|
||||||
|
"extra": {},
|
||||||
|
"version": 0.4
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user