From 07ad5c1dc13c2650c265ccee2813971a08f1b714 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 4 Jul 2026 22:40:52 +0200 Subject: [PATCH] Add LTX audio workflow + Text (Prompt) node New workflow/workflow_ltx_audio_api.json: Audio Wave + Segments + LoadImage (subject) + Text (Prompt) holding the LTX audio system prompt -> Judge in chat mode (json_output, model 3.5-9b) -> LTX beats JSON on the analysis output. Added a tiny CalibratorText node ('Text (Prompt)', STRING source) needed to feed the judge's forceInput system_prompt/user_prompt/axes sockets. README points to the workflow. Co-Authored-By: Claude Opus 4.8 --- README.md | 7 +++ __init__.py | 10 +++- nodes/text.py | 27 +++++++++++ workflow/workflow_ltx_audio_api.json | 71 ++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 nodes/text.py create mode 100644 workflow/workflow_ltx_audio_api.json diff --git a/README.md b/README.md index 608dc06..2db527f 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,13 @@ seg1: slow dreamy intro # by segment number (also "1:" or "S1:") cinematic, moody grade # no prefix = global (applies throughout) ``` +**Ready-made workflow:** `workflow/workflow_ltx_audio_api.json` wires it all up — +`Audio Wave + Segments` + `LoadImage` (subject) + `Text (Prompt)` (the LTX system prompt +from `prompts/ltx_audio_relay_system_prompt.txt`) → the judge in chat mode +(`json_output=true`, model `3.5-9b`). Load it, upload your audio + set your subject image, +and the judge's `analysis` output is the LTX beats JSON. (`Text (Prompt)` is a small STRING +node this pack adds to feed the judge's `system_prompt`/`user_prompt`/`axes` sockets.) + ### Interactive: `Audio Wave + Segments` Same outputs, but with an in-node waveform widget: **upload** an audio clip, **play** it, diff --git a/__init__.py b/__init__.py index e09c11e..aca7ef4 100644 --- a/__init__.py +++ b/__init__.py @@ -16,9 +16,15 @@ from .nodes.audio_wave_segments import ( NODE_CLASS_MAPPINGS as _WAVE_CLASSES, NODE_DISPLAY_NAME_MAPPINGS as _WAVE_NAMES, ) +from .nodes.text import ( + NODE_CLASS_MAPPINGS as _TEXT_CLASSES, + NODE_DISPLAY_NAME_MAPPINGS as _TEXT_NAMES, +) -NODE_CLASS_MAPPINGS = {**_JUDGE_CLASSES, **_RECEPTOR_CLASSES, **_AUDIO_CLASSES, **_WAVE_CLASSES} -NODE_DISPLAY_NAME_MAPPINGS = {**_JUDGE_NAMES, **_RECEPTOR_NAMES, **_AUDIO_NAMES, **_WAVE_NAMES} +NODE_CLASS_MAPPINGS = {**_JUDGE_CLASSES, **_RECEPTOR_CLASSES, **_AUDIO_CLASSES, + **_WAVE_CLASSES, **_TEXT_CLASSES} +NODE_DISPLAY_NAME_MAPPINGS = {**_JUDGE_NAMES, **_RECEPTOR_NAMES, **_AUDIO_NAMES, + **_WAVE_NAMES, **_TEXT_NAMES} WEB_DIRECTORY = "./web" # serves web/audio_wave.js (the waveform + segment widget) __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] diff --git a/nodes/text.py b/nodes/text.py new file mode 100644 index 0000000..a641ba4 --- /dev/null +++ b/nodes/text.py @@ -0,0 +1,27 @@ +""" +Text (Prompt) node — a plain multiline STRING source. + +Needed because the judge's system_prompt / user_prompt / axes / reference_description +are input sockets (forceInput): wire this node into them to type a value. Handy for the +LTX system prompt, custom axes, etc. +""" + +from __future__ import annotations + + +class CalibratorText: + CATEGORY = "prompt_calibrator" + FUNCTION = "out" + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("text",) + + @classmethod + def INPUT_TYPES(cls): + return {"required": {"text": ("STRING", {"default": "", "multiline": True})}} + + def out(self, text): + return (text,) + + +NODE_CLASS_MAPPINGS = {"CalibratorText": CalibratorText} +NODE_DISPLAY_NAME_MAPPINGS = {"CalibratorText": "Text (Prompt)"} diff --git a/workflow/workflow_ltx_audio_api.json b/workflow/workflow_ltx_audio_api.json new file mode 100644 index 0000000..92a24c5 --- /dev/null +++ b/workflow/workflow_ltx_audio_api.json @@ -0,0 +1,71 @@ +{ + "1": { + "class_type": "LoadImage", + "inputs": { + "image": "reference.png" + }, + "_meta": { + "title": "Reference subject image (put in ComfyUI/input/)" + } + }, + "2": { + "class_type": "AudioWaveSegments", + "inputs": { + "audio": "audio.wav", + "fps": 24, + "max_segments": 6, + "notes": "", + "segments_json": "[]" + }, + "_meta": { + "title": "Audio Wave + Segments (upload + click segments)" + } + }, + "3": { + "class_type": "CalibratorText", + "inputs": { + "text": "You are an expert prompt engineer for the LTX-2.3 video model, generating a\nmulti-segment \"Prompt Relay\" timeline TIMED TO AUDIO. Output ONLY valid JSON — no\nmarkdown, no code fences, no text before or after.\n\nYou are given:\n- IMAGE 1 = the REFERENCE subject/scene. Use it ONLY for persistent visual appearance\n (subject's look, setting, lighting, color palette, style) in global_prompt.\n- IMAGE 2 = the AUDIO ENERGY WAVEFORM (energy envelope with segment boundaries and beat\n markers). Read the overall energy shape and where it peaks.\n- An \"AUDIO GUIDE\" text block: per-segment start, duration_s, frames (already snapped to\n 8n+1), energy level, stage hint, per-segment NOTE, and GLOBAL NOTES.\n\nMatch this schema exactly:\n\n{\n \"fps\": 24,\n \"global_prompt\": \"string\",\n \"negative_prompt\": \"string\",\n \"beats\": [\n {\n \"segment\": 1,\n \"duration_s\": 5,\n \"frames\": 121,\n \"camera\": \"string\",\n \"delta\": \"string\",\n \"sound\": \"string\",\n \"dialogue\": \"string\",\n \"prompt\": \"string\"\n }\n ]\n}\n\nRules:\n- fps: take from the AUDIO GUIDE.\n- global_prompt: one present-tense paragraph with ALL persistent elements taken from\n IMAGE 1 — subject's physical appearance (age, hair, clothing as visual cues), setting,\n lighting, color palette, style — plus the sustained ambient/audio layer. No camera\n moves. Never repeated in beats.\n- negative_prompt: \"shaky, glitchy, low quality, worst quality, deformed, distorted,\n disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, ugly, static\".\n- beats: EXACTLY ONE beat per AUDIO GUIDE segment, in order. Copy that segment's\n \"segment\", \"duration_s\", and \"frames\" verbatim (do not recompute).\n- Beat 1: \"delta\" = the static visible starting state only; \"camera\" = \"static hold\".\n- Every later beat: \"delta\" = ONLY what CHANGES from the previous beat. Never repeat the\n global_prompt or earlier beats.\n- MOTION FROM AUDIO: drive each beat's motion from its energy — higher energy => faster\n motion and bigger camera moves; lower energy => slower, settle. Follow the energy\n envelope across the timeline (establish -> build -> peak -> settle), putting the release\n on the final segment.\n- NOTES OVERRIDE: if a segment has a NOTE, that instruction OVERRIDES the energy default\n for that beat (e.g. NOTE \"slow\" on a HIGH-energy segment => slow, deliberate motion).\n Apply GLOBAL NOTES throughout.\n- \"camera\" starts with a camera verb (push in, pan, arc, handheld drift, hold).\n- \"sound\" = short sound cue; \"dialogue\" = spoken line in quotes or \"\".\n- \"prompt\" = the final assembled beat text in order: camera, then delta action, then any\n dialogue/sound — present tense, concrete on-screen verbs, no emotion labels. Ready to paste.\n" + }, + "_meta": { + "title": "LTX audio system prompt" + } + }, + "4": { + "class_type": "QwenVLImageJudge", + "inputs": { + "reference_image": [ + "1", + 0 + ], + "mode": "chat", + "profile": "general", + "model_select": "(manual — use model_path below)", + "model_path": "3.5-9b", + "precision": "bf16", + "max_new_tokens": 3072, + "temperature": 0.4, + "swap_eval": false, + "enable_thinking": false, + "json_output": true, + "keep_loaded": true, + "auto_download": true, + "report_dir": "/media/p5/Comfyui/output/ltx", + "run_tag": "ltx", + "generated_image": [ + "2", + 0 + ], + "system_prompt": [ + "3", + 0 + ], + "user_prompt": [ + "2", + 1 + ] + }, + "_meta": { + "title": "Judge (chat + json_output) -> LTX beats JSON" + } + } +} \ No newline at end of file