diff --git a/README.md b/README.md index 25fde3c..a5f83cf 100644 --- a/README.md +++ b/README.md @@ -152,13 +152,13 @@ Runs the super-resolution. Outputs: **`AUDIO`** (48 kHz) and **`IMAGE`** (spectr ### UniverSR Load Video Audio -Extracts a video's audio track (native rate/channels, via `ffmpeg`) and keeps a reference to the -source video for remuxing. Outputs **`AUDIO`** and **`UNIVERSR_VIDEO`**, and previews the video inline. +Upload or pick a video, extract its audio track (native rate/channels, via `ffmpeg`), and keep a +reference to the source video for remuxing. The clip **previews inline in the node** — with an upload +button and drag-and-drop, just like a normal video loader. Outputs **`UNIVERSR_VIDEO`** and **`AUDIO`**. | Input | Type | Default | Description | |---|---|---|---| -| `video_path` | string | `""` | Absolute path to a video. Takes priority over `video`. | -| `video` *(opt.)* | choice | — | Pick a file from ComfyUI's `input/` folder (used when `video_path` is empty). | +| `video` | upload / choice | — | Drop or upload a video, or pick one from ComfyUI's `input/` folder. | | `start_time` *(opt.)* | float | `0.0` | Trim start, seconds. | | `duration` *(opt.)* | float | `0.0` | Trim length, seconds (`0` = to end). | diff --git a/__init__.py b/__init__.py index 3ef44aa..edd2b59 100644 --- a/__init__.py +++ b/__init__.py @@ -17,4 +17,8 @@ try: except Exception as e: # video nodes are optional (need ffmpeg/soundfile) print(f"[ComfyUI-UniverSR] Failed to load video nodes: {e}") -__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] +# Serves web/js/UniverSRVideo.js — the inline video preview + upload widget. +# (./web + web/js/ + ../../../scripts imports mirrors the FoleyTune layout exactly.) +WEB_DIRECTORY = "./web" + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] diff --git a/example_workflows/universr_video.json b/example_workflows/universr_video.json index e111db1..426f8e5 100644 --- a/example_workflows/universr_video.json +++ b/example_workflows/universr_video.json @@ -12,11 +12,11 @@ "mode": 0, "inputs": [], "outputs": [ - {"name": "audio", "type": "AUDIO", "links": [1], "slot_index": 0}, - {"name": "video", "type": "UNIVERSR_VIDEO", "links": [2], "slot_index": 1} + {"name": "video", "type": "UNIVERSR_VIDEO", "links": [2], "slot_index": 0}, + {"name": "audio", "type": "AUDIO", "links": [1], "slot_index": 1} ], "properties": {"Node name for S&R": "UniverSRLoadVideoAudio"}, - "widgets_values": ["/path/to/video.mp4", "(none)", 0.0, 0.0] + "widgets_values": ["video.mp4", 0.0, 0.0] }, { "id": 2, @@ -72,8 +72,8 @@ } ], "links": [ - [1, 1, 0, 3, 0, "AUDIO"], - [2, 1, 1, 4, 0, "UNIVERSR_VIDEO"], + [1, 1, 1, 3, 0, "AUDIO"], + [2, 1, 0, 4, 0, "UNIVERSR_VIDEO"], [3, 2, 0, 3, 1, "UNIVERSR_MODEL"], [4, 3, 0, 4, 1, "AUDIO"] ], diff --git a/nodes_video.py b/nodes_video.py index 902235b..eb4bfb2 100644 --- a/nodes_video.py +++ b/nodes_video.py @@ -1,16 +1,18 @@ """Video helper nodes for ComfyUI-UniverSR. -Adapted from the HunyuanVideo-FoleyTune video loader/combiner, but trimmed to -what audio super-resolution needs: pull the audio track out of a video, run it -through the UniverSR sampler, then mux the enhanced track back onto the video. +Modelled directly on the HunyuanVideo-FoleyTune video loader/combiner (same +upload widget, drag-drop, and inline preview via web/js/UniverSRVideo.js), but +the loader outputs the video's **audio** alongside a video reference instead of +visual features — so you can super-resolve the audio and remux it back: - UniverSR Load Video Audio -> AUDIO + UNIVERSR_VIDEO (ffmpeg audio extract + preview) + UniverSR Load Video Audio -> UNIVERSR_VIDEO + AUDIO (ffmpeg audio extract + preview) UniverSR Video Combiner -> STRING (output path) (ffmpeg mux, no video re-encode) ffmpeg must be on PATH. Audio is read through a WAV pipe with soundfile, avoiding torchaudio's fragile torchcodec backend (same reasoning as the SR node). """ +import hashlib import io import os import re @@ -37,7 +39,7 @@ def _ffmpeg() -> str: if not exe: raise RuntimeError( "ffmpeg was not found on PATH. Install it (e.g. `apt install ffmpeg`, " - "`brew install ffmpeg`, or a conda/static build) to use the UniverSR video nodes." + "`brew install ffmpeg`, or `conda install -c conda-forge ffmpeg`) to use the video nodes." ) return exe @@ -85,45 +87,59 @@ def _write_temp_wav(audio: dict) -> str: return tmp +def _temp_preview_symlink(path: str) -> str: + """Link `path` into ComfyUI's temp/ dir for an inline preview; return the temp filename.""" + temp_dir = folder_paths.get_temp_directory() + os.makedirs(temp_dir, exist_ok=True) + ext = os.path.splitext(path)[1] or ".mp4" + name = f"universr_preview_{hashlib.md5(path.encode()).hexdigest()[:8]}{ext}" + dst = os.path.join(temp_dir, name) + if os.path.islink(dst) or os.path.exists(dst): + try: + os.unlink(dst) + except OSError: + pass + try: + os.symlink(os.path.abspath(path), dst) + except OSError: + shutil.copy(path, dst) # filesystems without symlink support + return name + + +def _list_input_videos() -> list: + if not HAS_FOLDER_PATHS: + return [] + try: + in_dir = folder_paths.get_input_directory() + return sorted( + f for f in os.listdir(in_dir) + if os.path.isfile(os.path.join(in_dir, f)) + and f.rsplit(".", 1)[-1].lower() in VIDEO_EXTENSIONS + ) + except Exception: + return [] + + # --------------------------------------------------------------------------- # -# Load Video Audio +# Load Video Audio (mirrors FoleyTuneVideoLoaderUpload; outputs video + audio) # --------------------------------------------------------------------------- # class UniverSRLoadVideoAudio: - """Extract a video's audio track and keep a reference to the source video. + """Upload/select a video, extract its audio track, and keep a reference to remux later. - Outputs AUDIO (feed it to UniverSR Super-Resolution) and UNIVERSR_VIDEO - (feed it, with the enhanced audio, to UniverSR Video Combiner). + Outputs the video reference (-> UniverSR Video Combiner) and the AUDIO + (-> UniverSR Super-Resolution). The video previews inline in the node. """ - DESCRIPTION = "Extract audio from a video for super-resolution, keeping a handle to remux later." + DESCRIPTION = "Load a video: outputs its audio (to super-resolve) and a reference (to remux)." CATEGORY = "audio/UniverSR" @classmethod def INPUT_TYPES(cls): - files = [] - if HAS_FOLDER_PATHS: - try: - in_dir = folder_paths.get_input_directory() - files = sorted( - f for f in os.listdir(in_dir) - if os.path.isfile(os.path.join(in_dir, f)) - and f.rsplit(".", 1)[-1].lower() in VIDEO_EXTENSIONS - ) - except Exception: - files = [] return { "required": { - "video_path": ("STRING", { - "default": "", - "placeholder": "/path/to/video.mp4 (or pick from 'video' below)", - "tooltip": "Absolute path to a video file. Takes priority over the 'video' dropdown.", - }), + "video": (_list_input_videos(), {"video_upload": True}), }, "optional": { - "video": (files or ["(none)"], { - "video_upload": True, - "tooltip": "Pick a video from the ComfyUI input/ folder (used when video_path is empty).", - }), "start_time": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360000.0, "step": 0.1, "tooltip": "Trim start in seconds."}), "duration": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360000.0, "step": 0.1, @@ -131,72 +147,49 @@ class UniverSRLoadVideoAudio: }, } - RETURN_TYPES = ("AUDIO", "UNIVERSR_VIDEO") - RETURN_NAMES = ("audio", "video") + RETURN_TYPES = ("UNIVERSR_VIDEO", "AUDIO") + RETURN_NAMES = ("video", "audio") FUNCTION = "load" OUTPUT_NODE = True - def _resolve_path(self, video_path, video): - video_path = (video_path or "").strip() - if video_path: - if not os.path.isfile(video_path): - raise FileNotFoundError(f"Video not found: {video_path}") - return os.path.abspath(video_path) - if video and video != "(none)" and HAS_FOLDER_PATHS: - return os.path.abspath(folder_paths.get_annotated_filepath(video)) - raise ValueError("No video given — set video_path or pick a file in 'video'.") + def load(self, video, start_time=0.0, duration=0.0): + video_path = folder_paths.get_annotated_filepath(video) + if not video_path or not os.path.isfile(video_path): + raise FileNotFoundError(f"Video not found: {video}") - def load(self, video_path, video="(none)", start_time=0.0, duration=0.0): - path = self._resolve_path(video_path, video) - waveform, sr = _extract_audio(path, start_time, duration) + waveform, sr = _extract_audio(video_path, start_time, duration) dur = waveform.shape[-1] / max(sr, 1) - print(f"[UniverSR] Loaded audio from {os.path.basename(path)}: " + print(f"[UniverSR] Loaded audio from {os.path.basename(video_path)}: " f"{waveform.shape[1]}ch @ {sr} Hz ({dur:.2f}s)") audio = {"waveform": waveform, "sample_rate": sr} - info = {"video_path": path, "start_time": float(start_time), "duration": float(duration), - "source_sr": sr, "source_channels": int(waveform.shape[1])} + info = {"video_path": os.path.abspath(video_path), "start_time": float(start_time), + "duration": float(duration), "source_sr": sr, "source_channels": int(waveform.shape[1])} - ui = self._preview(path) - return {"ui": ui, "result": (audio, info)} - - def _preview(self, path): - """Symlink (or copy) the source video into temp/ for an inline preview.""" - if not HAS_FOLDER_PATHS: - return {} - try: - import hashlib - temp_dir = folder_paths.get_temp_directory() - os.makedirs(temp_dir, exist_ok=True) - ext = os.path.splitext(path)[1] or ".mp4" - name = f"universr_preview_{hashlib.md5(path.encode()).hexdigest()[:8]}{ext}" - dst = os.path.join(temp_dir, name) - if os.path.islink(dst) or os.path.exists(dst): - os.unlink(dst) - try: - os.symlink(os.path.abspath(path), dst) - except OSError: - shutil.copy(path, dst) # filesystems without symlink support - return {"gifs": [{"filename": name, "subfolder": "", "type": "temp", - "format": f"video/{ext.lstrip('.')}"}]} - except Exception as e: - print(f"[UniverSR] Video preview skipped: {e}") - return {} + temp_name = _temp_preview_symlink(video_path) + ext = (os.path.splitext(video_path)[1] or ".mp4").lstrip(".") + return {"ui": {"gifs": [{"filename": temp_name, "subfolder": "", "type": "temp", + "format": f"video/{ext}"}]}, + "result": (info, audio)} @classmethod - def IS_CHANGED(cls, video_path, video="(none)", start_time=0.0, duration=0.0): + def IS_CHANGED(cls, video, start_time=0.0, duration=0.0): try: - p = (video_path or "").strip() - if not p and video and video != "(none)" and HAS_FOLDER_PATHS: - p = folder_paths.get_annotated_filepath(video) - mtime = os.path.getmtime(p) if p and os.path.isfile(p) else 0 + p = folder_paths.get_annotated_filepath(video) + m = os.path.getmtime(p) if os.path.isfile(p) else 0 except Exception: - mtime = 0 - return f"{video_path}:{video}:{start_time}:{duration}:{mtime}" + m = 0 + return f"{video}:{start_time}:{duration}:{m}" + + @classmethod + def VALIDATE_INPUTS(cls, video, **kwargs): + if not folder_paths.exists_annotated_filepath(video): + return f"Invalid video file: {video}" + return True # --------------------------------------------------------------------------- # -# Video Combiner +# Video Combiner (mirrors FoleyTuneVideoCombiner) # --------------------------------------------------------------------------- # class UniverSRVideoCombiner: """Mux audio onto the source video (no video re-encode) and save the result.""" diff --git a/web/js/UniverSRVideo.js b/web/js/UniverSRVideo.js new file mode 100644 index 0000000..abce023 --- /dev/null +++ b/web/js/UniverSRVideo.js @@ -0,0 +1,157 @@ +import { app } from "../../../scripts/app.js"; +import { api } from "../../../scripts/api.js"; + +// Inline video preview + upload widget for the UniverSR video nodes. +// Adapted from HunyuanVideo-FoleyTune's FoleyTuneVideo.js. + +const VIDEO_EXTENSIONS = ["webm", "mp4", "mkv", "gif", "mov", "avi", "flv", "wmv", "m4v", "mpg", "mpeg", "ts"]; + +function fitHeight(node) { + node.setSize([node.size[0], node.computeSize([node.size[0], node.size[1]])[1]]); + node?.graph?.setDirtyCanvas(true); +} + +function addVideoPreview(nodeType) { + const onNodeCreated = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function () { + onNodeCreated?.apply(this, arguments); + + const node = this; + const container = document.createElement("div"); + container.style.width = "100%"; + + const videoEl = document.createElement("video"); + videoEl.controls = true; + videoEl.loop = true; + videoEl.muted = true; + videoEl.style.width = "100%"; + videoEl.onmouseenter = () => { videoEl.muted = false; }; + videoEl.onmouseleave = () => { videoEl.muted = true; }; + container.appendChild(videoEl); + + const previewWidget = this.addDOMWidget("videopreview", "preview", container, { + serialize: false, + hideOnZoom: false, + getValue() { return container.value; }, + setValue(v) { container.value = v; }, + }); + + previewWidget.videoEl = videoEl; + previewWidget.aspectRatio = null; + + previewWidget.computeSize = function (width) { + if (this.aspectRatio && !container.hidden) { + const height = (node.size[0] - 20) / this.aspectRatio + 10; + return [width, Math.max(height, 0)]; + } + return [width, -4]; + }; + + videoEl.addEventListener("loadedmetadata", () => { + previewWidget.aspectRatio = videoEl.videoWidth / videoEl.videoHeight; + container.hidden = false; + fitHeight(node); + }); + + videoEl.addEventListener("error", () => { + container.hidden = true; + fitHeight(node); + }); + + node._universrVideoPreview = previewWidget; + + const onExecuted = node.onExecuted; + node.onExecuted = function (output) { + onExecuted?.apply(this, arguments); + if (output?.gifs?.[0]) { + const g = output.gifs[0]; + const params = new URLSearchParams({ + filename: g.filename, + type: g.type || "temp", + subfolder: g.subfolder || "", + }); + videoEl.src = api.apiURL("/view?" + params.toString()); + } + }; + }; +} + +function addUploadWidget(nodeType) { + const onNodeCreated = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function () { + onNodeCreated?.apply(this, arguments); + + const node = this; + const pathWidget = this.widgets.find((w) => w.name === "video"); + if (!pathWidget) return; + + const fileInput = document.createElement("input"); + fileInput.type = "file"; + fileInput.accept = "video/*,image/gif"; + fileInput.style.display = "none"; + document.body.appendChild(fileInput); + + async function uploadFile(file) { + const body = new FormData(); + body.append("image", file); + body.append("overwrite", "true"); + const resp = await api.fetchApi("/upload/image", { method: "POST", body }); + if (resp.ok) { + const data = await resp.json(); + if (!pathWidget.options.values.includes(data.name)) { + pathWidget.options.values.push(data.name); + } + pathWidget.value = data.name; + pathWidget.callback?.(data.name); + } + } + + fileInput.onchange = () => { + if (fileInput.files.length) uploadFile(fileInput.files[0]); + }; + + const uploadWidget = this.addWidget("button", "choose video to upload", null, () => { + fileInput.click(); + }); + uploadWidget.serialize = false; + + this.onDragOver = (e) => !!e?.dataTransfer?.types?.includes?.("Files"); + this.onDragDrop = async (e) => { + const file = e?.dataTransfer?.files?.[0]; + if (!file) return false; + const ext = file.name.split(".").pop()?.toLowerCase(); + if (!VIDEO_EXTENSIONS.includes(ext)) return false; + await uploadFile(file); + return true; + }; + + function showPreview(filename) { + if (!filename) return; + const pw = node._universrVideoPreview; + if (!pw) return; + const params = new URLSearchParams({ filename, type: "input", subfolder: "" }); + pw.videoEl.src = api.apiURL("/view?" + params.toString()); + } + + const origCallback = pathWidget.callback; + pathWidget.callback = function (value) { + origCallback?.apply(this, arguments); + showPreview(value); + }; + + requestAnimationFrame(() => showPreview(pathWidget.value)); + }; +} + +app.registerExtension({ + name: "UniverSR.VideoNodes", + async beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData?.name === "UniverSRLoadVideoAudio") { + addVideoPreview(nodeType); + addUploadWidget(nodeType); + } + if (nodeData?.name === "UniverSRVideoCombiner") { + addVideoPreview(nodeType); + } + }, +});