"""ComfyUI inference adapter for the official SPEED runtime.""" from __future__ import annotations from contextlib import nullcontext import importlib import logging from pathlib import Path import sys import types import torch logger = logging.getLogger("Tween") _SPEED_NAMESPACE = "_tween_speed_upstream" def _cuda_bf16_supported(device: torch.device) -> bool: if device.type != "cuda": return False with torch.cuda.device(device): return torch.cuda.is_bf16_supported() def _namespace_package(name: str, path: Path) -> None: if name in sys.modules: return package = types.ModuleType(name) package.__path__ = [str(path)] package.__package__ = name sys.modules[name] = package def load_speed_model_class(source_root: str): """Import SpeedDiT without adding the upstream repository to sys.path.""" root = Path(source_root).resolve() model_file = root / "src" / "models" / "model.py" if not model_file.is_file(): raise RuntimeError(f"Invalid SPEED source directory: missing {model_file}") _namespace_package(_SPEED_NAMESPACE, root) _namespace_package(f"{_SPEED_NAMESPACE}.src", root / "src") _namespace_package(f"{_SPEED_NAMESPACE}.src.models", root / "src" / "models") module = importlib.import_module(f"{_SPEED_NAMESPACE}.src.models.model") return module.SpeedDiT class SpeedVFIModel: """Midpoint interpolation wrapper around the official SPEED SpeedDiT.""" def __init__(self, checkpoint_path: str, source_root: str, precision: str = "auto", device: str = "cpu"): SpeedDiT = load_speed_model_class(source_root) self.model = SpeedDiT( hidden_dim=768, head_dim=64, depths=(2, 6, 4), patch_sizes=(64, 32, 16), ) self.precision = precision self.device = str(device) self._seed = 0 self._generator = None self._generator_device = None self._load_checkpoint(checkpoint_path) self.model.requires_grad_(False).eval() self.to(device) def _load_checkpoint(self, checkpoint_path: str) -> None: checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) state_dict = checkpoint.get("model", checkpoint) if isinstance(checkpoint, dict) else checkpoint if not isinstance(state_dict, dict): raise TypeError(f"SPEED checkpoint does not contain a state dict: {checkpoint_path}") if state_dict and all(key.startswith("module.") for key in state_dict): state_dict = {key[len("module."):]: value for key, value in state_dict.items()} self.model.load_state_dict(state_dict, strict=True) def _autocast_dtype(self, device: torch.device): if self.precision == "fp32" or device.type != "cuda": return None if self.precision == "fp16": return torch.float16 if self.precision == "bf16": if not _cuda_bf16_supported(device): raise RuntimeError( "SPEED BF16 precision requires a CUDA GPU with BF16 support; " "select auto or fp16 on this GPU" ) return torch.bfloat16 if _cuda_bf16_supported(device): return torch.bfloat16 return torch.float16 def to(self, device): target = torch.device(device) self.device = str(target) # Match the official runtime: retain FP32 weights and use autocast for # CUDA inference. This also avoids mixed-dtype timestep embedding bugs. self.model.to(device=target, dtype=torch.float32) # Keep the generator alive across CPU offloading. Recreating it on every # pair batch would restart the noise stream whenever keep_device=False. # _get_generator replaces it automatically if inference changes device. return self def set_seed(self, seed: int) -> None: self._seed = int(seed) self._generator = None self._generator_device = None def reset_seed(self) -> None: self.set_seed(self._seed) def clear_cache(self) -> None: rope = getattr(self.model, "rope_embedder", None) cache = getattr(rope, "rope_cache", None) if cache is not None: cache.clear() def _get_generator(self, device: torch.device) -> torch.Generator: device_name = str(device) if self._generator is None or self._generator_device != device_name: self._generator = torch.Generator(device=device) self._generator.manual_seed(self._seed) self._generator_device = device_name return self._generator @torch.no_grad() def interpolate_batch(self, frames0, frames1, time_step=0.5): if abs(float(time_step) - 0.5) > 1e-6: raise ValueError("SPEED's released checkpoint supports midpoint interpolation only") device = next(self.model.parameters()).device frame0 = frames0.to(device=device, dtype=torch.float32, non_blocking=True).mul(2).sub(1) frame1 = frames1.to(device=device, dtype=torch.float32, non_blocking=True).mul(2).sub(1) cond_frames = torch.cat((frame0, frame1), dim=0) noisy_frames = torch.randn( frame0.shape, generator=self._get_generator(device), device=device, dtype=torch.float32, ) timestep = torch.full( (frame0.shape[0],), 1000.0, device=device, dtype=torch.float32 ) autocast_dtype = self._autocast_dtype(device) autocast = ( torch.autocast(device_type="cuda", dtype=autocast_dtype) if autocast_dtype is not None else nullcontext() ) with autocast: prediction = self.model( noisy_frames=noisy_frames, cond_frames=cond_frames, timestep=timestep, ) return prediction.div(2).add(0.5).clamp_(0, 1).float()