feat: integrate training UI, BEATs model, and clean up legacy code
- Remove legacy distance-mode scanning (build_profile, _similarity, etc.) and hand-crafted intensity features — pipeline is now embedding-only - Integrate Microsoft BEATs as embedding option alongside wav2vec2/HuBERT - Add TrainDialog with positive class selector, model picker, video dir fallback, and live training stats - Add TrainWorker QThread with cancel support and proper lifecycle cleanup - Add source_path column to DB for robust source video tracking - Add get_export_folders/get_training_data/get_training_stats to DB - Wire source_path in all export DB writes (_on_clip_done, _on_auto_clip_done) - Cancel scan/train workers in closeEvent to prevent use-after-free crashes - Add setup_env.sh supporting both conda and python venv (CUDA 12.8) - Update requirements.txt with all actual dependencies - Update 8cut_train.py with --positive flag for new DB-driven training Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+347
-128
@@ -1,105 +1,359 @@
|
||||
"""Audio similarity scanning — MFCC + spectral contrast profile matching."""
|
||||
"""Audio scanning — embedding-based classifier for audio event detection."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import numpy as np
|
||||
import librosa
|
||||
|
||||
from .paths import _log
|
||||
|
||||
_N_MFCC = 13 # coefficients 0-12; we drop C0 → 12 usable
|
||||
_SR = 16000 # lower sr = faster, no quality loss for style matching
|
||||
_HOP_LENGTH = 1024 # STFT hop (~64ms frames at 16kHz)
|
||||
_N_FFT = 2048 # STFT window
|
||||
_SR = 16000 # lower sr = faster
|
||||
_WINDOW = 8.0 # seconds
|
||||
_N_FEATURES = 62 # (12 mfcc + 12 delta + 7 sc) * 2 (mean + std)
|
||||
_MODEL_DIR = os.path.join(os.path.expanduser("~"), ".8cut_models")
|
||||
_W2V_CACHE_DIR = os.path.join(os.path.expanduser("~"), ".8cut_cache", "w2v")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Embedding extraction (lazy-loaded)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_w2v_model = None
|
||||
_w2v_device = None
|
||||
_w2v_model_name = None
|
||||
|
||||
# Supported embedding models — name → embed_dim
|
||||
_EMBED_MODELS = {
|
||||
"WAV2VEC2_BASE": 768,
|
||||
"WAV2VEC2_LARGE": 1024,
|
||||
"WAV2VEC2_LARGE_LV60K":1024,
|
||||
"HUBERT_BASE": 768,
|
||||
"HUBERT_LARGE": 1024,
|
||||
"HUBERT_XLARGE": 1280,
|
||||
"BEATS": 768,
|
||||
}
|
||||
_DEFAULT_EMBED_MODEL = "WAV2VEC2_BASE"
|
||||
|
||||
_BEATS_CHECKPOINT = os.path.join(
|
||||
os.path.expanduser("~"), ".cache", "huggingface", "hub",
|
||||
"models--lpepino--beats_ckpts", "snapshots",
|
||||
"5b53b0404df452a3a607d7e67687227730e5bad1", "BEATs_iter3_plus_AS2M.pt",
|
||||
)
|
||||
|
||||
|
||||
def _extract_features_from_signal(y: np.ndarray, sr: int = _SR) -> np.ndarray:
|
||||
"""Compute feature matrix (31 x T) from a raw audio signal.
|
||||
def _get_w2v_model(model_name: str | None = None):
|
||||
"""Lazy-load an embedding model. Reloads if model_name differs from cached."""
|
||||
global _w2v_model, _w2v_device, _w2v_model_name
|
||||
if model_name is None:
|
||||
model_name = _DEFAULT_EMBED_MODEL
|
||||
if _w2v_model is None or _w2v_model_name != model_name:
|
||||
import torch
|
||||
_w2v_device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
Features per frame: 12 MFCCs (skip C0) + 12 delta MFCCs + 7 spectral contrast.
|
||||
if model_name == "BEATS":
|
||||
from .beats_model import BEATs, BEATsConfig
|
||||
checkpoint = torch.load(_BEATS_CHECKPOINT, map_location=_w2v_device,
|
||||
weights_only=False)
|
||||
cfg = BEATsConfig(checkpoint['cfg'])
|
||||
_w2v_model = BEATs(cfg)
|
||||
_w2v_model.load_state_dict(checkpoint['model'])
|
||||
_w2v_model.to(_w2v_device)
|
||||
else:
|
||||
import torchaudio
|
||||
bundle = getattr(torchaudio.pipelines, model_name)
|
||||
_w2v_model = bundle.get_model().to(_w2v_device)
|
||||
|
||||
_w2v_model.eval()
|
||||
_w2v_model_name = model_name
|
||||
_log(f"audio_scan: {model_name} loaded on {_w2v_device}")
|
||||
return _w2v_model, _w2v_device
|
||||
|
||||
|
||||
def _embed_dim(model_name: str | None = None) -> int:
|
||||
"""Return embedding dimension for a model name."""
|
||||
if model_name is None:
|
||||
model_name = _DEFAULT_EMBED_MODEL
|
||||
return _EMBED_MODELS.get(model_name, 768)
|
||||
|
||||
|
||||
def _w2v_cache_path(video_path: str, hop: float, window: float,
|
||||
model_name: str | None = None) -> str:
|
||||
"""Return cache file path for a video's embeddings (includes model name)."""
|
||||
if model_name is None:
|
||||
model_name = _DEFAULT_EMBED_MODEL
|
||||
abspath = os.path.abspath(video_path)
|
||||
mtime = os.path.getmtime(abspath)
|
||||
key = f"{abspath}|{mtime}|{hop}|{window}|{model_name}"
|
||||
h = hashlib.sha256(key.encode()).hexdigest()[:16]
|
||||
return os.path.join(_W2V_CACHE_DIR, f"{h}.npz")
|
||||
|
||||
|
||||
def _extract_w2v_windows(y: np.ndarray, sr: int = _SR,
|
||||
hop: float = 1.0, window: float = _WINDOW,
|
||||
video_path: str | None = None,
|
||||
cancel_flag: object = None,
|
||||
model_name: str | None = None,
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Extract embeddings for all sliding windows using a torchaudio model.
|
||||
|
||||
If video_path is given, results are cached to disk for fast re-scans.
|
||||
Returns (timestamps, embeddings) where embeddings is (N, D).
|
||||
"""
|
||||
S = np.abs(librosa.stft(y, n_fft=_N_FFT, hop_length=_HOP_LENGTH)) ** 2
|
||||
mel_S = librosa.feature.melspectrogram(S=S, sr=sr, hop_length=_HOP_LENGTH)
|
||||
mfcc = librosa.feature.mfcc(S=librosa.power_to_db(mel_S), sr=sr, n_mfcc=_N_MFCC)
|
||||
mfcc = mfcc[1:] # drop C0 (energy) — dominates cosine sim, kills discrimination
|
||||
delta = librosa.feature.delta(mfcc)
|
||||
sc = librosa.feature.spectral_contrast(S=S, sr=sr, hop_length=_HOP_LENGTH)
|
||||
return np.vstack([mfcc, delta, sc]) # (31, T)
|
||||
edim = _embed_dim(model_name)
|
||||
|
||||
|
||||
def _aggregate(feature_matrix: np.ndarray) -> np.ndarray:
|
||||
"""Collapse a (31, T) feature matrix into a (62,) vector via mean + std."""
|
||||
return np.concatenate([
|
||||
feature_matrix.mean(axis=1),
|
||||
feature_matrix.std(axis=1),
|
||||
])
|
||||
|
||||
|
||||
def _extract_features(path: str, sr: int = _SR) -> np.ndarray:
|
||||
"""Load audio from a file and return a 62-dim feature vector."""
|
||||
y, _ = librosa.load(path, sr=sr, mono=True)
|
||||
feat = _extract_features_from_signal(y, sr)
|
||||
return _aggregate(feat)
|
||||
|
||||
|
||||
def build_profile(clip_paths: list[str]) -> dict | None:
|
||||
"""Extract features from reference clips.
|
||||
|
||||
Returns dict with:
|
||||
- mean_vector: averaged feature vector across all clips (62,)
|
||||
- clip_vectors: list of individual feature vectors
|
||||
Returns None if no clips could be loaded.
|
||||
"""
|
||||
vectors = []
|
||||
for p in clip_paths:
|
||||
# Try loading from cache
|
||||
cache_file = None
|
||||
if video_path:
|
||||
try:
|
||||
vec = _extract_features(p)
|
||||
vectors.append(vec)
|
||||
cache_file = _w2v_cache_path(video_path, hop, window, model_name)
|
||||
if os.path.exists(cache_file):
|
||||
data = np.load(cache_file)
|
||||
_log(f"audio_scan: cache hit ({cache_file})")
|
||||
return data["timestamps"], data["embeddings"]
|
||||
except Exception as e:
|
||||
_log(f"audio_scan: skip {p}: {e}")
|
||||
if not vectors:
|
||||
return None
|
||||
arr = np.stack(vectors)
|
||||
return {
|
||||
"mean_vector": arr.mean(axis=0),
|
||||
"clip_vectors": vectors,
|
||||
}
|
||||
_log(f"audio_scan: cache read failed: {e}")
|
||||
|
||||
win_samples = int(window * sr)
|
||||
hop_samples = int(hop * sr)
|
||||
n_windows = max(0, (len(y) - win_samples) // hop_samples + 1)
|
||||
|
||||
if n_windows == 0:
|
||||
return np.array([]), np.empty((0, edim))
|
||||
|
||||
import torch
|
||||
model, device = _get_w2v_model(model_name)
|
||||
is_beats = (model_name or _DEFAULT_EMBED_MODEL) == "BEATS"
|
||||
batch_size = 16
|
||||
timestamps = np.arange(n_windows) * hop
|
||||
embeddings = []
|
||||
|
||||
for batch_start in range(0, n_windows, batch_size):
|
||||
if cancel_flag and getattr(cancel_flag, '_cancel', False):
|
||||
return np.array([]), np.empty((0, edim))
|
||||
batch_end = min(batch_start + batch_size, n_windows)
|
||||
chunks = []
|
||||
for i in range(batch_start, batch_end):
|
||||
start = i * hop_samples
|
||||
chunks.append(y[start:start + win_samples])
|
||||
with torch.no_grad():
|
||||
waveforms = torch.from_numpy(np.stack(chunks)).float().to(device)
|
||||
if is_beats:
|
||||
padding_mask = torch.zeros_like(waveforms, dtype=torch.bool)
|
||||
features, _ = model.extract_features(waveforms, padding_mask=padding_mask)
|
||||
else:
|
||||
features, _ = model(waveforms)
|
||||
batch_emb = features.mean(dim=1).cpu().numpy()
|
||||
embeddings.append(batch_emb)
|
||||
|
||||
result_ts = timestamps
|
||||
result_emb = np.vstack(embeddings)
|
||||
|
||||
# Save to cache
|
||||
if cache_file:
|
||||
try:
|
||||
os.makedirs(_W2V_CACHE_DIR, exist_ok=True)
|
||||
np.savez(cache_file, timestamps=result_ts, embeddings=result_emb)
|
||||
_log(f"audio_scan: w2v cache saved ({cache_file})")
|
||||
except Exception as e:
|
||||
_log(f"audio_scan: cache write failed: {e}")
|
||||
|
||||
return result_ts, result_emb
|
||||
|
||||
|
||||
def _similarity(a: np.ndarray, b: np.ndarray) -> float:
|
||||
"""Euclidean-distance-based similarity in (0, 1].
|
||||
def _extract_w2v_targeted(y: np.ndarray, sr: int, gt_intense: list[float],
|
||||
gt_soft: list[float], tolerance: float = 12.0,
|
||||
neg_margin: float = 120.0,
|
||||
model_name: str | None = None,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""Extract embeddings only near positives and distant negatives.
|
||||
|
||||
1/(1+dist): identical → 1.0, very different → near 0.
|
||||
Returns (timestamps, embeddings, labels) where labels: 1=pos, -1=neg, 0=ambig.
|
||||
"""
|
||||
return float(1.0 / (1.0 + np.linalg.norm(a - b)))
|
||||
edim = _embed_dim(model_name)
|
||||
duration = len(y) / sr
|
||||
win_samples = int(_WINDOW * sr)
|
||||
all_gt = list(gt_intense) + list(gt_soft)
|
||||
|
||||
# Positive windows: every second near intense markers
|
||||
pos_times = set()
|
||||
for gt in gt_intense:
|
||||
for offset in range(-int(tolerance), int(tolerance) + 1):
|
||||
t = gt + offset
|
||||
if 0 <= t <= duration - _WINDOW:
|
||||
pos_times.add(int(t))
|
||||
|
||||
# Negative windows: every 4s, far from any marker
|
||||
neg_times = set()
|
||||
for t in range(0, int(duration - _WINDOW), 4):
|
||||
if min((abs(t - g) for g in all_gt), default=9999) > neg_margin:
|
||||
neg_times.add(t)
|
||||
|
||||
all_times = sorted(pos_times | neg_times)
|
||||
# Filter out windows that go past the end
|
||||
valid_times = [t for t in all_times if int(t * sr) + win_samples <= len(y)]
|
||||
|
||||
if not valid_times:
|
||||
return np.array([]), np.zeros((0, edim)), np.array([], dtype=int)
|
||||
|
||||
import torch
|
||||
model, device = _get_w2v_model(model_name)
|
||||
batch_size = 16
|
||||
timestamps_list: list[float] = []
|
||||
embeddings_list: list[np.ndarray] = []
|
||||
|
||||
is_beats = (model_name or _DEFAULT_EMBED_MODEL) == "BEATS"
|
||||
|
||||
for batch_start in range(0, len(valid_times), batch_size):
|
||||
batch_end = min(batch_start + batch_size, len(valid_times))
|
||||
chunks = []
|
||||
for t in valid_times[batch_start:batch_end]:
|
||||
start = int(t * sr)
|
||||
chunks.append(y[start:start + win_samples])
|
||||
timestamps_list.append(float(t))
|
||||
with torch.no_grad():
|
||||
waveforms = torch.from_numpy(np.stack(chunks)).float().to(device)
|
||||
if is_beats:
|
||||
padding_mask = torch.zeros_like(waveforms, dtype=torch.bool)
|
||||
features, _ = model.extract_features(waveforms, padding_mask=padding_mask)
|
||||
else:
|
||||
features, _ = model(waveforms)
|
||||
batch_emb = features.mean(dim=1).cpu().numpy()
|
||||
embeddings_list.append(batch_emb)
|
||||
|
||||
timestamps = np.array(timestamps_list)
|
||||
embeddings = np.vstack(embeddings_list)
|
||||
|
||||
labels = np.zeros(len(timestamps), dtype=int)
|
||||
for i, t in enumerate(timestamps):
|
||||
di = min((abs(t - g) for g in gt_intense), default=9999)
|
||||
da = min((abs(t - g) for g in all_gt), default=9999)
|
||||
if di < tolerance:
|
||||
labels[i] = 1
|
||||
elif da > neg_margin:
|
||||
labels[i] = -1
|
||||
return timestamps, embeddings, labels
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Classifier mode — train / save / load / scan
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def train_classifier(video_infos: list[tuple[str, list[float], list[float]]],
|
||||
model_path: str | None = None,
|
||||
tolerance: float = 12.0,
|
||||
neg_margin: float = 120.0,
|
||||
embed_model: str | None = None) -> dict:
|
||||
"""Train a classifier from labeled videos.
|
||||
|
||||
Args:
|
||||
video_infos: list of (video_path, intense_times, soft_times)
|
||||
model_path: if given, save model to this path
|
||||
tolerance/neg_margin: labeling parameters
|
||||
embed_model: embedding model name (e.g. "HUBERT_BASE", "BEATS"), defaults to WAV2VEC2_BASE
|
||||
|
||||
Returns:
|
||||
dict with 'classifier', 'embed_model', and metadata, or None on failure.
|
||||
"""
|
||||
from sklearn.ensemble import GradientBoostingClassifier
|
||||
|
||||
all_X, all_y = [], []
|
||||
|
||||
for vi, (vpath, gt_intense, gt_soft) in enumerate(video_infos):
|
||||
_log(f"audio_scan: training [{vi+1}/{len(video_infos)}] {os.path.basename(vpath)}")
|
||||
y, _ = librosa.load(vpath, sr=_SR, mono=True)
|
||||
|
||||
timestamps, embeddings, labels = _extract_w2v_targeted(
|
||||
y, _SR, gt_intense, gt_soft, tolerance, neg_margin,
|
||||
model_name=embed_model,
|
||||
)
|
||||
if len(timestamps) == 0:
|
||||
continue
|
||||
# Per-video z-score normalize
|
||||
vid_mean = embeddings.mean(axis=0)
|
||||
vid_std = np.maximum(embeddings.std(axis=0), 1e-6)
|
||||
normed = (embeddings - vid_mean) / vid_std
|
||||
for i in range(len(labels)):
|
||||
if labels[i] == 1:
|
||||
all_X.append(normed[i])
|
||||
all_y.append(1)
|
||||
elif labels[i] == -1:
|
||||
all_X.append(normed[i])
|
||||
all_y.append(0)
|
||||
|
||||
if not all_X:
|
||||
_log("audio_scan: no training samples collected")
|
||||
return None
|
||||
|
||||
X = np.stack(all_X)
|
||||
y_arr = np.array(all_y)
|
||||
n_pos = (y_arr == 1).sum()
|
||||
n_neg = (y_arr == 0).sum()
|
||||
_log(f"audio_scan: training set — {n_pos} positive, {n_neg} negative")
|
||||
|
||||
if n_pos == 0 or n_neg == 0:
|
||||
_log(f"audio_scan: need both classes — {n_pos} pos, {n_neg} neg")
|
||||
return None
|
||||
|
||||
# Subsample negatives for balance
|
||||
rng = np.random.RandomState(42)
|
||||
pos_idx = np.where(y_arr == 1)[0]
|
||||
neg_idx = np.where(y_arr == 0)[0]
|
||||
n_neg_sample = min(len(neg_idx), len(pos_idx) * 3)
|
||||
neg_sample = rng.choice(neg_idx, n_neg_sample, replace=False)
|
||||
train_idx = np.concatenate([pos_idx, neg_sample])
|
||||
rng.shuffle(train_idx)
|
||||
|
||||
clf = GradientBoostingClassifier(
|
||||
n_estimators=200, max_depth=5, learning_rate=0.1, random_state=42,
|
||||
)
|
||||
clf.fit(X[train_idx], y_arr[train_idx])
|
||||
_log("audio_scan: classifier trained")
|
||||
|
||||
model = {"classifier": clf, "n_features": X.shape[1],
|
||||
"embed_model": embed_model or _DEFAULT_EMBED_MODEL}
|
||||
|
||||
if model_path:
|
||||
import joblib
|
||||
parent = os.path.dirname(model_path)
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
joblib.dump(model, model_path)
|
||||
_log(f"audio_scan: model saved to {model_path}")
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def load_classifier(model_path: str) -> dict | None:
|
||||
"""Load a saved classifier model."""
|
||||
if not os.path.exists(model_path):
|
||||
return None
|
||||
import joblib
|
||||
return joblib.load(model_path)
|
||||
|
||||
|
||||
def default_model_path(profile_name: str = "default") -> str:
|
||||
"""Return the default path for a profile's classifier model."""
|
||||
return os.path.join(_MODEL_DIR, f"{profile_name}.joblib")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scanning
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def scan_video(
|
||||
video_path: str,
|
||||
profile: dict,
|
||||
mode: str = "average",
|
||||
threshold: float = 0.05,
|
||||
model: dict = None,
|
||||
threshold: float = 0.30,
|
||||
hop: float = 1.0,
|
||||
window: float = _WINDOW,
|
||||
cancel_flag: object = None,
|
||||
) -> list[tuple[float, float, float]]:
|
||||
"""Slide a window across the video audio and score against the profile.
|
||||
"""Scan a video for matching audio regions using a trained classifier.
|
||||
|
||||
Pre-computes STFT once for the whole file, then uses vectorized
|
||||
cumulative-sum sliding window for speed.
|
||||
|
||||
Args:
|
||||
video_path: path to video/audio file
|
||||
profile: dict from build_profile()
|
||||
mode: "average" (compare to mean) or "nearest" (max over all clips)
|
||||
threshold: minimum similarity to include (0-1, default 0.05)
|
||||
hop: step size in seconds
|
||||
window: window size in seconds (default 8s)
|
||||
cancel_flag: object with _cancel bool attribute; checked periodically
|
||||
|
||||
Returns:
|
||||
list of (start_time, end_time, score) for regions above threshold
|
||||
Returns list of (start_time, end_time, score) above threshold.
|
||||
"""
|
||||
if model is None:
|
||||
_log("audio_scan: no model provided")
|
||||
return []
|
||||
|
||||
_log(f"audio_scan: loading {video_path}")
|
||||
y, sr = librosa.load(video_path, sr=_SR, mono=True)
|
||||
duration = len(y) / sr
|
||||
@@ -108,68 +362,33 @@ def scan_video(
|
||||
if cancel_flag and getattr(cancel_flag, '_cancel', False):
|
||||
return []
|
||||
|
||||
# Compute features for the entire file at once (one STFT)
|
||||
feat = _extract_features_from_signal(y, sr) # (31, T)
|
||||
n_feats, T = feat.shape
|
||||
fps = sr / _HOP_LENGTH # frames per second
|
||||
win_frames = int(window * fps)
|
||||
hop_frames = int(hop * fps)
|
||||
clf = model["classifier"]
|
||||
embed_model = model.get("embed_model")
|
||||
|
||||
if win_frames > T:
|
||||
_log(f"audio_scan: extracting embeddings ({embed_model or 'default'})...")
|
||||
timestamps, window_vectors = _extract_w2v_windows(
|
||||
y, sr, hop=hop, window=window, video_path=video_path,
|
||||
cancel_flag=cancel_flag, model_name=embed_model,
|
||||
)
|
||||
if len(timestamps) == 0:
|
||||
_log("audio_scan: video shorter than window")
|
||||
return []
|
||||
|
||||
_log(f"audio_scan: scanning {T} frames, win={win_frames}, hop={hop_frames}")
|
||||
# Per-video z-score normalize
|
||||
vid_mean = window_vectors.mean(axis=0)
|
||||
vid_std = np.maximum(window_vectors.std(axis=0), 1e-6)
|
||||
normed = (window_vectors - vid_mean) / vid_std
|
||||
|
||||
# Vectorized sliding window via cumulative sums
|
||||
cumsum = np.zeros((n_feats, T + 1))
|
||||
cumsum[:, 1:] = np.cumsum(feat, axis=1)
|
||||
cumsq = np.zeros((n_feats, T + 1))
|
||||
cumsq[:, 1:] = np.cumsum(feat ** 2, axis=1)
|
||||
|
||||
starts = np.arange(0, T - win_frames + 1, hop_frames)
|
||||
ends = starts + win_frames
|
||||
|
||||
sums = cumsum[:, ends] - cumsum[:, starts] # (31, n_windows)
|
||||
sq_sums = cumsq[:, ends] - cumsq[:, starts]
|
||||
means = sums / win_frames
|
||||
stds = np.sqrt(np.maximum(sq_sums / win_frames - means ** 2, 0) + 1e-10)
|
||||
|
||||
window_vectors = np.vstack([means, stds]).T # (n_windows, 62)
|
||||
_log(f"audio_scan: classifying {len(normed)} windows...")
|
||||
|
||||
if cancel_flag and getattr(cancel_flag, '_cancel', False):
|
||||
return []
|
||||
|
||||
# Score all windows
|
||||
if mode == "nearest":
|
||||
# Compare each window to every clip vector, take max
|
||||
clip_vecs = np.stack(profile["clip_vectors"]) # (n_clips, 62)
|
||||
results = []
|
||||
# Process in batches to check cancel_flag periodically
|
||||
batch = 500
|
||||
for i in range(0, len(window_vectors), batch):
|
||||
if cancel_flag and getattr(cancel_flag, '_cancel', False):
|
||||
_log("audio_scan: cancelled")
|
||||
return results
|
||||
chunk = window_vectors[i:i + batch]
|
||||
# cdist: (batch, n_clips) distances
|
||||
dists = np.linalg.norm(chunk[:, None, :] - clip_vecs[None, :, :], axis=2)
|
||||
scores = 1.0 / (1.0 + dists.min(axis=1)) # min dist = max similarity
|
||||
for j, score in enumerate(scores):
|
||||
if score >= threshold:
|
||||
idx = i + j
|
||||
start_t = starts[idx] / fps
|
||||
results.append((start_t, start_t + window, float(score)))
|
||||
else:
|
||||
# Average mode: compare to mean vector
|
||||
ref = profile["mean_vector"]
|
||||
dists = np.linalg.norm(window_vectors - ref, axis=1)
|
||||
scores = 1.0 / (1.0 + dists)
|
||||
mask = scores >= threshold
|
||||
results = [
|
||||
(starts[i] / fps, starts[i] / fps + window, float(scores[i]))
|
||||
for i in np.nonzero(mask)[0]
|
||||
]
|
||||
|
||||
probs = clf.predict_proba(normed)[:, 1]
|
||||
mask = probs >= threshold
|
||||
results = [
|
||||
(timestamps[i], timestamps[i] + window, float(probs[i]))
|
||||
for i in np.nonzero(mask)[0]
|
||||
]
|
||||
_log(f"audio_scan: {len(results)} regions above threshold {threshold}")
|
||||
return results
|
||||
|
||||
Reference in New Issue
Block a user