feat: add CivitAI stats fetch service

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

stats_service.py

@@ -0,0 +1,142 @@
+"""Service for fetching CivitAI stats and storing them in the local DB."""
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+import aiohttp
+
+try:
+    from .stats_db import StatsDB
+except ImportError:
+    from stats_db import StatsDB
+
+logger = logging.getLogger(__name__)
+
+_CIVITAI_API = "https://civitai.com/api/v1"
+_RATE_LIMIT_DELAY = 1.5  # seconds between requests
+
+
+def extract_version_stats(model_data: dict) -> list[tuple[str, dict]]:
+    """Extract (sha256, stats_dict) pairs from a CivitAI model response.
+
+    Each model has multiple versions; each version has files with hashes.
+    Returns one entry per version that has a SHA256 hash.
+    """
+    model_id = model_data.get("id")
+    results = []
+
+    for version in model_data.get("modelVersions", []):
+        version_id = version.get("id")
+        sha256 = None
+        for f in version.get("files", []):
+            sha256 = (f.get("hashes") or {}).get("SHA256")
+            if sha256:
+                break
+        if not sha256:
+            continue
+
+        stats = version.get("stats") or {}
+        results.append((
+            sha256.lower(),
+            {
+                "civitai_model_id": model_id,
+                "civitai_version_id": version_id,
+                "download_count": stats.get("downloadCount", 0),
+                "rating": stats.get("rating", 0),
+                "rating_count": stats.get("ratingCount", 0),
+                "thumbs_up_count": stats.get("thumbsUpCount", 0),
+            },
+        ))
+
+    return results
+
+
+class StatsFetchService:
+    """Fetches CivitAI stats for models and stores them in StatsDB."""
+
+    def __init__(self, db: StatsDB, api_key: str | None = None):
+        self.db = db
+        self._api_key = api_key
+        self._session: aiohttp.ClientSession | None = None
+
+    async def _get_session(self) -> aiohttp.ClientSession:
+        if self._session is None or self._session.closed:
+            headers = {}
+            if self._api_key:
+                headers["Authorization"] = f"Bearer {self._api_key}"
+            timeout = aiohttp.ClientTimeout(total=30)
+            self._session = aiohttp.ClientSession(
+                headers=headers, timeout=timeout
+            )
+        return self._session
+
+    async def _fetch_model(self, model_id: int) -> dict | None:
+        """Fetch a single model's data from CivitAI API."""
+        url = f"{_CIVITAI_API}/models/{model_id}"
+        session = await self._get_session()
+        try:
+            async with session.get(url) as resp:
+                if resp.status == 429:
+                    logger.warning("[LM-Remote] CivitAI rate limited, backing off")
+                    await asyncio.sleep(5)
+                    return None
+                if resp.status != 200:
+                    logger.debug("[LM-Remote] CivitAI returned %d for model %d",
+                                 resp.status, model_id)
+                    return None
+                return await resp.json()
+        except Exception as exc:
+            logger.warning("[LM-Remote] CivitAI fetch failed for model %d: %s",
+                           model_id, exc)
+            return None
+
+    async def fetch_stats_for_models(
+        self,
+        models: list[dict],
+        progress_callback: Any = None,
+    ) -> int:
+        """Fetch stats from CivitAI for a list of models.
+
+        Args:
+            models: list of dicts with 'sha256' and 'civitai_model_id' keys.
+            progress_callback: optional async callable(current, total).
+
+        Returns:
+            Number of model versions successfully updated.
+        """
+        # Deduplicate by model_id (multiple versions share the same model)
+        seen_model_ids: set[int] = set()
+        unique_models: list[dict] = []
+        for m in models:
+            mid = m.get("civitai_model_id")
+            if mid and mid not in seen_model_ids:
+                seen_model_ids.add(mid)
+                unique_models.append(m)
+
+        total = len(unique_models)
+        updated = 0
+
+        for i, model in enumerate(unique_models):
+            model_id = model["civitai_model_id"]
+            data = await self._fetch_model(model_id)
+            if data:
+                rows = extract_version_stats(data)
+                if rows:
+                    self.db.upsert_batch(rows)
+                    updated += len(rows)
+
+            if progress_callback:
+                await progress_callback(i + 1, total)
+
+            # Rate limiting between requests
+            if i < total - 1:
+                await asyncio.sleep(_RATE_LIMIT_DELAY)
+
+        return updated
+
+    async def close(self) -> None:
+        if self._session and not self._session.closed:
+            await self._session.close()
+            self._session = None