claude

CLAUDE.md

@@ -0,0 +1,63 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+**Turbo Sorter Pro v12.5** - A Python-based image management and sorting system with two web interfaces:
+- **Streamlit app** (port 8501): 5-tab workflow for image discovery, collision resolution, archive management, categorization, and gallery staging
+- **NiceGUI app** (port 8080): Real-time image tagging interface with hotkey support and batch operations
+
+## Running the Applications
+
+```bash
+# Install dependencies
+pip install -r requirements.txt
+
+# Run Streamlit interface
+streamlit run app.py --server.port=8501 --server.address=0.0.0.0
+
+# Run NiceGUI gallery interface
+python3 gallery_app.py
+
+# Run both (Docker production mode)
+./start.sh
+```
+
+## Architecture
+
+### Core Components
+
+- **`engine.py`** - `SorterEngine` class with 40+ static methods for all business logic. Central SQLite-based state management at `/app/sorter_database.db`. Handles profile management, image operations, staging, batch processing, and undo history.
+
+- **`app.py`** - Streamlit entry point. Initializes database, manages session state, renders 5-tab interface.
+
+- **`gallery_app.py`** - NiceGUI entry point with `AppState` class. Provides async image serving via FastAPI, hotkey-based tagging, and batch copy/move operations.
+
+### Streamlit Tab Modules
+
+| Tab | Module | Purpose |
+|-----|--------|---------|
+| 1. Discovery | `tab_time_discovery.py` | Time-sync matcher for sibling folders |
+| 2. ID Review | `tab_id_review.py` | Collision detection and ID harmonization |
+| 3. Unused | `tab_unused_review.py` | Archive review and restoration |
+| 4. Category Sorter | `tab_category_sorter.py` | Bulk categorization and renaming |
+| 5. Gallery Staged | `tab_gallery_sorter.py` | Interactive tagging interface |
+
+### Database Schema (SQLite)
+
+Key tables:
+- `profiles` - Workspace configurations with tab path mappings
+- `folder_ids` - Persistent folder identifiers
+- `staging_area` - Pending file operations
+- `processed_log` - Action history for undo
+- `folder_tags` - Per-folder image tags with metadata
+- `profile_categories` - Profile-specific category lists
+
+### Key Patterns
+
+- **Profile-based multi-tenancy**: Each workspace has isolated path configurations
+- **Soft deletes**: Files moved to `_DELETED` folder for undo support
+- **Parallel image loading**: `ThreadPoolExecutor` in `load_batch_parallel()`
+- **Session state**: Streamlit `st.session_state` for tab indices and history
+- **WebP compression**: PIL-based with configurable quality slider

gallery_app.py

@@ -1,5 +1,6 @@
 import os
 import math
+import shutil
 import asyncio
 from typing import Optional, List, Dict, Set
 from nicegui import ui, app, run
@@ -32,6 +33,7 @@ class AppState:
         self.next_index = 1
         self.hovered_image = None  # Track currently hovered image for keyboard shortcuts
         self.category_hotkeys: Dict[str, str] = {}  # Maps hotkey -> category name
+        self.hotkey_by_category: Dict[str, str] = {}  # Reverse mapping: category -> hotkey
         
         # Undo Stack
         self.undo_stack: List[Dict] = []  # Stores last actions for undo
@@ -42,17 +44,26 @@ class AppState:
         # Batch Settings
         self.batch_mode = "Copy"
         self.cleanup_mode = "Keep"
+        self.applying_global = False  # Loading state for global apply
         
         # Data Caches
         self.all_images: List[str] = []
         self.staged_data: Dict = {}
         self.green_dots: Set[int] = set()
         self.index_map: Dict[int, str] = {}
+        self._cached_tagged: Set[str] = set()  # Cached set of tagged image paths
+        self._cached_untagged: Set[str] = set()  # Cached set of untagged image paths
+        self._committed_files: Dict[str, Set[str]] = {}  # category -> set of filenames on disk
         
         # UI Containers (populated later)
         self.sidebar_container = None
         self.grid_container = None
         self.pagination_container = None
+        # Sub-containers for partial refresh
+        self.number_grid_container = None
+        self.category_list_container = None
+        self.index_display_container = None
+        self.stats_container = None
 
     def load_active_profile(self):
         """Load paths from active profile."""
@@ -92,13 +103,14 @@ class AppState:
         return cats
 
     def get_filtered_images(self) -> List[str]:
-        """Get images based on current filter mode."""
+        """Get images based on current filter mode using cached sets."""
         if self.filter_mode == "all":
             return self.all_images
         elif self.filter_mode == "tagged":
-            return [img for img in self.all_images if img in self.staged_data]
+            # Use cached set for O(1) lookups
+            return [img for img in self.all_images if img in self._cached_tagged]
         elif self.filter_mode == "untagged":
-            return [img for img in self.all_images if img not in self.staged_data]
+            return [img for img in self.all_images if img in self._cached_untagged]
         return self.all_images
 
     @property
@@ -116,9 +128,9 @@ class AppState:
         return filtered[start : start + self.page_size]
     
     def get_stats(self) -> Dict:
-        """Get image statistics for display."""
+        """Get image statistics for display using cached counts."""
         total = len(self.all_images)
-        tagged = len([img for img in self.all_images if img in self.staged_data])
+        tagged = len(self._cached_tagged)
         return {"total": total, "tagged": tagged, "untagged": total - tagged}
 
 state = AppState()
@@ -129,19 +141,31 @@ state = AppState()
 
 @app.get('/thumbnail')
 async def get_thumbnail(path: str, size: int = 400, q: int = 50):
-    """Serve WebP thumbnail with dynamic quality."""
+    """Serve WebP thumbnail with dynamic quality and caching."""
     if not os.path.exists(path):
         return Response(status_code=404)
     img_bytes = await run.cpu_bound(SorterEngine.compress_for_web, path, q, size)
-    return Response(content=img_bytes, media_type="image/webp") if img_bytes else Response(status_code=500)
+    if img_bytes:
+        return Response(
+            content=img_bytes,
+            media_type="image/webp",
+            headers={"Cache-Control": "max-age=86400, immutable"}
+        )
+    return Response(status_code=500)
 
 @app.get('/full_res')
 async def get_full_res(path: str):
-    """Serve full resolution image."""
+    """Serve full resolution image with caching."""
     if not os.path.exists(path):
         return Response(status_code=404)
     img_bytes = await run.cpu_bound(SorterEngine.compress_for_web, path, 90, None)
-    return Response(content=img_bytes, media_type="image/webp") if img_bytes else Response(status_code=500)
+    if img_bytes:
+        return Response(
+            content=img_bytes,
+            media_type="image/webp",
+            headers={"Cache-Control": "max-age=86400, immutable"}
+        )
+    return Response(status_code=500)
 
 # ==========================================
 # CORE LOGIC
@@ -162,6 +186,9 @@ def load_images():
     # Clear staging area when loading a new folder
     SorterEngine.clear_staging_area()
 
+    # Clear committed files cache for all categories (new folder = new output dir)
+    state._committed_files.clear()
+
     state.all_images = SorterEngine.get_images(state.source_dir, recursive=True)
 
     # Restore previously saved tags for this folder and profile
@@ -173,16 +200,25 @@ def load_images():
     if state.page >= state.total_pages:
         state.page = 0
 
-    refresh_staged_info()
+    refresh_staged_info(full_scan=True)
     refresh_ui()
 
-def refresh_staged_info():
-    """Update staged data and index maps."""
+def refresh_staged_info(full_scan: bool = False):
+    """Update staged data and index maps.
+
+    Args:
+        full_scan: If True, rescan disk for committed files. Otherwise use cache.
+    """
     state.staged_data = SorterEngine.get_staged_data()
+    staged_keys = set(state.staged_data.keys())
+
+    # Update cached tagged/untagged sets
+    state._cached_tagged = staged_keys
+    all_set = set(state.all_images)
+    state._cached_untagged = all_set - staged_keys
 
     # Update green dots (pages with staged images)
     state.green_dots.clear()
-    staged_keys = set(state.staged_data.keys())
     for idx, img_path in enumerate(state.all_images):
         if img_path in staged_keys:
             state.green_dots.add(idx // state.page_size)
@@ -197,11 +233,18 @@ def refresh_staged_info():
             if idx is not None:
                 state.index_map[idx] = orig_path
 
-    # Add committed images from disk
+    # Add committed images from disk (use cache unless full_scan requested)
     cat_path = os.path.join(state.output_dir, state.active_cat)
+    if full_scan or state.active_cat not in state._committed_files:
+        # Scan disk and cache the results
+        state._committed_files[state.active_cat] = set()
         if os.path.exists(cat_path):
             for filename in os.listdir(cat_path):
                 if filename.startswith(state.active_cat):
+                    state._committed_files[state.active_cat].add(filename)
+
+    # Build index map from cached committed files
+    for filename in state._committed_files.get(state.active_cat, set()):
         idx = _extract_index(filename)
         if idx is not None and idx not in state.index_map:
             state.index_map[idx] = os.path.join(cat_path, filename)
@@ -213,6 +256,20 @@ def _extract_index(filename: str) -> Optional[int]:
     except (ValueError, IndexError):
         return None
 
+def _add_to_undo_stack(entry: Dict):
+    """Add entry to undo stack with size limit."""
+    state.undo_stack.append(entry)
+    if len(state.undo_stack) > 50:
+        state.undo_stack.pop(0)
+
+def _remove_hotkey_for_category(category: str):
+    """Remove any hotkey assigned to the given category."""
+    to_remove = [hk for hk, c in state.category_hotkeys.items() if c == category]
+    for hk in to_remove:
+        del state.category_hotkeys[hk]
+        if hasattr(state, 'hotkey_by_category'):
+            state.hotkey_by_category.pop(category, None)
+
 # ==========================================
 # ACTIONS
 # ==========================================
@@ -232,15 +289,13 @@ def action_tag(img_path: str, manual_idx: Optional[int] = None):
         name = f"{state.active_cat}_{idx:03d}_{len(staged_names)+1}{ext}"
     
     # Save to undo stack
-    state.undo_stack.append({
+    _add_to_undo_stack({
         "action": "tag",
         "path": img_path,
         "category": state.active_cat,
         "name": name,
         "index": idx
     })
-    if len(state.undo_stack) > 50:  # Limit undo history
-        state.undo_stack.pop(0)
 
     SorterEngine.stage_image(img_path, state.active_cat, name)
 
@@ -249,36 +304,32 @@ def action_tag(img_path: str, manual_idx: Optional[int] = None):
         state.next_index = idx + 1
 
     refresh_staged_info()
-    refresh_ui()
+    refresh_ui_minimal()
 
 def action_untag(img_path: str):
     """Remove staging from an image."""
     # Save to undo stack
     if img_path in state.staged_data:
         info = state.staged_data[img_path]
-        state.undo_stack.append({
+        _add_to_undo_stack({
             "action": "untag",
             "path": img_path,
             "category": info['cat'],
             "name": info['name'],
             "index": _extract_index(info['name'])
         })
-        if len(state.undo_stack) > 50:
-            state.undo_stack.pop(0)
 
     SorterEngine.clear_staged_item(img_path)
     refresh_staged_info()
-    refresh_ui()
+    refresh_ui_minimal()
 
 def action_delete(img_path: str):
     """Delete image to trash."""
     # Save to undo stack
-    state.undo_stack.append({
+    _add_to_undo_stack({
         "action": "delete",
         "path": img_path
     })
-    if len(state.undo_stack) > 50:
-        state.undo_stack.pop(0)
 
     SorterEngine.delete_to_trash(img_path)
     load_images()
@@ -305,7 +356,6 @@ def action_undo():
         # Undo delete = restore from trash
         trash_path = os.path.join(os.path.dirname(last["path"]), "_DELETED", os.path.basename(last["path"]))
         if os.path.exists(trash_path):
-            import shutil
             shutil.move(trash_path, last["path"])
             ui.notify(f"Restored: {os.path.basename(last['path'])}", type='info')
         else:
@@ -339,7 +389,13 @@ def action_apply_page():
 
 async def action_apply_global():
     """Apply all staged changes globally."""
+    if state.applying_global:
+        ui.notify("Global apply already in progress", type='warning')
+        return
+
+    state.applying_global = True
     ui.notify("Starting global apply... This may take a while.", type='info')
+    try:
         await run.io_bound(
             SorterEngine.commit_global,
             state.output_dir,
@@ -350,6 +406,8 @@ async def action_apply_global():
         )
         load_images()
         ui.notify("Global apply complete!", type='positive')
+    finally:
+        state.applying_global = False
 
 # ==========================================
 # UI COMPONENTS
@@ -393,12 +451,8 @@ def open_zoom_dialog(path: str, title: Optional[str] = None, show_untag: bool =
 
 def open_hotkey_dialog(category: str):
     """Open dialog to set/change hotkey for a category."""
-    # Find current hotkey if any
-    current_hotkey = None
-    for hk, cat in state.category_hotkeys.items():
-        if cat == category:
-            current_hotkey = hk
-            break
+    # Use reverse mapping for O(1) lookup
+    current_hotkey = state.hotkey_by_category.get(category)
 
     with ui.dialog() as dialog, ui.card().classes('p-4 bg-gray-800'):
         ui.label(f'Set Hotkey for "{category}"').classes('font-bold text-white mb-2')
@@ -417,24 +471,25 @@ def open_hotkey_dialog(category: str):
             key = hotkey_input.value.lower().strip()
             if key and len(key) == 1 and key.isalpha():
                 # Remove old hotkey for this category
-                to_remove = [hk for hk, c in state.category_hotkeys.items() if c == category]
-                for hk in to_remove:
-                    del state.category_hotkeys[hk]
+                _remove_hotkey_for_category(category)
 
                 # Remove if another category had this hotkey
                 if key in state.category_hotkeys:
+                    old_cat = state.category_hotkeys[key]
                     del state.category_hotkeys[key]
+                    if hasattr(state, 'hotkey_by_category'):
+                        state.hotkey_by_category.pop(old_cat, None)
 
                 # Set new hotkey
                 state.category_hotkeys[key] = category
+                if hasattr(state, 'hotkey_by_category'):
+                    state.hotkey_by_category[category] = key
                 ui.notify(f'Hotkey "{key.upper()}" set for {category}', type='positive')
                 dialog.close()
                 render_sidebar()
             elif key == '':
                 # Clear hotkey
-                to_remove = [hk for hk, c in state.category_hotkeys.items() if c == category]
-                for hk in to_remove:
-                    del state.category_hotkeys[hk]
+                _remove_hotkey_for_category(category)
                 ui.notify(f'Hotkey cleared for {category}', type='info')
                 dialog.close()
                 render_sidebar()
@@ -451,14 +506,14 @@ def open_hotkey_dialog(category: str):
     
     dialog.open()
 
-def render_sidebar():
-    """Render category management sidebar."""
-    state.sidebar_container.clear()
+def render_number_grid():
+    """Render the 1-25 number grid for quick index selection."""
+    if state.number_grid_container:
+        state.number_grid_container.clear()
+    else:
+        return
 
-    with state.sidebar_container:
-        ui.label("🏷️ Category Manager").classes('text-xl font-bold mb-2 text-white')
-        
-        # Number grid (1-25)
+    with state.number_grid_container:
         with ui.grid(columns=5).classes('gap-1 mb-4 w-full'):
             for i in range(1, 26):
                 is_used = i in state.index_map
@@ -467,7 +522,6 @@ def render_sidebar():
                 def make_click_handler(num: int):
                     def handler():
                         if num in state.index_map:
-                            # Number is used - open preview
                             img_path = state.index_map[num]
                             is_staged = img_path in state.staged_data
                             open_zoom_dialog(
@@ -477,43 +531,39 @@ def render_sidebar():
                                 show_jump=True
                             )
                         else:
-                            # Number is free - set as next index
                             state.next_index = num
-                            render_sidebar()
+                            render_number_grid()
                     return handler
 
                 ui.button(str(i), on_click=make_click_handler(i)) \
                   .props(f'color={color} size=sm flat') \
                   .classes('w-full border border-gray-800')
 
-        # Category Manager (expanded)
-        ui.label("📂 Categories").classes('text-sm font-bold text-gray-400 mt-2')
+def render_category_list():
+    """Render the list of categories with hotkey buttons."""
+    if state.category_list_container:
+        state.category_list_container.clear()
+    else:
+        return
 
+    with state.category_list_container:
         categories = state.get_categories()
 
-        # Category list with hotkey buttons
         for cat in categories:
             is_active = cat == state.active_cat
-            hotkey = None
-            # Find if this category has a hotkey
-            for hk, cat_name in state.category_hotkeys.items():
-                if cat_name == cat:
-                    hotkey = hk
-                    break
+            hotkey = state.hotkey_by_category.get(cat)
 
             with ui.row().classes('w-full items-center no-wrap gap-1'):
-                # Category button
                 ui.button(
                     cat,
                     on_click=lambda c=cat: (
                         setattr(state, 'active_cat', c),
-                        refresh_staged_info(),
+                        refresh_staged_info(full_scan=(c not in state._committed_files)),
                         render_sidebar()
                     )
                 ).props(f'{"" if is_active else "flat"} color={"green" if is_active else "grey"} dense') \
                  .classes('flex-grow text-left')
 
-                # Hotkey badge/button
                 def make_hotkey_handler(category):
                     def handler():
                         open_hotkey_dialog(category)
@@ -544,27 +594,40 @@ def render_sidebar():
         # Delete category
         with ui.expansion('Danger Zone', icon='warning').classes('w-full text-red-400 mt-2'):
             def delete_category():
-                # Also remove any hotkey for this category
-                to_remove = [hk for hk, c in state.category_hotkeys.items() if c == state.active_cat]
-                for hk in to_remove:
-                    del state.category_hotkeys[hk]
+                _remove_hotkey_for_category(state.active_cat)
                 SorterEngine.delete_category(state.active_cat, state.profile_name)
                 refresh_staged_info()
                 render_sidebar()
 
             ui.button('DELETE CATEGORY', color='red', on_click=delete_category).classes('w-full')
 
+def render_sidebar():
+    """Render category management sidebar."""
+    state.sidebar_container.clear()
+
+    with state.sidebar_container:
+        ui.label("🏷️ Category Manager").classes('text-xl font-bold mb-2 text-white')
+
+        # Number grid container
+        state.number_grid_container = ui.column().classes('w-full')
+        render_number_grid()
+
+        # Category Manager
+        ui.label("📂 Categories").classes('text-sm font-bold text-gray-400 mt-2')
+        state.category_list_container = ui.column().classes('w-full')
+        render_category_list()
+
         ui.separator().classes('my-4 bg-gray-700')
 
         # Index counter
-        with ui.row().classes('w-full items-end no-wrap'):
+        state.index_display_container = ui.row().classes('w-full items-end no-wrap')
+        with state.index_display_container:
             ui.number(label="Next Index", min=1, precision=0) \
                 .bind_value(state, 'next_index') \
                 .classes('flex-grow').props('dark outlined')
 
             def reset_index():
                 state.next_index = (max(state.index_map.keys()) + 1) if state.index_map else 1
-                render_sidebar()
 
             ui.button('🔄', on_click=reset_index).props('flat color=white')
 
@@ -653,18 +716,28 @@ def render_image_card(img_path: str):
                     on_click=lambda p=img_path, i=local_idx: action_tag(p, int(i.value))
                 ).classes('w-2/3').props('color=green dense')
 
+def render_stats():
+    """Render only the stats labels (tagged/untagged counts)."""
+    if state.stats_container:
+        state.stats_container.clear()
+    else:
+        return
+
+    stats = state.get_stats()
+    with state.stats_container:
+        ui.label(f"📁 {stats['total']} images").classes('text-gray-400')
+        ui.label(f"🏷️ {stats['tagged']} tagged").classes('text-green-400')
+        ui.label(f"⬜ {stats['untagged']} untagged").classes('text-gray-500')
+
 def render_pagination():
     """Render pagination controls."""
     state.pagination_container.clear()
 
-    stats = state.get_stats()
-    
     with state.pagination_container:
         # Stats bar
         with ui.row().classes('w-full justify-center items-center gap-4 mb-2'):
-            ui.label(f"📁 {stats['total']} images").classes('text-gray-400')
-            ui.label(f"🏷️ {stats['tagged']} tagged").classes('text-green-400')
-            ui.label(f"⬜ {stats['untagged']} untagged").classes('text-gray-500')
+            state.stats_container = ui.row().classes('gap-4')
+            render_stats()
 
             # Filter toggle
             filter_colors = {"all": "grey", "tagged": "green", "untagged": "orange"}
@@ -738,6 +811,12 @@ def refresh_ui():
     render_pagination()
     render_gallery()
 
+def refresh_ui_minimal():
+    """Minimal refresh after tag/untag - only stats, number grid, and gallery."""
+    render_stats()
+    render_number_grid()
+    render_gallery()
+
 def handle_keyboard(e):
     """Handle keyboard navigation and shortcuts (fallback)."""
     if not e.action.keydown:
@@ -791,46 +870,6 @@ def handle_keyboard(e):
         refresh_ui()
         ui.notify(f"Filter: {state.filter_mode}", type='info')
 
-def process_key(key: str, ctrl: bool):
-    """Process keyboard input from JS event."""
-    # Navigation
-    if key == 'arrowleft' and state.page > 0:
-        set_page(state.page - 1)
-    elif key == 'arrowright' and state.page < state.total_pages - 1:
-        set_page(state.page + 1)
-    # Undo
-    elif key == 'z' and ctrl:
-        action_undo()
-    # Save
-    elif key == 's' and ctrl:
-        action_save_tags()
-    # Custom category hotkeys
-    elif not ctrl and len(key) == 1 and key.isalpha() and key in state.category_hotkeys:
-        state.active_cat = state.category_hotkeys[key]
-        refresh_staged_info()
-        refresh_ui()
-        ui.notify(f"Category: {state.active_cat}", type='info')
-    # Tag with number
-    elif key in '123456789' and not ctrl:
-        if state.hovered_image and state.hovered_image not in state.staged_data:
-            action_tag(state.hovered_image, int(key))
-    # Tag with next index
-    elif key == '0' and not ctrl:
-        if state.hovered_image and state.hovered_image not in state.staged_data:
-            action_tag(state.hovered_image)
-    # Untag (only if 'u' not assigned to category)
-    elif key == 'u' and not ctrl and 'u' not in state.category_hotkeys:
-        if state.hovered_image and state.hovered_image in state.staged_data:
-            action_untag(state.hovered_image)
-    # Filter (only if 'f' not assigned to category)
-    elif key == 'f' and not ctrl and 'f' not in state.category_hotkeys:
-        modes = ["all", "untagged", "tagged"]
-        current_idx = modes.index(state.filter_mode)
-        state.filter_mode = modes[(current_idx + 1) % 3]
-        state.page = 0
-        refresh_ui()
-        ui.notify(f"Filter: {state.filter_mode}", type='info')
-
 # ==========================================
 # MAIN LAYOUT
 # ==========================================
@@ -990,17 +1029,13 @@ build_header()
 build_sidebar()
 build_main_content()
 
-# JavaScript keyboard handler for Firefox compatibility
+# Prevent browser defaults for keyboard shortcuts (e.g., Ctrl+S save dialog)
 ui.add_body_html('''
 <script>
 document.addEventListener('keydown', function(e) {
-    // Skip if typing in input
     if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return;
-    
     const key = e.key.toLowerCase();
     const ctrl = e.ctrlKey || e.metaKey;
-    
-    // Prevent browser defaults for our shortcuts
     if (ctrl && (key === 's' || key === 'z')) {
         e.preventDefault();
     }