From 074e36f88341c5555791a5ba7b37919e11b1d24a Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Wed, 18 Mar 2026 22:17:25 +0100 Subject: [PATCH] Fix blocking I/O on event loop, cache graphviz, optimize DB sync Move all save_json/load_json/sync_to_db/DB calls off the event loop with asyncio.to_thread to prevent UI freezes. Cache graphviz SVG by DOT source hash (bounded LRU of 20). Replace DELETE-all/re-INSERT in sync_to_db with UPSERT + targeted DELETE. Add DB indexes, COUNT query, and reduce graph poll interval to 0.5s. Co-Authored-By: Claude Opus 4.6 --- db.py | 11 +++++++++ history_tree.py | 5 ++-- main.py | 9 ++++---- tab_projects_ng.py | 57 +++++++++++++++++++++++++++------------------- tab_timeline_ng.py | 50 ++++++++++++++++++++++++---------------- utils.py | 17 ++++++++++---- 6 files changed, 95 insertions(+), 54 deletions(-) diff --git a/db.py b/db.py index 85e60da..2e8cb19 100644 --- a/db.py +++ b/db.py @@ -47,6 +47,9 @@ CREATE TABLE IF NOT EXISTS history_trees ( tree_data TEXT NOT NULL DEFAULT '{}', updated_at REAL NOT NULL ); + +CREATE INDEX IF NOT EXISTS idx_data_files_project_id ON data_files(project_id); +CREATE INDEX IF NOT EXISTS idx_sequences_data_file_id ON sequences(data_file_id); """ @@ -146,6 +149,14 @@ class ProjectDB: ).fetchall() return [dict(r) for r in rows] + def count_data_files(self, project_id: int) -> int: + """Return the number of data files for a project.""" + row = self.conn.execute( + "SELECT COUNT(*) AS cnt FROM data_files WHERE project_id = ?", + (project_id,), + ).fetchone() + return row["cnt"] + def get_data_file(self, project_id: int, name: str) -> dict | None: row = self.conn.execute( "SELECT id, project_id, name, data_type, top_level, created_at, updated_at " diff --git a/history_tree.py b/history_tree.py index 4e1f911..77db105 100644 --- a/history_tree.py +++ b/history_tree.py @@ -129,9 +129,10 @@ class HistoryTree: while current and current in self.nodes: if current in visited: break + if current in node_to_branch: + break # this node and all ancestors already assigned visited.add(current) - if current not in node_to_branch: - node_to_branch[current] = b_name + node_to_branch[current] = b_name current = self.nodes[current].get('parent') # Per-branch color palette (bg, border) — cycles for many branches diff --git a/main.py b/main.py index 159ce7b..c98ff18 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +import asyncio import copy import json import logging @@ -270,11 +271,11 @@ def index(): current_val = pane_state.file_path.name if pane_state.file_path else None - def on_select(e): + async def on_select(e): if not e.value: return fp = pane_state.current_dir / e.value - data, mtime = load_json(fp) + data, mtime = await asyncio.to_thread(load_json, fp) pane_state.data_cache = data pane_state.last_mtime = mtime pane_state.loaded_file = str(fp) @@ -289,12 +290,12 @@ def index(): on_change=on_select, ).classes('w-full') - def load_file(file_name: str): + async def load_file(file_name: str): """Load a JSON file and refresh the main content.""" fp = state.current_dir / file_name if state.loaded_file == str(fp): return - data, mtime = load_json(fp) + data, mtime = await asyncio.to_thread(load_json, fp) state.data_cache = data state.last_mtime = mtime state.loaded_file = str(fp) diff --git a/tab_projects_ng.py b/tab_projects_ng.py index 0f41844..4dd825f 100644 --- a/tab_projects_ng.py +++ b/tab_projects_ng.py @@ -1,3 +1,4 @@ +import asyncio import json import logging import sqlite3 @@ -42,13 +43,13 @@ def render_projects_tab(state: AppState): name_input = ui.input('Project Name', placeholder='my_project').classes('w-full') desc_input = ui.input('Description (optional)', placeholder='A short description').classes('w-full') - def create_project(): + async def create_project(): name = name_input.value.strip() if not name: ui.notify('Please enter a project name', type='warning') return try: - state.db.create_project(name, str(state.current_dir), desc_input.value.strip()) + await asyncio.to_thread(state.db.create_project, name, str(state.current_dir), desc_input.value.strip()) name_input.set_value('') desc_input.set_value('') ui.notify(f'Created project "{name}"', type='positive') @@ -59,10 +60,12 @@ def render_projects_tab(state: AppState): ui.button('Create Project', icon='add', on_click=create_project).classes('w-full') # --- Active project indicator --- + # Fetch once and reuse in render_project_list + _cached_projects = state.db.list_projects() + if state.current_project: # Check if active project actually exists in the database - projects_list = state.db.list_projects() - project_exists = any(p['name'] == state.current_project for p in projects_list) + project_exists = any(p['name'] == state.current_project for p in _cached_projects) if project_exists: ui.label(f'Active Project: {state.current_project}').classes( 'text-bold text-primary q-pa-sm') @@ -98,7 +101,9 @@ def render_projects_tab(state: AppState): # --- Project list --- @ui.refreshable def render_project_list(): + nonlocal _cached_projects projects = state.db.list_projects() + _cached_projects = projects if not projects: ui.label('No projects yet. Create one above.').classes('text-caption q-pa-md') return @@ -114,8 +119,8 @@ def render_projects_tab(state: AppState): if proj['description']: ui.label(proj['description']).classes('text-caption') ui.label(f'Path: {proj["folder_path"]}').classes('text-caption') - files = state.db.list_data_files(proj['id']) - ui.label(f'{len(files)} data file(s)').classes('text-caption') + file_count = state.db.count_data_files(proj['id']) + ui.label(f'{file_count} data file(s)').classes('text-caption') with ui.row().classes('q-gutter-xs'): if not is_active: @@ -151,7 +156,7 @@ def render_projects_tab(state: AppState): if new_name and new_name.strip() and new_name.strip() != name: new_name = new_name.strip() try: - state.db.rename_project(name, new_name) + await asyncio.to_thread(state.db.rename_project, name, new_name) if state.current_project == name: state.current_project = new_name state.config['current_project'] = new_name @@ -179,7 +184,7 @@ def render_projects_tab(state: AppState): if not Path(new_path).is_dir(): ui.notify(f'Warning: "{new_path}" does not exist', type='warning') - state.db.update_project_path(name, new_path) + await asyncio.to_thread(state.db.update_project_path, name, new_path) ui.notify(f'Path updated to "{new_path}"', type='positive') render_project_list.refresh() @@ -192,8 +197,8 @@ def render_projects_tab(state: AppState): ui.button('Import Folder', icon='folder_open', on_click=import_folder).props('flat dense') - def delete_proj(name=proj['name']): - state.db.delete_project(name) + async def delete_proj(name=proj['name']): + await asyncio.to_thread(state.db.delete_project, name) if state.current_project == name: state.current_project = '' state.config['current_project'] = '' @@ -211,7 +216,7 @@ def render_projects_tab(state: AppState): render_project_content() -def _import_folder(state: AppState, project_id: int, project_name: str, refresh_fn): +async def _import_folder(state: AppState, project_id: int, project_name: str, refresh_fn): """Bulk import all .json files from current directory into a project.""" json_files = sorted(state.current_dir.glob('*.json')) json_files = [f for f in json_files if f.name not in ( @@ -221,19 +226,23 @@ def _import_folder(state: AppState, project_id: int, project_name: str, refresh_ ui.notify('No JSON files in current directory', type='warning') return - imported = 0 - skipped = 0 - for jf in json_files: - file_name = jf.stem - existing = state.db.get_data_file(project_id, file_name) - if existing: - skipped += 1 - continue - try: - state.db.import_json_file(project_id, jf) - imported += 1 - except Exception as e: - logger.warning(f"Failed to import {jf}: {e}") + def _do_import(): + imported = 0 + skipped = 0 + for jf in json_files: + file_name = jf.stem + existing = state.db.get_data_file(project_id, file_name) + if existing: + skipped += 1 + continue + try: + state.db.import_json_file(project_id, jf) + imported += 1 + except Exception as e: + logger.warning(f"Failed to import {jf}: {e}") + return imported, skipped + + imported, skipped = await asyncio.to_thread(_do_import) msg = f'Imported {imported} file(s)' if skipped: diff --git a/tab_timeline_ng.py b/tab_timeline_ng.py index 0721d09..0ffa39c 100644 --- a/tab_timeline_ng.py +++ b/tab_timeline_ng.py @@ -1,4 +1,5 @@ -import copy +import asyncio +import hashlib import json import time @@ -13,7 +14,7 @@ def _delete_nodes(htree, data, file_path, node_ids): """Delete nodes with backup, branch cleanup, re-parenting, and head fallback.""" if 'history_tree_backup' not in data: data['history_tree_backup'] = [] - data['history_tree_backup'].append(copy.deepcopy(htree.to_dict())) + data['history_tree_backup'].append(json.loads(json.dumps(htree.to_dict()))) data['history_tree_backup'] = data['history_tree_backup'][-10:] # Save deleted node parents before removal (needed for branch re-pointing) deleted_parents = {} @@ -48,7 +49,6 @@ def _delete_nodes(htree, data, file_path, node_ids): else: htree.head_id = None data[KEY_HISTORY_TREE] = htree.to_dict() - save_json(file_path, data) def _render_selection_picker(all_nodes, htree, state, refresh_fn): @@ -154,11 +154,12 @@ def _render_batch_delete(htree, data, file_path, state, refresh_fn): f'{count} node{"s" if count != 1 else ""} selected for deletion.' ).classes('text-warning q-mt-md') - def do_batch_delete(): + async def do_batch_delete(): current_valid = state.timeline_selected_nodes & set(htree.nodes.keys()) _delete_nodes(htree, data, file_path, current_valid) + await asyncio.to_thread(save_json, file_path, data) if state.db_enabled and state.current_project and state.db: - sync_to_db(state.db, state.current_project, file_path, data) + await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, data) state.timeline_selected_nodes = set() ui.notify( f'Deleted {len(current_valid)} node{"s" if len(current_valid) != 1 else ""}!', @@ -319,13 +320,13 @@ def _render_node_manager(all_nodes, htree, data, file_path, restore_fn, refresh_ # Rename rename_input = ui.input('Rename Label').classes('col').props('dense') - def rename_node(): + async def rename_node(): if sel_id in htree.nodes and rename_input.value: htree.nodes[sel_id]['note'] = rename_input.value data[KEY_HISTORY_TREE] = htree.to_dict() - save_json(file_path, data) + await asyncio.to_thread(save_json, file_path, data) if state and state.db_enabled and state.current_project and state.db: - sync_to_db(state.db, state.current_project, file_path, data) + await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, data) ui.notify('Label updated', type='positive') refresh_fn() @@ -336,11 +337,12 @@ def _render_node_manager(all_nodes, htree, data, file_path, restore_fn, refresh_ 'w-full q-mt-sm').style('border-left: 3px solid var(--negative)'): ui.label('Deleting a node cannot be undone.').classes('text-warning') - def delete_selected(): + async def delete_selected(): if sel_id in htree.nodes: _delete_nodes(htree, data, file_path, {sel_id}) + await asyncio.to_thread(save_json, file_path, data) if state and state.db_enabled and state.current_project and state.db: - sync_to_db(state.db, state.current_project, file_path, data) + await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, data) # Reset selection if branch was removed if selected['branch'] not in htree.branches: selected['branch'] = next(iter(htree.branches), None) @@ -427,8 +429,8 @@ def render_timeline_tab(state: AppState): state.timeline_selected_nodes.discard(nid) render_timeline.refresh() - def _restore_and_refresh(node): - _restore_node(data, node, htree, file_path, state) + async def _restore_and_refresh(node): + await _restore_node(data, node, htree, file_path, state) # Refresh all tabs (batch, raw, timeline) so they pick up the restored data state._render_main.refresh() @@ -463,15 +465,25 @@ def render_timeline_tab(state: AppState): selected['node_id'] = node_id render_timeline.refresh() - graph_timer = ui.timer(0.2, _poll_graph_click) + graph_timer = ui.timer(0.5, _poll_graph_click) + + +_graphviz_svg_cache: dict[str, str] = {} +_GRAPHVIZ_CACHE_MAX = 20 def _render_graphviz(dot_source: str, selected_node_id: str | None = None): """Render graphviz DOT source as interactive SVG with click-to-select.""" try: import graphviz - src = graphviz.Source(dot_source) - svg = src.pipe(format='svg').decode('utf-8') + cache_key = hashlib.md5(dot_source.encode()).hexdigest() + svg = _graphviz_svg_cache.get(cache_key) + if svg is None: + src = graphviz.Source(dot_source) + svg = src.pipe(format='svg').decode('utf-8') + if len(_graphviz_svg_cache) >= _GRAPHVIZ_CACHE_MAX: + _graphviz_svg_cache.pop(next(iter(_graphviz_svg_cache))) + _graphviz_svg_cache[cache_key] = svg sel_escaped = json.dumps(selected_node_id or '')[1:-1] # strip quotes, get JS-safe content @@ -529,9 +541,9 @@ def _render_graphviz(dot_source: str, selected_node_id: str | None = None): ui.label(f'Graph rendering error: {e}').classes('text-negative') -def _restore_node(data, node, htree, file_path, state: AppState): +async def _restore_node(data, node, htree, file_path, state: AppState): """Restore a history node as the current version (full replace, not merge).""" - node_data = copy.deepcopy(node.get('data', {})) + node_data = json.loads(json.dumps(node.get('data', {}))) # Preserve the history tree before clearing preserved_tree = data.get(KEY_HISTORY_TREE) preserved_backup = data.get('history_tree_backup') @@ -544,9 +556,9 @@ def _restore_node(data, node, htree, file_path, state: AppState): data['history_tree_backup'] = preserved_backup htree.head_id = node['id'] data[KEY_HISTORY_TREE] = htree.to_dict() - save_json(file_path, data) + await asyncio.to_thread(save_json, file_path, data) if state.db_enabled and state.current_project and state.db: - sync_to_db(state.db, state.current_project, file_path, data) + await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, data) label = f"{node.get('note', 'Step')} ({node['id'][:4]})" state.restored_indicator = label ui.notify('Restored!', type='positive') diff --git a/utils.py b/utils.py index 35e4f9f..ce187c0 100644 --- a/utils.py +++ b/utils.py @@ -184,11 +184,11 @@ def sync_to_db(db, project_name: str, file_path: Path, data: dict) -> None: # Use a single transaction for atomicity db.conn.execute("BEGIN IMMEDIATE") try: + now = time.time() df = db.get_data_file(proj["id"], file_name) top_level = {k: v for k, v in data.items() if k not in (KEY_BATCH_DATA, KEY_HISTORY_TREE)} if not df: - now = time.time() cur = db.conn.execute( "INSERT INTO data_files (project_id, name, data_type, top_level, created_at, updated_at) " "VALUES (?, ?, ?, ?, ?, ?)", @@ -198,7 +198,6 @@ def sync_to_db(db, project_name: str, file_path: Path, data: dict) -> None: else: df_id = df["id"] # Update top_level metadata - now = time.time() db.conn.execute( "UPDATE data_files SET top_level = ?, updated_at = ? WHERE id = ?", (json.dumps(top_level), now, df_id), @@ -207,23 +206,31 @@ def sync_to_db(db, project_name: str, file_path: Path, data: dict) -> None: # Sync sequences batch_data = data.get(KEY_BATCH_DATA, []) if isinstance(batch_data, list): - db.conn.execute("DELETE FROM sequences WHERE data_file_id = ?", (df_id,)) + new_seq_nums = set() for item in batch_data: if not isinstance(item, dict): continue seq_num = int(item.get(KEY_SEQUENCE_NUMBER, 0)) - now = time.time() + new_seq_nums.add(seq_num) db.conn.execute( "INSERT INTO sequences (data_file_id, sequence_number, data, updated_at) " "VALUES (?, ?, ?, ?) " "ON CONFLICT(data_file_id, sequence_number) DO UPDATE SET data=excluded.data, updated_at=excluded.updated_at", (df_id, seq_num, json.dumps(item), now), ) + # Remove sequences that no longer exist + if new_seq_nums: + placeholders = ','.join('?' * len(new_seq_nums)) + db.conn.execute( + f"DELETE FROM sequences WHERE data_file_id = ? AND sequence_number NOT IN ({placeholders})", + (df_id, *new_seq_nums), + ) + else: + db.conn.execute("DELETE FROM sequences WHERE data_file_id = ?", (df_id,)) # Sync history tree history_tree = data.get(KEY_HISTORY_TREE) if history_tree and isinstance(history_tree, dict): - now = time.time() db.conn.execute( "INSERT INTO history_trees (data_file_id, tree_data, updated_at) " "VALUES (?, ?, ?) "