Ethanfel/Comfyui-JSON-Manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix blocking I/O on event loop, cache graphviz, optimize DB sync

Browse Source 
Move all save_json/load_json/sync_to_db/DB calls off the event loop
with asyncio.to_thread to prevent UI freezes. Cache graphviz SVG by
DOT source hash (bounded LRU of 20). Replace DELETE-all/re-INSERT in
sync_to_db with UPSERT + targeted DELETE. Add DB indexes, COUNT query,
and reduce graph poll interval to 0.5s.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ethanfel
2026-03-18 22:17:25 +01:00
parent b36200faaa
commit 074e36f883
6 changed files with 95 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db.py
+11
View File
@@ -47,6 +47,9 @@ CREATE TABLE IF NOT EXISTS history_trees (
tree_data TEXT NOT NULL DEFAULT '{}',
updated_at REAL NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_data_files_project_id ON data_files(project_id);
CREATE INDEX IF NOT EXISTS idx_sequences_data_file_id ON sequences(data_file_id);
"""
@@ -146,6 +149,14 @@ class ProjectDB:
).fetchall()
return [dict(r) for r in rows]
def count_data_files(self, project_id: int) -> int:
"""Return the number of data files for a project."""
row = self.conn.execute(
"SELECT COUNT(*) AS cnt FROM data_files WHERE project_id = ?",
(project_id,),
).fetchone()
return row["cnt"]
def get_data_file(self, project_id: int, name: str) -> dict | None:
row = self.conn.execute(
"SELECT id, project_id, name, data_type, top_level, created_at, updated_at "
history_tree.py
+3 -2
View File
@@ -129,9 +129,10 @@ class HistoryTree:
while current and current in self.nodes:
if current in visited:
break
if current in node_to_branch:
break # this node and all ancestors already assigned
visited.add(current)
if current not in node_to_branch:
node_to_branch[current] = b_name
node_to_branch[current] = b_name
current = self.nodes[current].get('parent')
# Per-branch color palette (bg, border) — cycles for many branches
main.py
+5 -4
View File
@@ -1,3 +1,4 @@
import asyncio
import copy
import json
import logging
@@ -270,11 +271,11 @@ def index():
current_val = pane_state.file_path.name if pane_state.file_path else None
def on_select(e):
async def on_select(e):
if not e.value:
return
fp = pane_state.current_dir / e.value
data, mtime = load_json(fp)
data, mtime = await asyncio.to_thread(load_json, fp)
pane_state.data_cache = data
pane_state.last_mtime = mtime
pane_state.loaded_file = str(fp)
@@ -289,12 +290,12 @@ def index():
on_change=on_select,
).classes('w-full')
def load_file(file_name: str):
async def load_file(file_name: str):
"""Load a JSON file and refresh the main content."""
fp = state.current_dir / file_name
if state.loaded_file == str(fp):
return
data, mtime = load_json(fp)
data, mtime = await asyncio.to_thread(load_json, fp)
state.data_cache = data
state.last_mtime = mtime
state.loaded_file = str(fp)
tab_projects_ng.py
+33 -24
View File
@@ -1,3 +1,4 @@
import asyncio
import json
import logging
import sqlite3
@@ -42,13 +43,13 @@ def render_projects_tab(state: AppState):
name_input = ui.input('Project Name', placeholder='my_project').classes('w-full')
desc_input = ui.input('Description (optional)', placeholder='A short description').classes('w-full')
def create_project():
async def create_project():
name = name_input.value.strip()
if not name:
ui.notify('Please enter a project name', type='warning')
return
try:
state.db.create_project(name, str(state.current_dir), desc_input.value.strip())
await asyncio.to_thread(state.db.create_project, name, str(state.current_dir), desc_input.value.strip())
name_input.set_value('')
desc_input.set_value('')
ui.notify(f'Created project "{name}"', type='positive')
@@ -59,10 +60,12 @@ def render_projects_tab(state: AppState):
ui.button('Create Project', icon='add', on_click=create_project).classes('w-full')
# --- Active project indicator ---
# Fetch once and reuse in render_project_list
_cached_projects = state.db.list_projects()
if state.current_project:
# Check if active project actually exists in the database
projects_list = state.db.list_projects()
project_exists = any(p['name'] == state.current_project for p in projects_list)
project_exists = any(p['name'] == state.current_project for p in _cached_projects)
if project_exists:
ui.label(f'Active Project: {state.current_project}').classes(
'text-bold text-primary q-pa-sm')
@@ -98,7 +101,9 @@ def render_projects_tab(state: AppState):
# --- Project list ---
@ui.refreshable
def render_project_list():
nonlocal _cached_projects
projects = state.db.list_projects()
_cached_projects = projects
if not projects:
ui.label('No projects yet. Create one above.').classes('text-caption q-pa-md')
return
@@ -114,8 +119,8 @@ def render_projects_tab(state: AppState):
if proj['description']:
ui.label(proj['description']).classes('text-caption')
ui.label(f'Path: {proj["folder_path"]}').classes('text-caption')
files = state.db.list_data_files(proj['id'])
ui.label(f'{len(files)} data file(s)').classes('text-caption')
file_count = state.db.count_data_files(proj['id'])
ui.label(f'{file_count} data file(s)').classes('text-caption')
with ui.row().classes('q-gutter-xs'):
if not is_active:
@@ -151,7 +156,7 @@ def render_projects_tab(state: AppState):
if new_name and new_name.strip() and new_name.strip() != name:
new_name = new_name.strip()
try:
state.db.rename_project(name, new_name)
await asyncio.to_thread(state.db.rename_project, name, new_name)
if state.current_project == name:
state.current_project = new_name
state.config['current_project'] = new_name
@@ -179,7 +184,7 @@ def render_projects_tab(state: AppState):
if not Path(new_path).is_dir():
ui.notify(f'Warning: "{new_path}" does not exist',
type='warning')
state.db.update_project_path(name, new_path)
await asyncio.to_thread(state.db.update_project_path, name, new_path)
ui.notify(f'Path updated to "{new_path}"', type='positive')
render_project_list.refresh()
@@ -192,8 +197,8 @@ def render_projects_tab(state: AppState):
ui.button('Import Folder', icon='folder_open',
on_click=import_folder).props('flat dense')
def delete_proj(name=proj['name']):
state.db.delete_project(name)
async def delete_proj(name=proj['name']):
await asyncio.to_thread(state.db.delete_project, name)
if state.current_project == name:
state.current_project = ''
state.config['current_project'] = ''
@@ -211,7 +216,7 @@ def render_projects_tab(state: AppState):
render_project_content()
def _import_folder(state: AppState, project_id: int, project_name: str, refresh_fn):
async def _import_folder(state: AppState, project_id: int, project_name: str, refresh_fn):
"""Bulk import all .json files from current directory into a project."""
json_files = sorted(state.current_dir.glob('*.json'))
json_files = [f for f in json_files if f.name not in (
@@ -221,19 +226,23 @@ def _import_folder(state: AppState, project_id: int, project_name: str, refresh_
ui.notify('No JSON files in current directory', type='warning')
return
imported = 0
skipped = 0
for jf in json_files:
file_name = jf.stem
existing = state.db.get_data_file(project_id, file_name)
if existing:
skipped += 1
continue
try:
state.db.import_json_file(project_id, jf)
imported += 1
except Exception as e:
logger.warning(f"Failed to import {jf}: {e}")
def _do_import():
imported = 0
skipped = 0
for jf in json_files:
file_name = jf.stem
existing = state.db.get_data_file(project_id, file_name)
if existing:
skipped += 1
continue
try:
state.db.import_json_file(project_id, jf)
imported += 1
except Exception as e:
logger.warning(f"Failed to import {jf}: {e}")
return imported, skipped
imported, skipped = await asyncio.to_thread(_do_import)
msg = f'Imported {imported} file(s)'
if skipped:
tab_timeline_ng.py
+31 -19
View File
@@ -1,4 +1,5 @@
import copy
import asyncio
import hashlib
import json
import time
@@ -13,7 +14,7 @@ def _delete_nodes(htree, data, file_path, node_ids):
"""Delete nodes with backup, branch cleanup, re-parenting, and head fallback."""
if 'history_tree_backup' not in data:
data['history_tree_backup'] = []
data['history_tree_backup'].append(copy.deepcopy(htree.to_dict()))
data['history_tree_backup'].append(json.loads(json.dumps(htree.to_dict())))
data['history_tree_backup'] = data['history_tree_backup'][-10:]
# Save deleted node parents before removal (needed for branch re-pointing)
deleted_parents = {}
@@ -48,7 +49,6 @@ def _delete_nodes(htree, data, file_path, node_ids):
else:
htree.head_id = None
data[KEY_HISTORY_TREE] = htree.to_dict()
save_json(file_path, data)
def _render_selection_picker(all_nodes, htree, state, refresh_fn):
@@ -154,11 +154,12 @@ def _render_batch_delete(htree, data, file_path, state, refresh_fn):
f'{count} node{"s" if count != 1 else ""} selected for deletion.'
).classes('text-warning q-mt-md')
def do_batch_delete():
async def do_batch_delete():
current_valid = state.timeline_selected_nodes & set(htree.nodes.keys())
_delete_nodes(htree, data, file_path, current_valid)
await asyncio.to_thread(save_json, file_path, data)
if state.db_enabled and state.current_project and state.db:
sync_to_db(state.db, state.current_project, file_path, data)
await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, data)
state.timeline_selected_nodes = set()
ui.notify(
f'Deleted {len(current_valid)} node{"s" if len(current_valid) != 1 else ""}!',
@@ -319,13 +320,13 @@ def _render_node_manager(all_nodes, htree, data, file_path, restore_fn, refresh_
# Rename
rename_input = ui.input('Rename Label').classes('col').props('dense')
def rename_node():
async def rename_node():
if sel_id in htree.nodes and rename_input.value:
htree.nodes[sel_id]['note'] = rename_input.value
data[KEY_HISTORY_TREE] = htree.to_dict()
save_json(file_path, data)
await asyncio.to_thread(save_json, file_path, data)
if state and state.db_enabled and state.current_project and state.db:
sync_to_db(state.db, state.current_project, file_path, data)
await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, data)
ui.notify('Label updated', type='positive')
refresh_fn()
@@ -336,11 +337,12 @@ def _render_node_manager(all_nodes, htree, data, file_path, restore_fn, refresh_
'w-full q-mt-sm').style('border-left: 3px solid var(--negative)'):
ui.label('Deleting a node cannot be undone.').classes('text-warning')
def delete_selected():
async def delete_selected():
if sel_id in htree.nodes:
_delete_nodes(htree, data, file_path, {sel_id})
await asyncio.to_thread(save_json, file_path, data)
if state and state.db_enabled and state.current_project and state.db:
sync_to_db(state.db, state.current_project, file_path, data)
await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, data)
# Reset selection if branch was removed
if selected['branch'] not in htree.branches:
selected['branch'] = next(iter(htree.branches), None)
@@ -427,8 +429,8 @@ def render_timeline_tab(state: AppState):
state.timeline_selected_nodes.discard(nid)
render_timeline.refresh()
def _restore_and_refresh(node):
_restore_node(data, node, htree, file_path, state)
async def _restore_and_refresh(node):
await _restore_node(data, node, htree, file_path, state)
# Refresh all tabs (batch, raw, timeline) so they pick up the restored data
state._render_main.refresh()
@@ -463,15 +465,25 @@ def render_timeline_tab(state: AppState):
selected['node_id'] = node_id
render_timeline.refresh()
graph_timer = ui.timer(0.2, _poll_graph_click)
graph_timer = ui.timer(0.5, _poll_graph_click)
_graphviz_svg_cache: dict[str, str] = {}
_GRAPHVIZ_CACHE_MAX = 20
def _render_graphviz(dot_source: str, selected_node_id: str | None = None):
"""Render graphviz DOT source as interactive SVG with click-to-select."""
try:
import graphviz
src = graphviz.Source(dot_source)
svg = src.pipe(format='svg').decode('utf-8')
cache_key = hashlib.md5(dot_source.encode()).hexdigest()
svg = _graphviz_svg_cache.get(cache_key)
if svg is None:
src = graphviz.Source(dot_source)
svg = src.pipe(format='svg').decode('utf-8')
if len(_graphviz_svg_cache) >= _GRAPHVIZ_CACHE_MAX:
_graphviz_svg_cache.pop(next(iter(_graphviz_svg_cache)))
_graphviz_svg_cache[cache_key] = svg
sel_escaped = json.dumps(selected_node_id or '')[1:-1] # strip quotes, get JS-safe content
@@ -529,9 +541,9 @@ def _render_graphviz(dot_source: str, selected_node_id: str | None = None):
ui.label(f'Graph rendering error: {e}').classes('text-negative')
def _restore_node(data, node, htree, file_path, state: AppState):
async def _restore_node(data, node, htree, file_path, state: AppState):
"""Restore a history node as the current version (full replace, not merge)."""
node_data = copy.deepcopy(node.get('data', {}))
node_data = json.loads(json.dumps(node.get('data', {})))
# Preserve the history tree before clearing
preserved_tree = data.get(KEY_HISTORY_TREE)
preserved_backup = data.get('history_tree_backup')
@@ -544,9 +556,9 @@ def _restore_node(data, node, htree, file_path, state: AppState):
data['history_tree_backup'] = preserved_backup
htree.head_id = node['id']
data[KEY_HISTORY_TREE] = htree.to_dict()
save_json(file_path, data)
await asyncio.to_thread(save_json, file_path, data)
if state.db_enabled and state.current_project and state.db:
sync_to_db(state.db, state.current_project, file_path, data)
await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, data)
label = f"{node.get('note', 'Step')} ({node['id'][:4]})"
state.restored_indicator = label
ui.notify('Restored!', type='positive')
utils.py
+12 -5
View File
@@ -184,11 +184,11 @@ def sync_to_db(db, project_name: str, file_path: Path, data: dict) -> None:
# Use a single transaction for atomicity
db.conn.execute("BEGIN IMMEDIATE")
try:
now = time.time()
df = db.get_data_file(proj["id"], file_name)
top_level = {k: v for k, v in data.items()
if k not in (KEY_BATCH_DATA, KEY_HISTORY_TREE)}
if not df:
now = time.time()
cur = db.conn.execute(
"INSERT INTO data_files (project_id, name, data_type, top_level, created_at, updated_at) "
"VALUES (?, ?, ?, ?, ?, ?)",
@@ -198,7 +198,6 @@ def sync_to_db(db, project_name: str, file_path: Path, data: dict) -> None:
else:
df_id = df["id"]
# Update top_level metadata
now = time.time()
db.conn.execute(
"UPDATE data_files SET top_level = ?, updated_at = ? WHERE id = ?",
(json.dumps(top_level), now, df_id),
@@ -207,23 +206,31 @@ def sync_to_db(db, project_name: str, file_path: Path, data: dict) -> None:
# Sync sequences
batch_data = data.get(KEY_BATCH_DATA, [])
if isinstance(batch_data, list):
db.conn.execute("DELETE FROM sequences WHERE data_file_id = ?", (df_id,))
new_seq_nums = set()
for item in batch_data:
if not isinstance(item, dict):
continue
seq_num = int(item.get(KEY_SEQUENCE_NUMBER, 0))
now = time.time()
new_seq_nums.add(seq_num)
db.conn.execute(
"INSERT INTO sequences (data_file_id, sequence_number, data, updated_at) "
"VALUES (?, ?, ?, ?) "
"ON CONFLICT(data_file_id, sequence_number) DO UPDATE SET data=excluded.data, updated_at=excluded.updated_at",
(df_id, seq_num, json.dumps(item), now),
)
# Remove sequences that no longer exist
if new_seq_nums:
placeholders = ','.join('?' * len(new_seq_nums))
db.conn.execute(
f"DELETE FROM sequences WHERE data_file_id = ? AND sequence_number NOT IN ({placeholders})",
(df_id, *new_seq_nums),
)
else:
db.conn.execute("DELETE FROM sequences WHERE data_file_id = ?", (df_id,))
# Sync history tree
history_tree = data.get(KEY_HISTORY_TREE)
if history_tree and isinstance(history_tree, dict):
now = time.time()
db.conn.execute(
"INSERT INTO history_trees (data_file_id, tree_data, updated_at) "
"VALUES (?, ?, ?) "