From d3955c489b0ae4ff82adff76d9b6c00a828a7220 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Thu, 19 Mar 2026 10:00:51 +0100 Subject: [PATCH] Write slim history tree to JSON when DB is enabled The JSON file was storing full snapshot data in every history node, causing files to grow to multi-GB. Now when DB is enabled: - sync_to_db receives full tree (extracts snapshots to DB) - save_json receives slim tree (no snapshot data) - JSON file stays small, DB is authoritative for snapshots When DB is disabled, JSON still stores full snapshots (only store). Co-Authored-By: Claude Opus 4.6 --- tab_batch_ng.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tab_batch_ng.py b/tab_batch_ng.py index d5ba849..a8d9888 100644 --- a/tab_batch_ng.py +++ b/tab_batch_ng.py @@ -377,18 +377,28 @@ def render_batch_processor(state: AppState): except ValueError as e: ui.notify(f'Save failed: {e}', type='negative') return - data[KEY_HISTORY_TREE] = htree.to_dict() - t1 = time.perf_counter() - save_snapshot = json.loads(json.dumps(data)) - await asyncio.to_thread(save_json, file_path, save_snapshot) - logger.info("save_and_snap save_json %.3fs", time.perf_counter() - t1) if state.db_enabled and state.current_project and state.db: + # DB path: sync full tree (with snapshots) to DB, then + # write slim tree (no snapshots) to JSON and memory + full_tree = htree.to_dict() + data[KEY_HISTORY_TREE] = full_tree t1 = time.perf_counter() - await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, save_snapshot) + db_snapshot = json.loads(json.dumps(data)) + await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, db_snapshot) logger.info("save_and_snap sync_to_db %.3fs", time.perf_counter() - t1) - # Free snapshot data from memory — it's persisted in DB now htree.strip_snapshots() data[KEY_HISTORY_TREE] = htree.to_dict() + t1 = time.perf_counter() + slim_snapshot = json.loads(json.dumps(data)) + await asyncio.to_thread(save_json, file_path, slim_snapshot) + logger.info("save_and_snap save_json %.3fs", time.perf_counter() - t1) + else: + # No DB: write full tree (with snapshots) to JSON + data[KEY_HISTORY_TREE] = htree.to_dict() + t1 = time.perf_counter() + save_snapshot = json.loads(json.dumps(data)) + await asyncio.to_thread(save_json, file_path, save_snapshot) + logger.info("save_and_snap save_json %.3fs", time.perf_counter() - t1) state.restored_indicator = None commit_input.set_value('') logger.info("save_and_snap END (%.3fs)", time.perf_counter() - t_ss) @@ -856,14 +866,19 @@ def _render_mass_update(batch_list, data, file_path, state: AppState, refresh_li except ValueError as e: ui.notify(f'Mass update failed: {e}', type='negative') return - data[KEY_HISTORY_TREE] = htree.to_dict() - save_snapshot = json.loads(json.dumps(data)) - await asyncio.to_thread(save_json, file_path, save_snapshot) if state.db_enabled and state.current_project and state.db: - await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, save_snapshot) - # Free snapshot data from memory after persisting + full_tree = htree.to_dict() + data[KEY_HISTORY_TREE] = full_tree + db_snapshot = json.loads(json.dumps(data)) + await asyncio.to_thread(sync_to_db, state.db, state.current_project, file_path, db_snapshot) htree.strip_snapshots() data[KEY_HISTORY_TREE] = htree.to_dict() + slim_snapshot = json.loads(json.dumps(data)) + await asyncio.to_thread(save_json, file_path, slim_snapshot) + else: + data[KEY_HISTORY_TREE] = htree.to_dict() + save_snapshot = json.loads(json.dumps(data)) + await asyncio.to_thread(save_json, file_path, save_snapshot) ui.notify(f'Updated {len(targets)} sequences', type='positive') if refresh_list: refresh_list.refresh()