From 1ac091579db88452d5ac12b80a7d4fe59e2c7083 Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Sat, 21 Mar 2026 13:42:30 +0100 Subject: [PATCH] feat: add purge and vacuum actions to admin database page Rename /admin/db-size to /admin/database. Add purge action to NULL out run_track_activity_jsonl for older rows with radio buttons showing savings per retention level. Add vacuum button to run VACUUM FULL and reclaim disk space. Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend_multi_user/src/app.py | 98 ++++++- .../templates/admin/database.html | 244 ++++++++++++++++++ .../templates/admin/db_size.html | 121 --------- .../templates/admin/index.html | 6 +- 4 files changed, 342 insertions(+), 127 deletions(-) create mode 100644 frontend_multi_user/templates/admin/database.html delete mode 100644 frontend_multi_user/templates/admin/db_size.html diff --git a/frontend_multi_user/src/app.py b/frontend_multi_user/src/app.py index 6a952715..8dbee3c7 100644 --- a/frontend_multi_user/src/app.py +++ b/frontend_multi_user/src/app.py @@ -2006,6 +2006,83 @@ def _get_database_size_info(self) -> dict[str, Any]: info["error"] = str(e) return info + def _get_purge_activity_info(self) -> dict[str, Any]: + """Compute how much space each retention level would free by NULLing run_track_activity_jsonl.""" + from sqlalchemy import text + info: dict[str, Any] = {"error": None, "total_rows": 0, "rows_with_data": 0, "total_data_mb": 0.0, "options": []} + try: + with self.db.engine.connect() as conn: + row = conn.execute(text( + "SELECT count(*), " + "count(run_track_activity_jsonl), " + "coalesce(sum(octet_length(run_track_activity_jsonl)), 0) " + "FROM task_item" + )).fetchone() + if row: + info["total_rows"] = row[0] + info["rows_with_data"] = row[1] + info["total_data_mb"] = round(row[2] / (1024 * 1024), 2) + + for keep_n in [10, 25, 50, 100, 250, 500]: + result = conn.execute(text( + "SELECT coalesce(sum(octet_length(run_track_activity_jsonl)), 0), count(*) " + "FROM task_item " + "WHERE run_track_activity_jsonl IS NOT NULL " + "AND id NOT IN (" + " SELECT id FROM task_item " + " ORDER BY timestamp_created DESC " + " LIMIT :keep_n" + ")" + ), {"keep_n": keep_n}).fetchone() + if result: + info["options"].append({ + "keep_n": keep_n, + "purgeable_rows": result[1], + "savings_bytes": result[0], + "savings_mb": round(result[0] / (1024 * 1024), 2), + }) + except Exception as e: + logger.exception("Failed to query purge activity info") + info["error"] = str(e) + return info + + def _purge_activity_data(self, keep_n: int) -> dict[str, Any]: + """NULL out run_track_activity_jsonl for all rows except the latest keep_n.""" + from sqlalchemy import text + result: dict[str, Any] = {"error": None, "purged_rows": 0} + try: + with self.db.engine.connect() as conn: + row = conn.execute(text( + "UPDATE task_item " + "SET run_track_activity_jsonl = NULL, run_track_activity_bytes = NULL " + "WHERE run_track_activity_jsonl IS NOT NULL " + "AND id NOT IN (" + " SELECT id FROM task_item " + " ORDER BY timestamp_created DESC " + " LIMIT :keep_n" + ")" + ), {"keep_n": keep_n}) + result["purged_rows"] = row.rowcount + conn.commit() + except Exception as e: + logger.exception("Failed to purge activity data") + result["error"] = str(e) + return result + + def _vacuum_task_item(self) -> dict[str, Any]: + """Run VACUUM FULL on task_item to reclaim disk space.""" + from sqlalchemy import text + result: dict[str, Any] = {"error": None} + try: + with self.db.engine.connect() as conn: + conn.execution_options(isolation_level="AUTOCOMMIT").execute( + text("VACUUM FULL task_item") + ) + except Exception as e: + logger.exception("Failed to vacuum task_item") + result["error"] = str(e) + return result + def _build_reconciliation_report(self, max_tasks: int, tolerance_usd: float) -> tuple[list[dict[str, Any]], dict[str, Any]]: tasks = ( PlanItem.query @@ -2977,13 +3054,28 @@ def admin_reconciliation(): refresh_seconds=refresh_seconds, ) - @self.app.route('/admin/db-size') + @self.app.route('/admin/database', methods=['GET', 'POST']) @admin_required - def admin_db_size(): + def admin_database(): + purge_result = None + vacuum_result = None + if request.method == 'POST': + action = request.form.get('action', '') + if action == 'purge': + keep_n = int(request.form.get('keep_n', '50') or '50') + if keep_n not in (10, 25, 50, 100, 250, 500): + keep_n = 50 + purge_result = self._purge_activity_data(keep_n) + elif action == 'vacuum': + vacuum_result = self._vacuum_task_item() size_info = self._get_database_size_info() + purge_info = self._get_purge_activity_info() return self.admin.index_view.render( - "admin/db_size.html", + "admin/database.html", size_info=size_info, + purge_info=purge_info, + purge_result=purge_result, + vacuum_result=vacuum_result, ) @self.app.route('/ping/stream') diff --git a/frontend_multi_user/templates/admin/database.html b/frontend_multi_user/templates/admin/database.html new file mode 100644 index 00000000..532fe6ec --- /dev/null +++ b/frontend_multi_user/templates/admin/database.html @@ -0,0 +1,244 @@ +{% extends 'admin/master.html' %} + +{% block head_css %} + {{ super() }} + +{% endblock %} + +{% block body %} +
+

Database Size

+ + {% if size_info.error %} +
+ Error querying database size: {{ size_info.error }} +
+ {% else %} +
+
+
{{ size_info.total_mb }} MB
+
Total database size
+
+
+
Database: {{ size_info.database_name }}
+
+
+ + {% if size_info.tables %} +

Per-table breakdown

+ + + + + + + + + + + + {% for t in size_info.tables %} + + + + + + + + {% endfor %} + +
TableTotal (MB)Data (MB)Indexes (MB)
{{ t.name }}{{ t.total_mb }}{{ t.table_mb }}{{ t.index_mb }} + {% if size_info.tables[0].total_bytes > 0 %} +
+ {% endif %} +
+ {% endif %} + + {% if purge_result %} + {% if purge_result.error %} +
Purge failed: {{ purge_result.error }}
+ {% else %} +
Purged run_track_activity_jsonl from {{ purge_result.purged_rows }} row(s).
+ {% endif %} + {% endif %} + + {% if purge_info and not purge_info.error %} +
+

Purge run_track_activity_jsonl

+
+ {{ purge_info.rows_with_data }} of {{ purge_info.total_rows }} tasks have activity data + ({{ purge_info.total_data_mb }} MB total) +
+
+ + +
+ {% for opt in purge_info.options %} +
+ + + + {% if opt.purgeable_rows > 0 %} + purge {{ opt.purgeable_rows }} rows, save {{ opt.savings_mb }} MB + {% else %} + nothing to purge + {% endif %} + +
+ {% endfor %} +
+ +
+
+ {% elif purge_info and purge_info.error %} +
Failed to load purge info: {{ purge_info.error }}
+ {% endif %} + + {% if vacuum_result %} + {% if vacuum_result.error %} +
Vacuum failed: {{ vacuum_result.error }}
+ {% else %} +
VACUUM FULL task_item completed. Disk space reclaimed.
+ {% endif %} + {% endif %} + +
+

Vacuum

+

+ Run VACUUM FULL task_item to rewrite the table and reclaim disk space. + This locks the table briefly. +

+
+ + + +
+
+ + {% endif %} +
+{% endblock %} diff --git a/frontend_multi_user/templates/admin/db_size.html b/frontend_multi_user/templates/admin/db_size.html deleted file mode 100644 index 2a2866c0..00000000 --- a/frontend_multi_user/templates/admin/db_size.html +++ /dev/null @@ -1,121 +0,0 @@ -{% extends 'admin/master.html' %} - -{% block head_css %} - {{ super() }} - -{% endblock %} - -{% block body %} -
-

Database Size

- - {% if size_info.error %} -
- Error querying database size: {{ size_info.error }} -
- {% else %} -
-
-
{{ size_info.total_mb }} MB
-
Total database size
-
-
-
Database: {{ size_info.database_name }}
-
-
- - {% if size_info.tables %} -

Per-table breakdown

- - - - - - - - - - - - {% for t in size_info.tables %} - - - - - - - - {% endfor %} - -
TableTotal (MB)Data (MB)Indexes (MB)
{{ t.name }}{{ t.total_mb }}{{ t.table_mb }}{{ t.index_mb }} - {% if size_info.tables[0].total_bytes > 0 %} -
- {% endif %} -
- {% endif %} - {% endif %} -
-{% endblock %} diff --git a/frontend_multi_user/templates/admin/index.html b/frontend_multi_user/templates/admin/index.html index 4fd14ebd..19f98f38 100644 --- a/frontend_multi_user/templates/admin/index.html +++ b/frontend_multi_user/templates/admin/index.html @@ -75,9 +75,9 @@

Primary links

Reconciliation and drift detection
Compare billed usage cost vs tracked inference cost per task.
- - Database Size -
View PostgreSQL disk space usage and per-table breakdown.
+ + Database +
View PostgreSQL disk space usage, per-table breakdown, and maintenance actions.
 {% endblock %}