Merge pull request #68 from mattip/download-data

add a dump button to admin page and manage.py import_sqlite_dump file

Author: mattip

codespeed/admin_views.py

@@ -0,0 +1,176 @@
+import os
+import sqlite3
+import tempfile
+from datetime import datetime, date
+
+from django.contrib.admin.views.decorators import staff_member_required
+from django.http import FileResponse
+
+from django.db import connection
+
+SIZE_LIMIT = 95 * 1024 * 1024  # 95 MB
+
+# Schema for the exported SQLite file, in FK-safe creation order.
+# Column names must match Django's generated PostgreSQL column names exactly.
+_SCHEMA = """
+PRAGMA foreign_keys = OFF;
+
+CREATE TABLE codespeed_project (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    repo_type TEXT NOT NULL,
+    repo_path TEXT NOT NULL,
+    repo_user TEXT NOT NULL,
+    repo_pass TEXT NOT NULL,
+    commit_browsing_url TEXT NOT NULL,
+    track INTEGER NOT NULL,
+    default_branch TEXT NOT NULL
+);
+
+CREATE TABLE codespeed_branch (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    project_id INTEGER NOT NULL,
+    display_on_comparison_page INTEGER NOT NULL
+);
+
+CREATE TABLE codespeed_executable (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    description TEXT NOT NULL,
+    project_id INTEGER NOT NULL
+);
+
+CREATE TABLE codespeed_environment (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    cpu TEXT NOT NULL,
+    memory TEXT NOT NULL,
+    os TEXT NOT NULL,
+    kernel TEXT NOT NULL
+);
+
+CREATE TABLE codespeed_benchmark (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    parent_id INTEGER,
+    source TEXT NOT NULL,
+    data_type TEXT NOT NULL,
+    description TEXT NOT NULL,
+    units_title TEXT NOT NULL,
+    units TEXT NOT NULL,
+    lessisbetter INTEGER NOT NULL,
+    default_on_comparison INTEGER NOT NULL
+);
+
+CREATE TABLE codespeed_revision (
+    id INTEGER PRIMARY KEY,
+    commitid TEXT NOT NULL,
+    tag TEXT NOT NULL,
+    date TEXT,
+    message TEXT NOT NULL,
+    project_id INTEGER,
+    author TEXT NOT NULL,
+    branch_id INTEGER NOT NULL
+);
+
+CREATE TABLE codespeed_result (
+    id INTEGER PRIMARY KEY,
+    value REAL NOT NULL,
+    std_dev REAL,
+    val_min REAL,
+    val_max REAL,
+    q1 REAL,
+    q3 REAL,
+    suite_version TEXT NOT NULL,
+    date TEXT,
+    revision_id INTEGER NOT NULL,
+    executable_id INTEGER NOT NULL,
+    benchmark_id INTEGER NOT NULL,
+    environment_id INTEGER NOT NULL
+);
+"""
+
+_SMALL_TABLES = [
+    'codespeed_project',
+    'codespeed_branch',
+    'codespeed_executable',
+    'codespeed_environment',
+    'codespeed_benchmark',
+    'codespeed_revision',
+]
+
+
+def _conv(v):
+    """Convert PostgreSQL Python types to SQLite-safe scalars."""
+    if isinstance(v, (datetime, date)):
+        return v.isoformat()
+    return v
+
+
+def _build_sqlite(path):
+    lite = sqlite3.connect(path)
+    try:
+        lite.executescript(_SCHEMA)
+        lite.commit()
+
+        with connection.cursor() as pg:
+            for table in _SMALL_TABLES:
+                pg.execute(f'SELECT * FROM {table}')
+                cols = [d[0] for d in pg.description]
+                rows = [tuple(_conv(v) for v in row) for row in pg.fetchall()]
+                if rows:
+                    ph = ', '.join(['?'] * len(cols))
+                    lite.executemany(
+                        f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({ph})",
+                        rows,
+                    )
+            lite.commit()
+
+            # Result table: newest-first, stop at size cap
+            pg.execute(
+                'SELECT * FROM codespeed_result ORDER BY date DESC NULLS LAST'
+            )
+            cols = [d[0] for d in pg.description]
+            ph = ', '.join(['?'] * len(cols))
+            insert_sql = (
+                f"INSERT INTO codespeed_result ({', '.join(cols)}) VALUES ({ph})"
+            )
+            while True:
+                batch = pg.fetchmany(5000)
+                if not batch:
+                    break
+                lite.executemany(
+                    insert_sql,
+                    [tuple(_conv(v) for v in row) for row in batch],
+                )
+                lite.commit()
+                if os.path.getsize(path) > SIZE_LIMIT:
+                    break
+    finally:
+        lite.close()
+
+
+@staff_member_required
+def download_db(request):
+    fd, path = tempfile.mkstemp(suffix='.sqlite3')
+    os.close(fd)
+    try:
+        _build_sqlite(path)
+        f = open(path, 'rb')
+        os.unlink(path)  # unlink now; data survives until f is closed
+        today = datetime.today().strftime('%Y-%m-%d')
+        response = FileResponse(
+            f,
+            as_attachment=True,
+            filename=f'codespeed-{today}.sqlite3',
+        )
+        response.set_cookie(
+            'codespeed_download_ready', '1',
+            max_age=60, path='/', samesite='Lax',
+        )
+        return response
+    except Exception:
+        if os.path.exists(path):
+            os.unlink(path)
+        raise

codespeed/management/__init__.py

@@ -0,0 +1,95 @@
+"""
+Import a codespeed SQLite snapshot into the running database.
+
+Rows that already exist (by primary key or unique constraint) are silently
+skipped, so the command is safe to re-run and merges cleanly into existing
+data.
+
+Usage:
+    python manage.py import_sqlite_dump codespeed-YYYY-MM-DD.sqlite3
+"""
+import sqlite3
+from datetime import datetime
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import connection
+
+# Columns that are stored as 0/1 integers in SQLite but need Python bools
+# for PostgreSQL's boolean type.
+_BOOL_COLS = {
+    'codespeed_project':   {'track'},
+    'codespeed_branch':    {'display_on_comparison_page'},
+    'codespeed_benchmark': {'lessisbetter', 'default_on_comparison'},
+}
+
+# Columns stored as ISO strings in SQLite that must become datetime objects
+# for PostgreSQL's timestamp type.
+_DT_COLS = {
+    'codespeed_revision': {'date'},
+    'codespeed_result':   {'date'},
+}
+
+# Insertion order respects FK dependencies.
+_TABLES = [
+    'codespeed_project',
+    'codespeed_branch',
+    'codespeed_executable',
+    'codespeed_environment',
+    'codespeed_benchmark',
+    'codespeed_revision',
+    'codespeed_result',
+]
+
+
+def _adapt(value, col, bool_cols, dt_cols):
+    if value is None:
+        return None
+    if col in bool_cols:
+        return bool(value)
+    if col in dt_cols and isinstance(value, str):
+        return datetime.fromisoformat(value)
+    return value
+
+
+class Command(BaseCommand):
+    help = 'Import a SQLite snapshot into the running PostgreSQL database'
+
+    def add_arguments(self, parser):
+        parser.add_argument('sqlite_file', help='Path to the SQLite dump file')
+
+    def handle(self, *args, **options):
+        sqlite_file = options['sqlite_file']
+        try:
+            src = sqlite3.connect(sqlite_file)
+        except Exception as e:
+            raise CommandError(f'Cannot open {sqlite_file}: {e}')
+
+        src.row_factory = sqlite3.Row
+
+        with connection.cursor() as cur:
+            for table in _TABLES:
+                bool_cols = _BOOL_COLS.get(table, set())
+                dt_cols = _DT_COLS.get(table, set())
+
+                rows = src.execute(f'SELECT * FROM {table}').fetchall()
+                if not rows:
+                    self.stdout.write(f'  {table}: 0 rows')
+                    continue
+
+                cols = list(rows[0].keys())
+                col_list = ', '.join(cols)
+                placeholders = ', '.join(['%s'] * len(cols))
+                sql = (
+                    f'INSERT INTO {table} ({col_list}) '
+                    f'VALUES ({placeholders}) '
+                    f'ON CONFLICT DO NOTHING'
+                )
+                data = [
+                    tuple(_adapt(row[c], c, bool_cols, dt_cols) for c in cols)
+                    for row in rows
+                ]
+                cur.executemany(sql, data)
+                self.stdout.write(f'  {table}: {len(data)} rows')
+
+        src.close()
+        self.stdout.write(self.style.SUCCESS('Import complete'))

codespeed/management/commands/__init__.py

@@ -95,8 +95,6 @@
                 ]
 DEF_EXECUTABLES = [
                    {'name': 'pypy3.11-jit-64', 'project': 'PyPy3.11'},
-                   {'name': 'pypy3.10-jit-64', 'project': 'PyPy3.10'},
-                   {'name': 'pypy3.9-jit-64', 'project': 'PyPy3.9'},
                   ]
 DEF_ENVIRONMENT = 'benchmarker'
 CHART_ORIENTATION = 'horizontal'

codespeed/management/commands/import_sqlite_dump.py

@@ -0,0 +1,73 @@
+{% extends "admin/index.html" %}
+{% load i18n %}
+
+{% block content %}
+<div id="content-main">
+  {% include "admin/app_list.html" with app_list=app_list show_changelinks=True %}
+
+  <div class="app-datatools module">
+    <table>
+      <caption>Data tools</caption>
+      <thead class="visually-hidden">
+        <tr>
+          <th scope="col">{% trans "Action" %}</th>
+          <th scope="col"></th>
+          <th scope="col">{% trans "Link" %}</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr class="model-sqlitesnapshot">
+          <th scope="row" id="datatools-sqlitesnapshot">SQLite snapshot</th>
+          <td></td>
+          <td>
+            <a id="download-db-btn" href="{% url 'admin-download-db' %}" class="changelink">Download</a>
+            <span id="download-db-status" style="display:none; margin-left:1em; color:#666; font-style:italic;"></span>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+</div>
+<script>
+(function () {
+    var btn = document.getElementById('download-db-btn');
+    var status = document.getElementById('download-db-status');
+    if (!btn) { return; }
+
+    btn.addEventListener('click', function (e) {
+        e.preventDefault();
+        btn.style.display = 'none';
+        status.style.display = '';
+        status.textContent = 'Generating… (0s)';
+
+        var start = Date.now();
+        var timer = setInterval(function () {
+            var sec = Math.floor((Date.now() - start) / 1000);
+            status.textContent = 'Generating… (' + sec + 's)';
+        }, 1000);
+
+        window.location.href = btn.href;
+
+        var poll = setInterval(function () {
+            var ready = document.cookie.split(';').some(function (c) {
+                return c.trim().indexOf('codespeed_download_ready=') === 0;
+            });
+            if (ready) {
+                clearInterval(timer);
+                clearInterval(poll);
+                document.cookie = 'codespeed_download_ready=; max-age=0; path=/';
+                status.style.display = 'none';
+                btn.style.display = '';
+            }
+        }, 500);
+
+        setTimeout(function () {
+            clearInterval(timer);
+            clearInterval(poll);
+            status.style.display = 'none';
+            btn.style.display = '';
+        }, 90000);
+    });
+}());
+</script>
+{% endblock %}

speed_pypy/settings.py

@@ -4,7 +4,10 @@
 from django.urls import include, re_path
 from django.contrib import admin
 
+from codespeed import admin_views
+
 urlpatterns = [
+    re_path(r'^admin/download-db/$', admin_views.download_db, name='admin-download-db'),
     re_path(r'^admin/', admin.site.urls),
     re_path(r'^', include('codespeed.urls'))
 ]