Database schema and sqlc configs

Map out what a database could look like to serve the frontend.

I went with SQLC , good balance of safety and speed, coupled with
SQLite is an embedded database, meaning no extra service to run
in the singular container

Author: Luke Hinds

codegate.db

@@ -19,8 +19,11 @@ transformers = ">=4.46.3"
 structlog = ">=24.4.0"
 litellm = "^1.52.16"
 llama_cpp_python = ">=0.3.2"
-
 cryptography = "^44.0.0"
+sqlalchemy = "^2.0.28"
+greenlet = "^3.0.3"
+aiosqlite = "^0.19.0"
+
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
 pytest-cov = ">=4.1.0"

poetry.lock

@@ -0,0 +1,92 @@
+-- name: CreatePrompt :one
+INSERT INTO prompts (
+    id,
+    timestamp,
+    provider,
+    system_prompt,
+    user_prompt,
+    type,
+    status
+) VALUES (?, ?, ?, ?, ?, ?, ?) RETURNING *;
+
+-- name: GetPrompt :one
+SELECT * FROM prompts WHERE id = ?;
+
+-- name: ListPrompts :many
+SELECT * FROM prompts 
+ORDER BY timestamp DESC 
+LIMIT ? OFFSET ?;
+
+-- name: CreateOutput :one
+INSERT INTO outputs (
+    id,
+    prompt_id,
+    timestamp,
+    output,
+    status
+) VALUES (?, ?, ?, ?, ?) RETURNING *;
+
+-- name: GetOutput :one
+SELECT * FROM outputs WHERE id = ?;
+
+-- name: GetOutputsByPromptId :many
+SELECT * FROM outputs 
+WHERE prompt_id = ? 
+ORDER BY timestamp DESC;
+
+-- name: CreateAlert :one
+INSERT INTO alerts (
+    id,
+    prompt_id,
+    output_id,
+    code_snippet,
+    trigger_string,
+    trigger_type,
+    trigger_category,
+    timestamp
+) VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING *;
+
+-- name: GetAlert :one
+SELECT * FROM alerts WHERE id = ?;
+
+-- name: ListAlertsByPrompt :many
+SELECT * FROM alerts 
+WHERE prompt_id = ? 
+ORDER BY timestamp DESC;
+
+-- name: GetSettings :one
+SELECT * FROM settings ORDER BY id LIMIT 1;
+
+-- name: UpsertSettings :one
+INSERT INTO settings (
+    id,
+    ip,
+    port,
+    llm_model,
+    system_prompt,
+    other_settings
+) VALUES (?, ?, ?, ?, ?, ?)
+ON CONFLICT(id) DO UPDATE SET
+    ip = excluded.ip,
+    port = excluded.port,
+    llm_model = excluded.llm_model,
+    system_prompt = excluded.system_prompt,
+    other_settings = excluded.other_settings
+RETURNING *;
+
+-- name: GetPromptWithOutputsAndAlerts :many
+SELECT 
+    p.*,
+    o.id as output_id,
+    o.output,
+    o.status as output_status,
+    a.id as alert_id,
+    a.code_snippet,
+    a.trigger_string,
+    a.trigger_type,
+    a.trigger_category
+FROM prompts p
+LEFT JOIN outputs o ON p.id = o.prompt_id
+LEFT JOIN alerts a ON p.id = a.prompt_id
+WHERE p.id = ?
+ORDER BY o.timestamp DESC, a.timestamp DESC;

pyproject.toml

@@ -0,0 +1,54 @@
+-- Schema for codegate database using SQLite
+
+-- Prompts table
+CREATE TABLE prompts (
+    id TEXT PRIMARY KEY,  -- UUID stored as TEXT
+    timestamp DATETIME NOT NULL,
+    provider TEXT,       -- VARCHAR(255)
+    system_prompt TEXT,
+    user_prompt TEXT NOT NULL,
+    type TEXT NOT NULL, -- VARCHAR(50) (e.g. "fim", "chat")
+    status TEXT NOT NULL  -- VARCHAR(50)
+);
+
+-- Outputs table
+CREATE TABLE outputs (
+    id TEXT PRIMARY KEY,  -- UUID stored as TEXT
+    prompt_id TEXT NOT NULL,
+    timestamp DATETIME NOT NULL,
+    output TEXT NOT NULL,
+    status TEXT NOT NULL, -- VARCHAR(50)
+    FOREIGN KEY (prompt_id) REFERENCES prompts(id)
+);
+
+-- Alerts table
+CREATE TABLE alerts (
+    id TEXT PRIMARY KEY,  -- UUID stored as TEXT
+    prompt_id TEXT NOT NULL,
+    output_id TEXT NOT NULL,
+    code_snippet TEXT NOT NULL,  -- VARCHAR(255)
+    trigger_string TEXT NOT NULL, -- VARCHAR(255)
+    trigger_type TEXT NOT NULL,   -- VARCHAR(50)
+    trigger_category TEXT,
+    timestamp DATETIME NOT NULL,
+    FOREIGN KEY (prompt_id) REFERENCES prompts(id),
+    FOREIGN KEY (output_id) REFERENCES outputs(id)
+);
+
+-- Settings table
+CREATE TABLE settings (
+    id TEXT PRIMARY KEY,  -- UUID stored as TEXT
+    ip TEXT,             -- VARCHAR(45)
+    port INTEGER,
+    llm_model TEXT,      -- VARCHAR(255)
+    system_prompt TEXT,
+    other_settings TEXT  -- JSON stored as TEXT
+);
+
+-- Create indexes for foreign keys and frequently queried columns
+CREATE INDEX idx_outputs_prompt_id ON outputs(prompt_id);
+CREATE INDEX idx_alerts_prompt_id ON alerts(prompt_id);
+CREATE INDEX idx_alerts_output_id ON alerts(output_id);
+CREATE INDEX idx_prompts_timestamp ON prompts(timestamp);
+CREATE INDEX idx_outputs_timestamp ON outputs(timestamp);
+CREATE INDEX idx_alerts_timestamp ON alerts(timestamp);

sql/queries/queries.sql

@@ -0,0 +1,19 @@
+version: "2"
+plugins:
+  - name: "python"
+    wasm:
+      url: "https://downloads.sqlc.dev/plugin/sqlc-gen-python_1.2.0.wasm"
+      sha256: "a6c5d174c407007c3717eea36ff0882744346e6ba991f92f71d6ab2895204c0e"
+
+sql:
+  - engine: "sqlite"
+    schema: "sql/schema"
+    queries: "sql/queries"
+    codegen:
+      - plugin: "python"
+        out: "src/codegate/db"
+        options:
+          package: "codegate.db"
+          emit_sync_querier: true
+          emit_async_querier: true
+          query_parameter_limit: 5

sql/schema/schema.sql

@@ -9,7 +9,7 @@
 
 from codegate.codegate_logging import LogFormat, LogLevel, setup_logging
 from codegate.config import Config, ConfigurationError
-from codegate.server import init_app
+from codegate.server import create_app
 
 
 def validate_port(ctx: click.Context, param: click.Parameter, value: int) -> int:
@@ -164,7 +164,7 @@ def serve(
             },
         )
 
-        app = init_app()
+        app = create_app()
 
         import uvicorn
 

sqlc.yaml

@@ -0,0 +1,47 @@
+import asyncio
+from pathlib import Path
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine
+
+
+async def init_db():
+    """Initialize the database with the schema."""
+    # Get the absolute path to the schema file
+    current_dir = Path(__file__).parent
+    schema_path = current_dir.parent.parent.parent / "sql" / "schema" / "schema.sql"
+
+    if not schema_path.exists():
+        raise FileNotFoundError(f"Schema file not found at {schema_path}")
+
+    # Read the schema
+    with open(schema_path, "r") as f:
+        schema = f.read()
+
+    # Create the database engine with proper async SQLite URL
+    db_path = Path("codegate.db").absolute()
+    engine = create_async_engine(
+        f"sqlite+aiosqlite:///{db_path}",
+        echo=True,  # Set to False in production
+        isolation_level="AUTOCOMMIT",  # Required for SQLite
+    )
+
+    try:
+        # Execute the schema
+        async with engine.begin() as conn:
+            # Split the schema into individual statements and execute each one
+            statements = [stmt.strip() for stmt in schema.split(";") if stmt.strip()]
+            for statement in statements:
+                # Use SQLAlchemy text() to create executable SQL statements
+                await conn.execute(text(statement))
+    finally:
+        await engine.dispose()
+
+
+def init_db_sync():
+    """Synchronous wrapper for init_db."""
+    asyncio.run(init_db())
+
+
+if __name__ == "__main__":
+    init_db_sync()

src/codegate/cli.py

@@ -0,0 +1,47 @@
+# Code generated by sqlc. DO NOT EDIT.
+# versions:
+#   sqlc v1.27.0
+import dataclasses
+from typing import Any, Optional
+
+
+@dataclasses.dataclass()
+class Alert:
+    id: Any
+    prompt_id: Any
+    output_id: Any
+    code_snippet: Any
+    trigger_string: Any
+    trigger_type: Any
+    trigger_category: Optional[Any]
+    timestamp: Any
+
+
+@dataclasses.dataclass()
+class Output:
+    id: Any
+    prompt_id: Any
+    timestamp: Any
+    output: Any
+    status: Any
+
+
+@dataclasses.dataclass()
+class Prompt:
+    id: Any
+    timestamp: Any
+    provider: Optional[Any]
+    system_prompt: Optional[Any]
+    user_prompt: Any
+    type: Any
+    status: Any
+
+
+@dataclasses.dataclass()
+class Setting:
+    id: Any
+    ip: Optional[Any]
+    port: Optional[Any]
+    llm_model: Optional[Any]
+    system_prompt: Optional[Any]
+    other_settings: Optional[Any]