DQX Studio: Support for Lakebase storage backend, declarative storage with destroy protection, observer-aligned error/warning metrics

Makefile

@@ -118,19 +118,77 @@ app-test:
 	  $(if $(COV),--cov=src/databricks_labs_dqx_app/backend --cov-report=term-missing --cov-report=xml:coverage-app.xml)
 
 # Grant Unity Catalog permissions after bundle deploy.
+#
 # Usage: make app-grant-permissions PROFILE=my-profile
+#        make app-grant-permissions PROFILE=my-profile TARGET=dev \
+#                                   BUNDLE_VARS='--var=catalog_name=foo'
+#
+# BUNDLE_VARS must match the overrides used at deploy time. The script
+# reads bundle variables to discover which catalog / schema / volume to
+# grant on, so an unforwarded override would point the GRANTs at the
+# wrong objects.
 app-grant-permissions:
 	@test -n "$(PROFILE)" || (echo "Usage: make app-grant-permissions PROFILE=<databricks-profile> [TARGET=<bundle-target>]"; exit 1)
-	app/scripts/post_deploy_grants.sh -p $(PROFILE) $(if $(TARGET),-t $(TARGET))
+	app/scripts/post_deploy_grants.sh -p $(PROFILE) $(if $(TARGET),-t $(TARGET)) $(if $(BUNDLE_VARS),-- $(BUNDLE_VARS))
 
-# Full deploy: build, bundle deploy, grant permissions, and start the app.
+# Adopt pre-existing storage resources into bundle management.
+#
+# Use ONCE per target on workspaces where the schemas / volume /
+# Lakebase instance already exist (e.g. from a previous bootstrap-
+# script deploy, or from manual creation). The app's Postgres schema
+# inside ``databricks_postgres`` is not bundle-managed and does not
+# need binding — the app re-creates it on first connection if missing.
+# Without binding, ``databricks bundle deploy`` would try to CREATE
+# them and fail with "already exists" / "Instance name is not unique".
+#
+# Bind is idempotent at the CLI level — re-binding the same resource
+# is a no-op. Skip this target on fresh workspaces; ``bundle deploy``
+# creates the resources directly.
+#
+# Usage: make app-bind PROFILE=my-profile TARGET=dev
+#        make app-bind PROFILE=my-profile TARGET=dev \
+#                      BUNDLE_VARS='--var=lakebase_instance_name=<fresh-name>'
+#
+# BUNDLE_VARS forwards arbitrary ``--var key=value`` arguments through
+# to the bundle CLI. Use the same override here as you intend to pass
+# to ``make app-deploy`` — the bind step reads the resolved bundle
+# variables to know which instance/schema name to bind to, so an
+# override applied only at deploy time would bind the wrong resource.
+app-bind:
+	@test -n "$(PROFILE)" || (echo "Usage: make app-bind PROFILE=<databricks-profile> TARGET=<bundle-target>"; exit 1)
+	@test -n "$(TARGET)" || (echo "Usage: make app-bind PROFILE=<databricks-profile> TARGET=<bundle-target>"; exit 1)
+	app/scripts/bind_resources.sh -p $(PROFILE) -t $(TARGET) $(if $(BUNDLE_VARS),-- $(BUNDLE_VARS))
+
+# Full deploy: build, bundle deploy (creates storage on fresh
+# workspaces, updates managed resources otherwise), grant permissions
+# to the app SP, and start the app. Run ``make app-bind`` once before
+# the FIRST deploy on a workspace where the storage was previously
+# provisioned out-of-band — otherwise the bundle will try to CREATE
+# the existing resources and fail.
+#
 # Usage: make app-deploy PROFILE=my-profile TARGET=dev
+#        make app-deploy PROFILE=my-profile TARGET=dev \
+#                        BUNDLE_VARS='--var=lakebase_instance_name=<fresh-name>'
+#
+# BUNDLE_VARS forwards arbitrary ``--var key=value`` arguments to
+# every step of the deploy: ``bundle deploy``, ``post_deploy_grants.sh``,
+# and ``bundle run``. Threading the same overrides through all three is
+# load-bearing — the grants script re-runs ``bundle validate`` to
+# discover the deployed catalog / schema / volume, and without the
+# overrides it would issue GRANTs on the bundle defaults instead of
+# the resources actually deployed.
+#
+# The common use case is overriding ``lakebase_instance_name`` when the
+# original name is locked by Lakebase's ~7-day soft-delete retention
+# after a manual delete (the deploy errors out with "Instance name is
+# not unique" otherwise). Per-deploy CLI overrides keep ``databricks.yml``
+# clean.
 app-deploy: app-build
 	@test -n "$(PROFILE)" || (echo "Usage: make app-deploy PROFILE=<databricks-profile> TARGET=<bundle-target>"; exit 1)
 	@test -n "$(TARGET)" || (echo "Usage: make app-deploy PROFILE=<databricks-profile> TARGET=<bundle-target>"; exit 1)
-	cd app && databricks bundle deploy -p $(PROFILE) -t $(TARGET)
-	app/scripts/post_deploy_grants.sh -p $(PROFILE) -t $(TARGET)
-	cd app && databricks bundle run $(APP_NAME) -p $(PROFILE) -t $(TARGET)
+	cd app && databricks bundle deploy -p $(PROFILE) -t $(TARGET) $(BUNDLE_VARS)
+	app/scripts/post_deploy_grants.sh -p $(PROFILE) -t $(TARGET) $(if $(BUNDLE_VARS),-- $(BUNDLE_VARS))
+	cd app && databricks bundle run $(APP_NAME) -p $(PROFILE) -t $(TARGET) $(BUNDLE_VARS)
 
 APP_NAME ?= dqx-studio
 
@@ -166,4 +224,4 @@ lock-dependencies:
 	perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple"|g' uv.lock
 
 .DEFAULT: all
-.PHONY: all clean dev lint fmt test integration e2e perf anomaly coverage combine-coverage docs-build docs-serve-dev docs-install docs-serve docs-clean app-install app-build app-start-dev app-stop-dev app-check app-test app-grant-permissions app-deploy fork-sync build lock-dependencies lock-app-dependencies
+.PHONY: all clean dev lint fmt test integration e2e perf anomaly coverage combine-coverage docs-build docs-serve-dev docs-install docs-serve docs-clean app-install app-build app-start-dev app-stop-dev app-check app-test app-grant-permissions app-bind app-deploy fork-sync build lock-dependencies lock-app-dependencies

app/.build-constraints.txt

@@ -101,15 +101,15 @@ markupsafe==3.0.3 \
     --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \
     --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50
     # via jinja2
-packaging==26.1 \
-    --hash=sha256:5d9c0669c6285e491e0ced2eee587eaf67b670d94a19e94e3984a481aba6802f \
-    --hash=sha256:f042152b681c4bfac5cae2742a55e103d27ab2ec0f3d88037136b6bfe7c9c5de
+packaging==26.2 \
+    --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \
+    --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661
     # via
     #   dunamai
     #   hatchling
-pathspec==1.0.4 \
-    --hash=sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645 \
-    --hash=sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723
+pathspec==1.1.1 \
+    --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \
+    --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189
     # via hatchling
 pluggy==1.6.0 \
     --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \
@@ -119,9 +119,9 @@ tomlkit==0.14.0 \
     --hash=sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680 \
     --hash=sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064
     # via uv-dynamic-versioning
-trove-classifiers==2026.1.14.14 \
-    --hash=sha256:00492545a1402b09d4858605ba190ea33243d361e2b01c9c296ce06b5c3325f3 \
-    --hash=sha256:1f9553927f18d0513d8e5ff80ab8980b8202ce37ecae0e3274ed2ef11880e74d
+trove-classifiers==2026.4.28.13 \
+    --hash=sha256:8f4b1eb4e16296b57d612965444f87a83861cc989a0451ac97fe4265ddef03b8 \
+    --hash=sha256:c85bb8a53c3de7330d1699b844ed9fb809a602a09ac15dc79ad6d1a509be0676
     # via hatchling
 uv-dynamic-versioning==0.14.0 \
     --hash=sha256:574fbc07e87ace45c01d55967ad3b864871257b98ff5b8ac87c261227ac8db5b \

app/CLAUDE.md

@@ -36,36 +36,72 @@ RBAC is enforced — routes use `require_role(*roles)` from `backend/dependencie
 
 ## Internal Storage
 
-App uses a dedicated catalog selected at install time, with two schemas (managed by `MigrationRunner` in `backend/migrations/`):
+App uses a **hybrid backend** — analytical/append tables in Delta, OLTP
+tables in Lakebase Postgres. Both backends are managed by their own
+migration runner in `backend/migrations/`. Schemas, volume, and Lakebase
+instance are declared as bundle resources in `databricks.yml` with
+`lifecycle.prevent_destroy: true`, so `databricks bundle destroy` cannot
+drop them — see "Bundle conventions" below. The app's `dqx_studio`
+Postgres schema (inside the `databricks_postgres` admin database on the
+Lakebase instance) is created at startup, not provisioned by the bundle,
+but is protected transitively by the instance-level guard.
 
 ```
 {user_catalog}
- ├── dqx_app                          ← main schema (SP-managed)
- │   ├── dq_app_settings              ← key/value app configuration
- │   ├── dq_quality_rules             ← active/approved rules
- │   ├── dq_quality_rules_history     ← rule change audit log
- │   ├── dq_role_mappings             ← role → workspace group mappings (RBAC)
- │   ├── dq_comments                  ← comment threads on rules/runs
- │   ├── dq_profiling_results         ← profiler run results (suggestions in generated_rules_json)
- │   ├── dq_validation_runs           ← dryrun + scheduled run history
- │   ├── dq_quarantine_records        ← invalid rows captured by runs
- │   ├── dq_metrics                   ← per-run quality metrics for trend tracking
- │   ├── dq_schedule_configs          ← per-schedule config (cron/interval, target rules)
- │   ├── dq_schedule_configs_history  ← schedule config change audit log
- │   ├── dq_schedule_runs             ← scheduler last/next run state (survives restarts)
- │   └── dq_migrations                ← migration version tracker
- └── dqx_app_tmp                      ← temp views created via OBO for profiler/dryrun jobs
+ ├── dqx_studio                       ← main schema (SP-managed)
+ │   ├── dq_profiling_results         (Delta) profiler run results
+ │   ├── dq_validation_runs           (Delta) dryrun + scheduled run history
+ │   ├── dq_quarantine_records        (Delta) invalid rows captured by runs
+ │   ├── dq_metrics                   (Delta) per-run quality metrics for trend tracking
+ │   ├── dq_app_settings              (OLTP*) key/value app configuration
+ │   ├── dq_quality_rules             (OLTP*) active/approved rules
+ │   ├── dq_quality_rules_history     (OLTP*) rule change audit log
+ │   ├── dq_role_mappings             (OLTP*) role → workspace group mappings (RBAC)
+ │   ├── dq_comments                  (OLTP*) comment threads on rules/runs
+ │   ├── dq_schedule_configs          (OLTP*) per-schedule config (cron/interval, target rules)
+ │   ├── dq_schedule_configs_history  (OLTP*) schedule config change audit log
+ │   ├── dq_schedule_runs             (OLTP*) scheduler last/next run state (survives restarts)
+ │   └── dq_migrations                (Delta) Delta migration version tracker
+ ├── dqx_studio_tmp                   ← temp views created via OBO for profiler/dryrun jobs
+ └── dqx_studio.wheels (volume)       ← DQX + task-runner wheels uploaded at app startup
+
+Lakebase database (when enabled, default = `dqx-studio-lakebase`):
+ └── dqx_studio                       (database)
+     └── public                        (schema, configurable via DQX_LAKEBASE_SCHEMA)
+         ├── dq_app_settings, dq_role_mappings, dq_quality_rules,
+         │   dq_quality_rules_history, dq_comments, dq_schedule_configs,
+         │   dq_schedule_configs_history, dq_schedule_runs
+         └── dq_migrations             (Postgres migration version tracker)
 ```
 
-A separate UC volume (`{catalog}.dqx_app.wheels` by default) holds the DQX + task-runner wheels uploaded at app startup.
+`(OLTP*)` = lives in **Lakebase Postgres** when
+`lakebase_instance_name` is set, otherwise **Delta** (the
+`v2: Delta OLTP fallback` migration).
 
 ## Key Decisions
 
-- **No config.yaml** — all settings stored in Delta tables
-- **Dedicated catalog** — user selects at install, `dqx_app` schema created by the app
-- **Rule promotion** — export rules then deploy separately to prod; or save directly to prod checks table
-- **Internal storage** — Delta table (preferred); Lakebase also supported as option at install time
-- **Target environments** — Dev, UAT/QA (prod-like data); app is not intended for production rule execution
+- **No config.yaml** — all settings stored in Delta or Lakebase tables.
+- **Dedicated catalog** — user selects at install; `dqx_studio` and `dqx_studio_tmp` schemas are declared as bundle resources and created by `databricks bundle deploy`.
+- **Hybrid storage** — high-volume append tables in Delta; transactional/low-latency tables in Lakebase Postgres.
+- **Rule promotion** — export rules then deploy separately to prod; or save directly to prod checks table.
+- **Target environments** — Dev, UAT/QA (prod-like data); app is not intended for production rule execution.
+
+## Bundle conventions
+
+Stateful resources declared in `databricks.yml`:
+
+- `resources.schemas.main_schema` — `dqx_studio` schema
+- `resources.schemas.tmp_schema` — `dqx_studio_tmp` schema
+- `resources.volumes.wheels` — wheels volume
+- `resources.database_instances.lakebase` — Lakebase Postgres instance (autoscaling)
+
+Each carries `lifecycle.prevent_destroy: true` (Databricks CLI 0.268+), which blocks `databricks bundle destroy` and any deploy that would force-replace the resource. To intentionally tear something down: drop the flag, `databricks bundle deployment unbind <key> -t <target>`, then destroy.
+
+The app connects to the always-present `databricks_postgres` admin database on the Lakebase instance (set as the default `lakebase_database_name`) and creates its own `dqx_studio` Postgres schema there on first start. No DAB resource is needed to provision a per-app logical database; the bundle stays fully declarative. We deliberately do not use `database_catalogs` because it also creates a Unity Catalog catalog and therefore requires `CREATE CATALOG` on the metastore — a permission most app deployers don't hold.
+
+For workspaces where the schemas / volume / Lakebase instance already exist (e.g. created out-of-band before this layout existed), run `make app-bind PROFILE=... TARGET=...` once per target to adopt them — otherwise `databricks bundle deploy` errors out with "already exists" / "Instance name is not unique".
+
+Privileges on UC objects for the auto-created app SP are still reapplied with `scripts/post_deploy_grants.sh` after each deploy, because the app SP's UUID isn't known at bundle-write time.
 
 ## Architecture