chore: Extensions architecture POC

.github/CODEOWNERS

@@ -30,3 +30,13 @@
 
 **/*.geojson @villebro @rusackas
 /superset-frontend/plugins/legacy-plugin-chart-country-map/ @villebro @rusackas
+
+# Notify PMC members of changes to extension-related files
+
+/superset-core/ @michael-s-molina @villebro
+/superset-cli/ @michael-s-molina @villebro
+/superset/core/ @michael-s-molina @villebro
+/superset/extensions/ @michael-s-molina @villebro
+/superset-frontend/src/packages/superset-core/ @michael-s-molina @villebro
+/superset-frontend/src/core/ @michael-s-molina @villebro
+/superset-frontend/src/extensions/ @michael-s-molina @villebro

.github/actions/change-detector/action.yml

@@ -1,24 +1,27 @@
-name: 'Change Detector'
-description: 'Detects file changes for pull request and push events'
+name: Change Detector
+description: Detects file changes for pull request and push events
 inputs:
   token:
-    description: 'GitHub token for authentication'
+    description: GitHub token for authentication
     required: true
 outputs:
   python:
-    description: 'Whether Python-related files were changed'
+    description: Whether Python-related files were changed
     value: ${{ steps.change-detector.outputs.python }}
   frontend:
-    description: 'Whether frontend-related files were changed'
+    description: Whether frontend-related files were changed
     value: ${{ steps.change-detector.outputs.frontend }}
   docker:
-    description: 'Whether docker-related files were changed'
+    description: Whether docker-related files were changed
     value: ${{ steps.change-detector.outputs.docker }}
   docs:
-    description: 'Whether docs-related files were changed'
+    description: Whether docs-related files were changed
     value: ${{ steps.change-detector.outputs.docs }}
+  superset-cli:
+    description: Whether superset-cli package-related files were changed
+    value: ${{ steps.change-detector.outputs.superset-cli }}
 runs:
-  using: 'composite'
+  using: composite
   steps:
     - name: Detect file changes
       id: change-detector

.github/workflows/superset-app-cli.yml

@@ -0,0 +1,67 @@
+name: Superset App CLI tests
+
+on:
+  push:
+    branches:
+      - "master"
+      - "[0-9].[0-9]*"
+  pull_request:
+    types: [synchronize, opened, reopened, ready_for_review]
+
+# cancel previous workflow jobs for PRs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test-load-examples:
+    runs-on: ubuntu-24.04
+    env:
+      PYTHONPATH: ${{ github.workspace }}
+      SUPERSET_CONFIG: tests.integration_tests.superset_test_config
+      REDIS_PORT: 16379
+      SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:[email protected]:15432/superset
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_USER: superset
+          POSTGRES_PASSWORD: superset
+        ports:
+          # Use custom ports for services to avoid accidentally connecting to
+          # GitHub action runner's default installations
+          - 15432:5432
+      redis:
+        image: redis:7-alpine
+        ports:
+          - 16379:6379
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          submodules: recursive
+      - name: Check for file changes
+        id: check
+        uses: ./.github/actions/change-detector/
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup Python
+        if: steps.check.outputs.python
+        uses: ./.github/actions/setup-backend/
+      - name: Setup Postgres
+        if: steps.check.outputs.python
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: setup-postgres
+      - name: superset init
+        if: steps.check.outputs.python
+        run: |
+          pip install -e .
+          superset db upgrade
+          superset load_test_users
+      - name: superset load_examples
+        if: steps.check.outputs.python
+        run: |
+          # load examples without test data
+          superset load_examples --load-big-data

.github/workflows/superset-cli.yml

@@ -1,4 +1,4 @@
-name: Superset CLI tests
+name: Superset CLI Package Tests
 
 on:
   push:
@@ -14,54 +14,51 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test-load-examples:
+  test-superset-cli-package:
     runs-on: ubuntu-24.04
-    env:
-      PYTHONPATH: ${{ github.workspace }}
-      SUPERSET_CONFIG: tests.integration_tests.superset_test_config
-      REDIS_PORT: 16379
-      SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:[email protected]:15432/superset
-    services:
-      postgres:
-        image: postgres:16-alpine
-        env:
-          POSTGRES_USER: superset
-          POSTGRES_PASSWORD: superset
-        ports:
-          # Use custom ports for services to avoid accidentally connecting to
-          # GitHub action runner's default installations
-          - 15432:5432
-      redis:
-        image: redis:7-alpine
-        ports:
-          - 16379:6379
+    strategy:
+      matrix:
+        python-version: ["previous", "current", "next"]
+    defaults:
+      run:
+        working-directory: superset-cli
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v4
         with:
           persist-credentials: false
           submodules: recursive
+
       - name: Check for file changes
         id: check
         uses: ./.github/actions/change-detector/
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Setup Python
-        if: steps.check.outputs.python
+        if: steps.check.outputs.superset-cli
         uses: ./.github/actions/setup-backend/
-      - name: Setup Postgres
-        if: steps.check.outputs.python
-        uses: ./.github/actions/cached-dependencies
         with:
-          run: setup-postgres
-      - name: superset init
-        if: steps.check.outputs.python
-        run: |
-          pip install -e .
-          superset db upgrade
-          superset load_test_users
-      - name: superset load_examples
-        if: steps.check.outputs.python
+          python-version: ${{ matrix.python-version }}
+          requirements-type: dev
+
+      - name: Run pytest with coverage
+        if: steps.check.outputs.superset-cli
         run: |
-          # load examples without test data
-          superset load_examples --load-big-data
+          pytest --cov=superset_cli --cov-report=xml --cov-report=term-missing --cov-report=html -v --tb=short
+
+      - name: Upload coverage reports to Codecov
+        if: steps.check.outputs.superset-cli
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: superset-cli
+          name: superset-cli-coverage
+          fail_ci_if_error: false
+
+      - name: Upload HTML coverage report
+        if: steps.check.outputs.superset-cli
+        uses: actions/upload-artifact@v4
+        with:
+          name: superset-cli-coverage-html
+          path: htmlcov/

.gitignore

@@ -43,7 +43,7 @@ _modules
 _static
 build
 app.db
-apache_superset.egg-info/
+*.egg-info/
 changelog.sh
 dist
 dump.rdb

.pre-commit-config.yaml

@@ -23,7 +23,9 @@ repos:
     rev: v1.15.0
     hooks:
       - id: mypy
+        name: mypy (main)
         args: [--check-untyped-defs]
+        exclude: ^superset-cli/
         additional_dependencies: [
             types-simplejson,
             types-python-dateutil,
@@ -38,6 +40,10 @@ repos:
             types-paramiko,
             types-Markdown,
           ]
+      - id: mypy
+        name: mypy (superset-cli)
+        args: [--check-untyped-defs]
+        files: ^superset-cli/
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
@@ -54,25 +60,25 @@ repos:
         args: ["--markdown-linebreak-ext=md"]
   - repo: local
     hooks:
-    - id: eslint-frontend
-      name: eslint (frontend)
-      entry: ./scripts/eslint.sh
-      language: system
-      pass_filenames: true
-      files: ^superset-frontend/.*\.(js|jsx|ts|tsx)$
-    - id: eslint-docs
-      name: eslint (docs)
-      entry: bash -c 'cd docs && FILES=$(echo "$@" | sed "s|docs/||g") && yarn eslint --fix --ext .js,.jsx,.ts,.tsx --quiet $FILES'
-      language: system
-      pass_filenames: true
-      files: ^docs/.*\.(js|jsx|ts|tsx)$
-    - id: type-checking-frontend
-      name: Type-Checking (Frontend)
-      entry: ./scripts/check-type.js package=superset-frontend excludeDeclarationDir=cypress-base
-      language: system
-      files: ^superset-frontend\/.*\.(js|jsx|ts|tsx)$
-      exclude: ^superset-frontend/cypress-base\/
-      require_serial: true
+      - id: eslint-frontend
+        name: eslint (frontend)
+        entry: ./scripts/eslint.sh
+        language: system
+        pass_filenames: true
+        files: ^superset-frontend/.*\.(js|jsx|ts|tsx)$
+      - id: eslint-docs
+        name: eslint (docs)
+        entry: bash -c 'cd docs && FILES=$(echo "$@" | sed "s|docs/||g") && yarn eslint --fix --ext .js,.jsx,.ts,.tsx --quiet $FILES'
+        language: system
+        pass_filenames: true
+        files: ^docs/.*\.(js|jsx|ts|tsx)$
+      - id: type-checking-frontend
+        name: Type-Checking (Frontend)
+        entry: ./scripts/check-type.js package=superset-frontend excludeDeclarationDir=cypress-base
+        language: system
+        files: ^superset-frontend\/.*\.(js|jsx|ts|tsx)$
+        exclude: ^superset-frontend/cypress-base\/
+        require_serial: true
   # blacklist unsafe functions like make_url (see #19526)
   - repo: https://github.com/skorokithakis/blacklist-pre-commit-hook
     rev: e2f070289d8eddcaec0b580d3bde29437e7c8221
@@ -94,21 +100,21 @@ repos:
         args: [--fix]
   - repo: local
     hooks:
-    - id: pylint
-      name: pylint with custom Superset plugins
-      entry: bash
-      language: system
-      types: [python]
-      exclude: ^(tests/|superset/migrations/|scripts/|RELEASING/|docker/)
-      args:
-        - -c
-        - |
-          TARGET_BRANCH=${GITHUB_BASE_REF:-master}
-          git fetch origin "$TARGET_BRANCH"
-          BASE=$(git merge-base origin/"$TARGET_BRANCH" HEAD)
-          files=$(git diff --name-only --diff-filter=ACM "$BASE"..HEAD | grep '^superset/.*\.py$' || true)
-          if [ -n "$files" ]; then
-            pylint --rcfile=.pylintrc --load-plugins=superset.extensions.pylint --reports=no $files
-          else
-            echo "No Python files to lint."
-          fi
+      - id: pylint
+        name: pylint with custom Superset plugins
+        entry: bash
+        language: system
+        types: [python]
+        exclude: ^(tests/|superset/migrations/|scripts/|RELEASING/|docker/)
+        args:
+          - -c
+          - |
+            TARGET_BRANCH=${GITHUB_BASE_REF:-master}
+            git fetch origin "$TARGET_BRANCH"
+            BASE=$(git merge-base origin/"$TARGET_BRANCH" HEAD)
+            files=$(git diff --name-only --diff-filter=ACM "$BASE"..HEAD | grep '^superset/.*\.py$' || true)
+            if [ -n "$files" ]; then
+              pylint --rcfile=.pylintrc --load-plugins=superset.extensions.pylint --reports=no $files
+            else
+              echo "No Python files to lint."
+            fi

.rat-excludes

@@ -32,6 +32,8 @@ apache_superset.egg-info
 # json and csv in general cannot have comments
 .*json
 .*csv
+# jinja templates often need to be as-is
+.*j2
 # Generated doc files
 env/*
 docs/.htaccess*

Dockerfile

@@ -219,6 +219,10 @@ FROM python-common AS lean
 
 # Install Python dependencies using docker/pip-install.sh
 COPY requirements/base.txt requirements/
+
+# Copy superset-core package needed for editable install in base.txt
+COPY superset-core superset-core
+
 RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
     /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt
 # Install the superset package
@@ -241,6 +245,11 @@ RUN /app/docker/apt-install.sh \
 
 # Copy development requirements and install them
 COPY requirements/*.txt requirements/
+
+# Copy local packages needed for editable installs in development.txt
+COPY superset-core superset-core
+COPY superset-cli superset-cli
+
 # Install Python dependencies using docker/pip-install.sh
 RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
     /app/docker/pip-install.sh --requires-build-essential -r requirements/development.txt

pyproject.toml

@@ -35,6 +35,7 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
+    "apache-superset-core>=0.0.1, <0.2",
     "backoff>=1.8.0",
     "celery>=5.3.6, <6.0.0",
     "click>=8.0.3",
@@ -101,6 +102,7 @@ dependencies = [
     "tabulate>=0.9.0, <1.0",
     "typing-extensions>=4, <5",
     "waitress; sys_platform == 'win32'",
+    "watchdog>=6.0.0",
     "wtforms>=2.3.3, <4",
     "wtforms-json",
     "xlsxwriter>=3.0.7, <3.1",
@@ -190,6 +192,7 @@ doris = ["pydoris>=1.0.0, <2.0.0"]
 oceanbase = ["oceanbase_py>=0.0.1"]
 ydb = ["ydb-sqlalchemy>=0.1.2"]
 development = [
+    "apache-superset-cli>=0.0.1, <0.2",
     "docker",
     "flask-testing",
     "freezegun",
@@ -221,7 +224,7 @@ documentation = "https://superset.apache.org/docs/intro"
 combine_as_imports = true
 include_trailing_comma = true
 line_length = 88
-known_first_party = "superset"
+known_first_party = "superset, apache-superset-core, apache-superset-cli"
 known_third_party = "alembic, apispec, backoff, celery, click, colorama, cron_descriptor, croniter, cryptography, dateutil, deprecation, flask, flask_appbuilder, flask_babel, flask_caching, flask_compress, flask_jwt_extended, flask_login, flask_migrate, flask_sqlalchemy, flask_talisman, flask_testing, flask_wtf, freezegun, geohash, geopy, holidays, humanize, isodate, jinja2, jwt, markdown, markupsafe, marshmallow, msgpack, nh3, numpy, pandas, parameterized, parsedatetime, pgsanity, polyline, prison, progress, pyarrow, sqlalchemy_bigquery, pyhive, pyparsing, pytest, pytest_mock, pytz, redis, requests, selenium, setuptools, shillelagh, simplejson, slack, sqlalchemy, sqlalchemy_utils, typing_extensions, urllib3, werkzeug, wtforms, wtforms_json, yaml"
 multi_line_output = 3
 order_by_type = false
@@ -424,3 +427,7 @@ python-geohash = "0"
 # TODO REMOVE THESE DEPS FROM CODEBASE
 paramiko = "3"  # GPL
 pyxlsb = "1"  # GPL
+
+[tool.uv.sources]
+apache-superset-core = { path = "./superset-core", editable = true }
+apache-superset-cli = { path = "./superset-cli", editable = true }