🔥 Added API Docs

.github/workflows/docs-release.yml

@@ -13,23 +13,38 @@ jobs:
       group: databrickslabs-protected-runner-group
       labels: linux-ubuntu-latest
     environment: release
-    defaults:
-      run:
-        working-directory: docs/dqx
+
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          cache: 'pip'
+          cache-dependency-path: '**/pyproject.toml'
+          python-version: '3.11'
+
+      - name: Install Hatch
+        run: |
+          pip install hatch==1.9.4
+
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: yarn
           cache-dependency-path: docs/dqx/yarn.lock # need to put the lockfile path explicitly
 
       - name: Install dependencies
-        run: yarn install --frozen-lockfile
+        run: yarn --cwd docs/dqx install --frozen-lockfile
+
+
+      - name: Generate API docs
+        run: hatch run docs:pydoc-markdown
+
       - name: Build website
-        run: yarn build
+        run: yarn --cwd docs/dqx build
 
       - name: Upload Build Artifact
         uses: actions/upload-pages-artifact@v3

.gitignore

@@ -159,3 +159,13 @@ dev/cleanup.py
 .python-version
 .databricks-login.json
 .local-dev
+
+# Cursor IDE specific files
+.cursorrules
+.cursor/
+
+
+# docgen
+docs/dqx/docs/reference/api
+!docs/dqx/docs/reference/api/index.mdx
+

Makefile

@@ -1,6 +1,6 @@
 all: clean dev lint fmt test integration coverage e2e
 
-clean:
+clean: docs-clean
 	rm -fr .venv clean htmlcov .mypy_cache .pytest_cache .ruff_cache .coverage coverage.xml
 	rm -fr **/*.pyc
 
@@ -31,14 +31,22 @@ e2e:
 coverage:
 	hatch run coverage; open htmlcov/index.html
 
-docs-build:
+docs-build: 
+	hatch run docs:pydoc-markdown
 	yarn --cwd docs/dqx build
 
 docs-serve-dev:
+	hatch run docs:pydoc-markdown
 	yarn --cwd docs/dqx start
 
 docs-install:
 	yarn --cwd docs/dqx install
 
 docs-serve: docs-build
-	yarn --cwd docs/dqx serve
+	hatch run docs:pydoc-markdown
+	yarn --cwd docs/dqx serve
+
+docs-clean:
+	rm -rf docs/dqx/build
+	rm -rf docs/dqx/.docusaurus docs/dqx/.cache
+	find docs/dqx/docs/reference/api -mindepth 1 -not -name 'index.mdx' -exec rm -rf {} +

docs/dqx/docs/dev/contributing.mdx

@@ -45,7 +45,7 @@ Here are the example steps to submit your first contribution:
 3. `git checkout main` (or `gcm` if you're using [ohmyzsh](https://ohmyz.sh/)).
 4. `git pull` (or `gl` if you're using [ohmyzsh](https://ohmyz.sh/)).
 5. `git checkout -b FEATURENAME` (or `gcb FEATURENAME` if you're using [ohmyzsh](https://ohmyz.sh/)).
-6. .. do the work
+6. .. do the work and make sure [Definition of Done (DoD) items](#definition-of-done) are fulfilled.
 7. `make fmt` (Note: If you have an issue with `make fmt`, ensure your IDE folder is ignored in .gitignore. Already added for .idea/ and .cursor/)
 8. `make lint`
 9. .. fix if any issues are reported
@@ -72,6 +72,34 @@ Here are the example steps to submit your first contribution:
     request description to [automatically link it](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue)
     to an existing issue.
 
+## Definition of Done
+
+Please ensure the following DoDs are met before submitting your pull request:
+- [ ] **Code formatting** — Code is formatted consistently with the project style (`make fmt`).
+- [ ] **Linting & checks** — Code passes linting and static analysis (`make lint`).
+- [ ] **Testing** — Unit, integration and e2e tests cover changes and all tests pass (`make test`, `make integration`, `make e2e`).
+- [ ] **Docstrings** — Public functions, classes, and modules include [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). Docstrings are clean and consistent ([guidance](#writing-docstrings)).
+- [ ] **Documentation** — If user-facing behavior changes, ensure docs are updated. See [docs authoring](/docs/dev/docs_authoring).
+- [ ] **Commit signing** — Commits are GPG-signed (e.g., `git commit -S -a -m "your message"`).
+- [ ] **Pull request requirements** — PR is linked to an existing issue and includes a clear description of the changes.
+- [ ] **Backward compatibility** — Check that changes don't break existing APIs or document breaking changes clearly in the PR.
+- [ ] **Security considerations** — Sensitive data (keys, passwords, etc.) are not hardcoded or exposed.
+- [ ] **Performance considerations** — New code does not introduce obvious performance bottlenecks. Benchmarks added if performance is a concern.
+
+Alternatively, you may open a [Draft PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if your work is not yet ready for submission.
+This is a good way to gather early feedback without needing to meet all Definition of Done (DoD) requirements.
+
+<Admonition type="warning" title="Integration and e2e tests">
+* Draft PRs — Integration and end-to-end (e2e) tests are not run automatically on Draft PRs. These tests will only execute once the PR is marked as "Ready for Review".
+* Forked PRs — No checks are run automatically for PRs opened from forks.
+    Project maintainers must approve it.
+    Integration and end-to-end (e2e) tests cannot be run for PRs opened from forks.
+    In these cases, project maintainers will run the tests manually before merging the code.
+    The contributor should ensure that the tests are passing in their local environment before submitting the PR.
+
+In all cases, the full test suite is executed on code merged into `main` branch as part of the nightly CI/CD pipeline.
+</Admonition>
+
 ## Local Setup
 
 This section provides a step-by-step guide for setting up and starting work on the project. These steps will help you set up your project environment and dependencies for efficient development.
@@ -297,4 +325,36 @@ Example:
 ```python
 def viz_type(self) -> str:
     return self.viz.get("type", "UNKNOWN")
-```
+```
+
+## Writing Docstrings
+
+Use [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) format for the docstrings,
+so that they are rendered correctly in the API docs.
+
+Example:
+```python
+def method(self, arg1: str, arg2: int) -> str:
+    """
+    Short method description in Markdown format.
+
+    Very long method description in Markdown format. Referring to *arg1* and *arg2* in the narrative text.
+
+    Args:
+        arg1 (str): Argument 1 description.
+        arg2 (int): Argument 2 description.
+
+    Returns:
+        str: Return value description.
+    """
+    return "Hello, world!"
+```
+
+<Admonition type="warning" title="Special Cases">
+* Avoid using backticks around object names in docstrings (e.g., \`arg1\`), as this can cause issues when rendering API documentation. Instead, use italics (e.g., \*arg1\*) to emphasize object names.
+* Double curly braces are not allowed in the description. Mask them with backslashes, e.g.: `{{`.
+If you need a code example, use triple backticks, e.g.:
+```python
+print("Hello, world!")
+```
+</Admonition>

docs/dqx/docs/dev/docs_authoring.mdx

@@ -14,6 +14,8 @@ We also use [MDX](https://mdxjs.com/) to write markdown files that include JSX c
 
 For styling, we use [Tailwind CSS](https://tailwindcss.com/), a utility-first CSS framework for rapidly building custom designs.
 
+API docs are generated using [pydoc-markdown](https://github.com/pypa/pydoc-markdown).
+
 ## Writing Documentation
 
 Most of the documentation is written in markdown files with the `.mdx` extension. 
@@ -32,7 +34,6 @@ brew install node
 npm install --global yarn
 ```
 
-
 ## Setup 
 
 To set up the documentation locally, follow these steps:
@@ -115,7 +116,6 @@ make docs-build
 
 It will throw an error on any unresolved link. 
 
-
 ## Content alignment and structure of folders
 
 When writing documentation, make sure to align the content with the existing documentation.
@@ -138,4 +138,16 @@ No need for:
 - Deep technical details
 - Implementation details
 
-Or any other details that are not necessary for the **end-user**.
+Or any other details that are not necessary for the **end-user**.
+
+## API Documentation
+
+The API docs are generated in the `docs/dqx/docs/reference/api` directory.
+
+To generate API docs, run the following command:
+```shell
+hatch run docs:pydoc-markdown
+```
+
+This command is run also as part of make docs build and server.
+The command will generate the API documentation from the Python codebase using pydoc-markdown.

docs/dqx/docs/reference/api/index.mdx

@@ -0,0 +1,8 @@
+---
+title: API Reference
+sidebar_position: 8
+---
+
+# API Reference
+
+This section contains the API reference for the Databricks Labs DQX project.

docs/dqx/docs/reference/engine.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 1
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/docs/reference/profiler.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 2
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/docs/reference/quality_rules.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 3
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/docs/reference/testing.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 4
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/package.json

@@ -15,8 +15,8 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "3.7.0",
-    "@docusaurus/preset-classic": "3.7.0",
+    "@docusaurus/core": "^3.8.1",
+    "@docusaurus/preset-classic": "^3.8.1",
     "@mdx-js/react": "^3.0.0",
     "@radix-ui/react-slot": "^1.1.1",
     "class-variance-authority": "^0.7.1",
@@ -28,12 +28,13 @@
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
     "tailwind-merge": "^2.6.0",
-    "tailwindcss-animate": "^1.0.7"
+    "tailwindcss-animate": "^1.0.7",
+    "tsx": "^4.20.3"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "3.7.0",
-    "@docusaurus/tsconfig": "3.7.0",
-    "@docusaurus/types": "3.7.0",
+    "@docusaurus/module-type-aliases": "^3.8.1",
+    "@docusaurus/tsconfig": "^3.8.1",
+    "@docusaurus/types": "^3.8.1",
     "@tailwindcss/typography": "^0.5.16",
     "autoprefixer": "^10.4.20",
     "postcss": "^8.5.1",

docs/dqx/sidebars.ts

@@ -1,4 +1,4 @@
-import type {SidebarsConfig} from '@docusaurus/plugin-content-docs';
+import type { SidebarsConfig } from '@docusaurus/plugin-content-docs';
 
 // This runs in Node.js - Don't use client-side code here (browser APIs, JSX...)
 
@@ -14,8 +14,8 @@ import type {SidebarsConfig} from '@docusaurus/plugin-content-docs';
  */
 const sidebars: SidebarsConfig = {
   // By default, Docusaurus generates a sidebar from the docs folder structure
-  tutorialSidebar: [{type: 'autogenerated', dirName: '.'}],
-
+  tutorialSidebar: [
+    { type: 'autogenerated', dirName: '.' }],
 };
 
 export default sidebars;

docs/dqx/src/css/custom.css

@@ -58,4 +58,22 @@ button {
 
 .header-github-link:hover::before {
   background-color: var(--ifm-navbar-link-hover-color);
-}
+}
+
+/* TOC sidebar header (desktop and mobile) */
+.theme-doc-toc-desktop .table-of-contents::before,
+.theme-doc-toc-desktop .tableOfContents::before {
+  content: "📄 On this page:";
+  display: block;
+  font-weight: 600;
+  margin-bottom: 0.5rem;
+  color: var(--ifm-color-content);
+}
+
+.theme-doc-toc-mobile .menu__list::before {
+  content: "📄 On this page:";
+  display: block;
+  font-weight: 600;
+  margin: 0.5rem 0 0.25rem 0.5rem;
+  color: var(--ifm-color-content);
+}