🔥 Added API Docs

.github/workflows/docs-release.yml

@@ -13,23 +13,38 @@ jobs:
       group: databrickslabs-protected-runner-group
       labels: linux-ubuntu-latest
     environment: release
-    defaults:
-      run:
-        working-directory: docs/dqx
+
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          cache: 'pip'
+          cache-dependency-path: '**/pyproject.toml'
+          python-version: '3.11'
+
+      - name: Install Hatch
+        run: |
+          pip install hatch==1.9.4
+
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: yarn
           cache-dependency-path: docs/dqx/yarn.lock # need to put the lockfile path explicitly
 
       - name: Install dependencies
-        run: yarn install --frozen-lockfile
+        run: yarn --cwd docs/dqx install --frozen-lockfile
+
+
+      - name: Generate API docs
+        run: hatch run docs:pydoc-markdown
+
       - name: Build website
-        run: yarn build
+        run: yarn --cwd docs/dqx build
 
       - name: Upload Build Artifact
         uses: actions/upload-pages-artifact@v3

.gitignore

@@ -159,3 +159,13 @@ dev/cleanup.py
 .python-version
 .databricks-login.json
 .local-dev
+
+# Cursor IDE specific files
+.cursorrules
+.cursor/
+
+
+# docgen
+docs/dqx/docs/reference/api
+!docs/dqx/docs/reference/api/index.mdx
+

Makefile

@@ -1,6 +1,6 @@
 all: clean dev lint fmt test integration coverage e2e
 
-clean:
+clean: docs-clean
 	rm -fr .venv clean htmlcov .mypy_cache .pytest_cache .ruff_cache .coverage coverage.xml
 	rm -fr **/*.pyc
 
@@ -31,14 +31,22 @@ e2e:
 coverage:
 	hatch run coverage; open htmlcov/index.html
 
-docs-build:
+docs-build: 
+	hatch run docs:pydoc-markdown
 	yarn --cwd docs/dqx build
 
 docs-serve-dev:
+	hatch run docs:pydoc-markdown
 	yarn --cwd docs/dqx start
 
 docs-install:
 	yarn --cwd docs/dqx install
 
 docs-serve: docs-build
-	yarn --cwd docs/dqx serve
+	hatch run docs:pydoc-markdown
+	yarn --cwd docs/dqx serve
+
+docs-clean:
+	rm -rf docs/dqx/build
+	rm -rf docs/dqx/.docusaurus docs/dqx/.cache
+	find docs/dqx/docs/reference/api -mindepth 1 -not -name 'index.mdx' -exec rm -rf {} +

docs/dqx/docs/dev/contributing.mdx

@@ -45,7 +45,7 @@ Here are the example steps to submit your first contribution:
 3. `git checkout main` (or `gcm` if you're using [ohmyzsh](https://ohmyz.sh/)).
 4. `git pull` (or `gl` if you're using [ohmyzsh](https://ohmyz.sh/)).
 5. `git checkout -b FEATURENAME` (or `gcb FEATURENAME` if you're using [ohmyzsh](https://ohmyz.sh/)).
-6. .. do the work
+6. .. do the work. Please make sure to document all your public methods, functions and classes using [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). We generate API documentation automatically using the docstrings.
 7. `make fmt` (Note: If you have an issue with `make fmt`, ensure your IDE folder is ignored in .gitignore. Already added for .idea/ and .cursor/)
 8. `make lint`
 9. .. fix if any issues are reported
@@ -297,4 +297,45 @@ Example:
 ```python
 def viz_type(self) -> str:
     return self.viz.get("type", "UNKNOWN")
-```
+```
+
+## Writing API docs
+
+API docs are generated using [pydoc-markdown](https://github.com/pypa/pydoc-markdown).
+
+To generate API docs, run the following command:
+```shell
+hatch run docs:pydoc-markdown
+```
+
+The API docs are generated in the `docs/dqx/docs/reference/api` directory.
+
+Use [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) format for the docstrings,
+so that they are rendered correctly in the API docs.
+
+Example:
+```python
+def method(self, arg1: str, arg2: int) -> str:
+    """
+    Short method description in Markdown format.
+
+    Very long method description in Markdown format. Referring to *arg1* and *arg2* in the narrative text.
+
+    Args:
+        arg1 (str): Argument 1 description.
+        arg2 (int): Argument 2 description.
+
+    Returns:
+        str: Return value description.
+    """
+    return "Hello, world!"
+```
+
+<Admonition type="warning" title="Special Cases">
+* Avoid using backticks around object names in the docstring (e.g., \`arg1\`), as this can cause issues when rendering API documentation. Instead, use italics (e.g., \*arg1\*) to emphasize object names.
+* Double curly braces are not allowed in the description. Mask them with backslashes, e.g.: `{{`.
+If you need a code example, use triple backticks, e.g.:
+```python
+print("Hello, world!")
+```
+</Admonition>

docs/dqx/docs/reference/api/index.mdx

@@ -0,0 +1,8 @@
+---
+title: API Reference
+sidebar_position: 8
+---
+
+# API Reference
+
+This section contains the API reference for the Databricks Labs DQX project.

docs/dqx/docs/reference/engine.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 1
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/docs/reference/profiler.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 2
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/docs/reference/quality_rules.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 3
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/docs/reference/testing.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 4
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/package.json

@@ -15,8 +15,8 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "3.7.0",
-    "@docusaurus/preset-classic": "3.7.0",
+    "@docusaurus/core": "^3.8.1",
+    "@docusaurus/preset-classic": "^3.8.1",
     "@mdx-js/react": "^3.0.0",
     "@radix-ui/react-slot": "^1.1.1",
     "class-variance-authority": "^0.7.1",
@@ -28,12 +28,13 @@
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
     "tailwind-merge": "^2.6.0",
-    "tailwindcss-animate": "^1.0.7"
+    "tailwindcss-animate": "^1.0.7",
+    "tsx": "^4.20.3"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "3.7.0",
-    "@docusaurus/tsconfig": "3.7.0",
-    "@docusaurus/types": "3.7.0",
+    "@docusaurus/module-type-aliases": "^3.8.1",
+    "@docusaurus/tsconfig": "^3.8.1",
+    "@docusaurus/types": "^3.8.1",
     "@tailwindcss/typography": "^0.5.16",
     "autoprefixer": "^10.4.20",
     "postcss": "^8.5.1",

docs/dqx/sidebars.ts

@@ -1,4 +1,4 @@
-import type {SidebarsConfig} from '@docusaurus/plugin-content-docs';
+import type { SidebarsConfig } from '@docusaurus/plugin-content-docs';
 
 // This runs in Node.js - Don't use client-side code here (browser APIs, JSX...)
 
@@ -14,8 +14,8 @@ import type {SidebarsConfig} from '@docusaurus/plugin-content-docs';
  */
 const sidebars: SidebarsConfig = {
   // By default, Docusaurus generates a sidebar from the docs folder structure
-  tutorialSidebar: [{type: 'autogenerated', dirName: '.'}],
-
+  tutorialSidebar: [
+    { type: 'autogenerated', dirName: '.' }],
 };
 
 export default sidebars;

docs/dqx/src/css/custom.css

@@ -58,4 +58,22 @@ button {
 
 .header-github-link:hover::before {
   background-color: var(--ifm-navbar-link-hover-color);
-}
+}
+
+/* TOC sidebar header (desktop and mobile) */
+.theme-doc-toc-desktop .table-of-contents::before,
+.theme-doc-toc-desktop .tableOfContents::before {
+  content: "📄 On this page:";
+  display: block;
+  font-weight: 600;
+  margin-bottom: 0.5rem;
+  color: var(--ifm-color-content);
+}
+
+.theme-doc-toc-mobile .menu__list::before {
+  content: "📄 On this page:";
+  display: block;
+  font-weight: 600;
+  margin: 0.5rem 0 0.25rem 0.5rem;
+  color: var(--ifm-color-content);
+}