Docs API: Add CrateDB functions extractor

Author: amotl

CHANGES.md

@@ -3,7 +3,7 @@
 ## Unreleased
 - MCP: Add subsystem providing a few server and client utilities through
   the `ctk query mcp {list,inquire,launch}` subcommands.
-- Docs API: Added CrateDB settings extractor
+- Docs API: Added extractors for CrateDB functions and settings
 
 ## 2025/01/31 v0.0.31
 - Fixed connectivity for `jobstats collect`

cratedb_toolkit/docs/cli.py

@@ -19,10 +19,27 @@ def cli(ctx: click.Context, verbose: bool, debug: bool):
     return boot_click(ctx, verbose, debug)
 
 
+def help_functions():
+    """
+    Extract CrateDB SQL function definitions by scraping relevant documentation pages.
+
+    Examples
+    ========
+
+    # Extract functions to JSON (default)
+    ctk docs functions
+
+    # Extract functions to Markdown
+    ctk docs functions --format markdown
+
+    # Specify custom output file
+    ctk docs functions --format markdown --output cratedb-functions.md
+    """  # noqa: E501
+
+
 def help_settings():
     """
-    This tool scrapes the CrateDB documentation to extract configuration settings,
-    their default values, descriptions, and runtime configurability status.
+    Extract CrateDB configuration settings by scraping relevant documentation pages.
 
     Examples
     ========
@@ -37,10 +54,37 @@ def help_settings():
     ctk docs settings --format sql
 
     # Specify custom output file
-    ctk docs settings --format markdown --output cratedb_reference.md
+    ctk docs settings --format markdown --output cratedb-settings.md
     """  # noqa: E501
 
 
+@make_command(cli, "functions", help_functions)
+@click.option(
+    "--format",
+    "-f",
+    "format_",
+    type=click.Choice(["json", "yaml", "markdown", "sql"]),
+    default="json",
+    help="Output format (json, yaml, markdown or sql)",
+)
+@click.option("--output", "-o", default=None, help="Output file name")
+def functions(format_: str, output: str):
+    """
+    Extract CrateDB functions from documentation.
+
+    Output in JSON, Markdown, or SQL format.
+    """
+    from .functions import FunctionsExtractor
+
+    try:
+        extractor = FunctionsExtractor()
+        extractor.acquire().render(format_).write(output)
+    except Exception as e:
+        msg = f"Failed to extract functions: {e}"
+        logger.error(msg)
+        raise click.ClickException(msg) from e
+
+
 @make_command(cli, "settings", help_settings)
 @click.option(
     "--format",

cratedb_toolkit/docs/functions.py

@@ -0,0 +1,132 @@
+import dataclasses
+import datetime as dt
+import logging
+from typing import Any, Dict, Optional
+
+import docutils.nodes
+import requests
+from docutils import nodes
+from docutils.examples import internals
+from docutils.parsers.rst.directives import register_directive
+from docutils.parsers.rst.directives.admonitions import Note
+from docutils.parsers.rst.roles import normalized_role_options, register_canonical_role  # type: ignore[attr-defined]
+
+from cratedb_toolkit.docs.util import GenericProcessor
+
+logger = logging.getLogger(__name__)
+
+
+DOCS_URL = "https://github.com/crate/crate/raw/refs/heads/5.10/docs/general/builtins/scalar-functions.rst"
+
+
+@dataclasses.dataclass
+class Function:
+    name: str
+    signature: str
+    category: str
+    description: str
+    # TODO: Parse `returns` and `example` from `description`.
+    returns: Optional[str] = None
+    example: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert the dataclass instance to a dictionary.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing all fields of the instance.
+        """
+        return dataclasses.asdict(self)
+
+
+@dataclasses.dataclass
+class FunctionRegistry:
+    meta: Dict[str, str] = dataclasses.field(default_factory=dict)
+    functions: Dict[str, Function] = dataclasses.field(default_factory=dict)
+
+    def register(self, function: Function):
+        """
+        Register a new function in the registry.
+
+        Adds a Function instance to the registry using its signature as the unique key.
+        Raises a ValueError if a function with the same signature is already registered.
+
+        Args:
+            function: A Function instance to be added to the registry.
+        """
+        if function.signature in self.functions:
+            raise ValueError(f"Function already registered: {function.signature}")
+        self.functions[function.signature] = function
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert the instance to a dictionary.
+
+        Returns:
+            dict: A dictionary containing the instance's fields and their values.
+        """
+        return dataclasses.asdict(self)
+
+
+def sphinx_ref_role(role, rawtext, text=None, lineno=None, inliner=None, options=None, content=None):
+    options = normalized_role_options(options)
+    text = nodes.unescape(text, True)  # type: ignore[attr-defined]
+    label = text.split(" ", 1)[0]
+    node = nodes.raw(rawtext, label, **options)
+    node.source, node.line = inliner.reporter.get_source_and_line(lineno)
+    return [node], []
+
+
+@dataclasses.dataclass
+class FunctionsExtractor(GenericProcessor):
+    """
+    Extract CrateDB functions from documentation.
+    Output in JSON, YAML, Markdown, or SQL format.
+    """
+
+    registry: FunctionRegistry = dataclasses.field(default_factory=FunctionRegistry)
+    thing: Dict[str, Dict[str, Any]] = dataclasses.field(default_factory=dict)
+    payload: Optional[str] = None
+
+    def acquire(self):
+        """
+        Extract and register CrateDB functions from online documentation.
+
+        Fetch documentation from a defined URL, and process its content to extract functions grouped
+        under categories. For each function section, it parses the title and description to create a
+        Function instance, updates the registry with metadata such as creation time and generator info.
+        If no functions are found, the method logs an error and terminates the program. The registry
+        is then converted to a dictionary and stored in the instance attribute 'thing'.
+
+        Returns:
+            FunctionsExtractor: The instance with an updated function registry.
+        """
+        register_canonical_role("ref", sphinx_ref_role)
+        register_directive("seealso", Note)
+        document, pub = internals(requests.get(DOCS_URL, timeout=10).text)
+
+        self.registry.meta["created"] = dt.datetime.now().isoformat()
+        self.registry.meta["generator"] = "CrateDB Toolkit"
+
+        item: docutils.nodes.Element
+        function: docutils.nodes.Element
+        for item in document:
+            if item.tagname == "section":
+                category_title = item.children[0].astext()
+                for function in item.children:  # type: ignore[assignment]
+                    if function.tagname == "section":
+                        function_title = function.children[0].astext()
+                        function_body = function.children[1].astext()
+                        fun = Function(
+                            name=function_title.split("(")[0],
+                            signature=function_title,
+                            category=category_title,
+                            description=function_body,
+                        )
+                        self.registry.register(fun)
+
+        if self.registry.functions:
+            self.thing = self.registry.to_dict()
+        else:
+            logger.error("No functions were extracted. Please check the script or documentation structure.")
+        return self

cratedb_toolkit/docs/util.py

@@ -8,7 +8,7 @@
 @dataclasses.dataclass
 class GenericProcessor:
     """
-    Extract CrateDB settings from documentation.
+    Extract CrateDB knowledge bites (e.g., settings, functions) from documentation.
     Output in JSON, YAML, Markdown, or SQL format.
     """
 

doc/docs-api.md

@@ -9,6 +9,47 @@ uv pip install 'cratedb-toolkit[docs-api]'
 
 ## Usage
 
+### CrateDB functions
+
+This tool extracts functions from CrateDB's documentation and outputs them
+in either JSON, YAML or Markdown formats.
+
+```shell
+ctk docs functions --help
+```
+
+:::{rubric} Example
+:::
+```shell
+ctk docs functions --format=json
+```
+```json
+{
+  "meta": {
+    "created": "2025-04-13T22:57:02.258806",
+    "generator": "CrateDB Toolkit"
+  },
+  "functions": {
+    "concat('first_arg', second_arg, [ parameter , ... ])": {
+      "name": "concat",
+      "signature": "concat('first_arg', second_arg, [ parameter , ... ])",
+      "category": "String functions",
+      "description": "Concatenates a variable number of arguments into a single string. It ignores\nNULL values.",
+      "returns": null,
+      "example": null
+    },
+    "concat_ws('separator', second_arg, [ parameter , ... ])": {
+      "name": "concat_ws",
+      "signature": "concat_ws('separator', second_arg, [ parameter , ... ])",
+      "category": "String functions",
+      "description": "Concatenates a variable number of arguments into a single string using a\nseparator defined by the first argument. If first argument is NULL the\nreturn value is NULL. Remaining NULL arguments are ignored.",
+      "returns": null,
+      "example": null
+    }
+  }
+}
+```
+
 ### CrateDB settings
 
 This tool extracts settings from CrateDB's documentation and outputs them

tests/docs/test_cli.py

@@ -91,3 +91,45 @@ def test_settings_yaml(tmp_path: Path):
     # Verify the outcome.
     data = yaml.safe_load(output_path.read_text())
     assert "whether or not to collect statistical information" in data["stats.enabled"]["purpose"]
+
+
+def test_functions_json(tmp_path: Path):
+    """
+    Verify `ctk docs functions`.
+    """
+
+    output_path = tmp_path / "cratedb-functions.json"
+
+    # Invoke command.
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        args=f"functions --format=json --output={output_path}",
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+
+    # Verify the outcome.
+    data = json.loads(output_path.read_text())
+    assert "substr('string' FROM 'pattern')" in data["functions"]
+
+
+def test_functions_markdown(tmp_path: Path):
+    """
+    Verify `ctk docs functions`.
+    """
+
+    output_path = tmp_path / "cratedb-functions.md"
+
+    # Invoke command.
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        args=f"functions --format=markdown --output={output_path}",
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+
+    # Verify the outcome.
+    data = output_path.read_text()
+    assert "substr('string' FROM 'pattern')" in data