Repository navigation
docs: propose Layer 3 execute_query MCP profile #529
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
938957e
08067ef
b5418ee
eca221a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| #!/usr/bin/env python3 | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
| """Publish executable schemas and an endpoint capture from the profile examples.""" | ||
|
|
||
| import argparse | ||
| import json | ||
| import re | ||
| from pathlib import Path | ||
|
|
||
| ROOT = Path(__file__).resolve().parents[1] | ||
| PROFILE = ROOT / "docs/proposals/execute-query-mcp.md" | ||
| DESTINATION = ROOT / "validation/mcp" | ||
| LICENSE = ( | ||
| "Licensed to the Apache Software Foundation (ASF) under one or more " | ||
| "contributor license agreements. See NOTICE for copyright ownership. " | ||
| "Licensed under the Apache License, Version 2.0; " | ||
| "http://www.apache.org/licenses/LICENSE-2.0 . " | ||
| "Unless required by applicable law or agreed to in writing, distributed " | ||
| "on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND." | ||
| ) | ||
|
|
||
|
|
||
| def generated_files() -> dict[str, str]: | ||
| blocks = [ | ||
| json.loads(block) | ||
| for block in re.findall(r"```json\n(.*?)\n```", PROFILE.read_text(encoding="utf-8"), re.S) | ||
| ] | ||
| schemas = [block for block in blocks if "$schema" in block] | ||
| if len(schemas) != 2: | ||
| raise ValueError("Expected exactly the input and output schemas in the profile") | ||
| inputs, outputs = schemas | ||
| for name in ("expression", "projection", "predicate", "query"): | ||
| if inputs["$defs"][name] != outputs["$defs"][name]: | ||
| raise ValueError(f"Request and replacement query definitions differ: {name}") | ||
| version = outputs["properties"]["contract_version"]["const"] | ||
| query = { | ||
| "$schema": inputs["$schema"], | ||
| "$ref": "#/$defs/query", | ||
| "$defs": {key: inputs["$defs"][key] for key in ("expression", "projection", "predicate", "query")}, | ||
| } | ||
| files = {} | ||
| for name, schema in (("input", inputs), ("output", outputs), ("query", query)): | ||
| schema = { | ||
| "$comment": LICENSE + " Generated by scripts/generate-mcp-profile.py; do not edit.", | ||
| "$id": f"urn:apache:ossie:execute-query:{name}:{version}", | ||
| **schema, | ||
| } | ||
| files[f"execute-query-{name}.schema.json"] = json.dumps(schema, indent=2) + "\n" | ||
| arguments = next(block for block in blocks if "data_source_id" in block) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: this binds to whichever example with 🤖 AI-generated review comment — agents make mistakes, so please verify before acting on it. |
||
| result = next(block for block in blocks if block.get("structuredContent", {}).get("status") == "success") | ||
| error = next(block for block in blocks if block.get("structuredContent", {}).get("status") == "error") | ||
| invalid = {"data_source_id": arguments["data_source_id"], "query": {"measures": ["total_reveneu"]}} | ||
| capture = { | ||
| "$comment": LICENSE + " Generated example capture, not a real engine execution.", | ||
| "protocol_version": "2026-07-28", | ||
| "tool": {"name": "execute_query", "inputSchema": inputs, "outputSchema": outputs}, | ||
| "cases": [ | ||
| {"name": "example success", "arguments": arguments, "result": result}, | ||
| {"name": "example semantic refusal", "arguments": invalid, "result": error}, | ||
| ], | ||
| } | ||
| files["example-capture.json"] = json.dumps(capture, indent=2) + "\n" | ||
| return files | ||
|
|
||
|
|
||
| def main() -> int: | ||
| parser = argparse.ArgumentParser(description=__doc__) | ||
| parser.add_argument("--check", action="store_true", help="Report drift without writing") | ||
| args = parser.parse_args() | ||
| changed = [] | ||
| for name, content in generated_files().items(): | ||
| path = DESTINATION / name | ||
| if path.exists() and path.read_text(encoding="utf-8") == content: | ||
| continue | ||
| changed.append(name) | ||
| if not args.check: | ||
| path.parent.mkdir(parents=True, exist_ok=True) | ||
| path.write_text(content, encoding="utf-8", newline="\n") | ||
| if args.check and changed: | ||
| print("Generated MCP profile files are out of date: " + ", ".join(changed)) | ||
| return 1 | ||
| print("MCP profile files " + ("updated" if changed else "are in sync")) | ||
| return 0 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| raise SystemExit(main()) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| <!-- | ||
| Licensed to the Apache Software Foundation (ASF) under one | ||
| or more contributor license agreements. See the NOTICE file | ||
| distributed with this work for additional information | ||
| regarding copyright ownership. The ASF licenses this file | ||
| to you under the Apache License, Version 2.0 (the | ||
| "License"); you may not use this file except in compliance | ||
| with the License. You may obtain a copy of the License at | ||
|
|
||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, | ||
| software distributed under the License is distributed on an | ||
| "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| KIND, either express or implied. See the License for the | ||
| specific language governing permissions and limitations | ||
| under the License. | ||
| --> | ||
|
|
||
| # MCP endpoint contract checks | ||
|
|
||
| Implementor tooling for the proposed | ||
| [`execute_query` profile](../../docs/proposals/execute-query-mcp.md), version | ||
| `0.4-draft`. It checks the **MCP tool declaration and captured observable | ||
| contract**, not a query compiler, an engine or an adopted Ossie standard. | ||
| An engine's integration tests can collect these captures without changing its | ||
| implementation language. | ||
|
|
||
| ## Reusable schemas | ||
|
|
||
| | Artifact | Purpose | | ||
| | --- | --- | | ||
| | `execute-query-input.schema.json` | Publish as the tool's `inputSchema`; accepts `{data_source_id, query}`. | | ||
| | `execute-query-output.schema.json` | Publish as `outputSchema` for structured results, including semantic errors and object-valued repair queries. | | ||
| | `execute-query-query.schema.json` | Validate the Layer 3 object independently; uses the same definitions as requests and repairs. | | ||
| | `call-result.schema.json` | Ossie constraints on a final MCP result: text/resource content, explicit status and optional structured data. Not the complete MCP protocol schema. | | ||
| | `capture.schema.json` | Offline test-harness capture format; not a new endpoint or RPC. | | ||
|
|
||
| The first three schemas and `example-capture.json` are generated from the | ||
| profile's schema/example blocks, preventing documentation and implementation | ||
| contracts from drifting. Their `urn:` identifiers are offline schema identities, | ||
| not download URLs. Resolve the provided files locally; never fetch captured | ||
| schema references over the network. | ||
|
|
||
| ```sh | ||
| python scripts/generate-mcp-profile.py # refresh generated files | ||
| python scripts/generate-mcp-profile.py --check # CI drift check | ||
| ``` | ||
|
|
||
| ## Checking an endpoint | ||
|
|
||
| Copy its `execute_query` entry from `tools/list` and capture final results from | ||
| your SDK/client. Strip JSON-RPC wrappers: `arguments` is the `tools/call` | ||
| argument object; `result` is the final `CallToolResult`. For a completed Task, | ||
| capture the final nested tool result, not the initial Task handle. | ||
|
|
||
| ```json | ||
| { | ||
| "protocol_version": "2026-07-28", | ||
| "tool": { | ||
| "name": "execute_query", | ||
| "inputSchema": {} | ||
| }, | ||
| "cases": [{ | ||
| "name": "replace with a real captured exchange", | ||
| "arguments": { | ||
| "data_source_id": "sales-model", | ||
| "query": {"measures": ["total_revenue"]} | ||
| }, | ||
| "result": {} | ||
| }] | ||
| } | ||
| ``` | ||
|
|
||
| The empty schemas/result above are placeholders, **not a conforming capture**. | ||
| Use `example-capture.json` for a complete generated illustration: | ||
|
|
||
| ```sh | ||
| uv run validation/validate_mcp.py validation/mcp/example-capture.json | ||
| uv run validation/validate_mcp.py path/to/endpoint-capture.json | ||
| ``` | ||
|
|
||
| With `jsonschema` already installed, `python` works in place of `uv run`. | ||
| The CLI reads at most 16 MiB, rejects duplicate JSON keys/non-finite constants, | ||
| prints a JSON report, and performs no network requests or query execution. | ||
| Do not commit captures containing private query results or credentials. | ||
|
|
||
| | Exit / status | Meaning within the reported scope | | ||
| | --- | --- | | ||
| | `0` / `passed` | Recognized declarations and supplied exchanges passed the observable contract checks. | | ||
| | `1` / `failed` | A concrete violation, malformed capture or invalid schema was found; JSON Pointer findings identify the surface. | | ||
| | `2` / `unverified` | Declaration equivalence, Markdown-only interpretation or an opaque encoding could not be established. Never treat this as a pass. | | ||
|
|
||
| The library entry point is `validate_capture(capture)`. Its report includes | ||
| `profile`, `scope`, `cases_checked`, `findings`, `unverified` and **`unchecked`**. | ||
| CI should require exit zero and review unchecked obligations separately. | ||
|
|
||
| ## What the checker establishes | ||
|
|
||
| - Tool name and valid schema declarations; recognizes the published input/output | ||
| schemas including local reference expansion, reordered set-like keywords and | ||
| annotation changes. It does **not** claim to prove arbitrary JSON Schema | ||
| equivalence: other schemas are explicitly unverified. | ||
| - Requests outside the input contract cannot receive success; result objects | ||
| match the published profile, repairs use the query schema, and `isError`, | ||
| structured status, source and model/revision consistency agree. | ||
| - Success descriptors refer to actual embedded resources, not link-only | ||
| substitutes; CSV headers, arity, materialized counts, completeness and metadata | ||
| agree. Null/empty/string escaping, exact numeric strings and locale-neutral | ||
| date/time encodings are checked. | ||
| - Preview rows/counts/types match embedded CSV, including sample membership, | ||
| duplicate rows/names, and first-row ordering when declared. Omitted previews | ||
| are valid. Binary payloads must be base64, with correct declared export MIME | ||
| types; this does not prove the file is valid XLSX/Parquet. | ||
|
|
||
| The report does **not** certify Foundation query answers, semantic-code triggers, | ||
| authorization/RLS/OLS, read-only enforcement, invisible retries/probes, live | ||
| transport/discovery, Task lifecycle, server budgets, natural-language prose | ||
| meaning or binary file fidelity. A captured source's real model identity needs | ||
| independent discovery/engine evidence. Passing one fixture proves only that | ||
| fixture, not every behavior of an endpoint. | ||
|
|
||
| Recommended integration cases: aggregation/scalar success; header-only zero | ||
| rows; invalid shapes; semantic refusals; unsupported capabilities; null/empty, | ||
| large integer/decimal and Unicode values; optional previews; resource limits; | ||
| and the final output from synchronous and Task execution. Use the official MCP | ||
| conformance tooling for protocol lifecycle, and Foundation semantic fixtures | ||
| for engine answers; neither replaces these Ossie-specific output checks. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| { | ||
| "$schema": "https://json-schema.org/draft/2020-12/schema", | ||
| "$id": "urn:apache:ossie:execute-query:call-result:0.4-draft", | ||
| "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See NOTICE for copyright ownership. Licensed under the Apache License, Version 2.0, http://www.apache.org/licenses/LICENSE-2.0 . Distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND. This is an Ossie final-result constraint, not a replacement for the full MCP protocol schema.", | ||
| "type": "object", | ||
| "required": ["resultType", "content", "isError"], | ||
| "properties": { | ||
| "_meta": {"type": "object"}, | ||
| "resultType": {"const": "complete"}, | ||
| "isError": {"type": "boolean"}, | ||
| "structuredContent": {"$ref": "urn:apache:ossie:execute-query:output:0.4-draft"}, | ||
| "content": { | ||
| "type": "array", "minItems": 1, | ||
| "contains": {"type": "object", "required": ["type", "text"], "properties": {"type": {"const": "text"}, "text": {"type": "string", "pattern": "\\S"}}}, | ||
| "items": { | ||
| "oneOf": [ | ||
| {"type": "object", "required": ["type", "text"], "properties": {"type": {"const": "text"}, "text": {"type": "string"}}}, | ||
| {"type": "object", "required": ["type", "resource"], "properties": {"type": {"const": "resource"}, "resource": {"$ref": "#/$defs/embedded"}}}, | ||
| {"type": "object", "required": ["type", "uri", "name"], "properties": {"type": {"const": "resource_link"}, "uri": {"type": "string", "format": "uri"}, "name": {"type": "string"}}}, | ||
| {"type": "object", "required": ["type", "data", "mimeType"], "properties": {"type": {"enum": ["image", "audio"]}, "data": {"type": "string"}, "mimeType": {"type": "string"}}} | ||
| ] | ||
| } | ||
| } | ||
| }, | ||
| "$defs": { | ||
| "embedded": { | ||
| "type": "object", "required": ["uri", "mimeType"], | ||
| "properties": { | ||
| "_meta": {"type": "object"}, | ||
| "uri": {"type": "string", "format": "uri"}, | ||
| "mimeType": {"type": "string"}, | ||
| "text": {"type": "string"}, | ||
| "blob": {"type": "string"} | ||
| }, | ||
| "oneOf": [ | ||
| {"required": ["text"], "not": {"required": ["blob"]}}, | ||
| {"required": ["blob"], "not": {"required": ["text"]}} | ||
| ] | ||
| } | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| { | ||
| "$schema": "https://json-schema.org/draft/2020-12/schema", | ||
| "$id": "urn:apache:ossie:execute-query:capture:0.4-draft", | ||
| "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See NOTICE for copyright ownership. Licensed under the Apache License, Version 2.0, http://www.apache.org/licenses/LICENSE-2.0 . Distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND. Capture format for offline endpoint contract checks; not an MCP RPC.", | ||
| "type": "object", "additionalProperties": false, | ||
| "required": ["protocol_version", "tool", "cases"], | ||
| "properties": { | ||
| "$comment": {"type": "string"}, | ||
| "protocol_version": {"const": "2026-07-28"}, | ||
| "tool": { | ||
| "type": "object", "required": ["name", "inputSchema"], | ||
| "properties": { | ||
| "name": {"const": "execute_query"}, | ||
| "title": {"type": "string"}, | ||
| "description": {"type": "string"}, | ||
| "_meta": {"type": "object"}, | ||
| "inputSchema": {"type": "object"}, | ||
| "outputSchema": {"type": "object"}, | ||
| "annotations": { | ||
| "type": "object", | ||
| "properties": { | ||
| "title": {"type": "string"}, | ||
| "readOnlyHint": {"type": "boolean"}, | ||
| "destructiveHint": {"type": "boolean"}, | ||
| "idempotentHint": {"type": "boolean"}, | ||
| "openWorldHint": {"type": "boolean"} | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "cases": { | ||
| "type": "array", "minItems": 1, | ||
| "items": { | ||
| "type": "object", "additionalProperties": false, | ||
| "required": ["arguments", "result"], | ||
| "properties": { | ||
| "name": {"type": "string"}, | ||
| "arguments": {"type": "object"}, | ||
| "result": {"$ref": "urn:apache:ossie:execute-query:call-result:0.4-draft"} | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: this breaks as soon as the doc gains another block with
$schema. Can we select the input/output schemas by an explicit marker instead, e.g. a ```json input-schema info string?🤖 AI-generated review comment — agents make mistakes, so please verify before acting on it.