Add 'remove additional props' processor

Author: ikonst

README.md

@@ -1,6 +1,10 @@
-`jschon-sort` sorts a JSON or YAML document according to its JSON Schema:
+Provides JSON tools:
+
+- `jschon-sort` sorts a JSON or YAML document according to its JSON Schema:
 object properties are ordered to match the order in which JSON Schema properties (that match them) are declared.
 
+- `jschon-remove-additional-props` removes properties not defined in the JSON Schema.
+
 The "jschon" name relates to it being based on the [jschon](https://github.com/marksparkza/jschon) library
 for JSON Schema handling.
 
@@ -28,13 +32,14 @@ jschon-sort --schema ../schema.json file.yaml
 
 ```python
 import jschon
-import jschon_sort
+import jschon_tools
 
 jschon.create_catalog('2020-12')
 ...
-sorted_doc_data = jschon_sort.sort_doc_by_schema(
+sorted_doc_data = jschon_tools.process_json_doc(
     schema_data=schema_data,
     doc_data=doc_data,
+    sort=True,
 )
 ```
 

jschon_sort/__init__.py

@@ -0,0 +1,5 @@
+from ._main import process_json_doc
+
+__all__ = [
+    'process_json_doc',
+]

jschon_sort/cli.py

@@ -1,4 +1,3 @@
-import copy
 import math
 from typing import cast
 from typing import Dict
@@ -11,6 +10,9 @@
 from jschon.json import JSONCompatible
 
 
+_END_SORT_KEY = (math.inf,)
+
+
 def _get_sort_keys_for_json_nodes(root_node: jschon.JSON) -> Mapping[jschon.JSONPointer, Tuple[int, ...]]:
     """
     Gets a mapping from JSON nodes (as JSON pointers) to sort keys (as tuples of integers) that match their position
@@ -39,20 +41,7 @@ def _recurse(node: jschon.JSON, node_sort_key: Tuple[int, ...]) -> None:
     return mapping
 
 
-def sort_doc_by_schema(*, doc_data: JSONCompatible, schema_data: Mapping[str, JSONCompatible]) -> JSONCompatible:
-    try:
-        root_schema = jschon.JSONSchema(schema_data)
-    except jschon.CatalogError:
-        # jschon only supports newer jsonschema drafts
-        schema_data = dict(schema_data)
-        schema_data['$schema'] = "https://json-schema.org/draft/2020-12/schema"
-        root_schema = jschon.JSONSchema(schema_data)
-
-    doc_json = jschon.JSON(doc_data)
-    res = root_schema.evaluate(doc_json)
-    if not res.valid:
-        raise ValueError('Document failed schema validation')
-
+def _get_sort_keys_for_json_doc(*, root_scope: jschon.jsonschema.Scope) -> Mapping[jschon.JSONPointer, Tuple[int, ...]]:
     schema_sort_keys_cache: Dict[jschon.URI, Mapping[jschon.JSONPointer, Tuple[int, ...]]] = {}
 
     def _get_sort_keys_for_schema(schema: jschon.JSONSchema) -> Mapping[jschon.JSONPointer, Tuple[int, ...]]:
@@ -73,11 +62,37 @@ def _traverse_scope(scope: jschon.jsonschema.Scope) -> None:
         for child in scope.iter_children():
             _traverse_scope(child)
 
-    _traverse_scope(res)
+    _traverse_scope(root_scope)
 
-    end_sort_key = (math.inf,)
+    return doc_sort_keys
+
+
+def _get_root_scope(doc_json: jschon.JSON, schema_data: Mapping[str, JSONCompatible]) -> jschon.jsonschema.Scope:
+    try:
+        root_schema = jschon.JSONSchema(schema_data)
+    except jschon.CatalogError:
+        # jschon only supports newer jsonschema drafts
+        schema_data = dict(schema_data)
+        schema_data['$schema'] = "https://json-schema.org/draft/2020-12/schema"
+        root_schema = jschon.JSONSchema(schema_data)
+    res = root_schema.evaluate(doc_json)
+    if not res.valid:
+        raise ValueError('Document failed schema validation')
+    return res
+
+
+def process_json_doc(
+    *,
+    doc_data: JSONCompatible,
+    schema_data: Mapping[str, JSONCompatible],
+    sort: bool = False,
+    remove_additional_props: bool = False,
+) -> JSONCompatible:
+    doc_json = jschon.JSON(doc_data)
+    root_scope = _get_root_scope(doc_json, schema_data=schema_data)
+    doc_sort_keys = _get_sort_keys_for_json_doc(root_scope=root_scope)
 
-    def _sort_json_node(node: JSONCompatible, json_node: jschon.JSON) -> JSONCompatible:
+    def _traverse_node(node: JSONCompatible, json_node: jschon.JSON) -> JSONCompatible:
         """
         @param node: the node being traversed (the data)
         @param json_node: the node being traversed (jschon's representation)
@@ -93,14 +108,18 @@ def _sort_json_node(node: JSONCompatible, json_node: jschon.JSON) -> JSONCompati
             v: JSONCompatible
             v_json: jschon.JSON
             for (k, v), v_json in zip(node.items(), object_data.values()):
-                properties.append((k, _sort_json_node(v, v_json)))
+                v = _traverse_node(v, v_json)
                 # Keys which don't map to the schema (e.g. undefined properties when additionalProperties is missing,
                 # defaulting to true) are assumed to come last (end_sort_key).
                 # As a tie breaker for multiple such undefined properties, we use the key's name.
                 # TODO: update jschon to add additional properties to res.children when appropriate
-                key_sort_keys[k] = doc_sort_keys.get(v_json.path, end_sort_key), k
+                sk = doc_sort_keys.get(v_json.path, _END_SORT_KEY)
+                if sk is not _END_SORT_KEY or not remove_additional_props:
+                    key_sort_keys[k] = sk, k
+                    properties.append((k, v))
 
-            properties.sort(key=lambda pair: key_sort_keys[pair[0]])
+            if sort:
+                properties.sort(key=lambda pair: key_sort_keys[pair[0]])
 
             # to maintain YAML round-trip data, copy node and re-populate
             node_copy = node.copy()
@@ -111,10 +130,10 @@ def _sort_json_node(node: JSONCompatible, json_node: jschon.JSON) -> JSONCompati
 
         elif isinstance(node, list):
             list_data = cast(Sequence[jschon.JSON], json_node.data)
-            return [_sort_json_node(node[idx], v_json) for idx, v_json in enumerate(list_data)]
+            return [_traverse_node(node[idx], v_json) for idx, v_json in enumerate(list_data)]
 
         return node
 
     # we recurse down both the "JSON" and the actual document, and mutate only the actual document
     # which is the primitive type that we can serialize back to JSON/YAML easily
-    return _sort_json_node(doc_data, doc_json)
+    return _traverse_node(doc_data, doc_json)

jschon_tools/__init__.py

@@ -10,7 +10,7 @@ class YamlIndent(NamedTuple):
     offset: int
 
 
-def create_yaml(*, indent: YamlIndent) -> ruyaml.main.YAML:
+def create_yaml_processor(*, indent: YamlIndent) -> ruyaml.main.YAML:
     def _null_representer(self: ruyaml.representer.BaseRepresenter, data: None) -> Any:
         return self.represent_scalar('tag:yaml.org,2002:null', 'null')
 

jschon_sort/_main.py jschon_tools/_main.pyjschon_sort/_main.py renamed to jschon_tools/_main.py

@@ -0,0 +1,97 @@
+import argparse
+import json
+from typing import Mapping
+from typing import Tuple
+
+import jschon
+
+from ._main import process_json_doc
+from ._yaml import create_yaml_processor
+from ._yaml import YamlIndent
+
+
+def _make_parser(*, prog: str, description: str) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=prog,
+        description=description,
+    )
+    parser.add_argument('path', help='path to the JSON / YAML document')
+    parser.add_argument(
+        '--schema', required=True, metavar='/path/to/schema.json', help='path to the JSON Schema document'
+    )
+    parser.add_argument(
+        '--dry-run',
+        '-n',
+        help='if set, result is not persisted back to the original file',
+        action='store_true',
+    )
+    parser.add_argument('--indent', type=int, default=4, help='indent size')
+    parser.add_argument(
+        '--yaml-indent',
+        type=lambda s: YamlIndent(*map(int, s.split(','))),
+        metavar='MAPPING,SEQUENCE,OFFSET',
+        default=YamlIndent(2, 4, 2),
+        help='YAML indent size',
+    )
+    return parser
+
+
+def _is_yaml_path(path: str) -> bool:
+    return path.endswith('.yaml') or path.endswith('.yml')
+
+
+def _load_doc_and_schema(
+    args: argparse.Namespace,
+) -> Tuple[jschon.json.JSONCompatible, Mapping[str, jschon.json.JSONCompatible]]:
+    with open(args.path) as f:
+        if _is_yaml_path(args.path):
+            yaml = create_yaml_processor(indent=args.yaml_indent)
+            doc_data = yaml.load(f)
+        else:
+            doc_data = json.load(f)
+
+    with open(args.schema) as f:
+        schema_data = json.load(f)
+
+    return doc_data, schema_data
+
+
+def _maybe_persist(doc_data: jschon.json.JSONCompatible, args: argparse.Namespace) -> None:
+    if args.dry_run:
+        return
+
+    if _is_yaml_path(args.path):
+        with open(args.path, 'w') as f:
+            yaml = create_yaml_processor(indent=args.yaml_indent)
+            yaml.dump(doc_data, f)
+    else:
+        with open(args.path, 'w') as f:
+            json.dump(doc_data, f, indent=args.indent)
+
+
+def sort_main() -> None:
+    jschon.create_catalog('2020-12')
+
+    parser = _make_parser(
+        prog='jschon-sort',
+        description="Sorts a JSON or YAML document to match a JSON Schema's order of properties",
+    )
+    args = parser.parse_args()
+
+    doc_data, schema_data = _load_doc_and_schema(args)
+    doc_data = process_json_doc(doc_data=doc_data, schema_data=schema_data, sort=True)
+    _maybe_persist(doc_data, args)
+
+
+def remove_additional_props_main() -> None:
+    jschon.create_catalog('2020-12')
+
+    parser = _make_parser(
+        prog='jschon-remove-additional-props',
+        description="Processes a JSON or YAML document to remove additional properties not defined in the schema",
+    )
+    args = parser.parse_args()
+
+    doc_data, schema_data = _load_doc_and_schema(args)
+    doc_data = process_json_doc(doc_data=doc_data, schema_data=schema_data, remove_additional_props=True)
+    _maybe_persist(doc_data, args)

jschon_sort/_yaml.py jschon_tools/_yaml.pyjschon_sort/_yaml.py renamed to jschon_tools/_yaml.py

@@ -1,6 +1,6 @@
 [metadata]
 name = jschon-sort
-version = 0.0.5
+version = 0.1.0
 description = Sorts a JSON or YAML document to match a JSON Schema's order of properties
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -26,10 +26,11 @@ exclude = tests*
 
 [options.entry_points]
 console_scripts =
-  jschon-sort = jschon_sort.cli:main
+  jschon-sort = jschon_tools.cli:sort_main
+  jschon-remove-additional-props = jschon_tools.cli:remove_additional_props_main
 
 [tool:pytest]
-addopts = --cov
+addopts = --cov .
 
 [coverage:run]
 branch = true