[RFC] Add 'optional' key to canonical-schema

.travis.yml

@@ -7,4 +7,5 @@ cache: yarn
 script:
   - bin/check_required_files_present
   - sh bin/check_versions
+  - bin/check_optional
   - yarn test

OPTIONAL-KEYS.txt

@@ -0,0 +1,3 @@
+floating-point
+big-integer
+unicode

README.md

@@ -40,7 +40,7 @@ is easier to understand with an example:
 
 ```json
 { "exercise": "foobar"
-, "version" : "1.0.0"
+, "version" : "1.1.0"
 , "comments":
     [ " Comments are always optional and can be used almost anywhere.      "
     , "                                                                    "
@@ -92,6 +92,15 @@ is easier to understand with an example:
               }
             , "expected"   : null
             }
+            { "description": "Foo'ing a very big number returns nothing"
+            , "optional"   : "big-ints"
+            , "comments"   : [ "Making this test case pass requires using BigInts." ]
+            , "property"   : "foo"
+            , "input"      : {
+                "word"       : "28948022309329048855892746252171976962977213799489202546401021394546514198529"
+              }
+            , "expected"   : null
+            }
           , { "description": "Bar'ing a name with numbers gives an error"
             , "property"   : "bar"
             , "input"      : {
@@ -118,6 +127,7 @@ There are also some conventions that must be followed:
   - If an error is expected (because the input is invalid, or any other reason), the value at `"expected"` should be an object containing exactly one property, `"error"`, whose value is a string.
     - The string should explain why the error would occur.
     - A particular track's implementation of the exercise **need not** necessarily check that the error includes that exact string as the cause, depending on what is idiomatic in the language (it may not be idiomatic to check strings for error messages).
+  - Test cases that only some tracks should implement, for example because it would unnecessarily increase the complexity of the exercise in some but not all languages, mark it with an `optional`-key. Multiple cases related to the same reason for optionality should have the same key. The decision that a test case is optional will often be made in the PR discussion, so don't worry about it too much while creating a PR.
 
 ### Test Data Versioning
 

bin/check_optional

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# This script checks that 'optional' fields in canonical-data.json are listed.
+
+optional_keys_file=OPTIONAL-KEYS.txt
+allowed_optional=$(jq -nR '[inputs]' $optional_keys_file)
+
+failed=0
+
+check_optional_for_file() {
+    json_file=$1
+    echo "Checking 'optional' fields in $json_file"
+
+    present_optional=$(jq '[ .cases[] | recurse(.cases[]?) | .optional | select(. != null) ]' $json_file)
+    invalid_optional=$(jq --null-input \
+                          --argjson allowed "$allowed_optional" \
+                          --argjson present "$present_optional" \
+                          --raw-output '$present - $allowed | join("\n")')
+
+    if [ ! -z "$invalid_optional" ]; then
+        echo "Invalid optional fields:"
+        echo "$invalid_optional" | perl -pe 's/^/ - /'
+        echo
+
+        failed=1
+    fi
+}
+
+for json_file in $(git diff --name-only master HEAD | grep '^exercises/.*/canonical-data\.json$'); do
+    check_optional_for_file $json_file
+done
+
+if [ $failed -gt 0 ]; then
+    echo "Allowed optional fields (see $optional_keys_file) are:"
+    cat $optional_keys_file | perl -pe 's/^/ - /'
+    exit 1
+fi
+
+exit 0

canonical-schema.json

@@ -27,7 +27,7 @@
    "self": { "vendor" : "io.exercism"
            , "name"   : "canonical-data"
            , "format" : "jsonschema"
-           , "version": "1-0-0"
+           , "version": "1-1-0"
            },
 
    "$ref": "#/definitions/canonicalData",
@@ -86,6 +86,7 @@
           , "required"   : ["description", "property", "input", "expected"]
           , "properties" :
                 { "description": { "$ref": "#/definitions/description" }
+                , "optional"   : { "$ref": "#/definitions/optional"    }
                 , "comments"   : { "$ref": "#/definitions/comments"    }
                 , "property"   : { "$ref": "#/definitions/property"    }
                 , "input"      : { "$ref": "#/definitions/input"       }
@@ -100,6 +101,7 @@
           , "required"   : ["description", "cases"]
           , "properties" :
                 { "description": { "$ref": "#/definitions/description" }
+                , "optional"   : { "$ref": "#/definitions/optional"    }
                 , "comments"   : { "$ref": "#/definitions/comments"    }
                 , "cases"      : { "$ref": "#/definitions/testGroup"   }
                 }
@@ -111,6 +113,12 @@
           , "type"       : "string"
           },
 
+      "optional":
+          { "description": "An identifier for similar optional test cases (kebab-case)"
+          , "type"       : "string"
+          , "pattern"    : "^[a-z]+(-[a-z]+)*$"
+          },
+
       "property":
           { "description": "A letters-only, lowerCamelCase property name"
           , "type"       : "string"