diff --git a/.travis.yml b/.travis.yml index 3ac8ff346a..79db16f5b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,4 +7,5 @@ cache: yarn script: - bin/check_required_files_present - sh bin/check_versions + - bin/check_optional - yarn test diff --git a/OPTIONAL-KEYS.txt b/OPTIONAL-KEYS.txt new file mode 100644 index 0000000000..a063b289f1 --- /dev/null +++ b/OPTIONAL-KEYS.txt @@ -0,0 +1,3 @@ +floating-point +big-integer +unicode diff --git a/README.md b/README.md index de43f1e339..95461f91bf 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ is easier to understand with an example: ```json { "exercise": "foobar" -, "version" : "1.0.0" +, "version" : "1.1.0" , "comments": [ " Comments are always optional and can be used almost anywhere. " , " " @@ -92,6 +92,15 @@ is easier to understand with an example: } , "expected" : null } + { "description": "Foo'ing a very big number returns nothing" + , "optional" : "big-ints" + , "comments" : [ "Making this test case pass requires using BigInts." ] + , "property" : "foo" + , "input" : { + "word" : "28948022309329048855892746252171976962977213799489202546401021394546514198529" + } + , "expected" : null + } , { "description": "Bar'ing a name with numbers gives an error" , "property" : "bar" , "input" : { @@ -118,6 +127,7 @@ There are also some conventions that must be followed: - If an error is expected (because the input is invalid, or any other reason), the value at `"expected"` should be an object containing exactly one property, `"error"`, whose value is a string. - The string should explain why the error would occur. - A particular track's implementation of the exercise **need not** necessarily check that the error includes that exact string as the cause, depending on what is idiomatic in the language (it may not be idiomatic to check strings for error messages). + - Test cases that only some tracks should implement, for example because it would unnecessarily increase the complexity of the exercise in some but not all languages, mark it with an `optional`-key. Multiple cases related to the same reason for optionality should have the same key. The decision that a test case is optional will often be made in the PR discussion, so don't worry about it too much while creating a PR. ### Test Data Versioning diff --git a/bin/check_optional b/bin/check_optional new file mode 100755 index 0000000000..7d9aaf272d --- /dev/null +++ b/bin/check_optional @@ -0,0 +1,39 @@ +#!/bin/sh + +# This script checks that 'optional' fields in canonical-data.json are listed. + +optional_keys_file=OPTIONAL-KEYS.txt +allowed_optional=$(jq -nR '[inputs]' $optional_keys_file) + +failed=0 + +check_optional_for_file() { + json_file=$1 + echo "Checking 'optional' fields in $json_file" + + present_optional=$(jq '[ .cases[] | recurse(.cases[]?) | .optional | select(. != null) ]' $json_file) + invalid_optional=$(jq --null-input \ + --argjson allowed "$allowed_optional" \ + --argjson present "$present_optional" \ + --raw-output '$present - $allowed | join("\n")') + + if [ ! -z "$invalid_optional" ]; then + echo "Invalid optional fields:" + echo "$invalid_optional" | perl -pe 's/^/ - /' + echo + + failed=1 + fi +} + +for json_file in $(git diff --name-only master HEAD | grep '^exercises/.*/canonical-data\.json$'); do + check_optional_for_file $json_file +done + +if [ $failed -gt 0 ]; then + echo "Allowed optional fields (see $optional_keys_file) are:" + cat $optional_keys_file | perl -pe 's/^/ - /' + exit 1 +fi + +exit 0 diff --git a/canonical-schema.json b/canonical-schema.json index 858edab6e1..bc8bca3381 100644 --- a/canonical-schema.json +++ b/canonical-schema.json @@ -27,7 +27,7 @@ "self": { "vendor" : "io.exercism" , "name" : "canonical-data" , "format" : "jsonschema" - , "version": "1-0-0" + , "version": "1-1-0" }, "$ref": "#/definitions/canonicalData", @@ -86,6 +86,7 @@ , "required" : ["description", "property", "input", "expected"] , "properties" : { "description": { "$ref": "#/definitions/description" } + , "optional" : { "$ref": "#/definitions/optional" } , "comments" : { "$ref": "#/definitions/comments" } , "property" : { "$ref": "#/definitions/property" } , "input" : { "$ref": "#/definitions/input" } @@ -100,6 +101,7 @@ , "required" : ["description", "cases"] , "properties" : { "description": { "$ref": "#/definitions/description" } + , "optional" : { "$ref": "#/definitions/optional" } , "comments" : { "$ref": "#/definitions/comments" } , "cases" : { "$ref": "#/definitions/testGroup" } } @@ -111,6 +113,12 @@ , "type" : "string" }, + "optional": + { "description": "An identifier for similar optional test cases (kebab-case)" + , "type" : "string" + , "pattern" : "^[a-z]+(-[a-z]+)*$" + }, + "property": { "description": "A letters-only, lowerCamelCase property name" , "type" : "string"