From dbf1c095a334a8dfbf5b839a3da7de4e7bae7155 Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Mon, 31 Aug 2020 20:51:29 +1200
Subject: [PATCH 01/12] empty fragments are valid pointers per RFC 6901 sec.6

---
 tests/draft2019-09/optional/format/json-pointer.json | 8 ++++----
 tests/draft6/optional/format/json-pointer.json       | 8 ++++----
 tests/draft7/optional/format/json-pointer.json       | 8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/draft2019-09/optional/format/json-pointer.json b/tests/draft2019-09/optional/format/json-pointer.json
index a0346b57..e75e5cc4 100644
--- a/tests/draft2019-09/optional/format/json-pointer.json
+++ b/tests/draft2019-09/optional/format/json-pointer.json
@@ -139,14 +139,14 @@
                 "valid": true
             },
             {
-                "description": "not a valid JSON-pointer (URI Fragment Identifier) #1",
+                "description": "valid JSON-pointer (URI Fragment Identifier) #1",
                 "data": "#",
-                "valid": false
+                "valid": true
             },
             {
-                "description": "not a valid JSON-pointer (URI Fragment Identifier) #2",
+                "description": "valid JSON-pointer (URI Fragment Identifier) #2",
                 "data": "#/",
-                "valid": false
+                "valid": true
             },
             {
                 "description": "not a valid JSON-pointer (URI Fragment Identifier) #3",
diff --git a/tests/draft6/optional/format/json-pointer.json b/tests/draft6/optional/format/json-pointer.json
index a0346b57..e75e5cc4 100644
--- a/tests/draft6/optional/format/json-pointer.json
+++ b/tests/draft6/optional/format/json-pointer.json
@@ -139,14 +139,14 @@
                 "valid": true
             },
             {
-                "description": "not a valid JSON-pointer (URI Fragment Identifier) #1",
+                "description": "valid JSON-pointer (URI Fragment Identifier) #1",
                 "data": "#",
-                "valid": false
+                "valid": true
             },
             {
-                "description": "not a valid JSON-pointer (URI Fragment Identifier) #2",
+                "description": "valid JSON-pointer (URI Fragment Identifier) #2",
                 "data": "#/",
-                "valid": false
+                "valid": true
             },
             {
                 "description": "not a valid JSON-pointer (URI Fragment Identifier) #3",
diff --git a/tests/draft7/optional/format/json-pointer.json b/tests/draft7/optional/format/json-pointer.json
index a0346b57..e75e5cc4 100644
--- a/tests/draft7/optional/format/json-pointer.json
+++ b/tests/draft7/optional/format/json-pointer.json
@@ -139,14 +139,14 @@
                 "valid": true
             },
             {
-                "description": "not a valid JSON-pointer (URI Fragment Identifier) #1",
+                "description": "valid JSON-pointer (URI Fragment Identifier) #1",
                 "data": "#",
-                "valid": false
+                "valid": true
             },
             {
-                "description": "not a valid JSON-pointer (URI Fragment Identifier) #2",
+                "description": "valid JSON-pointer (URI Fragment Identifier) #2",
                 "data": "#/",
-                "valid": false
+                "valid": true
             },
             {
                 "description": "not a valid JSON-pointer (URI Fragment Identifier) #3",

From 372381d4e48150b5c354f63d271ebb7571566519 Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Tue, 28 Dec 2021 15:16:38 +1300
Subject: [PATCH 02/12] add report runner script; iterate through tests and
 call test runner cli

---
 .gitignore    |  4 +++-
 report/run.sh | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 report/run.sh

diff --git a/.gitignore b/.gitignore
index 1333ed77..6a30f3a8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
-TODO
+# TODO
+
+.vscode
\ No newline at end of file
diff --git a/report/run.sh b/report/run.sh
new file mode 100644
index 00000000..e176b847
--- /dev/null
+++ b/report/run.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+_jq() {
+    echo ${1} | base64 --decode --ignore-garbage | jq -c ${2}
+}
+
+for d in ../tests/*/ ; do
+    for f in $d* ; do # $d contains the ending /
+        echo "$f"
+        json=$(jq -cr '.[] | @base64' < $f)
+        scenarioIndex=0
+        for j in $json
+        do
+            scenario=$(_jq $j '.description')
+            schemaUri="$f#/$scenarioIndex/schema"
+            cases=$(echo $j | base64 --decode --ignore-garbage | jq -c '.tests[] | @base64')
+            caseIndex=0
+            for c in $cases
+            do
+                case=$(_jq $c '.description')
+                instanceUri="$f#/$scenarioIndex/tests/$caseIndex/data"
+                valid=$(_jq $c '.valid')
+
+                ${1} --schema $schemaUri --instance $instanceUri
+                if [[ $? -ne 0 ]]
+                then
+                    echo "'$scenario' / '$case' : $?"
+                fi
+#               capture result
+                ((caseIndex++))
+            done
+            ((scenarioIndex++))
+        done
+    done
+done

From 80b971d3b9b7bb213bbd42820c1cbc4c59f68fba Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Tue, 28 Dec 2021 19:21:33 +1300
Subject: [PATCH 03/12] better flow and use of piping, no base64

---
 .gitignore    |  4 +++-
 report/run.sh | 22 +++++++++-------------
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/.gitignore b/.gitignore
index 6a30f3a8..2c919dac 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
 # TODO
 
-.vscode
\ No newline at end of file
+.vscode
+
+report/jq
\ No newline at end of file
diff --git a/report/run.sh b/report/run.sh
index e176b847..ca7d59a9 100644
--- a/report/run.sh
+++ b/report/run.sh
@@ -1,25 +1,21 @@
 #!/bin/sh
 
-_jq() {
-    echo ${1} | base64 --decode --ignore-garbage | jq -c ${2}
-}
-
-for d in ../tests/*/ ; do
-    for f in $d* ; do # $d contains the ending /
+for d in ../tests/*/
+do
+    for f in $d* # $d contains the ending /
+    do
         echo "$f"
-        json=$(jq -cr '.[] | @base64' < $f)
         scenarioIndex=0
-        for j in $json
+        jq -cr '.[]' < $f | while read j
         do
-            scenario=$(_jq $j '.description')
+            scenario=$(echo $j | jq '.description')
             schemaUri="$f#/$scenarioIndex/schema"
-            cases=$(echo $j | base64 --decode --ignore-garbage | jq -c '.tests[] | @base64')
             caseIndex=0
-            for c in $cases
+            echo $j | jq -c '.tests[]' | while read c
             do
-                case=$(_jq $c '.description')
+                case=$(echo $c | jq '.description')
                 instanceUri="$f#/$scenarioIndex/tests/$caseIndex/data"
-                valid=$(_jq $c '.valid')
+                valid=$(echo $c | jq '.valid')
 
                 ${1} --schema $schemaUri --instance $instanceUri
                 if [[ $? -ne 0 ]]

From 583c13381226f8b474e79ffd3643dd31c12b143b Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Wed, 29 Dec 2021 09:00:00 +1300
Subject: [PATCH 04/12] process response a bit; output messages

---
 report/run.sh | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/report/run.sh b/report/run.sh
index ca7d59a9..410301c9 100644
--- a/report/run.sh
+++ b/report/run.sh
@@ -2,27 +2,40 @@
 
 for d in ../tests/*/
 do
+    specVersion=$(basename $d)
     for f in $d* # $d contains the ending /
     do
         echo "$f"
         scenarioIndex=0
         jq -cr '.[]' < $f | while read j
         do
-            scenario=$(echo $j | jq '.description')
+            scenario=$(echo $j | jq -r '.description')
             schemaUri="$f#/$scenarioIndex/schema"
             caseIndex=0
             echo $j | jq -c '.tests[]' | while read c
             do
-                case=$(echo $c | jq '.description')
+                case=$(echo $c | jq -r '.description')
                 instanceUri="$f#/$scenarioIndex/tests/$caseIndex/data"
-                valid=$(echo $c | jq '.valid')
+                case "$(echo $c | jq '.valid')" in
+                    true) expected=0 ;;
+                    false) expected=127 ;;  # bash seems to interpret a -1 exit code as 127
+                esac
 
-                ${1} --schema $schemaUri --instance $instanceUri
-                if [[ $? -ne 0 ]]
+                ${1} --schema $schemaUri --instance $instanceUri --spec-version $specVersion
+                evaluated=$?
+
+                if [[ $evaluated -ne $expected ]]
                 then
-                    echo "'$scenario' / '$case' : $?"
+                    echo --schema $schemaUri --instance $instanceUri --spec-version $specVersion / $expected
+                    echo $evaluated
+                    case $evaluated in
+                        127) message="did not validate but should have" ;;
+                        0) message="validated but should not have" ;;
+                        1) message="an error occurred" ;;
+                        2) message="not supported" ;;
+                    esac
+                    echo $message
                 fi
-#               capture result
                 ((caseIndex++))
             done
             ((scenarioIndex++))

From 7d984dc57af446f158d575ce4c3de7dcf2b0a235 Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Tue, 4 Jan 2022 20:38:08 +1300
Subject: [PATCH 05/12] set up docker-compose to run suite (WIP)

---
 report/docker-compose.yml       | 10 ++++++++++
 report/implementations.json     |  6 ++++++
 report/run-report.sh            | 10 ++++++++++
 report/{run.sh => run-suite.sh} |  2 ++
 4 files changed, 28 insertions(+)
 create mode 100644 report/docker-compose.yml
 create mode 100644 report/implementations.json
 create mode 100644 report/run-report.sh
 rename report/{run.sh => run-suite.sh} (96%)

diff --git a/report/docker-compose.yml b/report/docker-compose.yml
new file mode 100644
index 00000000..6697137e
--- /dev/null
+++ b/report/docker-compose.yml
@@ -0,0 +1,10 @@
+version: "3.9"
+services:
+  test-suite:
+    image: ${IMPLEMENTATION:?Implementation not supplied}
+    volumes:
+      - .:/report
+      - ./${IMPLEMENTATION}:/output
+      - ../tests:/tests
+      - ../remotes:/remotes
+    command: ./report/run-suite.sh
diff --git a/report/implementations.json b/report/implementations.json
new file mode 100644
index 00000000..8213543a
--- /dev/null
+++ b/report/implementations.json
@@ -0,0 +1,6 @@
+[
+    {
+        "name": "JsonSchema.Net",
+        "docker-image": "file:///c/projects/json-everything/JsonSchema.TestSuiteRunner/docker/json-everything.img"
+    }
+]
\ No newline at end of file
diff --git a/report/run-report.sh b/report/run-report.sh
new file mode 100644
index 00000000..c99f40ae
--- /dev/null
+++ b/report/run-report.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+export IMPLEMENTATION=${1}
+
+echo "Running test suite with '${1}'"
+
+docker-compose run test-suite
+
+docker-compose logs --no-color > docker-compose.log
+[[ -z "${failed:-}" ]] || exit 1
\ No newline at end of file
diff --git a/report/run.sh b/report/run-suite.sh
similarity index 96%
rename from report/run.sh
rename to report/run-suite.sh
index 410301c9..ba5b619d 100644
--- a/report/run.sh
+++ b/report/run-suite.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+echo "Running test suite with implementation ${1}"
+
 for d in ../tests/*/
 do
     specVersion=$(basename $d)

From 257384b02f229df2d700fe18360918a6cdf60bbb Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Sat, 8 Jan 2022 16:46:27 +1300
Subject: [PATCH 06/12] integrate implementations file; add dockerfile to
 include jq; update scripts

---
 report/Dockerfile           |  6 ++++++
 report/docker-compose.yml   |  4 ++--
 report/implementations.json |  4 ++--
 report/run-report.sh        | 10 ++++++++--
 report/run-suite.sh         | 24 +++++++++++++++++++-----
 5 files changed, 37 insertions(+), 11 deletions(-)
 create mode 100644 report/Dockerfile

diff --git a/report/Dockerfile b/report/Dockerfile
new file mode 100644
index 00000000..74dfa91e
--- /dev/null
+++ b/report/Dockerfile
@@ -0,0 +1,6 @@
+ARG IMPLEMENTATION
+
+FROM ${IMPLEMENTATION}
+
+RUN apt-get update && \
+    apt-get install jq -y
\ No newline at end of file
diff --git a/report/docker-compose.yml b/report/docker-compose.yml
index 6697137e..04f9e282 100644
--- a/report/docker-compose.yml
+++ b/report/docker-compose.yml
@@ -1,10 +1,10 @@
 version: "3.9"
 services:
   test-suite:
-    image: ${IMPLEMENTATION:?Implementation not supplied}
+    image: test-run
     volumes:
       - .:/report
       - ./${IMPLEMENTATION}:/output
       - ../tests:/tests
       - ../remotes:/remotes
-    command: ./report/run-suite.sh
+    entrypoint: /report/run-suite.sh ${IMPLEMENTATION}
diff --git a/report/implementations.json b/report/implementations.json
index 8213543a..30b0f56c 100644
--- a/report/implementations.json
+++ b/report/implementations.json
@@ -1,6 +1,6 @@
 [
     {
-        "name": "JsonSchema.Net",
-        "docker-image": "file:///c/projects/json-everything/JsonSchema.TestSuiteRunner/docker/json-everything.img"
+        "docker-image": "json-everything",
+        "command": "dotnet json-everything.dll"
     }
 ]
\ No newline at end of file
diff --git a/report/run-report.sh b/report/run-report.sh
index c99f40ae..17d3f48c 100644
--- a/report/run-report.sh
+++ b/report/run-report.sh
@@ -4,7 +4,13 @@ export IMPLEMENTATION=${1}
 
 echo "Running test suite with '${1}'"
 
+docker image rm test-run -f
+
+docker build -t test-run --build-arg IMPLEMENTATION=${1} .
+
+docker create --name test-run test-run
+
 docker-compose run test-suite
 
-docker-compose logs --no-color > docker-compose.log
-[[ -z "${failed:-}" ]] || exit 1
\ No newline at end of file
+# docker-compose logs --no-color > docker-compose.log
+# [[ -z "${failed:-}" ]] || exit 1
\ No newline at end of file
diff --git a/report/run-suite.sh b/report/run-suite.sh
index ba5b619d..22e3d713 100644
--- a/report/run-suite.sh
+++ b/report/run-suite.sh
@@ -1,29 +1,41 @@
-#!/bin/sh
+#!/bin/bash
 
 echo "Running test suite with implementation ${1}"
 
+COMMAND=$(jq -r ".[] | select(.\"docker-image\"==\"${1}\") | .command" /report/implementations.json)
+
+if [[ "${COMMAND}" -eq "" ]]
+then
+    echo "Implementation either not found or not configured with a command to execute"
+    exit 1
+fi
+
 for d in ../tests/*/
 do
     specVersion=$(basename $d)
+    echo "$specVersion"
     for f in $d* # $d contains the ending /
     do
-        echo "$f"
+        filename=$(basename $f)
+        echo "  ${filename%%.*}"
         scenarioIndex=0
         jq -cr '.[]' < $f | while read j
         do
             scenario=$(echo $j | jq -r '.description')
             schemaUri="$f#/$scenarioIndex/schema"
             caseIndex=0
+            echo "    $scenario"
             echo $j | jq -c '.tests[]' | while read c
             do
                 case=$(echo $c | jq -r '.description')
                 instanceUri="$f#/$scenarioIndex/tests/$caseIndex/data"
                 case "$(echo $c | jq '.valid')" in
                     true) expected=0 ;;
-                    false) expected=127 ;;  # bash seems to interpret a -1 exit code as 127
+                    false) expected=255 ;;
                 esac
 
-                ${1} --schema $schemaUri --instance $instanceUri --spec-version $specVersion
+                echo "      $case"
+                ${COMMAND} --schema $schemaUri --instance $instanceUri --spec-version $specVersion
                 evaluated=$?
 
                 if [[ $evaluated -ne $expected ]]
@@ -31,7 +43,7 @@ do
                     echo --schema $schemaUri --instance $instanceUri --spec-version $specVersion / $expected
                     echo $evaluated
                     case $evaluated in
-                        127) message="did not validate but should have" ;;
+                        255) message="did not validate but should have" ;;
                         0) message="validated but should not have" ;;
                         1) message="an error occurred" ;;
                         2) message="not supported" ;;
@@ -42,5 +54,7 @@ do
             done
             ((scenarioIndex++))
         done
+
+        exit 0
     done
 done

From 81ac6a54d1ed7a54a817424e7309b3ace5d20288 Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Sat, 8 Jan 2022 20:23:33 +1300
Subject: [PATCH 07/12] generate report

---
 .gitignore                  |  2 +-
 report/implementations.json |  3 ++-
 report/report-template.yml  |  6 +++++
 report/run-suite.sh         | 53 +++++++++++++++++++++----------------
 4 files changed, 39 insertions(+), 25 deletions(-)
 create mode 100644 report/report-template.yml

diff --git a/.gitignore b/.gitignore
index 2c919dac..ed1a2137 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,4 @@
 
 .vscode
 
-report/jq
\ No newline at end of file
+report/*/
\ No newline at end of file
diff --git a/report/implementations.json b/report/implementations.json
index 30b0f56c..2f1a5b70 100644
--- a/report/implementations.json
+++ b/report/implementations.json
@@ -1,6 +1,7 @@
 [
     {
         "docker-image": "json-everything",
-        "command": "dotnet json-everything.dll"
+        "command": "dotnet json-everything.dll",
+        "versions": ["draft", "draft7", "draft2019-09", "draft2020-12"]
     }
 ]
\ No newline at end of file
diff --git a/report/report-template.yml b/report/report-template.yml
new file mode 100644
index 00000000..a0d73127
--- /dev/null
+++ b/report/report-template.yml
@@ -0,0 +1,6 @@
+# This file was generated by running `/report/run-report.sh ${IMPLEMENTATION}`
+
+# This is the compliance report for the indicated implementation
+
+implementation: ${IMPLEMENTATION}
+results:
diff --git a/report/run-suite.sh b/report/run-suite.sh
index 22e3d713..93708443 100644
--- a/report/run-suite.sh
+++ b/report/run-suite.sh
@@ -2,9 +2,13 @@
 
 echo "Running test suite with implementation ${1}"
 
-COMMAND=$(jq -r ".[] | select(.\"docker-image\"==\"${1}\") | .command" /report/implementations.json)
+command=$(jq -r ".[] | select(.\"docker-image\"==\"${1}\") | .command" /report/implementations.json)
+versions=$(jq -r ".[] | select(.\"docker-image\"==\"${1}\") | .versions" /report/implementations.json)
+output="/output/result.yml"
 
-if [[ "${COMMAND}" -eq "" ]]
+sed -e "s/\${IMPLEMENTATION}/${1}/" /report/report-template.yml > ${output}
+
+if [[ "${command}" -eq "" ]]
 then
     echo "Implementation either not found or not configured with a command to execute"
     exit 1
@@ -13,48 +17,51 @@ fi
 for d in ../tests/*/
 do
     specVersion=$(basename $d)
-    echo "$specVersion"
+
+    if ! [[ ${versions[@]} =~ $specVersion ]]
+    then
+        # record that the draft isn't supported
+        continue
+    fi
+
     for f in $d* # $d contains the ending /
     do
         filename=$(basename $f)
-        echo "  ${filename%%.*}"
         scenarioIndex=0
         jq -cr '.[]' < $f | while read j
         do
             scenario=$(echo $j | jq -r '.description')
             schemaUri="$f#/$scenarioIndex/schema"
             caseIndex=0
-            echo "    $scenario"
             echo $j | jq -c '.tests[]' | while read c
             do
                 case=$(echo $c | jq -r '.description')
                 instanceUri="$f#/$scenarioIndex/tests/$caseIndex/data"
                 case "$(echo $c | jq '.valid')" in
-                    true) expected=0 ;;
-                    false) expected=255 ;;
+                    true) expected="valid" ;;
+                    false) expected="invalid" ;;
                 esac
 
-                echo "      $case"
-                ${COMMAND} --schema $schemaUri --instance $instanceUri --spec-version $specVersion
+                ${command} --schema $schemaUri --instance $instanceUri --spec-version $specVersion
                 evaluated=$?
 
-                if [[ $evaluated -ne $expected ]]
-                then
-                    echo --schema $schemaUri --instance $instanceUri --spec-version $specVersion / $expected
-                    echo $evaluated
-                    case $evaluated in
-                        255) message="did not validate but should have" ;;
-                        0) message="validated but should not have" ;;
-                        1) message="an error occurred" ;;
-                        2) message="not supported" ;;
-                    esac
-                    echo $message
-                fi
+                case $evaluated in
+                    255) result="invalid" ;;
+                    0) result="valid" ;;
+                    1) result="error" ;;
+                    2) result="unsupported" ;;
+                esac
+
+                echo "  - draft: ${specVersion}" >> $output
+                echo "    file: ${filename%%.*}" >> $output
+                echo "    scenario: ${scenario}" >> $output
+                echo "    case: ${case}" >> $output
+                echo "    expected: ${expected}" >> $output
+                echo "    result: ${result}" >> $output
+
                 ((caseIndex++))
             done
             ((scenarioIndex++))
         done
-
-        exit 0
     done
 done

From 2e4bc377b5f0875f9db8285bfe43e7ac7be3b47e Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Sat, 8 Jan 2022 21:38:33 +1300
Subject: [PATCH 08/12] updated implementations file to use json-everything
 image from dockerhub

---
 report/implementations.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/report/implementations.json b/report/implementations.json
index 2f1a5b70..180c2b38 100644
--- a/report/implementations.json
+++ b/report/implementations.json
@@ -1,7 +1,7 @@
 [
     {
-        "docker-image": "json-everything",
+        "docker-image": "gregsdennis/json-everything",
         "command": "dotnet json-everything.dll",
-        "versions": ["draft", "draft7", "draft2019-09", "draft2020-12"]
+        "versions": ["draft6", "draft7", "draft2019-09", "draft2020-12"]
     }
 ]
\ No newline at end of file

From 159ed24b6c2aa580a9b9829e7f76c677a3464850 Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Sat, 8 Jan 2022 21:47:13 +1300
Subject: [PATCH 09/12] fix sed command to support slashes in docker image
 names

---
 report/run-suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/report/run-suite.sh b/report/run-suite.sh
index 93708443..387c2794 100644
--- a/report/run-suite.sh
+++ b/report/run-suite.sh
@@ -6,7 +6,7 @@ command=$(jq -r ".[] | select(.\"docker-image\"==\"${1}\") | .command" /report/i
 versions=$(jq -r ".[] | select(.\"docker-image\"==\"${1}\") | .versions" /report/implementations.json)
 output="/output/result.yml"
 
-sed -e "s/\${IMPLEMENTATION}/${1}/" /report/report-template.yml > ${output}
+sed -e "s~\${IMPLEMENTATION}~${1}~" /report/report-template.yml > ${output}
 
 if [[ "${command}" -eq "" ]]
 then

From d20cd1d2b567b5bc5981bb9d0845cc88bc8c9f87 Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Sat, 8 Jan 2022 22:24:47 +1300
Subject: [PATCH 10/12] add a readme for the runner

---
 report/README.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 report/README.md

diff --git a/report/README.md b/report/README.md
new file mode 100644
index 00000000..9c256f80
--- /dev/null
+++ b/report/README.md
@@ -0,0 +1,49 @@
+# Test Suite Runner
+
+This folder contains scripts and docker definitions that can run the suite against an implementation.  The implementations are configured in _implementations.json_.
+
+## Add your implementation
+
+Setting up your application requires several steps.
+
+### Create a CLI
+
+To add your implementation, you'll need to start by creating a CLI application that takes the following parameters:
+
+- `--schema <PATH#POINTER>` - The relative URI to the schema, e.g. `../tests/draft2019-09/additionalItems.json#/0/schema`.
+- `--instance <PATH#POINTER>` - The relative URI to the data instance, e.g. `../tests/draft2019-09/additionalItems.json#/0/tests/0/data`.
+- `--spec-version <VERSION>` - The draft/version for the schema, e.g. `draft2020-12`.  The value for this parameter will match a folder name under `/tests/` in this repo.
+<!-- Do we need this parameter? -->
+<!-- - `--validate-formats` - Indicates whether to validate formats. -->
+
+Internally, you may invoke your implementation as you see fit, setting any options you need to produce the correct outcome.
+
+Validation outcome is indicated by the CLI exit code as follows:
+
+- **-1** or **255** - The instance was determined to be invalid against the schema.
+- **0** - The instance was determined to be valid against the schema.
+- **1** - An application error occurred preventing the schema or instance from being processed.
+- **2** - The scenario is not supported.
+
+Each of these will be represented in the final report.
+
+### Package your CLI
+
+The CLI will need to be packaged into a Linux Docker image.
+
+At a minimum this docker image will need your CLI as well as any runtime needed.
+
+You'll need to push this to a public repository such as Docker Hub so that it can be pulled when the test suite runs.
+
+### Update this repository
+
+Lastly, update a file in this repo.
+
+_implementations.json_ (in this folder) needs an entry that includes the following:
+
+- `docker-image` - The name of your docker image.  The `latest` tag will be used.
+- `command` - The command to run your CLI.
+- `versions` - An array of versions/drafts supported by your implementations.  (To save time, unsupported versions will be skipped.)
+
+Once added, the runner will do the rest.
+

From 23dbed6a3076cdd630aa92ecf3ac0159d5a04a1f Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Sun, 9 Jan 2022 10:39:02 +1300
Subject: [PATCH 11/12] revert test changes

---
 tests/draft2019-09/optional/format/json-pointer.json | 8 ++++----
 tests/draft6/optional/format/json-pointer.json       | 8 ++++----
 tests/draft7/optional/format/json-pointer.json       | 8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/draft2019-09/optional/format/json-pointer.json b/tests/draft2019-09/optional/format/json-pointer.json
index e75e5cc4..a0346b57 100644
--- a/tests/draft2019-09/optional/format/json-pointer.json
+++ b/tests/draft2019-09/optional/format/json-pointer.json
@@ -139,14 +139,14 @@
                 "valid": true
             },
             {
-                "description": "valid JSON-pointer (URI Fragment Identifier) #1",
+                "description": "not a valid JSON-pointer (URI Fragment Identifier) #1",
                 "data": "#",
-                "valid": true
+                "valid": false
             },
             {
-                "description": "valid JSON-pointer (URI Fragment Identifier) #2",
+                "description": "not a valid JSON-pointer (URI Fragment Identifier) #2",
                 "data": "#/",
-                "valid": true
+                "valid": false
             },
             {
                 "description": "not a valid JSON-pointer (URI Fragment Identifier) #3",
diff --git a/tests/draft6/optional/format/json-pointer.json b/tests/draft6/optional/format/json-pointer.json
index e75e5cc4..a0346b57 100644
--- a/tests/draft6/optional/format/json-pointer.json
+++ b/tests/draft6/optional/format/json-pointer.json
@@ -139,14 +139,14 @@
                 "valid": true
             },
             {
-                "description": "valid JSON-pointer (URI Fragment Identifier) #1",
+                "description": "not a valid JSON-pointer (URI Fragment Identifier) #1",
                 "data": "#",
-                "valid": true
+                "valid": false
             },
             {
-                "description": "valid JSON-pointer (URI Fragment Identifier) #2",
+                "description": "not a valid JSON-pointer (URI Fragment Identifier) #2",
                 "data": "#/",
-                "valid": true
+                "valid": false
             },
             {
                 "description": "not a valid JSON-pointer (URI Fragment Identifier) #3",
diff --git a/tests/draft7/optional/format/json-pointer.json b/tests/draft7/optional/format/json-pointer.json
index e75e5cc4..a0346b57 100644
--- a/tests/draft7/optional/format/json-pointer.json
+++ b/tests/draft7/optional/format/json-pointer.json
@@ -139,14 +139,14 @@
                 "valid": true
             },
             {
-                "description": "valid JSON-pointer (URI Fragment Identifier) #1",
+                "description": "not a valid JSON-pointer (URI Fragment Identifier) #1",
                 "data": "#",
-                "valid": true
+                "valid": false
             },
             {
-                "description": "valid JSON-pointer (URI Fragment Identifier) #2",
+                "description": "not a valid JSON-pointer (URI Fragment Identifier) #2",
                 "data": "#/",
-                "valid": true
+                "valid": false
             },
             {
                 "description": "not a valid JSON-pointer (URI Fragment Identifier) #3",

From 7215368bf2c671f3596c7055ce51ee5f0662d50c Mon Sep 17 00:00:00 2001
From: Greg Dennis <gregsdennis@yahoo.com>
Date: Sun, 9 Jan 2022 11:34:50 +1300
Subject: [PATCH 12/12] fix command check; add optional tests

---
 report/run-suite.sh | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/report/run-suite.sh b/report/run-suite.sh
index 387c2794..8562754e 100644
--- a/report/run-suite.sh
+++ b/report/run-suite.sh
@@ -8,13 +8,13 @@ output="/output/result.yml"
 
 sed -e "s~\${IMPLEMENTATION}~${1}~" /report/report-template.yml > ${output}
 
-if [[ "${command}" -eq "" ]]
+if [[ "${command}" == "" ]]
 then
     echo "Implementation either not found or not configured with a command to execute"
     exit 1
 fi
 
-for d in ../tests/*/
+for d in /tests/*/
 do
     specVersion=$(basename $d)
 
@@ -24,9 +24,16 @@ do
         continue
     fi
 
-    for f in $d* # $d contains the ending /
+    for f in $(find $d) # $d contains the ending /
     do
         filename=$(basename $f)
+        if [[ $f == *"optional"* ]]
+        then
+            optional=true
+        else
+            optional=false
+        fi
+
         scenarioIndex=0
         jq -cr '.[]' < $f | while read j
         do
@@ -56,6 +63,7 @@ do
                 echo "    file: ${filename%%.*}" >> $output
                 echo "    scenario: ${scenario}" >> $output
                 echo "    case: ${case}" >> $output
+                echo "    optional: ${optional}" >> $output
                 echo "    expected: ${expected}" >> $output
                 echo "    result: ${result}" >> $output