test: Add GHA job testing learn-ocaml-client against other server(s)

* Adapt ./tests/runtests.sh so it recognizes USE_CLIENT_IMAGE=true

* Update .github/workflows/build-and-test.yml accordingly
  so learn-ocaml-client runs against each ${{ matrix.server_image }}

Author: erikmd

.github/workflows/build-and-test.yml

@@ -27,6 +27,33 @@ jobs:
       - name: Run learn-ocaml build on learn-ocaml-corpus
         run: "cd tests && bash -c ./runtests.sh"
 
+  client_using_other_server:
+    name: Build learn-ocaml-client and run quick tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        server_image:
+          - 'ocamlsf/learn-ocaml:0.12'
+          - 'learn-ocaml'  # use learn-ocaml image built from master
+    env:
+      USE_CLIENT_IMAGE: 'true'
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Build Docker images
+        run: "make docker-images"
+      # unneeded
+      # - name: Run learn-ocaml build on demo-repository
+      #   run: "docker run --rm -v $(pwd)/demo-repository:/repository learn-ocaml -- build"
+      - name: Pull server_image
+        if: ${{ matrix.server_image != 'learn-ocaml' }}
+        run: "docker pull ${{ matrix.server_image }} && docker tag ${{ matrix.server_image }} learn-ocaml"
+      - name: Print images metadata
+        run: "docker inspect -f '{{json .Config.Labels}}' learn-ocaml-client learn-ocaml | jq"
+      - name: Run learn-ocaml build on learn-ocaml-corpus
+        run: "cd tests && bash -c ./runtests.sh"
+
   build_extra_tests:
     name: Run dockerized tests
     runs-on: ubuntu-latest

tests/runtests.sh

@@ -2,13 +2,26 @@
 
 let count=0
 
-# If the first argument is "set", the script will replace all the expected 
+# If the first argument is "set", the script will replace all the expected
 # answers by the response of the server. It is useful when you need to
 # initialize a set of tests.
 
+# If the environment variable $USE_CLIENT_IMAGE is set to "true", then
+# the script will use both images learn-ocaml and learn-ocaml-client,
+# instead of the mere learn-ocaml image. It is useful for testing
+# cross-version compatibility.
+
 SETTER=0
 if [ "$1" == "set" ]; then SETTER=1; fi
 
+use_client_image () {
+    if [ "$USE_CLIENT_IMAGE" = "true" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
 # print in green $1
 green () {
     echo -e "\e[32m$1\e[0m"
@@ -19,6 +32,12 @@ red () {
     echo -e "\e[31m$1\e[0m"
 }
 
+if use_client_image; then
+    green "Running tests with BOTH images (learn-ocaml, learn-ocaml-client)"
+else
+    green "Running tests with single image learn-ocaml"
+fi
+
 # run a server in a docker container
 run_server () {
     SYNC="$(pwd)"/"$dir"/sync
@@ -28,7 +47,7 @@ run_server () {
     chmod o+w "$dir"/sync
 
     # Run the server in background
-    SERVERID=$(docker run --entrypoint '' -d \
+    SERVERID=$(docker run --entrypoint '' -d -p 8080:8080 \
       -v "$(pwd)"/"$dir":/home/learn-ocaml/actual \
       -v "$SYNC":/sync -v "$REPO":/repository \
       learn-ocaml /bin/sh \
@@ -50,6 +69,25 @@ learn-ocaml --sync-dir=/sync --repo=/repository build serve")
 	docker rm "$SERVERID" > /dev/null
 	exit 1
     fi
+
+    if use_client_image; then
+        # Run the client in background
+        CLIENTID=$(docker run --entrypoint '' -d -it --name=client \
+          -add-host host.docker.internal:host-gateway \
+          -v "$(pwd)"/"$dir":/home/learn-ocaml/actual \
+          learn-ocaml-client /bin/sh)
+
+        # Wait for the client to be initialized
+        sleep 2
+        if [ "$(docker ps -q -f name=client)" == "" ]; then
+            red "PROBLEM, client is not running.\n"
+
+            red "LOGS:"
+            docker logs "$CLIENTID"
+            docker rm "$CLIENTID" > /dev/null
+            exit 1
+        fi
+    fi
 }
 
 clean () {
@@ -58,6 +96,11 @@ clean () {
 
     docker kill "$SERVERID" > /dev/null
     docker rm   "$SERVERID" > /dev/null
+
+    if use_client_image; then
+        docker kill "$CLIENTID" > /dev/null
+        docker rm   "$CLIENTID" > /dev/null
+    fi
 }
 
 clean_fail () {
@@ -69,9 +112,13 @@ handle_file () {
     subdir="$1"
     tosend="$(basename "$2")"
     # Grade file
-    docker exec -i "$SERVERID" \
-	   learn-ocaml-client grade --json --id="$subdir" \
-	   /home/learn-ocaml/actual/"$subdir"/"$tosend" > res.json 2> stderr.txt
+    args=(learn-ocaml-client grade --json --id="$subdir"
+          /home/learn-ocaml/actual/"$subdir"/"$tosend")
+    if use_client_image; then
+        docker exec -i "$CLIENTID" "${args[@]}" > res.json 2> stderr.txt
+    else
+        docker exec -i "$SERVERID" "${args[@]}" > res.json 2> stderr.txt
+    fi
     if [ $? -ne 0 ]; then
 	red "NOT OK: $dir/$tosend"
 	cat stderr.txt
@@ -102,9 +149,13 @@ handle_subdir () {
     subdir="$(basename "$1")"
     pushd "$1" >/dev/null
 
-    #init config
-    docker exec -i "$SERVERID" \
-	   learn-ocaml-client init --server=http://localhost:8080 --token="$token"
+    # init config
+    args=(learn-ocaml-client init --token="$token")
+    if use_client_image; then
+        docker exec -i "$CLIENTID" "${args[@]}" --server="http://host.docker.internal:8080"
+    else
+        docker exec -i "$SERVERID" "${args[@]}" --server="http://localhost:8080"
+    fi
     # For each solution
     for tosend in $(find . -name "*.ml" -type f -printf "%f\n")
     do