Revise exports & baseline creation and testing using Kaleido

Author: archmoj

.circleci/config.yml

@@ -78,41 +78,90 @@ jobs:
           name: Run jasmine tests (part C)
           command: ./.circleci/test.sh no-gl-flaky-jasmine
 
-  stable-image:
-    docker:
-      - image: plotly/testbed:latest
+  make-baselines:
     parallelism: 4
-    working_directory: /var/www/streambed/image_server/plotly.js/
+    docker:
+      - image: circleci/python:3.8.9
+    working_directory: ~/plotly.js
     steps:
       - attach_workspace:
-          at: /var/www/streambed/image_server/
+          at: ~/
+      - run:
+          name: which pip3 version
+          command: which pip3 && pip3 --version
       - run:
-          name: Run and setup container
+          name: install kaleido v0.2.1
+          command: python3 -m pip install kaleido==0.2.1
+      - run:
+          name: install plotly.io v4.14.3
+          command: python3 -m pip install plotly==4.14.3
+      - run:
+          name: install fonts
           command: |
-            supervisord &
-            npm run docker -- setup
+            sudo cp -r .circleci/fonts/ /usr/share/
+            sudo fc-cache -f
+      - run:
+          name: create all png files
+          command: ./.circleci/test.sh make-baselines
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - plotly.js
+
+  test-baselines:
+    docker:
+      - image: circleci/node:12.22.1
+    working_directory: ~/plotly.js
+    steps:
+      - attach_workspace:
+          at: ~/
       - run:
-          name: Run image tests (part A)
-          command: ./.circleci/test.sh stable-image || { tar -cvf build/baselines.tar build/test_images/ ; exit 1; } ; find build -maxdepth 1 -type f -delete
+          name: compare pixels
+          command: ./.circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
       - store_artifacts:
           path: build
           destination: /
 
-  flaky-image:
+  make-exports:
     docker:
-      - image: plotly/testbed:latest
-    working_directory: /var/www/streambed/image_server/plotly.js/
+      - image: circleci/python:3.8.9
+    working_directory: ~/plotly.js
     steps:
       - attach_workspace:
-          at: /var/www/streambed/image_server/
+          at: ~/
       - run:
-          name: Run and setup container
-          command: |
-            supervisord &
-            npm run docker -- setup
+          name: which pip3 before upgrade
+          command: which pip3 && pip3 --version
+      - run:
+          name: upgrade pip3
+          command: python3 -m pip install --upgrade pip
+      - run:
+          name: which pip3 after upgrade
+          command: which pip3 && pip3 --version
+      - run:
+          name: install kaleido
+          command: python3 -m pip install kaleido
+      - run:
+          name: install plotly.io
+          command: python3 -m pip install plotly
+      - run:
+          name: create svg, jpg, jpeg, webp and pdf files
+          command: python3 test/image/make_exports.py
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - plotly.js
+
+  test-exports:
+    docker:
+      - image: circleci/node:12.22.1
+    working_directory: ~/plotly.js
+    steps:
+      - attach_workspace:
+          at: ~/
       - run:
-          name: Run image tests (part B)
-          command: ./.circleci/test.sh flaky-image ; find build -maxdepth 1 -type f -delete
+          name: test export sizes
+          command: node test/image/export_test.js ; find build -maxdepth 1 -type f -delete
       - store_artifacts:
           path: build
           destination: /
@@ -237,12 +286,18 @@ workflows:
       - no-gl-flaky-jasmine:
           requires:
             - install-and-cibuild
-      - stable-image:
+      - make-baselines:
           requires:
             - install-and-cibuild
-      - flaky-image:
+      - test-baselines:
+          requires:
+            - make-baselines
+      - make-exports:
           requires:
             - install-and-cibuild
+      - test-exports:
+          requires:
+            - make-exports
       - mock-validation:
           requires:
             - install-and-cibuild

.circleci/test.sh

@@ -76,16 +76,14 @@ case $1 in
         exit $EXIT_STATE
         ;;
 
-    stable-image)
-        SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | circleci tests split)
-        npm run test-image -- $SUITE --filter --skip-flaky || EXIT_STATE=$?
+    make-baselines)
+        SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
+        python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
-    flaky-image)
-        MAX_AUTO_RETRY=5
-        retry npm run test-image -- --just-flaky
-        npm run test-export     || EXIT_STATE=$?
+    test-image)
+        node test/image/compare_pixels_test.js || { tar -cvf build/baselines.tar build/test_images/*.png ; exit 1 ; } || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 

package-lock.json

@@ -36,12 +36,11 @@
     "no-bad-char": "eslint --no-ignore --no-eslintrc --no-inline-config --rule '{no-misleading-character-class: error}' $(find dist -type f -iname '*.js' | grep plotly)",
     "no-dup-keys": "eslint --no-ignore --no-eslintrc --no-inline-config --rule '{no-dupe-keys: error}' $(find dist -type f -iname '*.js' | grep plotly)",
     "no-es6-dist": "node tasks/no_es6_dist.js",
-    "docker": "node tasks/docker.js",
     "pretest": "node tasks/pretest.js",
     "test-jasmine": "karma start test/jasmine/karma.conf.js",
     "test-mock": "node tasks/test_mock.js",
-    "test-image": "node tasks/test_image.js",
-    "test-export": "node tasks/test_export.js",
+    "test-image": "node test/image/compare_pixels_test.js",
+    "test-export": "node test/image/export_test.js",
     "test-syntax": "node tasks/test_syntax.js && npm run find-strings -- --no-output",
     "test-bundle": "node tasks/test_bundle.js",
     "test-requirejs": "node tasks/test_requirejs.js",
@@ -50,7 +49,7 @@
     "start-test_dashboard": "node devtools/test_dashboard/server.js",
     "start-image_viewer": "node devtools/image_viewer/server.js",
     "start": "npm run start-test_dashboard",
-    "baseline": "node tasks/baseline.js",
+    "baseline": "node test/image/make_baseline.js",
     "noci-baseline": "npm run cibuild && ./tasks/noci_test.sh image && git checkout dist && echo 'Please do not commit unless the change was expected!'",
     "preversion": "check-node-version --node 12 --npm 6.14 && npm-link-check && npm ls --prod",
     "version": "npm run build && npm run no-bad-char && git add -A lib dist build src/version.js",
@@ -172,6 +171,7 @@
     "node-sass": "^6.0.0",
     "npm-link-check": "^4.0.0",
     "open": "^8.0.7",
+    "pixelmatch": "^5.2.1",
     "prepend-file": "^2.0.0",
     "prettysize": "^2.0.0",
     "read-last-lines": "^1.8.0",

package.json

@@ -24,25 +24,12 @@ test_jasmine () {
     npm run test-jasmine -- --tags=noCI,noCIdep --nowatch || EXIT_STATE=$?
 }
 
-# mapbox image tests take too much resources on CI
-#
-# since the update to [email protected], we must use orca
-# as mapbox-gl versions >0.22.1 aren't supported on [email protected] used in the
-# 'old' image server
-#
-# due to mathjax upgrade to v2.7.5, we are unable to test mathjax mocks in the
-# 'old' image server and we use orca
-#
+# having problem creating baselines for 2 mapbox mocks using kaleido
+# we must use orca
 test_image () {
     $root/../orca/bin/orca.js graph \
-        $root/test/image/mocks/mapbox_* \
-        $root/test/image/mocks/mathjax.json \
-        $root/test/image/mocks/ternary-mathjax.json \
-        $root/test/image/mocks/table_plain_birds.json \
-        $root/test/image/mocks/table_wrapped_birds.json \
-        $root/test/image/mocks/parcats_grid_subplots.json \
-        $root/test/image/mocks/legend_mathjax_title_and_items.json \
-        $root/test/image/mocks/table_latex_multitrace_scatter.json \
+        $root/test/image/mocks/mapbox_osm-style.json \
+        $root/test/image/mocks/mapbox_density0-legend.json \
         --mathjax $root/node_modules/mathjax/MathJax.js \
         --plotly $root/build/plotly.js \
         --mapbox-access-token "pk.eyJ1IjoicGxvdGx5LWpzLXRlc3RzIiwiYSI6ImNrNG9meTJmOTAxa3UzZm10dWdteDQ2eWMifQ.2REjOFyIrleMqwS8H8y1-A" \