diff --git a/.circleci/test.sh b/.circleci/test.sh index 148d18d19b9..c2e82b26442 100755 --- a/.circleci/test.sh +++ b/.circleci/test.sh @@ -5,17 +5,37 @@ set +e set +o pipefail EXIT_STATE=0 +MAX_AUTO_RETRY=5 + +# inspired by https://unix.stackexchange.com/a/82602 +retry () { + local n=0 + + until [ $n -ge $MAX_AUTO_RETRY ]; do + "$@" && break + n=$[$n+1] + echo '' + echo run $n of $MAX_AUTO_RETRY failed, trying again ... + echo '' + sleep 15 + done + + if [ $n -eq $MAX_AUTO_RETRY ]; then + EXIT_STATE=1 + fi +} case $1 in jasmine) - npm run test-jasmine -- --skip-tags=gl,noCI || EXIT_STATE=$? + npm run test-jasmine -- --skip-tags=gl,noCI,flaky || EXIT_STATE=$? exit $EXIT_STATE ;; jasmine2) - npm run test-jasmine -- --tags=gl --skip-tags=noCI || EXIT_STATE=$? - npm run test-bundle || EXIT_STATE=$? + npm run test-jasmine -- --tags=gl --skip-tags=noCI,flaky || EXIT_STATE=$? + retry npm run test-jasmine -- --tags=flaky --skip-tags=noCI + npm run test-bundle || EXIT_STATE=$? exit $EXIT_STATE ;; diff --git a/test/jasmine/tests/animate_test.js b/test/jasmine/tests/animate_test.js index a8dca8d0b53..4c43332de3a 100644 --- a/test/jasmine/tests/animate_test.js +++ b/test/jasmine/tests/animate_test.js @@ -720,7 +720,7 @@ describe('Animating multiple axes', function() { destroyGraphDiv(); }); - it('updates ranges of secondary axes', function(done) { + it('@flaky updates ranges of secondary axes', function(done) { Plotly.plot(gd, [ {y: [1, 2, 3]}, {y: [1, 2, 3], yaxis: 'y2'} diff --git a/test/jasmine/tests/gl2d_plot_interact_test.js b/test/jasmine/tests/gl2d_plot_interact_test.js index 8c1063f3c3d..af33ca0e2e1 100644 --- a/test/jasmine/tests/gl2d_plot_interact_test.js +++ b/test/jasmine/tests/gl2d_plot_interact_test.js @@ -250,7 +250,7 @@ describe('@gl Test gl2d plots', function() { }); } - it('should respond to drag interactions', function(done) { + it('@flaky should respond to drag interactions', function(done) { var _mock = Lib.extendDeep({}, mock); var relayoutCallback = jasmine.createSpy('relayoutCallback'); diff --git a/test/jasmine/tests/parcoords_test.js b/test/jasmine/tests/parcoords_test.js index cebffb84195..a1dc36f1130 100644 --- a/test/jasmine/tests/parcoords_test.js +++ b/test/jasmine/tests/parcoords_test.js @@ -774,7 +774,7 @@ describe('@gl parcoords', function() { }); - it('Calling `Plotly.animate` with patches targeting `dimensions` attributes should do the right thing', function(done) { + it('@flaky Calling `Plotly.animate` with patches targeting `dimensions` attributes should do the right thing', function(done) { Plotly.newPlot(gd, [{ type: 'parcoords', line: {color: 'blue'}, diff --git a/test/jasmine/tests/select_test.js b/test/jasmine/tests/select_test.js index 7d2af86b3e0..eb92824a06a 100644 --- a/test/jasmine/tests/select_test.js +++ b/test/jasmine/tests/select_test.js @@ -107,7 +107,7 @@ var BOXEVENTS = [1, 2, 1]; // assumes 5 points in the lasso path var LASSOEVENTS = [4, 2, 1]; -describe('Test select box and lasso in general:', function() { +describe('@flaky Test select box and lasso in general:', function() { var mock = require('@mocks/14.json'); var selectPath = [[93, 193], [143, 193]]; var lassoPath = [[316, 171], [318, 239], [335, 243], [328, 169]];