ENH: generate Dockerfiles with neurodocker + migrate to CircleCI 2.0

.circleci/config.yml

@@ -0,0 +1,190 @@
+version: 2
+jobs:
+
+  compare_base_dockerfiles:
+    docker:
+      - image: docker:17.09.0-ce-git
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - run:
+          name: Prune base Dockerfile in preparation for cache check
+          working_directory: /home/circleci/nipype/docker
+          command: |
+            mkdir -p /tmp/docker
+            # Use the sha256 sum of the pruned Dockerfile as the cache key.
+            ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned
+      - restore_cache:
+          # TODO: change this to 'master' after we are sure this works.
+          key: dftest-v5-enh/circleci-neurodocker-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }}
+      - run:
+          name: Determine how to get base image
+          command: |
+            GET_BASE="/tmp/docker/get_base_image.sh"
+
+            # This directory comes from the cache.
+            if [ -d /cache/base-dockerfile ]; then
+              echo "echo Pulling base image ..." > "$GET_BASE"
+              echo "docker pull kaczmarj/nipype:base" >> "$GET_BASE"
+            else
+              echo "echo Building base image ..." > "$GET_BASE"
+              echo "docker build -t kaczmarj/nipype:base - < /home/circleci/nipype/docker/Dockerfile.base" >> "$GET_BASE"
+            fi
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - docker/*
+
+
+  build_and_test:
+    parallelism: 4
+    machine:
+      # Ubuntu 14.04 with Docker 17.03.0-ce
+      image: circleci/classic:201703-01
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - attach_workspace:
+          at: /tmp
+      - run:
+          name: Get test dependencies
+          command: |
+            pip install --no-cache-dir codecov
+      - run:
+          name: Modify Nipype version if necessary
+          working_directory: /home/circleci/nipype
+          command: |
+            if [ "$CIRCLE_TAG" != "" ]; then
+              sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
+            fi
+      - run:
+          name: Get base image (pull or build)
+          no_output_timeout: 60m
+          # TODO: remove `docker pull` once once caching works.
+          command: |
+            # bash /tmp/docker/get_base_image.sh
+            docker pull kaczmarj/nipype:base
+      - run:
+          name: Build main image (py36)
+          no_output_timeout: 60m
+          working_directory: /home/circleci/nipype
+          command: |
+            e=1 && for i in {1..5}; do
+              docker build \
+                --rm=false \
+                --tag kaczmarj/nipype:latest \
+                --tag kaczmarj/nipype:py36 \
+                --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+                --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
+                --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \
+              && e=0 && break || sleep 15
+            done && [ "$e" -eq "0" ]
+      - run:
+          name: Build main image (py27)
+          no_output_timeout: 60m
+          working_directory: /home/circleci/nipype
+          command: |
+            e=1 && for i in {1..5}; do
+              docker build \
+                --rm=false \
+                --tag kaczmarj/nipype:py27 \
+                --build-arg PYTHON_VERSION_MAJOR=2 \
+                --build-arg PYTHON_VERSION_MINOR=7 \
+                --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+                --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
+                --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \
+              && e=0 && break || sleep 15
+            done && [ "$e" -eq "0" ]
+      - run:
+          name: Download test data
+          no_output_timeout: 20m
+          working_directory: /home/circleci/examples
+          environment:
+            OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage"
+          command: |
+            export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21"
+            curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj
+
+            export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
+            curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz
+
+            export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
+            curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz
+      - run:
+          name: Run tests
+          no_output_timeout: 4h
+          environment:
+            WORKDIR: /home/circleci/work
+          command: |
+            mkdir -p "$WORKDIR"
+            chmod -R 777 "$WORKDIR"
+            bash /home/circleci/nipype/.circleci/tests.sh
+      - store_artifacts:
+          path: /home/circleci/work/tests
+      - run:
+          name: Save Docker images to workspace
+          no_output_timeout: 60m
+          command: |
+            if [ "$CIRCLE_NODE_INDEX" -eq "0" ]; then
+              docker save kaczmarj/nipype:base \
+                          kaczmarj/nipype:latest \
+                          kaczmarj/nipype:py36 \
+                          kaczmarj/nipype:py27 > /tmp/docker/nipype-base-latest-py36-py27.tar
+            fi
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - docker/*
+
+
+  deploy:
+    docker:
+      - image: docker:17.09.0-ce-git
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - setup_remote_docker
+      - attach_workspace:
+          at: /tmp
+      - run:
+          name: Load saved Docker images.
+          no_output_timeout: 60m
+          command: |
+            docker load < /tmp/docker/nipype-base-latest-py36-py27.tar
+      - run:
+          name: Push to DockerHub
+          no_output_timeout: 120m
+          command: |
+            echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
+            docker push kaczmarj/nipype:base
+            docker push kaczmarj/nipype:latest
+            docker push kaczmarj/nipype:py36
+            docker push kaczmarj/nipype:py27
+      - run:
+          name: Prune base Dockerfile to update cache
+          working_directory: /home/circleci/nipype/docker
+          command: |
+            mkdir -p /tmp/docker
+            # Use the sha256 sum of the pruned Dockerfile as the cache key.
+            ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned
+      - save_cache:
+          paths:
+            - /tmp/docker/Dockerfile.base-pruned
+          key: dftest-v5-{{ .Branch }}-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }}
+
+
+workflows:
+  version: 2
+  build_test_deply:
+    jobs:
+      - compare_base_dockerfiles
+      - build_and_test:
+          requires:
+            - compare_base_dockerfiles
+      - deploy:
+          filters:
+            branches:
+              # TODO: change this to master after we are sure this works.
+              only: enh/circleci-neurodocker
+          requires:
+            - build_and_test

.circleci/tests.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Balance nipype testing workflows across CircleCI build nodes
+#
+
+# Setting      # $ help set
+set -e         # Exit immediately if a command exits with a non-zero status.
+set -u         # Treat unset variables as an error when substituting.
+set -x         # Print command traces before executing command.
+
+if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then
+  echo "These tests were designed to be run at 4x parallelism."
+  exit 1
+fi
+
+# TODO: change this image name
+DOCKER_IMAGE="kaczmarj/nipype"
+
+# These tests are manually balanced based on previous build timings.
+# They may need to be rebalanced in the future.
+case ${CIRCLE_NODE_INDEX} in
+  0)
+       docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data"  "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
+    exitcode=$?
+    ;;
+  1)
+       docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline
+    exitcode=$?
+    ;;
+  2)
+       docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
+    exitcode=$?
+    ;;
+  3)
+       docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow
+    exitcode=$?
+    ;;
+esac
+
+# Exit with error if any of the tests failed
+if [ "$exitcode" != "0" ]; then exit 1; fi
+
+codecov --file "${WORKDIR}/tests/coverage*.xml" \
+  --root "${HOME}/nipype/" --flags unittests -e CIRCLE_NODE_INDEX
+
+codecov --file "${WORKDIR}/tests/smoketest*.xml" \
+  --root "${HOME}/nipype/" --flags smoketests -e CIRCLE_NODE_INDEX