.gitlab-ci.yml

variables:
  RELEASE_BRANCH: master
  DRBD_VERSION_BASE: 9.3.0
  DRBD9_TESTS_REF: master
  BUILD_HELPERS_VERSION: 04d2651f29fb4166b22e58c1f336899cc7603ce4
  # we have to change the way GitLab CI clones the repository.
  # by default, it is shallow, which gives us a wrong commit count when
  # we do `git rev-list --count origin/master` below.
  GIT_DEPTH: 0
  GIT_STRATEGY: clone
  GIT_SUBMODULE_STRATEGY: normal
  # comma separated list of base images to exclude when testing
  EXCLUDE_BASE_IMAGES: rhel-7-drbd-k1160

stages:
  - check
  - build
  - test

.setup-lbbuild:
  before_script:
    - . gitlab-utils.sh
    - ci_prepare_tools
    - ci_fetch_rq
    - ci_fetch_lbbuildctl
    - git clone --branch ${DRBD9_TESTS_REF} https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/drbd/drbd9-tests.git
    - |
      kernels_from_vms() {
        python3 .gitlab/kernels-from-vms.py < drbd9-tests/virter/vms.toml
      }

      # parameters: [version]
      # select all "build targets" for the given drbd version,
      # ignoring some distros that always fail in CI anyway
      all_kernels() {
        local version="$1"
        lbbuildctl build drbd --ci -v $version --arch amd64 --debug debugSpecJson | \
          jq -r 'map(.image + " " + .aux_version) | unique[]' | \
          grep -v ^sles12 | grep -v ^rhel7 | grep -v ^sles15-sp0 | grep -v ^docker
      }

      # parameters: [version] [release] [rel-version] [kernels]
      # kernels should be a string with lines with the format: "[distribution] [kernel version]"
      drbd_build () {
        local version="$1"
        local release="$2"
        local rel_version="$3"
        local kernels="$4"

        set -x

        ./.gitlab/drbd-prepare-release.sh "$version" "$release" "$rel_version"
        make FORCE=1 PRESERVE_DEBIAN=1 REL_VERSION="$rel_version" tarball
        curl -isSf -u $LINBIT_REGISTRY_USER:$LINBIT_REGISTRY_PASSWORD \
          --upload-file drbd-*.tar.gz \
          $LINBIT_REGISTRY_URL/repository/lbbuild-upstream/

        local compat_rc=0  
        echo "$kernels" | parallel --colsep ' ' -k -j24 --joblog compat.log \
          lbbuildctl build drbd --arch amd64 --ci -v "$rel_version" --compat-only \
            -e LINBIT_REGISTRY_USER=$LINBIT_REGISTRY_USER \
            -e LINBIT_REGISTRY_PASSWORD=$LINBIT_REGISTRY_PASSWORD \
            -e LINBIT_REGISTRY_URL=$LINBIT_REGISTRY_URL \
            -d {1} -k {2} \
            || compat_rc=$?

        if [ "$compat_rc" -ne 0 ]; then
          echo "Error: compat build failed" >&2
          parse_joblog.py drbd compat.log
          exit $compat_rc
        fi

        pushd drbd/drbd-kernel-compat
        local compat_hs
        while read line; do
          kver=${line#* } # everything after " " -> the kernel version
          curl -fsSLO "$LINBIT_REGISTRY_URL/repository/drbd-compat/$rel_version/compat.h.$kver"
          compat_hs+=("compat.h.$kver")
        done <<< "$kernels"
        ./build_cocci_cache.sh ${compat_hs[@]}
        popd

        make -j $(nproc) -C drbd compat
        make FORCE=1 PRESERVE_DEBIAN=1 REL_VERSION="$rel_version" tarball
        curl -isSf -u $LINBIT_REGISTRY_USER:$LINBIT_REGISTRY_PASSWORD \
          --upload-file drbd-*.tar.gz \
          $LINBIT_REGISTRY_URL/repository/lbbuild-upstream/

        local build_rc=0
        echo "$kernels" | parallel --colsep ' ' -k -j24 --joblog parallel.log \
          lbbuildctl build drbd --arch amd64 --ci -v "$rel_version" \
            -e LINBIT_REGISTRY_USER=$LINBIT_REGISTRY_USER \
            -e LINBIT_REGISTRY_PASSWORD=$LINBIT_REGISTRY_PASSWORD \
            -e LINBIT_REGISTRY_URL=$LINBIT_REGISTRY_URL \
            -d {1} -k {2} \
            || build_rc=$?

        parse_joblog.py drbd parallel.log

        if [ "$build_rc" -ne 0 ]; then
          exit $build_rc
        fi

        set +x
      }

# rules clause determining whether to build
.rules-build:
  rules:
    - if: $CI_MERGE_REQUEST_ID
    - if: $CI_COMMIT_BRANCH == $RELEASE_BRANCH

check:coccicheck:
  stage: check
  extends: [ .rules-build ]
  image: docker.io/linbit/coccinelle:1.2
  script:
    - make coccicheck

check:pychecks:
  stage: check
  extends: [ .rules-build ]
  image: docker.io/library/python:3
  variables:
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  cache:
    paths:
      - .cache/pip
  script:
    - pip install tree-sitter tree-sitter-c
    - make pychecks

check:cocci-syntax:
  stage: check
  extends: [ .rules-build ]
  image: docker.io/linbit/coccinelle:1.2
  script:
    - make cocci-syntax

check:check-compat:
  stage: check
  extends: [ .rules-build ]
  image: docker.io/linbit/coccinelle:1.2
  script:
    - make check-compat

check:checkpatch.pl:
  stage: check
  rules:
    - if: $CI_MERGE_REQUEST_ID
  image: docker.io/library/perl:5
  script:
    - curl -sSLO https://raw.githubusercontent.com/torvalds/linux/master/scripts/checkpatch.pl
    - curl -sSLO https://raw.githubusercontent.com/torvalds/linux/master/scripts/spelling.txt
    - curl -sSLO https://raw.githubusercontent.com/torvalds/linux/master/scripts/const_structs.checkpatch
    - chmod +x checkpatch.pl
    - git fetch --no-tags origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
    - base=$(git merge-base remotes/origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} HEAD)
    - CHECKPATCH=./checkpatch.pl misc/checkpatch.sh commits "$base"

build:for-tests:
  stage: build
  extends: [ .setup-lbbuild ]
  rules:
    - !reference [.rules-build, rules]
    # Useful for building DRBD without opening a merge request
    - if: $CI_PIPELINE_SOURCE == 'api'
    # Used by the tests pipeline, if a DRBD branch build is required
    - if: $CI_PIPELINE_SOURCE == 'pipeline'
    # Useful for testing the downstream trigger used in the tests pipeline
    - if: $CI_PIPELINE_SOURCE == 'trigger'
  image: $LINBIT_DOCKER_REGISTRY/build-helpers:$BUILD_HELPERS_VERSION
  script:
    - if [ "$CI_COMMIT_BRANCH" = "$RELEASE_BRANCH" ]; then
    -   DRBD_VERSION=$DRBD_VERSION_BASE.latest
    -   DRBD_RELEASE=$(git rev-list --count "origin/$CI_COMMIT_BRANCH")
    - else
    -   DRBD_VERSION=$DRBD_VERSION_BASE.$CI_COMMIT_SHA
    -   DRBD_RELEASE=1
    - fi
    - echo DRBD_VERSION=$DRBD_VERSION >> build.env
    - |
      # If the build is conditional and some packages already exist, skip the build.
      # Then the purpose of this job is just to generate build.env.
      if [ "$DRBD_BUILD_CONDITIONAL" = true ]; then
        curl -fsSL "$LINBIT_REGISTRY_URL/service/rest/v1/search?repository=ci-yum&version=$DRBD_VERSION*" | \
            jq --exit-status '.items | length > 0' && exit 0
      fi
    - drbd_build $DRBD_VERSION $DRBD_RELEASE $DRBD_VERSION "$(kernels_from_vms)"
  artifacts:
    reports:
      dotenv: build.env
    paths:
      - drbd/drbd-kernel-compat/cocci_cache
      # Make the variables accessible through the API. See:
      # https://stackoverflow.com/questions/72202825/download-dotenv-job-artifact-via-gitlab-api
      - build.env
    when: always
    expire_in: 1 week
  needs: []

build:all:
  stage: build
  extends: [ .setup-lbbuild ]
  rules:
    - if: $CI_MERGE_REQUEST_ID
      when: manual
    - if: $CI_COMMIT_BRANCH == $RELEASE_BRANCH
      when: manual
    - if: $CI_PIPELINE_SOURCE == 'api'
      when: manual
    - if: $CI_PIPELINE_SOURCE == 'pipeline'
      when: manual
    - if: $CI_PIPELINE_SOURCE == 'trigger'
      when: manual
  allow_failure: true
  image: $LINBIT_DOCKER_REGISTRY/build-helpers:$BUILD_HELPERS_VERSION
  script:
    - DRBD_VERSION=$DRBD_VERSION_BASE.$CI_COMMIT_SHA
    - DRBD_RELEASE=1
    - echo DRBD_VERSION=$DRBD_VERSION >> build.env
    - drbd_build $DRBD_VERSION $DRBD_RELEASE $DRBD_VERSION "$(all_kernels $DRBD_VERSION)"
  artifacts:
    reports:
      dotenv: build.env
    paths:
      - drbd/drbd-kernel-compat/cocci_cache
    when: always
    expire_in: 1 week
  needs: []

.test:
  tags:
    - libvirt
  cache:
    paths:
      - download
  before_script:
    - |
      set -e
      run_test() {
        git clone --branch ${DRBD9_TESTS_REF} https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/drbd/drbd9-tests.git
        . drbd9-tests/virter/version.env

        git init build-helpers -q -b unused
        git -C build-helpers fetch -q --depth 1 $CI_SERVER_PROTOCOL://gitlab-ci-token:${CI_JOB_TOKEN}@$CI_SERVER_HOST:$CI_SERVER_PORT/linbit/build-helpers.git $BUILD_HELPERS_VERSION
        git -C build-helpers checkout -q $BUILD_HELPERS_VERSION
        . build-helpers/gitlab-utils.sh

        ci_prepare_tools
        ci_fetch_rq
        ci_fetch_binary virter virter-$VIRTER_VERSION https://github.com/LINBIT/virter/releases/download/$VIRTER_VERSION/virter-linux-amd64
        ci_fetch_binary vmshed vmshed-$VMSHED_VERSION https://github.com/LINBIT/vmshed/releases/download/$VMSHED_VERSION/vmshed-linux-amd64

        export DRBD_TEST_DOCKER_IMAGE=$LINBIT_DOCKER_REGISTRY/drbd9-tests:latest
        docker image pull $DRBD_TEST_DOCKER_IMAGE

        export DRBD_VERSION=$DRBD_VERSION
        export DRBD_UTILS_VERSION=9.0.0.latest-*
        ./drbd9-tests/virter/run-test.sh "$@" --exclude-base-image "$EXCLUDE_BASE_IMAGES"
      }
  dependencies:
    - build:for-tests
  artifacts:
    # provide a convenient name so that the downloaded artifacts can be identified
    name: $CI_PROJECT_NAME-$CI_JOB_ID
    paths:
      - tests-out/
    when: always
    reports:
      junit: tests-out/test-results/*.xml
  needs:
    - build:for-tests

test:default:
  stage: test
  extends: [ .test ]
  rules:
    - if: $CI_MERGE_REQUEST_ID
  script: run_test

test:rdma:
  stage: test
  extends: [ .test ]
  rules:
    - if: $CI_MERGE_REQUEST_ID
      changes:
        - drbd/drbd_transport_rdma.c
    - if: $CI_MERGE_REQUEST_ID
      when: manual
  allow_failure: true
  script: VMSHED_TEST_SELECTION=rdma_ci run_test --variant=rdma