azure-pipelines.yml

trigger:
  branches:
    include:
    - '*'  # must quote since "*" is a YAML reserved character; we want a string
  tags:
    include:
    - 'v*'

pr:
  branches:
    include:
    - '*'  # must quote since "*" is a YAML reserved character; we want a string

schedules:
- cron: "0 0 * * 6"
  displayName: Daily midnight build
  branches:
    include:
    - master
  always: true

variables:
  YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn

parameters:
  - name: initSteps
    type: stepList
    default:
      - task: NodeTool@0
        inputs:
          versionSpec: '12.x'
    
      - bash: npm install -g yarn
        displayName: 'Install Yarn'

      - task: Cache@2
        inputs:
          key: 'yarn | "$(Agent.OS)" | yarn.lock'
          restoreKeys: |
            yarn | "$(Agent.OS)"
            yarn
          path: $(YARN_CACHE_FOLDER)
        displayName: Cache Yarn packages
  
      - bash: yarn --frozen-lockfile
        displayName: "Install Deps"
  
      - task: UsePythonVersion@0
        inputs:
          versionSpec: '$(python.version)'
          architecture: 'x64'
        displayName: 'Use Python $(python.version)'
  
      - bash: python -m pip install --upgrade pip wheel setuptools jupyterlab numpy "pyarrow>=2" "black==20.8b1" flake8-black
        displayName: 'Install Python base dependencies'
        condition: and(succeeded(), ne(variables['python.version'], '2.7'))
  
      - bash: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
        displayName: "Install wasm-pack"

  - name: linuxInitSteps
    type: stepList
    default:
      - bash: sudo node scripts/install_tools.js
        displayName: "Linux System deps"

  - name: macInitSteps
    type: stepList
    default:
      - bash: brew install boost flatbuffers
        displayName: "Mac System deps"

  - name: windowsInitSteps
    type: stepList
    default:
      - script: |
          which python > python.txt
          set /p PYTHON=<python.txt
          ln -s %PYTHON% %PYTHON%$(python.version)
          python --version
          which python$(python.version)
        displayName: "Which python"

      - script: choco install boost-msvc-14.1 flatc
        displayName: "System deps"

  - name: webassemblySteps
    type: stepList
    default:
      - bash: yarn build_js --ci
        displayName: "Build"

      - bash: yarn test_js --quiet --ci
        displayName: "Test"
        env:
          PSP_DOCKER: 1

      - task: PublishTestResults@2
        condition: succeededOrFailed()
        inputs:
          testRunner: JUnit
          testResultsFiles: 'junit.xml'

  - name: pythonBuildSteps
    type: stepList
    default:
      - script: yarn build_python  --ci $(python_flag) $(manylinux_flag)
        displayName: 'build'

  - name: pythonBuildDockerSteps
    type: stepList
    default:
      - bash: yarn build_python  --ci $(python_flag) $(manylinux_flag)
        displayName: 'build'
        env:
          PSP_DOCKER: 1

  - name: pythonCoverageSteps
    type: stepList
    default:
      - task: PublishTestResults@2
        condition: succeededOrFailed()
        inputs:
          testResultsFiles: 'python/perspective/python_junit.xml'
          testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'

      - task: PublishCodeCoverageResults@1
        inputs: 
          codeCoverageTool: Cobertura
          summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/*coverage.xml'

  - name: pythonMacWheelSteps
    type: stepList
    default:
      # Build a python wheel for Mac 10.14
      - bash: yarn _wheel_python $(python_flag) --macos
        condition: and(succeeded(), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')))
        displayName: 'Build wheel'

      # Test the wheel
      - bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag) --macos
        condition: and(succeeded(), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')))
        displayName: 'Test Mac Wheel'

      # Save the artifact to Azure storage
      - task: PublishPipelineArtifact@1
        condition: and(succeeded(), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')))
        inputs:
          targetPath: '$(System.DefaultWorkingDirectory)/python/perspective/dist/'
          artifactName: '$(artifact_name)'

  - name: pythonLinuxWheelSteps
    type: stepList
    default:
      # Build a python wheel for Manylinux
      - bash: yarn _wheel_python --ci $(python_flag) $(manylinux_flag)
        condition: and(succeeded(), ne(variables['manylinux_flag'], ''), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')))
        displayName: 'Build Manylinux Wheel'
        env:
          PSP_DOCKER: 1

      # Test the wheel
      - bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag) $(manylinux_flag)
        condition: and(succeeded(), ne(variables['manylinux_flag'], ''), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')))
        displayName: 'Test Manylinux Wheel'

      # Save the artifact to Azure storage
      - task: PublishPipelineArtifact@1
        condition: and(succeeded(), ne(variables['manylinux_flag'], ''), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')))
        inputs:
          targetPath: '$(System.DefaultWorkingDirectory)/python/perspective/wheelhouse/'
          artifactName: '$(artifact_name)'
  
  - name: pythonWindowsWheelSteps
    type: stepList
    default:
      # Build a python wheel for Windows
      - bash: yarn _wheel_python $(python_flag)
        condition: and(succeeded(), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')))
        displayName: 'Build Windows Wheel'

      # Save the artifact to Azure storage
      - task: PublishPipelineArtifact@1
        condition: and(succeeded(), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')))
        inputs:
          targetPath: '$(System.DefaultWorkingDirectory)/python/perspective/dist/'
          artifactName: '$(artifact_name)'

jobs:
- job: 'WebAssembly'
  pool:
    vmImage: 'ubuntu-16.04'

  strategy:
    matrix:
      Base:
        # Do we care to test elsewhere?
        python.version: '3.9'

  steps:
    - ${{ parameters.initSteps }}
    - ${{ parameters.linuxInitSteps }}
    - ${{ parameters.webassemblySteps }}

    - task: PublishTestResults@2
      condition: succeeded()
      inputs:
        testRunner: JUnit
        testResultsFiles: 'junit.xml'

    - task: PublishPipelineArtifact@1
      condition: failed()
      inputs:
        targetPath: '$(System.DefaultWorkingDirectory)/packages/perspective-viewer/screenshots/'
        artifactName: "perspective-viewer-$(Build.BuildNumber)"

    - task: PublishPipelineArtifact@1
      condition: failed()
      inputs:
        targetPath: '$(System.DefaultWorkingDirectory)/packages/perspective-viewer-datagrid/screenshots/'
        artifactName: "perspective-viewer-datagrid-$(Build.BuildNumber)"

    - task: PublishPipelineArtifact@1
      condition: failed()
      inputs:
        targetPath: '$(System.DefaultWorkingDirectory)/packages/perspective-viewer-d3fc/screenshots/'
        artifactName: "perspective-viewer-d3fc-$(Build.BuildNumber)"

    - task: PublishPipelineArtifact@1
      condition: failed()
      inputs:
        targetPath: '$(System.DefaultWorkingDirectory)/packages/perspective-workspace/screenshots/'
        artifactName: "perspective-workspace-$(Build.BuildNumber)"


- job: 'Linux'
  pool:
    vmImage: 'ubuntu-16.04'

  strategy:
    matrix:
      # Python27ManyLinux2010:
      #   python.version: '2.7'
      #   python_flag: '--python2'
      #   manylinux_flag: '--manylinux2010'
      #   artifact_name: 'cp27-cp27m-manylinux2010_x86_64'
      Python37ManyLinux2010:
        python.version: '3.7'
        python_flag: ''
        manylinux_flag: '--manylinux2010'
        artifact_name: 'cp37-cp37m-manylinux2010_x86_64'

      ${{ if or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')) }}:
        Python36ManyLinux2010:
          python.version: '3.6'
          python_flag: '--python36'
          manylinux_flag: '--manylinux2010'
          artifact_name: 'cp36-cp36m-manylinux2010_x86_64'
        Python36ManyLinux2014:
          python.version: '3.6'
          python_flag: '--python36'
          manylinux_flag: '--manylinux2014'
          artifact_name: 'cp36-cp36m-manylinux2014_x86_64'
        # Python37ManyLinux2010 is always built
        Python37ManyLinux2014:
          python.version: '3.7'
          python_flag: ''
          manylinux_flag: '--manylinux2014'
          artifact_name: 'cp37-cp37m-manylinux2014_x86_64'
        Python38ManyLinux2010:
          python.version: '3.8'
          python_flag: '--python38'
          manylinux_flag: '--manylinux2010'
          artifact_name: 'cp38-cp38-manylinux2010_x86_64'
        Python38ManyLinux2014:
          python.version: '3.8'
          python_flag: '--python38'
          manylinux_flag: '--manylinux2014'
          artifact_name: 'cp38-cp38-manylinux2014_x86_64'
        Python39ManyLinux2010:
          python.version: '3.9'
          python_flag: '--python39'
          manylinux_flag: '--manylinux2010'
          artifact_name: 'cp39-cp39-manylinux2010_x86_64'
        Python39ManyLinux2014:
          python.version: '3.9'
          python_flag: '--python39'
          manylinux_flag: '--manylinux2014'
          artifact_name: 'cp39-cp39-manylinux2014_x86_64'

  steps:
    - ${{ parameters.initSteps }}
    - ${{ parameters.pythonBuildDockerSteps }}
    - ${{ parameters.pythonCoverageSteps }}
    - ${{ parameters.pythonLinuxWheelSteps }}


- job: 'Windows'
  pool:
    vmImage: 'vs2017-win2016'

  strategy:
    matrix:
      Python37:
        python.version: '3.7'
        python_flag: ''
        artifact_name: 'cp37-cp37m-win64_amd'

      ${{ if or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')) }}:
        Python36:
          python.version: '3.6'
          python_flag: '--python36'
          artifact_name: 'cp36-cp36m-win64_amd'
        Python38:
          python.version: '3.8'
          python_flag: '--python38'
          artifact_name: 'cp38-cp38m-win64_amd'
        Python39:
          python.version: '3.9'
          python_flag: '--python39'
          artifact_name: 'cp39-cp39m-win64_amd'

  steps:
    - ${{ parameters.initSteps }}
    - ${{ parameters.windowsInitSteps }}
    - ${{ parameters.pythonBuildSteps }}
    - ${{ parameters.pythonWindowsWheelSteps }}


- job: 'MacOS_Catalina'
  pool:
    vmImage: 'macos-10.15'

  strategy:
    matrix:

      Python39:
        python.version: '3.9'
        python_flag: '--python39'
        artifact_name: 'cp39-cp39-macosx_10_15_x86_64'

      ${{ if or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')) }}:
        Python36:
          python.version: '3.6'
          python_flag: '--python36'
          artifact_name: 'cp36-cp36m-macosx_10_15_x86_64'

        Python37:
          python.version: '3.7'
          python_flag: ''
          artifact_name: 'cp37-cp37m-macosx_10_15_x86_64'
    
        Python38:
          python.version: '3.8'
          python_flag: '--python38'
          artifact_name: 'cp38-cp38-macosx_10_15_x86_64'

  steps:
    - ${{ parameters.initSteps }}
    - ${{ parameters.macInitSteps }}
    - ${{ parameters.pythonBuildSteps }}
    - ${{ parameters.pythonCoverageSteps }}
    - ${{ parameters.pythonMacWheelSteps }}