From 77203a03177aa15d9c17dcf181240cd12a343046 Mon Sep 17 00:00:00 2001 From: Matt Cooper Date: Tue, 4 Dec 2018 08:47:48 -0500 Subject: [PATCH 1/5] unify macOS and Linux --- azure-pipelines.yml | 8 ++--- ci/azure/posix.yml | 85 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 ci/azure/posix.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 409b1ac8c9df3..3787c6d559ba0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,13 +1,11 @@ # Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml jobs: -# Mac and Linux could potentially use the same template -# except it isn't clear how to use a different build matrix -# for each, so for now they are separate -- template: ci/azure/macos.yml +# Mac and Linux use the same template +- template: ci/azure/posix.yml parameters: name: macOS vmImage: xcode9-macos10.13 -- template: ci/azure/linux.yml +- template: ci/azure/posix.yml parameters: name: Linux vmImage: ubuntu-16.04 diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml new file mode 100644 index 0000000000000..85515539fa277 --- /dev/null +++ b/ci/azure/posix.yml @@ -0,0 +1,85 @@ +parameters: + name: '' + vmImage: '' + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + matrix: + ${{ if eq(parameters.name, 'macOS') }}: + py35_np_120: + ENV_FILE: ci/deps/azure-macos-35.yaml + CONDA_PY: "35" + PATTERN: "not slow and not network" + + ${{ if eq(parameters.name, 'Linux') }}: + py27_np_120: + ENV_FILE: ci/deps/azure-27-compat.yaml + CONDA_PY: "27" + PATTERN: "not slow and not network" + + py37_locale: + ENV_FILE: ci/deps/azure-37-locale.yaml + CONDA_PY: "37" + PATTERN: "not slow and not network" + LOCALE_OVERRIDE: "zh_CN.UTF-8" + + py36_locale_slow: + ENV_FILE: ci/deps/azure-36-locale_slow.yaml + CONDA_PY: "36" + PATTERN: "not slow and not network" + LOCALE_OVERRIDE: "it_IT.UTF-8" + + steps: + - script: | + if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi + echo "Installing Miniconda" + ci/incremental/install_miniconda.sh + export PATH=$HOME/miniconda3/bin:$PATH + echo "Setting up Conda environment" + ci/incremental/setup_conda_environment.sh + displayName: 'Before Install' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + ci/incremental/build.sh + displayName: 'Build' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + ci/run_tests.sh + displayName: 'Test' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd + - task: PublishTestResults@2 + inputs: + testResultsFiles: 'test-data-*.xml' + testRunTitle: ${{ concat(parameters.name, '-$(CONDA_PY)') }} + - powershell: | + $junitXml = "test-data-single.xml" + $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"' + if ($matches[1] -eq 0) + { + Write-Host "No test failures in test-data-single" + } + else + { + # note that this will produce $LASTEXITCODE=1 + Write-Error "$($matches[1]) tests failed" + } + + $junitXmlMulti = "test-data-multiple.xml" + $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"' + if ($matches[1] -eq 0) + { + Write-Host "No test failures in test-data-multi" + } + else + { + # note that this will produce $LASTEXITCODE=1 + Write-Error "$($matches[1]) tests failed" + } + displayName: Check for test failures From bbb3379b0198604e8eb6a0c11b8884a7e90d4508 Mon Sep 17 00:00:00 2001 From: Matt Cooper Date: Tue, 4 Dec 2018 08:48:09 -0500 Subject: [PATCH 2/5] remove old macOS and Linux defs --- ci/azure/linux.yml | 79 ---------------------------------------------- ci/azure/macos.yml | 67 --------------------------------------- 2 files changed, 146 deletions(-) delete mode 100644 ci/azure/linux.yml delete mode 100644 ci/azure/macos.yml diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml deleted file mode 100644 index fe64307e9d08f..0000000000000 --- a/ci/azure/linux.yml +++ /dev/null @@ -1,79 +0,0 @@ -parameters: - name: '' - vmImage: '' - -jobs: -- job: ${{ parameters.name }} - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - maxParallel: 11 - matrix: - py27_np_120: - ENV_FILE: ci/deps/azure-27-compat.yaml - CONDA_PY: "27" - PATTERN: "not slow and not network" - - py37_locale: - ENV_FILE: ci/deps/azure-37-locale.yaml - CONDA_PY: "37" - PATTERN: "not slow and not network" - LOCALE_OVERRIDE: "zh_CN.UTF-8" - - py36_locale_slow: - ENV_FILE: ci/deps/azure-36-locale_slow.yaml - CONDA_PY: "36" - PATTERN: "not slow and not network" - LOCALE_OVERRIDE: "it_IT.UTF-8" - - steps: - - script: | - if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi - echo "Installing Miniconda"{ - ci/incremental/install_miniconda.sh - export PATH=$HOME/miniconda3/bin:$PATH - echo "Setting up Conda environment" - ci/incremental/setup_conda_environment.sh - displayName: 'Before Install' - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - ci/incremental/build.sh - displayName: 'Build' - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - ci/run_tests.sh - displayName: 'Test' - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - - task: PublishTestResults@2 - inputs: - testResultsFiles: 'test-data-*.xml' - testRunTitle: 'Linux' - - powershell: | - $junitXml = "test-data-single.xml" - $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"' - if ($matches[1] -eq 0) - { - Write-Host "No test failures in test-data-single" - } - else - { - # note that this will produce $LASTEXITCODE=1 - Write-Error "$($matches[1]) tests failed" - } - - $junitXmlMulti = "test-data-multiple.xml" - $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"' - if ($matches[1] -eq 0) - { - Write-Host "No test failures in test-data-multi" - } - else - { - # note that this will produce $LASTEXITCODE=1 - Write-Error "$($matches[1]) tests failed" - } - displayName: Check for test failures diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml deleted file mode 100644 index 98409576a5a87..0000000000000 --- a/ci/azure/macos.yml +++ /dev/null @@ -1,67 +0,0 @@ -parameters: - name: '' - vmImage: '' - -jobs: -- job: ${{ parameters.name }} - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - maxParallel: 11 - matrix: - py35_np_120: - ENV_FILE: ci/deps/azure-macos-35.yaml - CONDA_PY: "35" - PATTERN: "not slow and not network" - - steps: - - script: | - if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi - echo "Installing Miniconda" - ci/incremental/install_miniconda.sh - export PATH=$HOME/miniconda3/bin:$PATH - echo "Setting up Conda environment" - ci/incremental/setup_conda_environment.sh - displayName: 'Before Install' - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - ci/incremental/build.sh - displayName: 'Build' - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - ci/run_tests.sh - displayName: 'Test' - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - - task: PublishTestResults@2 - inputs: - testResultsFiles: 'test-data-*.xml' - testRunTitle: 'MacOS-35' - - powershell: | - $junitXml = "test-data-single.xml" - $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"' - if ($matches[1] -eq 0) - { - Write-Host "No test failures in test-data-single" - } - else - { - # note that this will produce $LASTEXITCODE=1 - Write-Error "$($matches[1]) tests failed" - } - - $junitXmlMulti = "test-data-multiple.xml" - $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"' - if ($matches[1] -eq 0) - { - Write-Host "No test failures in test-data-multi" - } - else - { - # note that this will produce $LASTEXITCODE=1 - Write-Error "$($matches[1]) tests failed" - } - displayName: Check for test failures From e2fb16fd48e63b83d9749753c6cd046aecaf2c9d Mon Sep 17 00:00:00 2001 From: Matt Cooper Date: Tue, 4 Dec 2018 08:50:53 -0500 Subject: [PATCH 3/5] unify Windows flavors --- azure-pipelines.yml | 8 +------- ci/azure/windows.yml | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3787c6d559ba0..1e4e43ac03815 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,17 +10,11 @@ jobs: name: Linux vmImage: ubuntu-16.04 -# Windows Python 2.7 needs VC 9.0 installed, and not sure -# how to make that a conditional task, so for now these are -# separate templates as well +# Windows Python 2.7 needs VC 9.0 installed, handled in the template - template: ci/azure/windows.yml parameters: name: Windows vmImage: vs2017-win2016 -- template: ci/azure/windows-py27.yml - parameters: - name: WindowsPy27 - vmImage: vs2017-win2016 - job: 'Checks_and_doc' pool: diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index b69c210ca27ba..cece002024936 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -7,18 +7,28 @@ jobs: pool: vmImage: ${{ parameters.vmImage }} strategy: - maxParallel: 11 matrix: py36_np14: ENV_FILE: ci/deps/azure-windows-36.yaml CONDA_PY: "36" + py27_np121: + ENV_FILE: ci/deps/azure-windows-27.yaml + CONDA_PY: "27" + steps: - task: CondaEnvironment@1 inputs: updateConda: no packageSpecs: '' + - powershell: | + $wc = New-Object net.webclient + $wc.Downloadfile("https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi", "VCForPython27.msi") + Start-Process "VCForPython27.msi" /qn -Wait + displayName: 'Install VC 9.0 only for Python 2.7' + condition: eq(variables.CONDA_PY, '27') + - script: | ci\\incremental\\setup_conda_environment.cmd displayName: 'Before Install' @@ -33,7 +43,7 @@ jobs: - task: PublishTestResults@2 inputs: testResultsFiles: 'test-data.xml' - testRunTitle: 'Windows 36' + testRunTitle: 'Windows-$(CONDA_PY)' - powershell: | $junitXml = "test-data.xml" $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"' From e9c66fd8bf762a4f05a133b8ff92e387d4cf111b Mon Sep 17 00:00:00 2001 From: Matt Cooper Date: Tue, 4 Dec 2018 08:51:09 -0500 Subject: [PATCH 4/5] remove old Windows def --- ci/azure/windows-py27.yml | 58 --------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 ci/azure/windows-py27.yml diff --git a/ci/azure/windows-py27.yml b/ci/azure/windows-py27.yml deleted file mode 100644 index 0d9aea816c4ad..0000000000000 --- a/ci/azure/windows-py27.yml +++ /dev/null @@ -1,58 +0,0 @@ -parameters: - name: '' - vmImage: '' - -jobs: -- job: ${{ parameters.name }} - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - maxParallel: 11 - matrix: - py36_np121: - ENV_FILE: ci/deps/azure-windows-27.yaml - CONDA_PY: "27" - - steps: - - task: CondaEnvironment@1 - inputs: - updateConda: no - packageSpecs: '' - - # Need to install VC 9.0 only for Python 2.7 - # Once we understand how to do tasks conditional on build matrix variables - # we could merge this into azure-windows.yml - - powershell: | - $wc = New-Object net.webclient - $wc.Downloadfile("https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi", "VCForPython27.msi") - Start-Process "VCForPython27.msi" /qn -Wait - displayName: 'Install VC 9.0' - - - script: | - ci\\incremental\\setup_conda_environment.cmd - displayName: 'Before Install' - - script: | - call activate pandas-dev - ci\\incremental\\build.cmd - displayName: 'Build' - - script: | - call activate pandas-dev - pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %* - displayName: 'Test' - - task: PublishTestResults@2 - inputs: - testResultsFiles: 'test-data.xml' - testRunTitle: 'Windows 27' - - powershell: | - $junitXml = "test-data.xml" - $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"' - if ($matches[1] -eq 0) - { - Write-Host "No test failures in test-data" - } - else - { - # note that this will produce $LASTEXITCODE=1 - Write-Error "$($matches[1]) tests failed" - } - displayName: Check for test failures From 87fb3f953401b250f24b79370c1b98c18f374aa5 Mon Sep 17 00:00:00 2001 From: Matt Cooper Date: Tue, 4 Dec 2018 08:53:08 -0500 Subject: [PATCH 5/5] fix format function --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 85515539fa277..374a82a5ed7d0 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -57,7 +57,7 @@ jobs: - task: PublishTestResults@2 inputs: testResultsFiles: 'test-data-*.xml' - testRunTitle: ${{ concat(parameters.name, '-$(CONDA_PY)') }} + testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} - powershell: | $junitXml = "test-data-single.xml" $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'