From 0c255787fdd4fbe7811bad03a50a2048c1a0d52e Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 09:43:26 -0500 Subject: [PATCH 01/14] circle -> azure --- .circleci/config.yml | 147 ------------------ azure-pipelines.yml | 8 +- ...le-27-compat.yaml => azure-27-compat.yaml} | 0 ...le-36-locale.yaml => azure-36-locale.yaml} | 0 ...le_slow.yaml => azure-36-locale_slow.yaml} | 0 ci/azure/linux.yml | 56 +++++++ ci/circle-35-ascii.yaml | 15 -- ci/install_circle.sh | 81 ---------- ci/run_circle.sh | 9 -- ci/show_circle.sh | 8 - 10 files changed, 60 insertions(+), 264 deletions(-) delete mode 100644 .circleci/config.yml rename ci/{circle-27-compat.yaml => azure-27-compat.yaml} (100%) rename ci/{circle-36-locale.yaml => azure-36-locale.yaml} (100%) rename ci/{circle-36-locale_slow.yaml => azure-36-locale_slow.yaml} (100%) create mode 100644 ci/azure/linux.yml delete mode 100644 ci/circle-35-ascii.yaml delete mode 100755 ci/install_circle.sh delete mode 100755 ci/run_circle.sh delete mode 100755 ci/show_circle.sh diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e947f30d285cd..0000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,147 +0,0 @@ -version: 2 -jobs: - - # -------------------------------------------------------------------------- - # 0. py27_compat - # -------------------------------------------------------------------------- - py27_compat: - docker: - - image: continuumio/miniconda:latest - # databases configuration - - image: circleci/postgres:9.6.5-alpine-ram - environment: - POSTGRES_USER: postgres - POSTGRES_DB: pandas_nosetest - - image: circleci/mysql:8-ram - environment: - MYSQL_USER: "root" - MYSQL_HOST: "localhost" - MYSQL_ALLOW_EMPTY_PASSWORD: "true" - MYSQL_DATABASE: "pandas_nosetest" - environment: - JOB: "2.7_COMPAT" - ENV_FILE: "ci/circle-27-compat.yaml" - LOCALE_OVERRIDE: "it_IT.UTF-8" - MINICONDA_DIR: /home/ubuntu/miniconda3 - steps: - - checkout - - run: - name: build - command: | - ./ci/install_circle.sh - ./ci/show_circle.sh - - run: - name: test - command: ./ci/run_circle.sh --skip-slow --skip-network - - # -------------------------------------------------------------------------- - # 1. py36_locale - # -------------------------------------------------------------------------- - py36_locale: - docker: - - image: continuumio/miniconda:latest - # databases configuration - - image: circleci/postgres:9.6.5-alpine-ram - environment: - POSTGRES_USER: postgres - POSTGRES_DB: pandas_nosetest - - image: circleci/mysql:8-ram - environment: - MYSQL_USER: "root" - MYSQL_HOST: "localhost" - MYSQL_ALLOW_EMPTY_PASSWORD: "true" - MYSQL_DATABASE: "pandas_nosetest" - - environment: - JOB: "3.6_LOCALE" - ENV_FILE: "ci/circle-36-locale.yaml" - LOCALE_OVERRIDE: "zh_CN.UTF-8" - MINICONDA_DIR: /home/ubuntu/miniconda3 - steps: - - checkout - - run: - name: build - command: | - ./ci/install_circle.sh - ./ci/show_circle.sh - - run: - name: test - command: ./ci/run_circle.sh --skip-slow --skip-network - - # -------------------------------------------------------------------------- - # 2. py36_locale_slow - # -------------------------------------------------------------------------- - py36_locale_slow: - docker: - - image: continuumio/miniconda:latest - # databases configuration - - image: circleci/postgres:9.6.5-alpine-ram - environment: - POSTGRES_USER: postgres - POSTGRES_DB: pandas_nosetest - - image: circleci/mysql:8-ram - environment: - MYSQL_USER: "root" - MYSQL_HOST: "localhost" - MYSQL_ALLOW_EMPTY_PASSWORD: "true" - MYSQL_DATABASE: "pandas_nosetest" - - environment: - JOB: "3.6_LOCALE_SLOW" - ENV_FILE: "ci/circle-36-locale_slow.yaml" - LOCALE_OVERRIDE: "zh_CN.UTF-8" - MINICONDA_DIR: /home/ubuntu/miniconda3 - steps: - - checkout - - run: - name: build - command: | - ./ci/install_circle.sh - ./ci/show_circle.sh - - run: - name: test - command: ./ci/run_circle.sh --only-slow --skip-network - - # -------------------------------------------------------------------------- - # 3. py35_ascii - # -------------------------------------------------------------------------- - py35_ascii: - docker: - - image: continuumio/miniconda:latest - # databases configuration - - image: circleci/postgres:9.6.5-alpine-ram - environment: - POSTGRES_USER: postgres - POSTGRES_DB: pandas_nosetest - - image: circleci/mysql:8-ram - environment: - MYSQL_USER: "root" - MYSQL_HOST: "localhost" - MYSQL_ALLOW_EMPTY_PASSWORD: "true" - MYSQL_DATABASE: "pandas_nosetest" - - environment: - JOB: "3.5_ASCII" - ENV_FILE: "ci/circle-35-ascii.yaml" - LOCALE_OVERRIDE: "C" - MINICONDA_DIR: /home/ubuntu/miniconda3 - steps: - - checkout - - run: - name: build - command: | - ./ci/install_circle.sh - ./ci/show_circle.sh - - run: - name: test - command: ./ci/run_circle.sh --skip-slow --skip-network - - -workflows: - version: 2 - build_and_test: - jobs: - - py27_compat - - py36_locale - - py36_locale_slow - - py35_ascii diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5d473bfc5a38c..373c22fdf8e62 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,10 +7,10 @@ jobs: parameters: name: macOS vmImage: xcode9-macos10.13 -# - template: ci/azure/linux.yml -# parameters: -# name: Linux -# vmImage: ubuntu-16.04 +- template: ci/azure/linux.yml + parameters: + name: Linux + vmImage: ubuntu-16.04 # Windows Python 2.7 needs VC 9.0 installed, and not sure # how to make that a conditional task, so for now these are diff --git a/ci/circle-27-compat.yaml b/ci/azure-27-compat.yaml similarity index 100% rename from ci/circle-27-compat.yaml rename to ci/azure-27-compat.yaml diff --git a/ci/circle-36-locale.yaml b/ci/azure-36-locale.yaml similarity index 100% rename from ci/circle-36-locale.yaml rename to ci/azure-36-locale.yaml diff --git a/ci/circle-36-locale_slow.yaml b/ci/azure-36-locale_slow.yaml similarity index 100% rename from ci/circle-36-locale_slow.yaml rename to ci/azure-36-locale_slow.yaml diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml new file mode 100644 index 0000000000000..56d624a3e5f20 --- /dev/null +++ b/ci/azure/linux.yml @@ -0,0 +1,56 @@ +parameters: + name: '' + vmImage: '' + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 11 + matrix: + py27_np_19: + ENV_FILE: ci/azure-27-compat.yml + CONDA_PY: "27" + CONDA_ENV: pandas + TEST_ARGS: "--skip-slow --skip-network" + + py36_locale: + ENV_FILE: ci/azure-36-locale.yml + CONDA_PY: "36" + CONDA_ENV: pandas + TEST_ARGS: "--skip-slow --skip-network" + LOCALE_OVERRIDE: "zh_CN.UTF-8" + + py36_locale_slow: + ENV_FILE: ci/azure-36-locale_slow.yml + CONDA_PY: "36" + CONDA_ENV: pandas + TEST_ARGS: "--only-slow --skip-network" + + steps: + - script: | + if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi + echo "Installing Miniconda" + ci/incremental/install_miniconda.sh + export PATH=$HOME/miniconda3/bin:$PATH + echo "Setting up Conda environment" + ci/incremental/setup_conda_environment.sh + displayName: 'Before Install' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/incremental/build.sh + displayName: 'Build' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/script_single.sh + ci/script_multi.sh + echo "[Test done]" + displayName: 'Test' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd + - task: PublishTestResults@2 + inputs: + testResultsFiles: '/tmp/*.xml' + testRunTitle: 'MacOS-35' \ No newline at end of file diff --git a/ci/circle-35-ascii.yaml b/ci/circle-35-ascii.yaml deleted file mode 100644 index 281ed59e2deff..0000000000000 --- a/ci/circle-35-ascii.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: pandas -channels: - - defaults -dependencies: - - cython>=0.28.2 - - nomkl - - numpy - - python-dateutil - - python=3.5* - - pytz - # universal - - pytest - - pytest-xdist - - pip: - - hypothesis>=3.58.0 diff --git a/ci/install_circle.sh b/ci/install_circle.sh deleted file mode 100755 index f8bcf6bcffc99..0000000000000 --- a/ci/install_circle.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -home_dir=$(pwd) -echo "[home_dir: $home_dir]" - -echo "[ls -ltr]" -ls -ltr - -apt-get update -y && apt-get install -y build-essential postgresql-client-9.6 - -echo "[update conda]" -conda config --set ssl_verify false || exit 1 -conda config --set always_yes true --set changeps1 false || exit 1 -conda update -q conda - -# add the pandas channel to take priority -# to add extra packages -echo "[add channels]" -conda config --add channels pandas || exit 1 -conda config --remove channels defaults || exit 1 -conda config --add channels defaults || exit 1 - -# Useful for debugging any issues with conda -conda info -a || exit 1 - -# support env variables passed -export ENVS_FILE=".envs" - -# make sure that the .envs file exists. it is ok if it is empty -touch $ENVS_FILE - -# assume all command line arguments are environmental variables -for var in "$@" -do - echo "export $var" >> $ENVS_FILE -done - -echo "[environmental variable file]" -cat $ENVS_FILE -source $ENVS_FILE - -# edit the locale override if needed -if [ -n "$LOCALE_OVERRIDE" ]; then - - apt-get update && apt-get -y install locales locales-all - - export LANG=$LOCALE_OVERRIDE - export LC_ALL=$LOCALE_OVERRIDE - - python -c "import locale; locale.setlocale(locale.LC_ALL, \"$LOCALE_OVERRIDE\")" || exit 1; - - echo "[Adding locale to the first line of pandas/__init__.py]" - rm -f pandas/__init__.pyc - sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, \"$LOCALE_OVERRIDE\")\n" - sed -i "$sedc" pandas/__init__.py - echo "[head -4 pandas/__init__.py]" - head -4 pandas/__init__.py - echo -fi - -# create envbuild deps -echo "[create env]" -time conda env create -q -n pandas --file="${ENV_FILE}" || exit 1 - -source activate pandas - -# remove any installed pandas package -# w/o removing anything else -echo -echo "[removing installed pandas]" -conda remove pandas -y --force -pip uninstall -y pandas - -# build but don't install -echo "[build em]" -time python setup.py build_ext --inplace || exit 1 - -echo -echo "[show environment]" - -conda list diff --git a/ci/run_circle.sh b/ci/run_circle.sh deleted file mode 100755 index fc2a8b849a354..0000000000000 --- a/ci/run_circle.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -echo "[running tests]" -export PATH="$MINICONDA_DIR/bin:$PATH" - -source activate pandas - -echo "pytest --strict --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml $@ pandas" -pytest --strict --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml $@ pandas diff --git a/ci/show_circle.sh b/ci/show_circle.sh deleted file mode 100755 index bfaa65c1d84f2..0000000000000 --- a/ci/show_circle.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -echo "[installed versions]" - -export PATH="$MINICONDA_DIR/bin:$PATH" -source activate pandas - -python -c "import pandas; pandas.show_versions();" From 799cbeaba3bec63583e3345d1bc8bdce81882b93 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 09:46:56 -0500 Subject: [PATCH 02/14] Update docs --- README.md | 8 -------- doc/source/contributing.rst | 15 +++------------ 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f26b9598bb5d3..d167fcb13c42b 100644 --- a/README.md +++ b/README.md @@ -45,14 +45,6 @@ - - - - - circleci build status - - - diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 65e151feeba67..dd9f717cff2ed 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -645,13 +645,11 @@ See :ref:`contributing.warnings` for more. Testing With Continuous Integration ----------------------------------- -The *pandas* test suite will run automatically on `Travis-CI `__, -`Appveyor `__, and `Circle CI `__ continuous integration -services, once your pull request is submitted. +The *pandas* test suite will run automatically on the `Travis-CI `__, +and `Azure-DevOps `__ services, once your pull request is submitted. However, if you wish to run the test suite on a branch prior to submitting the pull request, then the continuous integration services need to be hooked to your GitHub repository. Instructions are here -for `Travis-CI `__, -`Appveyor `__ , and `CircleCI `__. +for `Travis-CI `__. A pull-request will be considered for merging when you have an all 'green' build. If any tests are failing, then you will get a red 'X', where you can click through to see the individual failed tests. @@ -659,13 +657,6 @@ This is an example of a green build. .. image:: _static/ci.png -.. note:: - - Each time you push to *your* fork, a *new* run of the tests will be triggered on the CI. Appveyor will auto-cancel - any non-currently-running tests for that same pull-request. You can enable the auto-cancel feature for - `Travis-CI here `__ and - for `CircleCI here `__. - .. _contributing.tdd: From 10b598833d948a00cc635e7391a5c0e16d3d83d8 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 09:50:17 -0500 Subject: [PATCH 03/14] update locale setting --- ci/script_single.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/script_single.sh b/ci/script_single.sh index ed12ee35b9151..7e9703cc68997 100755 --- a/ci/script_single.sh +++ b/ci/script_single.sh @@ -5,8 +5,9 @@ echo "[script_single]" source activate pandas if [ -n "$LOCALE_OVERRIDE" ]; then + echo "Setting LC_ALL and LANG to $LOCALE_OVERRIDE" export LC_ALL="$LOCALE_OVERRIDE"; - echo "Setting LC_ALL to $LOCALE_OVERRIDE" + export LANG="$LOCALE_OVERRIDE"; pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))' python -c "$pycmd" From 72ab6b6ac76f3cce4c3fb7f45557c8d1ea078625 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 09:56:12 -0500 Subject: [PATCH 04/14] all yaml --- ci/{azure-macos-35.yml => azure-macos-35.yaml} | 0 ci/azure/linux.yml | 6 +++--- ci/azure/macos.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename ci/{azure-macos-35.yml => azure-macos-35.yaml} (100%) diff --git a/ci/azure-macos-35.yml b/ci/azure-macos-35.yaml similarity index 100% rename from ci/azure-macos-35.yml rename to ci/azure-macos-35.yaml diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml index 56d624a3e5f20..6cf360cc44c45 100644 --- a/ci/azure/linux.yml +++ b/ci/azure/linux.yml @@ -10,20 +10,20 @@ jobs: maxParallel: 11 matrix: py27_np_19: - ENV_FILE: ci/azure-27-compat.yml + ENV_FILE: ci/azure-27-compat.yaml CONDA_PY: "27" CONDA_ENV: pandas TEST_ARGS: "--skip-slow --skip-network" py36_locale: - ENV_FILE: ci/azure-36-locale.yml + ENV_FILE: ci/azure-36-locale.yaml CONDA_PY: "36" CONDA_ENV: pandas TEST_ARGS: "--skip-slow --skip-network" LOCALE_OVERRIDE: "zh_CN.UTF-8" py36_locale_slow: - ENV_FILE: ci/azure-36-locale_slow.yml + ENV_FILE: ci/azure-36-locale_slow.yaml CONDA_PY: "36" CONDA_ENV: pandas TEST_ARGS: "--only-slow --skip-network" diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml index 5bf8d18d6cbb9..9bfaef04ea2fa 100644 --- a/ci/azure/macos.yml +++ b/ci/azure/macos.yml @@ -10,7 +10,7 @@ jobs: maxParallel: 11 matrix: py35_np_110: - ENV_FILE: ci/azure-macos-35.yml + ENV_FILE: ci/azure-macos-35.yaml CONDA_PY: "35" CONDA_ENV: pandas TEST_ARGS: "--skip-slow --skip-network" From 80f374c47488c80c71ce8272e2ca32eee929b540 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 10:06:29 -0500 Subject: [PATCH 05/14] Ignore error --- ci/incremental/setup_conda_environment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index c716a39138644..a699b53b0d985 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -27,8 +27,8 @@ set -v # w/o removing anything else echo echo "[removing installed pandas]" -conda remove pandas -y --force -pip uninstall -y pandas +conda remove pandas -y --force || true +pip uninstall -y pandas || true echo echo "[no installed pandas]" From 31d2aead67c759de30ad3c82706f62cedc7f3a20 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 10:49:46 -0500 Subject: [PATCH 06/14] Try fix upload --- ci/azure/linux.yml | 4 ++-- doc/source/contributing.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml index 6cf360cc44c45..6a9176c20a0eb 100644 --- a/ci/azure/linux.yml +++ b/ci/azure/linux.yml @@ -52,5 +52,5 @@ jobs: source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - task: PublishTestResults@2 inputs: - testResultsFiles: '/tmp/*.xml' - testRunTitle: 'MacOS-35' \ No newline at end of file + testResultsFiles: '/tmp/{single,multiple}.xml' + testRunTitle: 'Linux' \ No newline at end of file diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index dd9f717cff2ed..106cee01f2f90 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -649,7 +649,7 @@ The *pandas* test suite will run automatically on the `Travis-CI `__ services, once your pull request is submitted. However, if you wish to run the test suite on a branch prior to submitting the pull request, then the continuous integration services need to be hooked to your GitHub repository. Instructions are here -for `Travis-CI `__. +for `Travis-CI `__. A pull-request will be considered for merging when you have an all 'green' build. If any tests are failing, then you will get a red 'X', where you can click through to see the individual failed tests. @@ -664,7 +664,7 @@ Test-driven development/code writing ------------------------------------ *pandas* is serious about testing and strongly encourages contributors to embrace -`test-driven development (TDD) `_. +`test-driven development (TDD) `_. This development process "relies on the repetition of a very short development cycle: first the developer writes an (initially failing) automated test case that defines a desired improvement or new function, then produces the minimum amount of code to pass that test." From 8141ee4295032ca86f1466538a126fdaabf95318 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 11:59:39 -0500 Subject: [PATCH 07/14] locale-gen, results --- ci/azure/linux.yml | 4 ++-- ci/incremental/setup_conda_environment.sh | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml index 6a9176c20a0eb..eacebb4d8933b 100644 --- a/ci/azure/linux.yml +++ b/ci/azure/linux.yml @@ -31,7 +31,7 @@ jobs: steps: - script: | if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi - echo "Installing Miniconda" + echo "Installing Miniconda"{ ci/incremental/install_miniconda.sh export PATH=$HOME/miniconda3/bin:$PATH echo "Setting up Conda environment" @@ -52,5 +52,5 @@ jobs: source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - task: PublishTestResults@2 inputs: - testResultsFiles: '/tmp/{single,multiple}.xml' + testResultsFiles: '/tmp/single.xml;/tmp/multiple.xml' testRunTitle: 'Linux' \ No newline at end of file diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index a699b53b0d985..1d402ad07ae56 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -34,6 +34,11 @@ echo echo "[no installed pandas]" conda list pandas +if "[ $LOCALE_OVERRIDE ]"; then + locale-gen "$LOCALE_OVERRIDE" + dpkg-reconfigure locales +fi + # # Install the compiler toolchain # if [[ $(uname) == Linux ]]; then # if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then From a711e27174951aa93221592ef08e2e65241c8cde Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 12:57:34 -0500 Subject: [PATCH 08/14] update test data location --- .travis.yml | 8 ++++---- ci/azure/linux.yml | 2 +- ci/azure/macos.yml | 2 +- ci/incremental/setup_conda_environment.sh | 2 +- ci/script_multi.sh | 12 ++++++------ ci/script_single.sh | 8 ++++---- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index c9bdb91283d42..204dd74bebcea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -121,10 +121,10 @@ after_success: after_script: - echo "after_script start" - source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - - if [ -e /tmp/single.xml ]; then - ci/print_skipped.py /tmp/single.xml; + - if [ -e test-data-single.xml ]; then + ci/print_skipped.py test-data-single.xml; fi - - if [ -e /tmp/multiple.xml ]; then - ci/print_skipped.py /tmp/multiple.xml; + - if [ -e test-data-multiple.xml ]; then + ci/print_skipped.py test-data-multiple.xml; fi - echo "after_script done" diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml index eacebb4d8933b..684c6de1a78f6 100644 --- a/ci/azure/linux.yml +++ b/ci/azure/linux.yml @@ -52,5 +52,5 @@ jobs: source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - task: PublishTestResults@2 inputs: - testResultsFiles: '/tmp/single.xml;/tmp/multiple.xml' + testResultsFiles: 'test-data-*.xml' testRunTitle: 'Linux' \ No newline at end of file diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml index 9bfaef04ea2fa..9796e37c3fc85 100644 --- a/ci/azure/macos.yml +++ b/ci/azure/macos.yml @@ -39,5 +39,5 @@ jobs: source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - task: PublishTestResults@2 inputs: - testResultsFiles: '/tmp/*.xml' + testResultsFiles: 'test-data-*.xml' testRunTitle: 'MacOS-35' diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index 1d402ad07ae56..50446d50de89a 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -34,7 +34,7 @@ echo echo "[no installed pandas]" conda list pandas -if "[ $LOCALE_OVERRIDE ]"; then +if "[ -n $LOCALE_OVERRIDE ]"; then locale-gen "$LOCALE_OVERRIDE" dpkg-reconfigure locales fi diff --git a/ci/script_multi.sh b/ci/script_multi.sh index 2b2d4d5488b91..c7ca9a04b588c 100755 --- a/ci/script_multi.sh +++ b/ci/script_multi.sh @@ -27,17 +27,17 @@ if [ "$DOC" ]; then echo "We are not running pytest as this is a doc-build" elif [ "$COVERAGE" ]; then - echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas - pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas + echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas + pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas elif [ "$SLOW" ]; then TEST_ARGS="--only-slow --skip-network" - echo pytest -r xX -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas - pytest -r xX -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas + echo pytest -r xX -m "not single and slow" -v --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas + pytest -r xX -m "not single and slow" -v --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas else - echo pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas - pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas # TODO: doctest + echo pytest -n 2 -r xX -m "not single" --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas + pytest -n 2 -r xX -m "not single" --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas # TODO: doctest fi diff --git a/ci/script_single.sh b/ci/script_single.sh index 7e9703cc68997..0e81757322a13 100755 --- a/ci/script_single.sh +++ b/ci/script_single.sh @@ -26,14 +26,14 @@ if [ "$DOC" ]; then echo "We are not running pytest as this is a doc-build" elif [ "$COVERAGE" ]; then - echo pytest -s -m "single" -r xXs --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas - pytest -s -m "single" -r xXs --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas + echo pytest -s -m "single" -r xXs --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas + pytest -s -m "single" -r xXs --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas echo pytest -s -r xXs --strict scripts pytest -s -r xXs --strict scripts else - echo pytest -m "single" -r xXs --junitxml=/tmp/single.xml --strict $TEST_ARGS pandas - pytest -m "single" -r xXs --junitxml=/tmp/single.xml --strict $TEST_ARGS pandas # TODO: doctest + echo pytest -m "single" -r xXs --junitxml=test-data-single.xml --strict $TEST_ARGS pandas + pytest -m "single" -r xXs --junitxml=test-data-single.xml --strict $TEST_ARGS pandas fi From c315e19a229ad23b1f4005d56fc5c551efcd3289 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 13:06:11 -0500 Subject: [PATCH 09/14] fixed quoting --- ci/incremental/setup_conda_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index 50446d50de89a..ce837c7948a30 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -34,7 +34,7 @@ echo echo "[no installed pandas]" conda list pandas -if "[ -n $LOCALE_OVERRIDE ]"; then +if [ -n "$LOCALE_OVERRIDE" ]; then locale-gen "$LOCALE_OVERRIDE" dpkg-reconfigure locales fi From a74ce981a6b29d0f89117285b6d02c6098b9388c Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 13:11:49 -0500 Subject: [PATCH 10/14] try sudo? --- ci/incremental/setup_conda_environment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index ce837c7948a30..9275d10abaa91 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -35,8 +35,8 @@ echo "[no installed pandas]" conda list pandas if [ -n "$LOCALE_OVERRIDE" ]; then - locale-gen "$LOCALE_OVERRIDE" - dpkg-reconfigure locales + sudo locale-gen "$LOCALE_OVERRIDE" + sudo dpkg-reconfigure locales fi # # Install the compiler toolchain From 0f7ce80b20ef178de3f1e3100cb6d3bb35e76daa Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 4 Oct 2018 13:18:04 -0500 Subject: [PATCH 11/14] try without dpkg --- ci/incremental/setup_conda_environment.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index 9275d10abaa91..f3ac99d5e7c5a 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -36,7 +36,6 @@ conda list pandas if [ -n "$LOCALE_OVERRIDE" ]; then sudo locale-gen "$LOCALE_OVERRIDE" - sudo dpkg-reconfigure locales fi # # Install the compiler toolchain From 74f4dccafb1e7296fa92ab8a4deff66d1ab98142 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 24 Oct 2018 21:25:43 -0500 Subject: [PATCH 12/14] Restore one --- .circleci/config.yml | 42 +++++++++++++++++++++ README.md | 8 ++++ ci/circle-36-locale.yaml | 35 +++++++++++++++++ ci/install_circle.sh | 81 ++++++++++++++++++++++++++++++++++++++++ ci/run_circle.sh | 9 +++++ 5 files changed, 175 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 ci/circle-36-locale.yaml create mode 100755 ci/install_circle.sh create mode 100755 ci/run_circle.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000000..b048afc7d6cfa --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,42 @@ +version: 2 +jobs: + + # -------------------------------------------------------------------------- + # 1. py36_locale + # -------------------------------------------------------------------------- + py36_locale: + docker: + - image: continuumio/miniconda:latest + # databases configuration + - image: circleci/postgres:9.6.5-alpine-ram + environment: + POSTGRES_USER: postgres + POSTGRES_DB: pandas_nosetest + - image: circleci/mysql:8-ram + environment: + MYSQL_USER: "root" + MYSQL_HOST: "localhost" + MYSQL_ALLOW_EMPTY_PASSWORD: "true" + MYSQL_DATABASE: "pandas_nosetest" + + environment: + JOB: "3.6_LOCALE" + ENV_FILE: "ci/circle-36-locale.yaml" + LOCALE_OVERRIDE: "zh_CN.UTF-8" + MINICONDA_DIR: /home/ubuntu/miniconda3 + steps: + - checkout + - run: + name: build + command: | + ./ci/install_circle.sh + ./ci/show_circle.sh + - run: + name: test + command: ./ci/run_circle.sh --skip-slow --skip-network + +workflows: + version: 2 + build_and_test: + jobs: + - py36_locale diff --git a/README.md b/README.md index 718e0ef76501a..b4dedecb4c697 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,14 @@ + + + + + circleci build status + + + diff --git a/ci/circle-36-locale.yaml b/ci/circle-36-locale.yaml new file mode 100644 index 0000000000000..ef97b85406709 --- /dev/null +++ b/ci/circle-36-locale.yaml @@ -0,0 +1,35 @@ +name: pandas +channels: + - defaults + - conda-forge +dependencies: + - beautifulsoup4 + - cython>=0.28.2 + - html5lib + - ipython + - jinja2 + - lxml + - matplotlib + - nomkl + - numexpr + - numpy + - openpyxl=2.5.5 + - psycopg2 + - pymysql + - pytables + - python-dateutil + - python=3.6* + - pytz + - s3fs + - scipy + - sqlalchemy + - xarray + - xlrd + - xlsxwriter + - xlwt + # universal + - pytest + - pytest-xdist + - moto + - pip: + - hypothesis>=3.58.0 diff --git a/ci/install_circle.sh b/ci/install_circle.sh new file mode 100755 index 0000000000000..f8bcf6bcffc99 --- /dev/null +++ b/ci/install_circle.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +home_dir=$(pwd) +echo "[home_dir: $home_dir]" + +echo "[ls -ltr]" +ls -ltr + +apt-get update -y && apt-get install -y build-essential postgresql-client-9.6 + +echo "[update conda]" +conda config --set ssl_verify false || exit 1 +conda config --set always_yes true --set changeps1 false || exit 1 +conda update -q conda + +# add the pandas channel to take priority +# to add extra packages +echo "[add channels]" +conda config --add channels pandas || exit 1 +conda config --remove channels defaults || exit 1 +conda config --add channels defaults || exit 1 + +# Useful for debugging any issues with conda +conda info -a || exit 1 + +# support env variables passed +export ENVS_FILE=".envs" + +# make sure that the .envs file exists. it is ok if it is empty +touch $ENVS_FILE + +# assume all command line arguments are environmental variables +for var in "$@" +do + echo "export $var" >> $ENVS_FILE +done + +echo "[environmental variable file]" +cat $ENVS_FILE +source $ENVS_FILE + +# edit the locale override if needed +if [ -n "$LOCALE_OVERRIDE" ]; then + + apt-get update && apt-get -y install locales locales-all + + export LANG=$LOCALE_OVERRIDE + export LC_ALL=$LOCALE_OVERRIDE + + python -c "import locale; locale.setlocale(locale.LC_ALL, \"$LOCALE_OVERRIDE\")" || exit 1; + + echo "[Adding locale to the first line of pandas/__init__.py]" + rm -f pandas/__init__.pyc + sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, \"$LOCALE_OVERRIDE\")\n" + sed -i "$sedc" pandas/__init__.py + echo "[head -4 pandas/__init__.py]" + head -4 pandas/__init__.py + echo +fi + +# create envbuild deps +echo "[create env]" +time conda env create -q -n pandas --file="${ENV_FILE}" || exit 1 + +source activate pandas + +# remove any installed pandas package +# w/o removing anything else +echo +echo "[removing installed pandas]" +conda remove pandas -y --force +pip uninstall -y pandas + +# build but don't install +echo "[build em]" +time python setup.py build_ext --inplace || exit 1 + +echo +echo "[show environment]" + +conda list diff --git a/ci/run_circle.sh b/ci/run_circle.sh new file mode 100755 index 0000000000000..fc2a8b849a354 --- /dev/null +++ b/ci/run_circle.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +echo "[running tests]" +export PATH="$MINICONDA_DIR/bin:$PATH" + +source activate pandas + +echo "pytest --strict --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml $@ pandas" +pytest --strict --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml $@ pandas From 608ba049136a436363564e28c9fd11f63b3d63ae Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 24 Oct 2018 21:27:07 -0500 Subject: [PATCH 13/14] 36 -> 37 --- ci/{azure-36-locale.yaml => azure-37-locale.yaml} | 0 ci/azure/linux.yml | 4 ++-- ci/show_circle.sh | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) rename ci/{azure-36-locale.yaml => azure-37-locale.yaml} (100%) create mode 100755 ci/show_circle.sh diff --git a/ci/azure-36-locale.yaml b/ci/azure-37-locale.yaml similarity index 100% rename from ci/azure-36-locale.yaml rename to ci/azure-37-locale.yaml diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml index 684c6de1a78f6..f34cba69a6195 100644 --- a/ci/azure/linux.yml +++ b/ci/azure/linux.yml @@ -16,8 +16,8 @@ jobs: TEST_ARGS: "--skip-slow --skip-network" py36_locale: - ENV_FILE: ci/azure-36-locale.yaml - CONDA_PY: "36" + ENV_FILE: ci/azure-37-locale.yaml + CONDA_PY: "37" CONDA_ENV: pandas TEST_ARGS: "--skip-slow --skip-network" LOCALE_OVERRIDE: "zh_CN.UTF-8" diff --git a/ci/show_circle.sh b/ci/show_circle.sh new file mode 100755 index 0000000000000..bfaa65c1d84f2 --- /dev/null +++ b/ci/show_circle.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +echo "[installed versions]" + +export PATH="$MINICONDA_DIR/bin:$PATH" +source activate pandas + +python -c "import pandas; pandas.show_versions();" From 5fc0d9691e8caad855af84467a18e8fe34d3c399 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 24 Oct 2018 21:30:00 -0500 Subject: [PATCH 14/14] move to circle --- .circleci/config.yml | 6 +++--- ci/{ => circle}/install_circle.sh | 0 ci/{ => circle}/run_circle.sh | 0 ci/{ => circle}/show_circle.sh | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename ci/{ => circle}/install_circle.sh (100%) rename ci/{ => circle}/run_circle.sh (100%) rename ci/{ => circle}/show_circle.sh (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index b048afc7d6cfa..5b10036818901 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,11 +29,11 @@ jobs: - run: name: build command: | - ./ci/install_circle.sh - ./ci/show_circle.sh + ./ci/circle/install_circle.sh + ./ci/circle/show_circle.sh - run: name: test - command: ./ci/run_circle.sh --skip-slow --skip-network + command: ./ci/circle/run_circle.sh --skip-slow --skip-network workflows: version: 2 diff --git a/ci/install_circle.sh b/ci/circle/install_circle.sh similarity index 100% rename from ci/install_circle.sh rename to ci/circle/install_circle.sh diff --git a/ci/run_circle.sh b/ci/circle/run_circle.sh similarity index 100% rename from ci/run_circle.sh rename to ci/circle/run_circle.sh diff --git a/ci/show_circle.sh b/ci/circle/show_circle.sh similarity index 100% rename from ci/show_circle.sh rename to ci/circle/show_circle.sh