From f08e4be2b457ba4f6f1158ec564f659958109047 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 26 Aug 2021 17:01:10 -0500 Subject: [PATCH 01/19] configure samples dir --- samples/samples | 1 - samples/{ => snippets}/__init__.py | 1 - samples/{tests => snippets}/conftest.py | 0 samples/snippets/noxfile.py | 273 ++++++++++++++++++ samples/{ => snippets}/read_gbq_legacy.py | 4 +- samples/{ => snippets}/read_gbq_simple.py | 0 .../read_gbq_test.py} | 4 +- samples/{ => snippets}/to_gbq_simple.py | 0 .../to_gbq_test.py} | 2 +- samples/tests/__init__.py | 4 - 10 files changed, 278 insertions(+), 11 deletions(-) delete mode 120000 samples/samples rename samples/{ => snippets}/__init__.py (99%) rename samples/{tests => snippets}/conftest.py (100%) create mode 100644 samples/snippets/noxfile.py rename samples/{ => snippets}/read_gbq_legacy.py (97%) rename samples/{ => snippets}/read_gbq_simple.py (100%) rename samples/{tests/test_read_gbq.py => snippets/read_gbq_test.py} (90%) rename samples/{ => snippets}/to_gbq_simple.py (100%) rename samples/{tests/test_to_gbq.py => snippets/to_gbq_test.py} (93%) delete mode 100644 samples/tests/__init__.py diff --git a/samples/samples b/samples/samples deleted file mode 120000 index d9d97bb8..00000000 --- a/samples/samples +++ /dev/null @@ -1 +0,0 @@ -docs/source/samples \ No newline at end of file diff --git a/samples/__init__.py b/samples/snippets/__init__.py similarity index 99% rename from samples/__init__.py rename to samples/snippets/__init__.py index edbca6c3..c9ab8506 100644 --- a/samples/__init__.py +++ b/samples/snippets/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2017 pandas-gbq Authors All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. - diff --git a/samples/tests/conftest.py b/samples/snippets/conftest.py similarity index 100% rename from samples/tests/conftest.py rename to samples/snippets/conftest.py diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py new file mode 100644 index 00000000..8bd8983b --- /dev/null +++ b/samples/snippets/noxfile.py @@ -0,0 +1,273 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys +from typing import Callable, Dict, List, Optional + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==19.10b0" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted( + [v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS] +) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get( + "INSTALL_LIBRARY_FROM_SOURCE", False +) in ("True", "true",) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir: str) -> List[str]: + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8", "flake8-import-order") + else: + session.install("flake8", "flake8-import-order", "flake8-annotations") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format( + session.python + ) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", + _get_repo_root() + "/scripts/readme-gen/readme_gen.py", + in_file, + ) diff --git a/samples/read_gbq_legacy.py b/samples/snippets/read_gbq_legacy.py similarity index 97% rename from samples/read_gbq_legacy.py rename to samples/snippets/read_gbq_legacy.py index ed2e4f40..2fa3c79c 100644 --- a/samples/read_gbq_legacy.py +++ b/samples/snippets/read_gbq_legacy.py @@ -6,11 +6,11 @@ import argparse -import pandas_gbq - def main(project_id): # [START bigquery_pandas_gbq_read_gbq_legacy] + import pandas_gbq + sql = """ SELECT country_name, alpha_2_code FROM [bigquery-public-data:utility_us.country_code_iso] diff --git a/samples/read_gbq_simple.py b/samples/snippets/read_gbq_simple.py similarity index 100% rename from samples/read_gbq_simple.py rename to samples/snippets/read_gbq_simple.py diff --git a/samples/tests/test_read_gbq.py b/samples/snippets/read_gbq_test.py similarity index 90% rename from samples/tests/test_read_gbq.py rename to samples/snippets/read_gbq_test.py index 1882ded0..8f4992d7 100644 --- a/samples/tests/test_read_gbq.py +++ b/samples/snippets/read_gbq_test.py @@ -4,8 +4,8 @@ """System tests for read_gbq code samples.""" -from .. import read_gbq_legacy -from .. import read_gbq_simple +from . import read_gbq_legacy +from . import read_gbq_simple def test_read_gbq_legacy(project_id): diff --git a/samples/to_gbq_simple.py b/samples/snippets/to_gbq_simple.py similarity index 100% rename from samples/to_gbq_simple.py rename to samples/snippets/to_gbq_simple.py diff --git a/samples/tests/test_to_gbq.py b/samples/snippets/to_gbq_test.py similarity index 93% rename from samples/tests/test_to_gbq.py rename to samples/snippets/to_gbq_test.py index c25b9059..f4208974 100644 --- a/samples/tests/test_to_gbq.py +++ b/samples/snippets/to_gbq_test.py @@ -4,7 +4,7 @@ """System tests for to_gbq code samples.""" -from .. import to_gbq_simple +from . import to_gbq_simple def test_to_gbq_simple(project_id, bigquery_client, random_dataset_id): diff --git a/samples/tests/__init__.py b/samples/tests/__init__.py deleted file mode 100644 index edbca6c3..00000000 --- a/samples/tests/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - From 483e7b35abd5209fa3fe8c4624d02868545a4f72 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 26 Aug 2021 16:23:15 -0500 Subject: [PATCH 02/19] add owlbot config --- .github/.OwlBot.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/.OwlBot.yaml diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml new file mode 100644 index 00000000..8642b5f3 --- /dev/null +++ b/.github/.OwlBot.yaml @@ -0,0 +1,19 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + image: gcr.io/repo-automation-bots/owlbot-python:latest + +begin-after-commit-hash: 1afeb53252641dc35a421fa5acc59e2f3229ad6d + From 6b6cfda7ba6b07425dbeb5c578db4ed1bbd5ee4d Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 26 Aug 2021 15:59:16 -0500 Subject: [PATCH 03/19] chore: use shared templates --- owlbot.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 owlbot.py diff --git a/owlbot.py b/owlbot.py new file mode 100644 index 00000000..cd787ee2 --- /dev/null +++ b/owlbot.py @@ -0,0 +1,56 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script is used to synthesize generated parts of this library.""" + +import pathlib + +import synthtool as s +from synthtool import gcp +from synthtool.languages import python + +REPO_ROOT = pathlib.Path(__file__).parent.absolute() + +common = gcp.CommonTemplates() + +# ---------------------------------------------------------------------------- +# Add templated files +# ---------------------------------------------------------------------------- + +extras = ["tqdm"] +templated_files = common.py_library( + unit_test_python_versions=["3.6", "3.7", "3.8", "3.9"], + system_test_python_versions=["3.8", "3.9"], + cov_level=100, + unit_test_extras=extras, + system_test_extras=extras, +) +s.move(templated_files, excludes=[ + # pandas-gbq was originally licensed BSD-3-Clause License + "LICENSE", +]) + +# ---------------------------------------------------------------------------- +# Samples templates +# ---------------------------------------------------------------------------- + +python.py_samples(skip_readmes=True) + +# ---------------------------------------------------------------------------- +# Final cleanup +# ---------------------------------------------------------------------------- + +s.shell.run(["nox", "-s", "blacken"], hide_output=False) +for noxfile in REPO_ROOT.glob("samples/**/noxfile.py"): + s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False) From 747e05cbfed65d806a0d81f8fbb977f6a8d473f3 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 14:14:31 -0600 Subject: [PATCH 04/19] fix root blacken --- owlbot.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/owlbot.py b/owlbot.py index cd787ee2..fa8481b9 100644 --- a/owlbot.py +++ b/owlbot.py @@ -41,6 +41,16 @@ "LICENSE", ]) +# ---------------------------------------------------------------------------- +# Fixup files +# ---------------------------------------------------------------------------- + +s.replace( + ["noxfile.py"], + r"[\"']google[\"']", + '"pandas_gbq"', +) + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From 7422264f0ff9c231e879d88599495528aa7c2fe5 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 14:17:01 -0600 Subject: [PATCH 05/19] Remove noxfile.py, it's generated --- noxfile.py | 115 ----------------------------------------------------- 1 file changed, 115 deletions(-) delete mode 100644 noxfile.py diff --git a/noxfile.py b/noxfile.py deleted file mode 100644 index e1564138..00000000 --- a/noxfile.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -"""Nox test automation configuration. - -See: https://nox.readthedocs.io/en/latest/ -""" - -import os -import os.path -import shutil - -import nox - - -supported_pythons = ["3.7", "3.8"] -system_test_pythons = ["3.7", "3.8"] -latest_python = "3.8" - -# Use a consistent version of black so CI is deterministic. -# Should match Stickler: https://stickler-ci.com/docs#black -black_package = "black==20.8b1" - - -@nox.session(python=latest_python) -def lint(session): - session.install(black_package, "flake8") - session.run("flake8", "pandas_gbq") - session.run("flake8", "tests") - session.run("black", "--check", ".") - - -@nox.session(python=latest_python) -def blacken(session): - session.install(black_package) - session.run("black", ".") - - -@nox.session(python=supported_pythons) -def unit(session): - session.install("pytest", "pytest-cov") - session.install( - "-e", - ".", - # Use dependencies versions from constraints file. This enables testing - # across a more full range of versions of the dependencies. - "-c", - os.path.join(".", "ci", "constraints-{}.pip".format(session.python)), - ) - session.run( - "pytest", - os.path.join(".", "tests", "unit"), - "-v", - "--cov=pandas_gbq", - "--cov=tests.unit", - "--cov-report", - "xml:/tmp/pytest-cov.xml", - *session.posargs - ) - - -@nox.session(python=latest_python) -def cover(session): - session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=73") - session.run("coverage", "erase") - - -@nox.session(python=latest_python) -def docs(session): - """Build the docs.""" - - session.install("-r", os.path.join("docs", "requirements-docs.txt")) - session.install("-e", ".") - - shutil.rmtree(os.path.join("docs", "source", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "source", "_build", "doctrees", ""), - os.path.join("docs", "source", ""), - os.path.join("docs", "source", "_build", "html", ""), - ) - - -@nox.session(python=system_test_pythons) -def system(session): - session.install("pytest", "pytest-cov") - session.install( - "-e", - ".", - # Use dependencies versions from constraints file. This enables testing - # across a more full range of versions of the dependencies. - "-c", - os.path.join(".", "ci", "constraints-{}.pip".format(session.python)), - ) - - # Skip local auth tests on CI. - additional_args = list(session.posargs) - if "CIRCLECI" in os.environ: - additional_args = additional_args + ["-m", "not local_auth"] - - session.run( - "pytest", - os.path.join(".", "tests", "system"), - os.path.join(".", "samples", "tests"), - "-v", - *additional_args - ) From 5fa4a1575c58c0f669bceda49bc4fd33d0d88e06 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 14:21:51 -0600 Subject: [PATCH 06/19] moved ci to testing (sort of) --- ci/config_auth.sh | 13 -------- ci/requirements-3.7-0.23.2.conda | 10 ------ ci/requirements-3.9-NIGHTLY.conda | 8 ----- ci/run_conda.sh | 32 ------------------- ci/run_pip.sh | 25 --------------- ci/run_tests.sh | 15 --------- .../constraints-3.7.txt | 0 .../constraints-3.8.txt | 0 .../constraints-3.9.txt | 0 9 files changed, 103 deletions(-) delete mode 100755 ci/config_auth.sh delete mode 100644 ci/requirements-3.7-0.23.2.conda delete mode 100644 ci/requirements-3.9-NIGHTLY.conda delete mode 100755 ci/run_conda.sh delete mode 100755 ci/run_pip.sh delete mode 100755 ci/run_tests.sh rename ci/constraints-3.7.pip => testing/constraints-3.7.txt (100%) rename ci/constraints-3.8.pip => testing/constraints-3.8.txt (100%) rename ci/constraints-3.9.pip => testing/constraints-3.9.txt (100%) diff --git a/ci/config_auth.sh b/ci/config_auth.sh deleted file mode 100755 index cde115c7..00000000 --- a/ci/config_auth.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -# Don't set -x, because we don't want to leak keys. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Write key to file if present. -if [ ! -z "$SERVICE_ACCOUNT_KEY" ] ; then - echo "$SERVICE_ACCOUNT_KEY" | base64 --decode > "$DIR"/service_account.json -fi diff --git a/ci/requirements-3.7-0.23.2.conda b/ci/requirements-3.7-0.23.2.conda deleted file mode 100644 index af4768ab..00000000 --- a/ci/requirements-3.7-0.23.2.conda +++ /dev/null @@ -1,10 +0,0 @@ -codecov -coverage -fastavro -flake8 -numpy==1.14.5 -google-cloud-bigquery==1.11.1 -pydata-google-auth -pytest -pytest-cov -tqdm==4.23.0 diff --git a/ci/requirements-3.9-NIGHTLY.conda b/ci/requirements-3.9-NIGHTLY.conda deleted file mode 100644 index 9dfe3f6b..00000000 --- a/ci/requirements-3.9-NIGHTLY.conda +++ /dev/null @@ -1,8 +0,0 @@ -pydata-google-auth -google-cloud-bigquery -google-cloud-bigquery-storage -pytest -pytest-cov -codecov -coverage -flake8 diff --git a/ci/run_conda.sh b/ci/run_conda.sh deleted file mode 100755 index e29da98a..00000000 --- a/ci/run_conda.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Install dependencies using Conda - -conda config --set always_yes yes --set changeps1 no -conda config --add channels pandas -conda config --add channels conda-forge -conda update -q conda -conda info -a -conda create -q -n test-environment python=$PYTHON -source activate test-environment -REQ="ci/requirements-${PYTHON}-${PANDAS}" -conda install -q --file "$REQ.conda"; - -if [[ "$PANDAS" == "NIGHTLY" ]]; then - conda install -q numpy pytz python-dateutil; - PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; - pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; -else - conda install -q pandas=$PANDAS; -fi - -python setup.py develop --no-deps - -# Run the tests -$DIR/run_tests.sh diff --git a/ci/run_pip.sh b/ci/run_pip.sh deleted file mode 100755 index 855b322e..00000000 --- a/ci/run_pip.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Install dependencies using Pip - -if [[ "$PANDAS" == "MASTER" ]]; then - PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; - pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; -else - pip install pandas==$PANDAS -fi - -# Install test requirements -pip install coverage pytest pytest-cov flake8 codecov - -REQ="ci/requirements-${PYTHON}-${PANDAS}" -pip install -r "$REQ.pip" -pip install -e . - -$DIR/run_tests.sh diff --git a/ci/run_tests.sh b/ci/run_tests.sh deleted file mode 100755 index 3b0113ba..00000000 --- a/ci/run_tests.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -if [ -f "$DIR/service_account.json" ]; then - export GOOGLE_APPLICATION_CREDENTIALS="$DIR/service_account.json" -fi - -# Install test requirements -pip install coverage pytest pytest-cov flake8 codecov -pytest -v -m "not local_auth" --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml tests diff --git a/ci/constraints-3.7.pip b/testing/constraints-3.7.txt similarity index 100% rename from ci/constraints-3.7.pip rename to testing/constraints-3.7.txt diff --git a/ci/constraints-3.8.pip b/testing/constraints-3.8.txt similarity index 100% rename from ci/constraints-3.8.pip rename to testing/constraints-3.8.txt diff --git a/ci/constraints-3.9.pip b/testing/constraints-3.9.txt similarity index 100% rename from ci/constraints-3.9.pip rename to testing/constraints-3.9.txt From f43cbdac47b52f835567c76cc80197658b803bd4 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 14:30:17 -0600 Subject: [PATCH 07/19] removed circleci config --- .circleci/config.yml | 75 -------------------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 1f9c1a42..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -version: 2 -jobs: - "lint": - docker: - - image: thekevjames/nox - environment: - # Resolve "Python 3 was configured to use ASCII as encoding for the environment" - LC_ALL: C.UTF-8 - LANG: C.UTF-8 - steps: - - checkout - - run: nox -s lint - "docs-presubmit": - docker: - - image: thekevjames/nox - environment: - # Resolve "Python 3 was configured to use ASCII as encoding for the environment" - LC_ALL: C.UTF-8 - LANG: C.UTF-8 - steps: - - checkout - - run: nox -s docs - - # Pip - "pip-3.7": - docker: - - image: thekevjames/nox - steps: - - checkout - - run: ci/config_auth.sh - - run: nox -s unit-3.7 - "pip-3.8": - docker: - - image: thekevjames/nox - steps: - - checkout - - run: ci/config_auth.sh - - run: nox -s unit-3.8 system-3.8 cover - - # Conda - "conda-3.7": - docker: - - image: continuumio/miniconda3 - environment: - PYTHON: "3.7" - PANDAS: "0.23.2" - steps: - - checkout - - run: ci/config_auth.sh - - run: ci/run_conda.sh - "conda-3.9-NIGHTLY": - docker: - - image: continuumio/miniconda3 - environment: - PYTHON: "3.9" - PANDAS: "NIGHTLY" - steps: - - checkout - - run: ci/config_auth.sh - - run: ci/run_conda.sh - -workflows: - version: 2 - build: - jobs: - - lint - - docs-presubmit - - "pip-3.7" - - "pip-3.8" - - "conda-3.7" - - "conda-3.9-NIGHTLY" \ No newline at end of file From 2d7957b794a3e097dce6c9ac4e416ca4e7dfe0bd Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 14:33:56 -0600 Subject: [PATCH 08/19] remove codecov.yml, we don't need it --- codecov.yml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 4c2ed9b1..00000000 --- a/codecov.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -coverage: - status: - project: - default: - target: '0' - enabled: no - patch: - default: - enabled: no - target: '50' - branches: null From df7e945d66d5801b13f1f1e39134ff992165716c Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 14:36:20 -0600 Subject: [PATCH 09/19] removed unused file --- .stickler.yml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .stickler.yml diff --git a/.stickler.yml b/.stickler.yml deleted file mode 100644 index 7bb34d25..00000000 --- a/.stickler.yml +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -linters: - black: - config: ./pyproject.toml - fixer: true \ No newline at end of file From cebc330d6156ec9bc2b642a712899876749a76a1 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 14:49:33 -0600 Subject: [PATCH 10/19] removed unused file --- CONTRIBUTING.md | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index a4f77914..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,35 +0,0 @@ -# Contributing - -We'd love to accept your patches and contributions to this project. There are -just a few small guidelines you need to follow. - -## Contributor License Agreement - -Contributions to this project must be accompanied by a Contributor License -Agreement (CLA). You (or your employer) retain the copyright to your -contribution; this simply gives us permission to use and redistribute your -contributions as part of the project. Head over to - to see your current agreements on file or -to sign a new one. - -You generally only need to submit a CLA once, so if you've already submitted one -(even if it was for a different project), you probably don't need to do it -again. - -## Code Reviews - -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. - -## Community Guidelines - -This project follows -[Google's Open Source Community Guidelines](https://opensource.google/conduct/). - -## Developer Tips - -See the [contributing guide in the pandas-gbq -docs](http://pandas-gbq.readthedocs.io/en/latest/contributing.html). - From 1335f2350f9ef95e2e12919003a6164c92775bbf Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 14:51:50 -0600 Subject: [PATCH 11/19] removed unused file --- .gitattributes | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 9afa4886..00000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -pandas_gbq/_version.py export-subst From f7ec2b6904afeb437e2a8ef032539cf0914f0965 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 15:39:19 -0600 Subject: [PATCH 12/19] added missing requirements files --- samples/snippets/requirements-test.txt | 12 ++++++++ samples/snippets/requirements.txt | 40 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 samples/snippets/requirements-test.txt create mode 100644 samples/snippets/requirements.txt diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt new file mode 100644 index 00000000..1d27f79b --- /dev/null +++ b/samples/snippets/requirements-test.txt @@ -0,0 +1,12 @@ +attrs==21.2.0 +google-cloud-testutils==1.1.0 +importlib-metadata==4.8.1 +iniconfig==1.1.1 +packaging==21.0 +pluggy==1.0.0 +py==1.10.0 +pyparsing==2.4.7 +pytest==6.2.5 +toml==0.10.2 +typing-extensions==3.10.0.2 +zipp==3.5.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt new file mode 100644 index 00000000..1fa0d05e --- /dev/null +++ b/samples/snippets/requirements.txt @@ -0,0 +1,40 @@ +cachetools==4.2.2 +certifi==2021.5.30 +cffi==1.14.6 +charset-normalizer==2.0.4 +google-api-core==2.0.1 +google-auth==2.0.2 +google-auth-oauthlib==0.4.6 +google-cloud-bigquery==2.26.0 +google-cloud-bigquery-storage==2.7.0 +google-cloud-core==2.0.0 +google-crc32c==1.1.2 +google-resumable-media==2.0.2 +googleapis-common-protos==1.53.0 +grpcio==1.39.0 +idna==3.2 +libcst==0.3.20 +mypy-extensions==0.4.3 +numpy==1.21.2 +oauthlib==3.1.1 +packaging==21.0 +pandas==1.3.2 +pandas-gbq==0.15.0 +proto-plus==1.19.0 +protobuf==3.17.3 +pyarrow==5.0.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.20 +pydata-google-auth==1.2.0 +pyparsing==2.4.7 +python-dateutil==2.8.2 +pytz==2021.1 +PyYAML==5.4.1 +requests==2.26.0 +requests-oauthlib==1.3.0 +rsa==4.7.2 +six==1.16.0 +typing-extensions==3.10.0.2 +typing-inspect==0.7.1 +urllib3==1.26.6 From c3eaadc94881614c4f717f3af8389eee8bb9b58d Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 15:43:04 -0600 Subject: [PATCH 13/19] don't test Python 3.6 --- samples/snippets/noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 8bd8983b..73a44c97 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -87,7 +87,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +ALL_VERSIONS = ["3.7", "3.8", "3.9"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] @@ -98,7 +98,7 @@ def get_pytest_env_vars() -> Dict[str, str]: INSTALL_LIBRARY_FROM_SOURCE = os.environ.get( "INSTALL_LIBRARY_FROM_SOURCE", False -) in ("True", "true",) +) in ("True", "true") # # Style Checks # From 37cd7535c18453028b79957e6c74f8f40a8e18f3 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 15:59:41 -0600 Subject: [PATCH 14/19] skip Python 2.7 and 3.6 --- samples/snippets/noxfile_config.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 samples/snippets/noxfile_config.py diff --git a/samples/snippets/noxfile_config.py b/samples/snippets/noxfile_config.py new file mode 100644 index 00000000..302669d7 --- /dev/null +++ b/samples/snippets/noxfile_config.py @@ -0,0 +1 @@ +TEST_CONFIG_OVERRIDE = dict(ignored_versions=["2.7", "3.6"]) From 1d812539f0d48867eba9ad7f45e9a98d7f1efb36 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 2 Sep 2021 16:16:48 -0600 Subject: [PATCH 15/19] flat is better than nested --- samples/{snippets => }/__init__.py | 0 samples/{snippets => }/conftest.py | 0 samples/{snippets => }/noxfile.py | 2 +- samples/{snippets => }/noxfile_config.py | 0 samples/{snippets => }/read_gbq_legacy.py | 0 samples/{snippets => }/read_gbq_simple.py | 0 samples/{snippets => }/read_gbq_test.py | 0 samples/{snippets => }/requirements-test.txt | 0 samples/{snippets => }/requirements.txt | 0 samples/{snippets => }/to_gbq_simple.py | 0 samples/{snippets => }/to_gbq_test.py | 0 11 files changed, 1 insertion(+), 1 deletion(-) rename samples/{snippets => }/__init__.py (100%) rename samples/{snippets => }/conftest.py (100%) rename samples/{snippets => }/noxfile.py (99%) rename samples/{snippets => }/noxfile_config.py (100%) rename samples/{snippets => }/read_gbq_legacy.py (100%) rename samples/{snippets => }/read_gbq_simple.py (100%) rename samples/{snippets => }/read_gbq_test.py (100%) rename samples/{snippets => }/requirements-test.txt (100%) rename samples/{snippets => }/requirements.txt (100%) rename samples/{snippets => }/to_gbq_simple.py (100%) rename samples/{snippets => }/to_gbq_test.py (100%) diff --git a/samples/snippets/__init__.py b/samples/__init__.py similarity index 100% rename from samples/snippets/__init__.py rename to samples/__init__.py diff --git a/samples/snippets/conftest.py b/samples/conftest.py similarity index 100% rename from samples/snippets/conftest.py rename to samples/conftest.py diff --git a/samples/snippets/noxfile.py b/samples/noxfile.py similarity index 99% rename from samples/snippets/noxfile.py rename to samples/noxfile.py index 73a44c97..4511ccad 100644 --- a/samples/snippets/noxfile.py +++ b/samples/noxfile.py @@ -87,7 +87,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.7", "3.8", "3.9"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/samples/snippets/noxfile_config.py b/samples/noxfile_config.py similarity index 100% rename from samples/snippets/noxfile_config.py rename to samples/noxfile_config.py diff --git a/samples/snippets/read_gbq_legacy.py b/samples/read_gbq_legacy.py similarity index 100% rename from samples/snippets/read_gbq_legacy.py rename to samples/read_gbq_legacy.py diff --git a/samples/snippets/read_gbq_simple.py b/samples/read_gbq_simple.py similarity index 100% rename from samples/snippets/read_gbq_simple.py rename to samples/read_gbq_simple.py diff --git a/samples/snippets/read_gbq_test.py b/samples/read_gbq_test.py similarity index 100% rename from samples/snippets/read_gbq_test.py rename to samples/read_gbq_test.py diff --git a/samples/snippets/requirements-test.txt b/samples/requirements-test.txt similarity index 100% rename from samples/snippets/requirements-test.txt rename to samples/requirements-test.txt diff --git a/samples/snippets/requirements.txt b/samples/requirements.txt similarity index 100% rename from samples/snippets/requirements.txt rename to samples/requirements.txt diff --git a/samples/snippets/to_gbq_simple.py b/samples/to_gbq_simple.py similarity index 100% rename from samples/snippets/to_gbq_simple.py rename to samples/to_gbq_simple.py diff --git a/samples/snippets/to_gbq_test.py b/samples/to_gbq_test.py similarity index 100% rename from samples/snippets/to_gbq_test.py rename to samples/to_gbq_test.py From 6d8c3fc4718bc62cd50296d130c4bc327eed207e Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Fri, 3 Sep 2021 07:31:38 -0600 Subject: [PATCH 16/19] flat is better than nested :) --- docs/{source => }/Makefile | 0 docs/README.rst | 10 - docs/{source => }/api.rst | 0 docs/changelog.md | 1 + docs/{source => }/contributing.rst | 0 docs/{source => }/howto/authentication.rst | 2 +- docs/{source => }/index.rst | 0 docs/{source => }/install.rst | 0 docs/{source => }/intro.rst | 0 docs/{source => }/privacy.rst | 0 docs/{source => }/reading.rst | 0 docs/samples | 1 + docs/source/_static/style.css | 9 - docs/source/_templates/layout.html | 8 - docs/source/changelog.md | 1 - docs/source/conf.py | 392 --------------------- docs/source/samples | 1 - docs/{source => }/writing.rst | 0 18 files changed, 3 insertions(+), 422 deletions(-) rename docs/{source => }/Makefile (100%) delete mode 100644 docs/README.rst rename docs/{source => }/api.rst (100%) create mode 120000 docs/changelog.md rename docs/{source => }/contributing.rst (100%) rename docs/{source => }/howto/authentication.rst (99%) rename docs/{source => }/index.rst (100%) rename docs/{source => }/install.rst (100%) rename docs/{source => }/intro.rst (100%) rename docs/{source => }/privacy.rst (100%) rename docs/{source => }/reading.rst (100%) create mode 120000 docs/samples delete mode 100644 docs/source/_static/style.css delete mode 100644 docs/source/_templates/layout.html delete mode 120000 docs/source/changelog.md delete mode 100644 docs/source/conf.py delete mode 120000 docs/source/samples rename docs/{source => }/writing.rst (100%) diff --git a/docs/source/Makefile b/docs/Makefile similarity index 100% rename from docs/source/Makefile rename to docs/Makefile diff --git a/docs/README.rst b/docs/README.rst deleted file mode 100644 index 8cd89d11..00000000 --- a/docs/README.rst +++ /dev/null @@ -1,10 +0,0 @@ -To build a local copy of the pandas-gbq docs, install the programs in -requirements-docs.txt and run 'make html'. If you use the conda package manager -these commands suffice:: - - git clone git@github.com:pydata/pandas-gbq.git - cd dask/docs - conda create -n pandas-gbq-docs --file requirements-docs.txt - source activate pandas-gbq-docs - make html - open build/html/index.html diff --git a/docs/source/api.rst b/docs/api.rst similarity index 100% rename from docs/source/api.rst rename to docs/api.rst diff --git a/docs/changelog.md b/docs/changelog.md new file mode 120000 index 00000000..04c99a55 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1 @@ +../CHANGELOG.md \ No newline at end of file diff --git a/docs/source/contributing.rst b/docs/contributing.rst similarity index 100% rename from docs/source/contributing.rst rename to docs/contributing.rst diff --git a/docs/source/howto/authentication.rst b/docs/howto/authentication.rst similarity index 99% rename from docs/source/howto/authentication.rst rename to docs/howto/authentication.rst index d49632de..877b1189 100644 --- a/docs/source/howto/authentication.rst +++ b/docs/howto/authentication.rst @@ -61,7 +61,7 @@ authentication methods: If pandas-gbq does not find cached credentials, it prompts you to open a web browser, where you can grant pandas-gbq permissions to access your cloud resources. These credentials are only used locally. See the - :doc:`privacy policy ` for details. + :doc:`privacy policy <../privacy>` for details. Authenticating with a Service Account diff --git a/docs/source/index.rst b/docs/index.rst similarity index 100% rename from docs/source/index.rst rename to docs/index.rst diff --git a/docs/source/install.rst b/docs/install.rst similarity index 100% rename from docs/source/install.rst rename to docs/install.rst diff --git a/docs/source/intro.rst b/docs/intro.rst similarity index 100% rename from docs/source/intro.rst rename to docs/intro.rst diff --git a/docs/source/privacy.rst b/docs/privacy.rst similarity index 100% rename from docs/source/privacy.rst rename to docs/privacy.rst diff --git a/docs/source/reading.rst b/docs/reading.rst similarity index 100% rename from docs/source/reading.rst rename to docs/reading.rst diff --git a/docs/samples b/docs/samples new file mode 120000 index 00000000..e804737e --- /dev/null +++ b/docs/samples @@ -0,0 +1 @@ +../samples \ No newline at end of file diff --git a/docs/source/_static/style.css b/docs/source/_static/style.css deleted file mode 100644 index 59a24072..00000000 --- a/docs/source/_static/style.css +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Copyright (c) 2017 pandas-gbq Authors All rights reserved. - * Use of this source code is governed by a BSD-style - * license that can be found in the LICENSE file. - */ - -@import url("theme.css"); - -a.internal em {font-style: normal} diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html deleted file mode 100644 index 74f6910c..00000000 --- a/docs/source/_templates/layout.html +++ /dev/null @@ -1,8 +0,0 @@ - - -{% extends "!layout.html" %} -{% set css_files = css_files + ["_static/style.css"] %} diff --git a/docs/source/changelog.md b/docs/source/changelog.md deleted file mode 120000 index 699cc9e7..00000000 --- a/docs/source/changelog.md +++ /dev/null @@ -1 +0,0 @@ -../../CHANGELOG.md \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index bfcc94ef..00000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,392 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# -*- coding: utf-8 -*- -# -# pandas-gbq documentation build configuration file, created by -# sphinx-quickstart on Wed Feb 8 10:52:12 2017. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import datetime -import os -import sys - -import pandas_gbq - -# sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", - "sphinx.ext.doctest", - "sphinx.ext.extlinks", - "sphinx.ext.todo", - "numpydoc", # used to parse numpy-style docstrings for autodoc - "IPython.sphinxext.ipython_console_highlighting", - "IPython.sphinxext.ipython_directive", - "sphinx.ext.intersphinx", - "sphinx.ext.coverage", - "sphinx.ext.ifconfig", - "recommonmark", -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = [".rst", ".md"] - -# The encoding of source files. -# -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = u"pandas-gbq" -copyright = u"2017-{}, PyData Development Team".format( - datetime.datetime.now().year -) -author = u"PyData Development Team" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = pandas_gbq.__version__ -# The full version, including alpha/beta/rc tags. -release = version - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# -# today = '' -# -# Else, today_fmt is used as the format for a strftime call. -# -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# Taken from docs.readthedocs.io: -# on_rtd is whether we are on readthedocs.io -on_rtd = os.environ.get("READTHEDOCS", None) == "True" - -if not on_rtd: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -# html_theme = 'alabaster' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. -# " v documentation" by default. -# -# html_title = u'pandas-gbq v0.1.0' - -# A shorter title for the navigation bar. Default is the same as html_title. -# -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# -# html_logo = None - -# The name of an image file (relative to this directory) to use as a favicon of -# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# -# html_extra_path = [] - -# If not None, a 'Last updated on:' timestamp is inserted at every page -# bottom, using the given strftime format. -# The empty string is equivalent to '%b %d, %Y'. -# -# html_last_updated_fmt = None - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# -# html_additional_pages = {} - -# If false, no module index is generated. -# -# html_domain_indices = True - -# If false, no index is generated. -# -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' -# -# html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# 'ja' uses this config value. -# 'zh' user can custom change `jieba` dictionary path. -# -# html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -# -# html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = "pandas-gbqdoc" - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ( - master_doc, - "pandas-gbq.tex", - u"pandas-gbq Documentation", - u"PyData Development Team", - "manual", - ) -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# -# latex_use_parts = False - -# If true, show page references after internal links. -# -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# -# latex_appendices = [] - -# It false, will not define \strong, \code, itleref, \crossref ... but only -# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added -# packages. -# -# latex_keep_old_macro_names = True - -# If false, no module index is generated. -# -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, "pandas-gbq", u"pandas-gbq Documentation", [author], 1) -] - -# If true, show URL addresses after external links. -# -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - "pandas-gbq", - u"pandas-gbq Documentation", - author, - "pandas-gbq", - "One line description of project.", - "Miscellaneous", - ) -] - -# Documents to append as an appendix to all manuals. -# -# texinfo_appendices = [] - -# If false, no module index is generated. -# -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# -# texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -# -# texinfo_no_detailmenu = False - - -# Configuration for intersphinx: -intersphinx_mapping = { - "https://docs.python.org/": None, - "https://pandas.pydata.org/pandas-docs/stable/": None, - "https://pydata-google-auth.readthedocs.io/en/latest/": None, - "https://google-auth.readthedocs.io/en/latest/": None, -} - -extlinks = { - "issue": ("https://github.com/pydata/pandas-gbq/issues/%s", "GH#"), - "pr": ("https://github.com/pydata/pandas-gbq/pull/%s", "GH#"), -} diff --git a/docs/source/samples b/docs/source/samples deleted file mode 120000 index 47920198..00000000 --- a/docs/source/samples +++ /dev/null @@ -1 +0,0 @@ -../../samples \ No newline at end of file diff --git a/docs/source/writing.rst b/docs/writing.rst similarity index 100% rename from docs/source/writing.rst rename to docs/writing.rst From 689a8466bafeccc6936fe53b07cb037dd623690d Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Fri, 3 Sep 2021 07:33:20 -0600 Subject: [PATCH 17/19] get docs working Also, don't test Python 3.6 --- owlbot.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/owlbot.py b/owlbot.py index fa8481b9..55284665 100644 --- a/owlbot.py +++ b/owlbot.py @@ -30,7 +30,7 @@ extras = ["tqdm"] templated_files = common.py_library( - unit_test_python_versions=["3.6", "3.7", "3.8", "3.9"], + unit_test_python_versions=["3.7", "3.8", "3.9"], system_test_python_versions=["3.8", "3.9"], cov_level=100, unit_test_extras=extras, @@ -39,6 +39,9 @@ s.move(templated_files, excludes=[ # pandas-gbq was originally licensed BSD-3-Clause License "LICENSE", + # It's unclear that the text in multiprocessing.rst is relavent to + # our users, and it messes up the document tree. + "docs/multiprocessing.rst" ]) # ---------------------------------------------------------------------------- @@ -51,6 +54,12 @@ '"pandas_gbq"', ) +s.replace( + ["noxfile.py"], + r"[\"']alabaster[\"']", + '"alabaster", "ipython"', +) + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From c3209a7a03a8c8f6db8832962d79167449d5ac0d Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Fri, 3 Sep 2021 08:38:27 -0600 Subject: [PATCH 18/19] blacken --- tests/__init__.py | 1 - tests/system/__init__.py | 1 - tests/system/test_gbq.py | 11 +++----- tests/system/test_read_gbq_with_bqstorage.py | 5 +--- tests/system/test_to_gbq.py | 4 +-- tests/unit/__init__.py | 1 - tests/unit/test_gbq.py | 28 +++++++++----------- tests/unit/test_load.py | 4 +-- tests/unit/test_schema.py | 5 +--- 9 files changed, 19 insertions(+), 41 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index edbca6c3..c9ab8506 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2017 pandas-gbq Authors All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. - diff --git a/tests/system/__init__.py b/tests/system/__init__.py index edbca6c3..c9ab8506 100644 --- a/tests/system/__init__.py +++ b/tests/system/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2017 pandas-gbq Authors All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. - diff --git a/tests/system/test_gbq.py b/tests/system/test_gbq.py index bbc0da64..83cda6b5 100644 --- a/tests/system/test_gbq.py +++ b/tests/system/test_gbq.py @@ -82,8 +82,7 @@ def get_schema( bqclient = gbq_connector.client table_ref = bigquery.TableReference( - bigquery.DatasetReference(bqclient.project, dataset_id), - table_id, + bigquery.DatasetReference(bqclient.project, dataset_id), table_id, ) try: @@ -330,9 +329,7 @@ def test_should_properly_handle_null_floats(self, project_id): def test_should_properly_handle_date(self, project_id): query = "SELECT DATE(2003, 1, 4) AS date_col" df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, + query, project_id=project_id, credentials=self.credentials, ) expected = DataFrame( { @@ -346,9 +343,7 @@ def test_should_properly_handle_date(self, project_id): def test_should_properly_handle_time(self, project_id): query = "SELECT TIME_ADD(TIME(3, 14, 15), INTERVAL 926589 MICROSECOND) AS time_col" df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, + query, project_id=project_id, credentials=self.credentials, ) expected = DataFrame( { diff --git a/tests/system/test_read_gbq_with_bqstorage.py b/tests/system/test_read_gbq_with_bqstorage.py index 8b9c7ecc..8440948a 100644 --- a/tests/system/test_read_gbq_with_bqstorage.py +++ b/tests/system/test_read_gbq_with_bqstorage.py @@ -34,10 +34,7 @@ def test_empty_results(method_under_test, query_string): See: https://github.com/pydata/pandas-gbq/issues/299 """ - df = method_under_test( - query_string, - use_bqstorage_api=True, - ) + df = method_under_test(query_string, use_bqstorage_api=True,) assert len(df.index) == 0 diff --git a/tests/system/test_to_gbq.py b/tests/system/test_to_gbq.py index 59435c33..ce8ce176 100644 --- a/tests/system/test_to_gbq.py +++ b/tests/system/test_to_gbq.py @@ -47,7 +47,5 @@ def test_float_round_trip( round_trip = bigquery_client.list_rows(table_id).to_dataframe() round_trip_floats = round_trip["float_col"].sort_values() pandas.testing.assert_series_equal( - round_trip_floats, - input_floats, - check_exact=True, + round_trip_floats, input_floats, check_exact=True, ) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index edbca6c3..c9ab8506 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2017 pandas-gbq Authors All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. - diff --git a/tests/unit/test_gbq.py b/tests/unit/test_gbq.py index 7476db3f..2ed7f639 100644 --- a/tests/unit/test_gbq.py +++ b/tests/unit/test_gbq.py @@ -218,8 +218,8 @@ def test_to_gbq_doesnt_run_query(mock_bigquery_client): def test_to_gbq_w_empty_df(mock_bigquery_client): import google.api_core.exceptions - mock_bigquery_client.get_table.side_effect = ( - google.api_core.exceptions.NotFound("my_table") + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" ) gbq.to_gbq(DataFrame(), "my_dataset.my_table", project_id="1234") mock_bigquery_client.create_table.assert_called_with(mock.ANY) @@ -234,8 +234,8 @@ def test_to_gbq_w_default_project(mock_bigquery_client): import google.api_core.exceptions from google.cloud.bigquery.table import TableReference - mock_bigquery_client.get_table.side_effect = ( - google.api_core.exceptions.NotFound("my_table") + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" ) gbq.to_gbq(DataFrame(), "my_dataset.my_table") @@ -254,8 +254,8 @@ def test_to_gbq_w_project_table(mock_bigquery_client): import google.api_core.exceptions from google.cloud.bigquery.table import TableReference - mock_bigquery_client.get_table.side_effect = ( - google.api_core.exceptions.NotFound("my_table") + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" ) gbq.to_gbq( DataFrame(), @@ -274,11 +274,11 @@ def test_to_gbq_w_project_table(mock_bigquery_client): def test_to_gbq_creates_dataset(mock_bigquery_client): import google.api_core.exceptions - mock_bigquery_client.get_table.side_effect = ( - google.api_core.exceptions.NotFound("my_table") + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" ) - mock_bigquery_client.get_dataset.side_effect = ( - google.api_core.exceptions.NotFound("my_dataset") + mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.NotFound( + "my_dataset" ) gbq.to_gbq(DataFrame([[1]]), "my_dataset.my_table", project_id="1234") mock_bigquery_client.create_dataset.assert_called_with(mock.ANY) @@ -370,8 +370,7 @@ def test_read_gbq_with_old_bq_raises_importerror(monkeypatch): monkeypatch.setattr(FEATURES, "_bigquery_installed_version", None) with pytest.raises(ImportError, match="google-cloud-bigquery"): gbq.read_gbq( - "SELECT 1", - project_id="my-project", + "SELECT 1", project_id="my-project", ) @@ -382,10 +381,7 @@ def test_read_gbq_with_verbose_old_pandas_no_warnings(monkeypatch, recwarn): mock.PropertyMock(return_value=False), ) gbq.read_gbq( - "SELECT 1", - project_id="my-project", - dialect="standard", - verbose=True, + "SELECT 1", project_id="my-project", dialect="standard", verbose=True, ) assert len(recwarn) == 0 diff --git a/tests/unit/test_load.py b/tests/unit/test_load.py index 353b8bd1..ffbce16d 100644 --- a/tests/unit/test_load.py +++ b/tests/unit/test_load.py @@ -60,9 +60,7 @@ def test_encode_chunk_with_floats(): csv_buffer, header=None, float_precision="round_trip" ) pandas.testing.assert_frame_equal( - round_trip, - input_df, - check_exact=True, + round_trip, input_df, check_exact=True, ) diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index bd04508e..67ffb32e 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -35,10 +35,7 @@ def module_under_test(): [{"name": "A", "type": "INTEGER"}], [{"name": "A", "type": "INT64"}], ), - ( - [{"name": "A", "type": "BOOL"}], - [{"name": "A", "type": "BOOLEAN"}], - ), + ([{"name": "A", "type": "BOOL"}], [{"name": "A", "type": "BOOLEAN"}],), ( # TODO: include sub-fields when struct uploads are supported. [{"name": "A", "type": "STRUCT"}], From 7ac33cafb9fe262feb2753e19e7a16de4969a7ff Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Fri, 3 Sep 2021 08:48:59 -0600 Subject: [PATCH 19/19] removed some added files to make the PR diff smaler for now --- .github/.OwlBot.yaml | 19 --- samples/noxfile.py | 273 ------------------------------------------- 2 files changed, 292 deletions(-) delete mode 100644 .github/.OwlBot.yaml delete mode 100644 samples/noxfile.py diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml deleted file mode 100644 index 8642b5f3..00000000 --- a/.github/.OwlBot.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - image: gcr.io/repo-automation-bots/owlbot-python:latest - -begin-after-commit-hash: 1afeb53252641dc35a421fa5acc59e2f3229ad6d - diff --git a/samples/noxfile.py b/samples/noxfile.py deleted file mode 100644 index 4511ccad..00000000 --- a/samples/noxfile.py +++ /dev/null @@ -1,273 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import os -from pathlib import Path -import sys -from typing import Callable, Dict, List, Optional - -import nox - - -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING -# DO NOT EDIT THIS FILE EVER! -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING - -BLACK_VERSION = "black==19.10b0" - -# Copy `noxfile_config.py` to your directory and modify it instead. - -# `TEST_CONFIG` dict is a configuration hook that allows users to -# modify the test configurations. The values here should be in sync -# with `noxfile_config.py`. Users will copy `noxfile_config.py` into -# their directory and modify it. - -TEST_CONFIG = { - # You can opt out from the test for specific Python versions. - "ignored_versions": [], - # Old samples are opted out of enforcing Python type hints - # All new samples should feature them - "enforce_type_hints": False, - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # If you need to use a specific version of pip, - # change pip_version_override to the string representation - # of the version number, for example, "20.2.4" - "pip_version_override": None, - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": {}, -} - - -try: - # Ensure we can import noxfile_config in the project's directory. - sys.path.append(".") - from noxfile_config import TEST_CONFIG_OVERRIDE -except ImportError as e: - print("No user noxfile_config found: detail: {}".format(e)) - TEST_CONFIG_OVERRIDE = {} - -# Update the TEST_CONFIG with the user supplied values. -TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) - - -def get_pytest_env_vars() -> Dict[str, str]: - """Returns a dict for pytest invocation.""" - ret = {} - - # Override the GCLOUD_PROJECT and the alias. - env_key = TEST_CONFIG["gcloud_project_env"] - # This should error out if not set. - ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] - - # Apply user supplied envs. - ret.update(TEST_CONFIG["envs"]) - return ret - - -# DO NOT EDIT - automatically generated. -# All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] - -# Any default versions that should be ignored. -IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] - -TESTED_VERSIONS = sorted( - [v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS] -) - -INSTALL_LIBRARY_FROM_SOURCE = os.environ.get( - "INSTALL_LIBRARY_FROM_SOURCE", False -) in ("True", "true") -# -# Style Checks -# - - -def _determine_local_import_names(start_dir: str) -> List[str]: - """Determines all import names that should be considered "local". - - This is used when running the linter to insure that import order is - properly checked. - """ - file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] - return [ - basename - for basename, extension in file_ext_pairs - if extension == ".py" - or os.path.isdir(os.path.join(start_dir, basename)) - and basename not in ("__pycache__") - ] - - -# Linting with flake8. -# -# We ignore the following rules: -# E203: whitespace before ‘:’ -# E266: too many leading ‘#’ for block comment -# E501: line too long -# I202: Additional newline in a section of imports -# -# We also need to specify the rules which are ignored by default: -# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] -FLAKE8_COMMON_ARGS = [ - "--show-source", - "--builtin=gettext", - "--max-complexity=20", - "--import-order-style=google", - "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", - "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", - "--max-line-length=88", -] - - -@nox.session -def lint(session: nox.sessions.Session) -> None: - if not TEST_CONFIG["enforce_type_hints"]: - session.install("flake8", "flake8-import-order") - else: - session.install("flake8", "flake8-import-order", "flake8-annotations") - - local_names = _determine_local_import_names(".") - args = FLAKE8_COMMON_ARGS + [ - "--application-import-names", - ",".join(local_names), - ".", - ] - session.run("flake8", *args) - - -# -# Black -# - - -@nox.session -def blacken(session: nox.sessions.Session) -> None: - session.install(BLACK_VERSION) - python_files = [path for path in os.listdir(".") if path.endswith(".py")] - - session.run("black", *python_files) - - -# -# Sample Tests -# - - -PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] - - -def _session_tests( - session: nox.sessions.Session, post_install: Callable = None -) -> None: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install( - "-r", "requirements-test.txt", "-c", "constraints-test.txt" - ) - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) - - -@nox.session(python=ALL_VERSIONS) -def py(session: nox.sessions.Session) -> None: - """Runs py.test for a sample using the specified version of Python.""" - if session.python in TESTED_VERSIONS: - _session_tests(session) - else: - session.skip( - "SKIPPED: {} tests are disabled for this sample.".format( - session.python - ) - ) - - -# -# Readmegen -# - - -def _get_repo_root() -> Optional[str]: - """ Returns the root folder of the project. """ - # Get root of this repository. Assume we don't have directories nested deeper than 10 items. - p = Path(os.getcwd()) - for i in range(10): - if p is None: - break - if Path(p / ".git").exists(): - return str(p) - # .git is not available in repos cloned via Cloud Build - # setup.py is always in the library's root, so use that instead - # https://github.com/googleapis/synthtool/issues/792 - if Path(p / "setup.py").exists(): - return str(p) - p = p.parent - raise Exception("Unable to detect repository root.") - - -GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) - - -@nox.session -@nox.parametrize("path", GENERATED_READMES) -def readmegen(session: nox.sessions.Session, path: str) -> None: - """(Re-)generates the readme for a sample.""" - session.install("jinja2", "pyyaml") - dir_ = os.path.dirname(path) - - if os.path.exists(os.path.join(dir_, "requirements.txt")): - session.install("-r", os.path.join(dir_, "requirements.txt")) - - in_file = os.path.join(dir_, "README.rst.in") - session.run( - "python", - _get_repo_root() + "/scripts/readme-gen/readme_gen.py", - in_file, - )