From 519074112775c19742522158f612b467cf590219 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 19:32:16 +0000 Subject: [PATCH 1/7] chore(python): exclude setup.py in renovate config (#861) Source-Link: https://github.com/googleapis/synthtool/commit/56da63e80c384a871356d1ea6640802017f213b4 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 --- .github/.OwlBot.lock.yaml | 2 +- renovate.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 2fa0f7c4f..b8dcb4a4a 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:94961fdc5c9ca6d13530a6a414a49d2f607203168215d074cdb0a1df9ec31c0b + digest: sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 diff --git a/renovate.json b/renovate.json index 566a70f3c..39b2a0ec9 100644 --- a/renovate.json +++ b/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } From 282e3b605e62c4f8d93debff20d40afbf2e718f5 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 6 Sep 2022 19:36:05 +0200 Subject: [PATCH 2/7] chore(deps): update dependency pytest to v7.1.3 (#862) * chore(deps): update all dependencies * revert Co-authored-by: Anthonios Partheniou --- samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 077bdf929..cbcfa2f4f 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==7.1.2 +pytest==7.1.3 mock==4.0.3 backoff==2.1.2 \ No newline at end of file From dd47bf0293d8687b7620fc5f28ffa068c1119dcb Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 12:39:15 -0400 Subject: [PATCH 3/7] docs(nodejs_mono_repo): update broken links in README (#864) Source-Link: https://github.com/googleapis/synthtool/commit/50db768f450a50d7c1fd62513c113c9bb96fd434 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:e09366bdf0fd9c8976592988390b24d53583dd9f002d476934da43725adbb978 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- samples/snippets/noxfile.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b8dcb4a4a..aa547962e 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 + digest: sha256:e09366bdf0fd9c8976592988390b24d53583dd9f002d476934da43725adbb978 diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 5fcb9d746..0398d72ff 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -207,8 +207,8 @@ def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: # check for presence of tests - test_list = glob.glob("*_test.py") + glob.glob("test_*.py") - test_list.extend(glob.glob("tests")) + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob("**/test_*.py", recursive=True) + test_list.extend(glob.glob("**/tests", recursive=True)) if len(test_list) == 0: print("No tests found, skipping directory.") From 9469f5dd5ca6d546a47efbc3d673a401ead9d632 Mon Sep 17 00:00:00 2001 From: cojenco Date: Thu, 15 Sep 2022 09:54:56 -0700 Subject: [PATCH 4/7] docs: clarify list_blobs usage (#866) --- google/cloud/storage/client.py | 3 ++- samples/snippets/storage_list_files.py | 1 + samples/snippets/storage_list_files_with_prefix.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 05a64f57f..e6c4f33c6 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -1269,7 +1269,8 @@ def list_blobs( Returns: Iterator of all :class:`~google.cloud.storage.blob.Blob` - in this bucket matching the arguments. + in this bucket matching the arguments. The RPC call + returns a response when the iterator is consumed. As part of the response, you'll also get back an iterator.prefixes entity that lists object names up to and including the requested delimiter. Duplicate entries are omitted from this list. diff --git a/samples/snippets/storage_list_files.py b/samples/snippets/storage_list_files.py index c6a80d9fa..5e80c833a 100644 --- a/samples/snippets/storage_list_files.py +++ b/samples/snippets/storage_list_files.py @@ -29,6 +29,7 @@ def list_blobs(bucket_name): # Note: Client.list_blobs requires at least package version 1.17.0. blobs = storage_client.list_blobs(bucket_name) + # Note: The call returns a response only when the iterator is consumed. for blob in blobs: print(blob.name) diff --git a/samples/snippets/storage_list_files_with_prefix.py b/samples/snippets/storage_list_files_with_prefix.py index f79413fb6..be7468cba 100644 --- a/samples/snippets/storage_list_files_with_prefix.py +++ b/samples/snippets/storage_list_files_with_prefix.py @@ -53,6 +53,7 @@ def list_blobs_with_prefix(bucket_name, prefix, delimiter=None): # Note: Client.list_blobs requires at least package version 1.17.0. blobs = storage_client.list_blobs(bucket_name, prefix=prefix, delimiter=delimiter) + # Note: The call returns a response only when the iterator is consumed. print("Blobs:") for blob in blobs: print(blob.name) From 05e07f248fc010d7a1b24109025e9230cb2a7259 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 19 Sep 2022 21:24:06 +0200 Subject: [PATCH 5/7] chore(deps): update dependency pandas to v1.5.0 (#867) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 6ec678121..56e800303 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-pubsub==2.13.6 google-cloud-storage==2.5.0 pandas===1.3.5; python_version == '3.7' -pandas==1.4.4; python_version >= '3.8' +pandas==1.5.0; python_version >= '3.8' From 2a6c162d511bf0afbffbc7ccfcac34499f3c2089 Mon Sep 17 00:00:00 2001 From: cojenco Date: Fri, 23 Sep 2022 13:43:29 -0700 Subject: [PATCH 6/7] tests: ensure config changes are propagated in sys tests (#870) * tests: ensure config changes are propagated in sys tests * refactor sys tests * refactor tests --- tests/system/_helpers.py | 20 +++++++---- tests/system/conftest.py | 16 +++++++++ tests/system/test_bucket.py | 15 ++++---- tests/system/test_kms_integration.py | 54 ++++++++++++---------------- 4 files changed, 60 insertions(+), 45 deletions(-) diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py index a1f8c38b9..12726db73 100644 --- a/tests/system/_helpers.py +++ b/tests/system/_helpers.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import time from google.api_core import exceptions @@ -63,16 +64,16 @@ def _no_retetion_period(bucket): retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy) -retry_no_event_based_hold = RetryInstanceState(_no_event_based_hold, max_tries=10) -retry_has_kms_key_name = RetryInstanceState(_has_kms_key_name, max_tries=10) +retry_no_event_based_hold = RetryInstanceState(_no_event_based_hold, max_tries=5) +retry_has_kms_key_name = RetryInstanceState(_has_kms_key_name, max_tries=5) retry_has_retention_expiration = RetryInstanceState( - _has_retention_expiration, max_tries=10 + _has_retention_expiration, max_tries=5 ) retry_no_retention_expiration = RetryInstanceState( - _no_retention_expiration, max_tries=10 + _no_retention_expiration, max_tries=5 ) -retry_has_retention_period = RetryInstanceState(_has_retetion_period, max_tries=10) -retry_no_retention_period = RetryInstanceState(_no_retetion_period, max_tries=10) +retry_has_retention_period = RetryInstanceState(_has_retetion_period, max_tries=5) +retry_no_retention_period = RetryInstanceState(_no_retetion_period, max_tries=5) def unique_name(prefix): @@ -106,3 +107,10 @@ def delete_bucket(bucket): retry = RetryErrors(errors, max_tries=15) retry(empty_bucket)(bucket) retry(bucket.delete)(force=True) + + +def await_config_changes_propagate(sec=3): + # Changes to the bucket will be readable immediately after writing, + # but configuration changes may take time to propagate. + # See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch + time.sleep(sec) diff --git a/tests/system/conftest.py b/tests/system/conftest.py index c42f62e99..c4c137007 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -165,6 +165,22 @@ def signing_bucket(storage_client, signing_bucket_name): _helpers.delete_bucket(bucket) +@pytest.fixture(scope="session") +def default_ebh_bucket_name(): + return _helpers.unique_name("gcp-systest-default-ebh") + + +@pytest.fixture(scope="session") +def default_ebh_bucket(storage_client, default_ebh_bucket_name): + bucket = storage_client.bucket(default_ebh_bucket_name) + bucket.default_event_based_hold = True + _helpers.retry_429_503(bucket.create)() + + yield bucket + + _helpers.delete_bucket(bucket) + + @pytest.fixture(scope="function") def buckets_to_delete(): buckets_to_delete = [] diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index 054a29018..9fe7aa648 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -682,16 +682,10 @@ def test_bucket_w_retention_period( def test_bucket_w_default_event_based_hold( storage_client, - buckets_to_delete, blobs_to_delete, + default_ebh_bucket, ): - bucket_name = _helpers.unique_name("w-def-ebh") - bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket_name) - buckets_to_delete.append(bucket) - - bucket.default_event_based_hold = True - bucket.patch() - + bucket = storage_client.get_bucket(default_ebh_bucket) assert bucket.default_event_based_hold assert bucket.retention_period is None assert bucket.retention_policy_effective_time is None @@ -725,6 +719,10 @@ def test_bucket_w_default_event_based_hold( assert bucket.retention_policy_effective_time is None assert not bucket.retention_policy_locked + # Changes to the bucket will be readable immediately after writing, + # but configuration changes may take time to propagate. + _helpers.await_config_changes_propagate() + blob.upload_from_string(payload) # https://github.com/googleapis/python-storage/issues/435 @@ -870,6 +868,7 @@ def test_ubla_set_unset_preserves_acls( # Clear UBLA bucket.iam_configuration.uniform_bucket_level_access_enabled = False bucket.patch() + _helpers.await_config_changes_propagate() # Query ACLs after clearing UBLA bucket.acl.reload() diff --git a/tests/system/test_kms_integration.py b/tests/system/test_kms_integration.py index 87c1a7c07..f047baced 100644 --- a/tests/system/test_kms_integration.py +++ b/tests/system/test_kms_integration.py @@ -137,10 +137,6 @@ def test_bucket_w_default_kms_key_name( file_data, ): blob_name = "default-kms-key-name" - override_blob_name = "override-default-kms-key-name" - alt_blob_name = "alt-default-kms-key-name" - cleartext_blob_name = "cleartext" - info = file_data["simple"] with open(info["path"], "rb") as file_obj: @@ -150,6 +146,10 @@ def test_bucket_w_default_kms_key_name( kms_bucket.patch() assert kms_bucket.default_kms_key_name == kms_key_name + # Changes to the bucket will be readable immediately after writing, + # but configuration changes may take time to propagate. + _helpers.await_config_changes_propagate() + defaulted_blob = kms_bucket.blob(blob_name) defaulted_blob.upload_from_filename(info["path"]) blobs_to_delete.append(defaulted_blob) @@ -159,34 +159,15 @@ def test_bucket_w_default_kms_key_name( # We don't know the current version of the key. assert defaulted_blob.kms_key_name.startswith(kms_key_name) - override_blob = kms_bucket.blob(override_blob_name, kms_key_name=alt_kms_key_name) - override_blob.upload_from_filename(info["path"]) - blobs_to_delete.append(override_blob) - - assert override_blob.download_as_bytes() == payload - # We don't know the current version of the key. - assert override_blob.kms_key_name.startswith(alt_kms_key_name) - + # Test changing the default KMS key. kms_bucket.default_kms_key_name = alt_kms_key_name kms_bucket.patch() + assert kms_bucket.default_kms_key_name == alt_kms_key_name - alt_blob = kms_bucket.blob(alt_blob_name) - alt_blob.upload_from_filename(info["path"]) - blobs_to_delete.append(alt_blob) - - assert alt_blob.download_as_bytes() == payload - # We don't know the current version of the key. - assert alt_blob.kms_key_name.startswith(alt_kms_key_name) - + # Test removing the default KMS key. kms_bucket.default_kms_key_name = None kms_bucket.patch() - - cleartext_blob = kms_bucket.blob(cleartext_blob_name) - cleartext_blob.upload_from_filename(info["path"]) - blobs_to_delete.append(cleartext_blob) - - assert cleartext_blob.download_as_bytes() == payload - assert cleartext_blob.kms_key_name is None + assert kms_bucket.default_kms_key_name is None def test_blob_rewrite_rotate_csek_to_cmek( @@ -240,9 +221,10 @@ def test_blob_upload_w_bucket_cmek_enabled( kms_bucket, blobs_to_delete, kms_key_name, - file_data, + alt_kms_key_name, ): blob_name = "test-blob" + override_blob_name = "override-default-kms-key-name" payload = b"DEADBEEF" alt_payload = b"NEWDEADBEEF" @@ -250,19 +232,29 @@ def test_blob_upload_w_bucket_cmek_enabled( kms_bucket.patch() assert kms_bucket.default_kms_key_name == kms_key_name + # Changes to the bucket will be readable immediately after writing, + # but configuration changes may take time to propagate. + _helpers.await_config_changes_propagate() + blob = kms_bucket.blob(blob_name) blob.upload_from_string(payload) blobs_to_delete.append(blob) _helpers.retry_429_harder(_helpers.retry_has_kms_key_name(blob.reload))() - # We don't know the current version of the key. assert blob.kms_key_name.startswith(kms_key_name) blob.upload_from_string(alt_payload, if_generation_match=blob.generation) - assert blob.download_as_bytes() == alt_payload + # Test the specific key is used to encrypt the object if you have both + # a default KMS key set on your bucket and a specific key included in your request. + override_blob = kms_bucket.blob(override_blob_name, kms_key_name=alt_kms_key_name) + override_blob.upload_from_string(payload) + blobs_to_delete.append(override_blob) + + assert override_blob.download_as_bytes() == payload + assert override_blob.kms_key_name.startswith(alt_kms_key_name) + kms_bucket.default_kms_key_name = None _helpers.retry_429_harder(kms_bucket.patch)() - assert kms_bucket.default_kms_key_name is None From 8d5a6c3557c48fb885e7e5966d1d906398edc399 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 23 Sep 2022 23:14:03 +0200 Subject: [PATCH 7/7] chore(deps): update dependency google-cloud-pubsub to v2.13.7 (#869) Co-authored-by: cojenco --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 56e800303..2910de3e1 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-pubsub==2.13.6 +google-cloud-pubsub==2.13.7 google-cloud-storage==2.5.0 pandas===1.3.5; python_version == '3.7' pandas==1.5.0; python_version >= '3.8'