diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 2fa0f7c4f..aa547962e 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:94961fdc5c9ca6d13530a6a414a49d2f607203168215d074cdb0a1df9ec31c0b + digest: sha256:e09366bdf0fd9c8976592988390b24d53583dd9f002d476934da43725adbb978 diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 05a64f57f..e6c4f33c6 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -1269,7 +1269,8 @@ def list_blobs( Returns: Iterator of all :class:`~google.cloud.storage.blob.Blob` - in this bucket matching the arguments. + in this bucket matching the arguments. The RPC call + returns a response when the iterator is consumed. As part of the response, you'll also get back an iterator.prefixes entity that lists object names up to and including the requested delimiter. Duplicate entries are omitted from this list. diff --git a/renovate.json b/renovate.json index 566a70f3c..39b2a0ec9 100644 --- a/renovate.json +++ b/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 5fcb9d746..0398d72ff 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -207,8 +207,8 @@ def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: # check for presence of tests - test_list = glob.glob("*_test.py") + glob.glob("test_*.py") - test_list.extend(glob.glob("tests")) + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob("**/test_*.py", recursive=True) + test_list.extend(glob.glob("**/tests", recursive=True)) if len(test_list) == 0: print("No tests found, skipping directory.") diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 077bdf929..cbcfa2f4f 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==7.1.2 +pytest==7.1.3 mock==4.0.3 backoff==2.1.2 \ No newline at end of file diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 6ec678121..2910de3e1 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-pubsub==2.13.6 +google-cloud-pubsub==2.13.7 google-cloud-storage==2.5.0 pandas===1.3.5; python_version == '3.7' -pandas==1.4.4; python_version >= '3.8' +pandas==1.5.0; python_version >= '3.8' diff --git a/samples/snippets/storage_list_files.py b/samples/snippets/storage_list_files.py index c6a80d9fa..5e80c833a 100644 --- a/samples/snippets/storage_list_files.py +++ b/samples/snippets/storage_list_files.py @@ -29,6 +29,7 @@ def list_blobs(bucket_name): # Note: Client.list_blobs requires at least package version 1.17.0. blobs = storage_client.list_blobs(bucket_name) + # Note: The call returns a response only when the iterator is consumed. for blob in blobs: print(blob.name) diff --git a/samples/snippets/storage_list_files_with_prefix.py b/samples/snippets/storage_list_files_with_prefix.py index f79413fb6..be7468cba 100644 --- a/samples/snippets/storage_list_files_with_prefix.py +++ b/samples/snippets/storage_list_files_with_prefix.py @@ -53,6 +53,7 @@ def list_blobs_with_prefix(bucket_name, prefix, delimiter=None): # Note: Client.list_blobs requires at least package version 1.17.0. blobs = storage_client.list_blobs(bucket_name, prefix=prefix, delimiter=delimiter) + # Note: The call returns a response only when the iterator is consumed. print("Blobs:") for blob in blobs: print(blob.name) diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py index a1f8c38b9..12726db73 100644 --- a/tests/system/_helpers.py +++ b/tests/system/_helpers.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import time from google.api_core import exceptions @@ -63,16 +64,16 @@ def _no_retetion_period(bucket): retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy) -retry_no_event_based_hold = RetryInstanceState(_no_event_based_hold, max_tries=10) -retry_has_kms_key_name = RetryInstanceState(_has_kms_key_name, max_tries=10) +retry_no_event_based_hold = RetryInstanceState(_no_event_based_hold, max_tries=5) +retry_has_kms_key_name = RetryInstanceState(_has_kms_key_name, max_tries=5) retry_has_retention_expiration = RetryInstanceState( - _has_retention_expiration, max_tries=10 + _has_retention_expiration, max_tries=5 ) retry_no_retention_expiration = RetryInstanceState( - _no_retention_expiration, max_tries=10 + _no_retention_expiration, max_tries=5 ) -retry_has_retention_period = RetryInstanceState(_has_retetion_period, max_tries=10) -retry_no_retention_period = RetryInstanceState(_no_retetion_period, max_tries=10) +retry_has_retention_period = RetryInstanceState(_has_retetion_period, max_tries=5) +retry_no_retention_period = RetryInstanceState(_no_retetion_period, max_tries=5) def unique_name(prefix): @@ -106,3 +107,10 @@ def delete_bucket(bucket): retry = RetryErrors(errors, max_tries=15) retry(empty_bucket)(bucket) retry(bucket.delete)(force=True) + + +def await_config_changes_propagate(sec=3): + # Changes to the bucket will be readable immediately after writing, + # but configuration changes may take time to propagate. + # See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch + time.sleep(sec) diff --git a/tests/system/conftest.py b/tests/system/conftest.py index c42f62e99..c4c137007 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -165,6 +165,22 @@ def signing_bucket(storage_client, signing_bucket_name): _helpers.delete_bucket(bucket) +@pytest.fixture(scope="session") +def default_ebh_bucket_name(): + return _helpers.unique_name("gcp-systest-default-ebh") + + +@pytest.fixture(scope="session") +def default_ebh_bucket(storage_client, default_ebh_bucket_name): + bucket = storage_client.bucket(default_ebh_bucket_name) + bucket.default_event_based_hold = True + _helpers.retry_429_503(bucket.create)() + + yield bucket + + _helpers.delete_bucket(bucket) + + @pytest.fixture(scope="function") def buckets_to_delete(): buckets_to_delete = [] diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index 054a29018..9fe7aa648 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -682,16 +682,10 @@ def test_bucket_w_retention_period( def test_bucket_w_default_event_based_hold( storage_client, - buckets_to_delete, blobs_to_delete, + default_ebh_bucket, ): - bucket_name = _helpers.unique_name("w-def-ebh") - bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket_name) - buckets_to_delete.append(bucket) - - bucket.default_event_based_hold = True - bucket.patch() - + bucket = storage_client.get_bucket(default_ebh_bucket) assert bucket.default_event_based_hold assert bucket.retention_period is None assert bucket.retention_policy_effective_time is None @@ -725,6 +719,10 @@ def test_bucket_w_default_event_based_hold( assert bucket.retention_policy_effective_time is None assert not bucket.retention_policy_locked + # Changes to the bucket will be readable immediately after writing, + # but configuration changes may take time to propagate. + _helpers.await_config_changes_propagate() + blob.upload_from_string(payload) # https://github.com/googleapis/python-storage/issues/435 @@ -870,6 +868,7 @@ def test_ubla_set_unset_preserves_acls( # Clear UBLA bucket.iam_configuration.uniform_bucket_level_access_enabled = False bucket.patch() + _helpers.await_config_changes_propagate() # Query ACLs after clearing UBLA bucket.acl.reload() diff --git a/tests/system/test_kms_integration.py b/tests/system/test_kms_integration.py index 87c1a7c07..f047baced 100644 --- a/tests/system/test_kms_integration.py +++ b/tests/system/test_kms_integration.py @@ -137,10 +137,6 @@ def test_bucket_w_default_kms_key_name( file_data, ): blob_name = "default-kms-key-name" - override_blob_name = "override-default-kms-key-name" - alt_blob_name = "alt-default-kms-key-name" - cleartext_blob_name = "cleartext" - info = file_data["simple"] with open(info["path"], "rb") as file_obj: @@ -150,6 +146,10 @@ def test_bucket_w_default_kms_key_name( kms_bucket.patch() assert kms_bucket.default_kms_key_name == kms_key_name + # Changes to the bucket will be readable immediately after writing, + # but configuration changes may take time to propagate. + _helpers.await_config_changes_propagate() + defaulted_blob = kms_bucket.blob(blob_name) defaulted_blob.upload_from_filename(info["path"]) blobs_to_delete.append(defaulted_blob) @@ -159,34 +159,15 @@ def test_bucket_w_default_kms_key_name( # We don't know the current version of the key. assert defaulted_blob.kms_key_name.startswith(kms_key_name) - override_blob = kms_bucket.blob(override_blob_name, kms_key_name=alt_kms_key_name) - override_blob.upload_from_filename(info["path"]) - blobs_to_delete.append(override_blob) - - assert override_blob.download_as_bytes() == payload - # We don't know the current version of the key. - assert override_blob.kms_key_name.startswith(alt_kms_key_name) - + # Test changing the default KMS key. kms_bucket.default_kms_key_name = alt_kms_key_name kms_bucket.patch() + assert kms_bucket.default_kms_key_name == alt_kms_key_name - alt_blob = kms_bucket.blob(alt_blob_name) - alt_blob.upload_from_filename(info["path"]) - blobs_to_delete.append(alt_blob) - - assert alt_blob.download_as_bytes() == payload - # We don't know the current version of the key. - assert alt_blob.kms_key_name.startswith(alt_kms_key_name) - + # Test removing the default KMS key. kms_bucket.default_kms_key_name = None kms_bucket.patch() - - cleartext_blob = kms_bucket.blob(cleartext_blob_name) - cleartext_blob.upload_from_filename(info["path"]) - blobs_to_delete.append(cleartext_blob) - - assert cleartext_blob.download_as_bytes() == payload - assert cleartext_blob.kms_key_name is None + assert kms_bucket.default_kms_key_name is None def test_blob_rewrite_rotate_csek_to_cmek( @@ -240,9 +221,10 @@ def test_blob_upload_w_bucket_cmek_enabled( kms_bucket, blobs_to_delete, kms_key_name, - file_data, + alt_kms_key_name, ): blob_name = "test-blob" + override_blob_name = "override-default-kms-key-name" payload = b"DEADBEEF" alt_payload = b"NEWDEADBEEF" @@ -250,19 +232,29 @@ def test_blob_upload_w_bucket_cmek_enabled( kms_bucket.patch() assert kms_bucket.default_kms_key_name == kms_key_name + # Changes to the bucket will be readable immediately after writing, + # but configuration changes may take time to propagate. + _helpers.await_config_changes_propagate() + blob = kms_bucket.blob(blob_name) blob.upload_from_string(payload) blobs_to_delete.append(blob) _helpers.retry_429_harder(_helpers.retry_has_kms_key_name(blob.reload))() - # We don't know the current version of the key. assert blob.kms_key_name.startswith(kms_key_name) blob.upload_from_string(alt_payload, if_generation_match=blob.generation) - assert blob.download_as_bytes() == alt_payload + # Test the specific key is used to encrypt the object if you have both + # a default KMS key set on your bucket and a specific key included in your request. + override_blob = kms_bucket.blob(override_blob_name, kms_key_name=alt_kms_key_name) + override_blob.upload_from_string(payload) + blobs_to_delete.append(override_blob) + + assert override_blob.download_as_bytes() == payload + assert override_blob.kms_key_name.startswith(alt_kms_key_name) + kms_bucket.default_kms_key_name = None _helpers.retry_429_harder(kms_bucket.patch)() - assert kms_bucket.default_kms_key_name is None