Updating recent changes in googleapis repo

.github/.OwlBot.lock.yaml

@@ -13,4 +13,4 @@
 # limitations under the License.
 docker:
   image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
-  digest: sha256:94961fdc5c9ca6d13530a6a414a49d2f607203168215d074cdb0a1df9ec31c0b
+  digest: sha256:e09366bdf0fd9c8976592988390b24d53583dd9f002d476934da43725adbb978

google/cloud/storage/client.py

@@ -1269,7 +1269,8 @@ def list_blobs(
 
         Returns:
             Iterator of all :class:`~google.cloud.storage.blob.Blob`
-            in this bucket matching the arguments.
+            in this bucket matching the arguments. The RPC call
+            returns a response when the iterator is consumed.
 
             As part of the response, you'll also get back an iterator.prefixes entity that lists object names
             up to and including the requested delimiter. Duplicate entries are omitted from this list.

renovate.json

@@ -5,7 +5,7 @@
     ":preserveSemverRanges",
     ":disableDependencyDashboard"
   ],
-  "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt"],
+  "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py"],
   "pip_requirements": {
     "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"]
   }

samples/snippets/noxfile.py

@@ -207,8 +207,8 @@ def _session_tests(
     session: nox.sessions.Session, post_install: Callable = None
 ) -> None:
     # check for presence of tests
-    test_list = glob.glob("*_test.py") + glob.glob("test_*.py")
-    test_list.extend(glob.glob("tests"))
+    test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob("**/test_*.py", recursive=True)
+    test_list.extend(glob.glob("**/tests", recursive=True))
 
     if len(test_list) == 0:
         print("No tests found, skipping directory.")

samples/snippets/requirements-test.txt

@@ -1,3 +1,3 @@
-pytest==7.1.2
+pytest==7.1.3
 mock==4.0.3
 backoff==2.1.2

samples/snippets/requirements.txt

@@ -1,4 +1,4 @@
-google-cloud-pubsub==2.13.6
+google-cloud-pubsub==2.13.7
 google-cloud-storage==2.5.0
 pandas===1.3.5; python_version == '3.7'
-pandas==1.4.4; python_version >= '3.8'
+pandas==1.5.0; python_version >= '3.8'

samples/snippets/storage_list_files.py

@@ -29,6 +29,7 @@ def list_blobs(bucket_name):
     # Note: Client.list_blobs requires at least package version 1.17.0.
     blobs = storage_client.list_blobs(bucket_name)
 
+    # Note: The call returns a response only when the iterator is consumed.
     for blob in blobs:
         print(blob.name)
 

samples/snippets/storage_list_files_with_prefix.py

@@ -53,6 +53,7 @@ def list_blobs_with_prefix(bucket_name, prefix, delimiter=None):
     # Note: Client.list_blobs requires at least package version 1.17.0.
     blobs = storage_client.list_blobs(bucket_name, prefix=prefix, delimiter=delimiter)
 
+    # Note: The call returns a response only when the iterator is consumed.
     print("Blobs:")
     for blob in blobs:
         print(blob.name)

tests/system/_helpers.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import time
 
 from google.api_core import exceptions
 
@@ -63,16 +64,16 @@ def _no_retetion_period(bucket):
 
 
 retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy)
-retry_no_event_based_hold = RetryInstanceState(_no_event_based_hold, max_tries=10)
-retry_has_kms_key_name = RetryInstanceState(_has_kms_key_name, max_tries=10)
+retry_no_event_based_hold = RetryInstanceState(_no_event_based_hold, max_tries=5)
+retry_has_kms_key_name = RetryInstanceState(_has_kms_key_name, max_tries=5)
 retry_has_retention_expiration = RetryInstanceState(
-    _has_retention_expiration, max_tries=10
+    _has_retention_expiration, max_tries=5
 )
 retry_no_retention_expiration = RetryInstanceState(
-    _no_retention_expiration, max_tries=10
+    _no_retention_expiration, max_tries=5
 )
-retry_has_retention_period = RetryInstanceState(_has_retetion_period, max_tries=10)
-retry_no_retention_period = RetryInstanceState(_no_retetion_period, max_tries=10)
+retry_has_retention_period = RetryInstanceState(_has_retetion_period, max_tries=5)
+retry_no_retention_period = RetryInstanceState(_no_retetion_period, max_tries=5)
 
 
 def unique_name(prefix):
@@ -106,3 +107,10 @@ def delete_bucket(bucket):
     retry = RetryErrors(errors, max_tries=15)
     retry(empty_bucket)(bucket)
     retry(bucket.delete)(force=True)
+
+
+def await_config_changes_propagate(sec=3):
+    # Changes to the bucket will be readable immediately after writing,
+    # but configuration changes may take time to propagate.
+    # See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch
+    time.sleep(sec)

tests/system/conftest.py

@@ -165,6 +165,22 @@ def signing_bucket(storage_client, signing_bucket_name):
     _helpers.delete_bucket(bucket)
 
 
+@pytest.fixture(scope="session")
+def default_ebh_bucket_name():
+    return _helpers.unique_name("gcp-systest-default-ebh")
+
+
+@pytest.fixture(scope="session")
+def default_ebh_bucket(storage_client, default_ebh_bucket_name):
+    bucket = storage_client.bucket(default_ebh_bucket_name)
+    bucket.default_event_based_hold = True
+    _helpers.retry_429_503(bucket.create)()
+
+    yield bucket
+
+    _helpers.delete_bucket(bucket)
+
+
 @pytest.fixture(scope="function")
 def buckets_to_delete():
     buckets_to_delete = []

tests/system/test_bucket.py

@@ -682,16 +682,10 @@ def test_bucket_w_retention_period(
 
 def test_bucket_w_default_event_based_hold(
     storage_client,
-    buckets_to_delete,
     blobs_to_delete,
+    default_ebh_bucket,
 ):
-    bucket_name = _helpers.unique_name("w-def-ebh")
-    bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket_name)
-    buckets_to_delete.append(bucket)
-
-    bucket.default_event_based_hold = True
-    bucket.patch()
-
+    bucket = storage_client.get_bucket(default_ebh_bucket)
     assert bucket.default_event_based_hold
     assert bucket.retention_period is None
     assert bucket.retention_policy_effective_time is None
@@ -725,6 +719,10 @@ def test_bucket_w_default_event_based_hold(
     assert bucket.retention_policy_effective_time is None
     assert not bucket.retention_policy_locked
 
+    # Changes to the bucket will be readable immediately after writing,
+    # but configuration changes may take time to propagate.
+    _helpers.await_config_changes_propagate()
+
     blob.upload_from_string(payload)
 
     # https://github.com/googleapis/python-storage/issues/435
@@ -870,6 +868,7 @@ def test_ubla_set_unset_preserves_acls(
     # Clear UBLA
     bucket.iam_configuration.uniform_bucket_level_access_enabled = False
     bucket.patch()
+    _helpers.await_config_changes_propagate()
 
     # Query ACLs after clearing UBLA
     bucket.acl.reload()

tests/system/test_kms_integration.py

@@ -137,10 +137,6 @@ def test_bucket_w_default_kms_key_name(
     file_data,
 ):
     blob_name = "default-kms-key-name"
-    override_blob_name = "override-default-kms-key-name"
-    alt_blob_name = "alt-default-kms-key-name"
-    cleartext_blob_name = "cleartext"
-
     info = file_data["simple"]
 
     with open(info["path"], "rb") as file_obj:
@@ -150,6 +146,10 @@ def test_bucket_w_default_kms_key_name(
     kms_bucket.patch()
     assert kms_bucket.default_kms_key_name == kms_key_name
 
+    # Changes to the bucket will be readable immediately after writing,
+    # but configuration changes may take time to propagate.
+    _helpers.await_config_changes_propagate()
+
     defaulted_blob = kms_bucket.blob(blob_name)
     defaulted_blob.upload_from_filename(info["path"])
     blobs_to_delete.append(defaulted_blob)
@@ -159,34 +159,15 @@ def test_bucket_w_default_kms_key_name(
     # We don't know the current version of the key.
     assert defaulted_blob.kms_key_name.startswith(kms_key_name)
 
-    override_blob = kms_bucket.blob(override_blob_name, kms_key_name=alt_kms_key_name)
-    override_blob.upload_from_filename(info["path"])
-    blobs_to_delete.append(override_blob)
-
-    assert override_blob.download_as_bytes() == payload
-    # We don't know the current version of the key.
-    assert override_blob.kms_key_name.startswith(alt_kms_key_name)
-
+    # Test changing the default KMS key.
     kms_bucket.default_kms_key_name = alt_kms_key_name
     kms_bucket.patch()
+    assert kms_bucket.default_kms_key_name == alt_kms_key_name
 
-    alt_blob = kms_bucket.blob(alt_blob_name)
-    alt_blob.upload_from_filename(info["path"])
-    blobs_to_delete.append(alt_blob)
-
-    assert alt_blob.download_as_bytes() == payload
-    # We don't know the current version of the key.
-    assert alt_blob.kms_key_name.startswith(alt_kms_key_name)
-
+    # Test removing the default KMS key.
     kms_bucket.default_kms_key_name = None
     kms_bucket.patch()
-
-    cleartext_blob = kms_bucket.blob(cleartext_blob_name)
-    cleartext_blob.upload_from_filename(info["path"])
-    blobs_to_delete.append(cleartext_blob)
-
-    assert cleartext_blob.download_as_bytes() == payload
-    assert cleartext_blob.kms_key_name is None
+    assert kms_bucket.default_kms_key_name is None
 
 
 def test_blob_rewrite_rotate_csek_to_cmek(
@@ -240,29 +221,40 @@ def test_blob_upload_w_bucket_cmek_enabled(
     kms_bucket,
     blobs_to_delete,
     kms_key_name,
-    file_data,
+    alt_kms_key_name,
 ):
     blob_name = "test-blob"
+    override_blob_name = "override-default-kms-key-name"
     payload = b"DEADBEEF"
     alt_payload = b"NEWDEADBEEF"
 
     kms_bucket.default_kms_key_name = kms_key_name
     kms_bucket.patch()
     assert kms_bucket.default_kms_key_name == kms_key_name
 
+    # Changes to the bucket will be readable immediately after writing,
+    # but configuration changes may take time to propagate.
+    _helpers.await_config_changes_propagate()
+
     blob = kms_bucket.blob(blob_name)
     blob.upload_from_string(payload)
     blobs_to_delete.append(blob)
 
     _helpers.retry_429_harder(_helpers.retry_has_kms_key_name(blob.reload))()
-    # We don't know the current version of the key.
     assert blob.kms_key_name.startswith(kms_key_name)
 
     blob.upload_from_string(alt_payload, if_generation_match=blob.generation)
-
     assert blob.download_as_bytes() == alt_payload
 
+    # Test the specific key is used to encrypt the object if you have both
+    # a default KMS key set on your bucket and a specific key included in your request.
+    override_blob = kms_bucket.blob(override_blob_name, kms_key_name=alt_kms_key_name)
+    override_blob.upload_from_string(payload)
+    blobs_to_delete.append(override_blob)
+
+    assert override_blob.download_as_bytes() == payload
+    assert override_blob.kms_key_name.startswith(alt_kms_key_name)
+
     kms_bucket.default_kms_key_name = None
     _helpers.retry_429_harder(kms_bucket.patch)()
-
     assert kms_bucket.default_kms_key_name is None