From 08711895ba037971ac2d60e1b3636c6cbf3bdc0d Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Thu, 19 Jun 2025 14:10:56 +0000 Subject: [PATCH 01/16] Adding support of single shot download --- google/cloud/storage/_media/_download.py | 2 + .../cloud/storage/_media/requests/download.py | 47 +++++++++++++------ google/cloud/storage/blob.py | 44 +++++++++++++++++ google/cloud/storage/client.py | 5 ++ google/cloud/storage/fileio.py | 6 ++- 5 files changed, 87 insertions(+), 17 deletions(-) diff --git a/google/cloud/storage/_media/_download.py b/google/cloud/storage/_media/_download.py index 349ddf30c..8fbaabd5a 100644 --- a/google/cloud/storage/_media/_download.py +++ b/google/cloud/storage/_media/_download.py @@ -169,6 +169,7 @@ def __init__( headers=None, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): super(Download, self).__init__( media_url, stream=stream, start=start, end=end, headers=headers, retry=retry @@ -182,6 +183,7 @@ def __init__( self._expected_checksum = None self._checksum_object = None self._object_generation = None + self._single_shot_download = single_shot_download def _prepare_request(self): """Prepare the contents of an HTTP request. diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index 6222148b3..384834465 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -132,13 +132,22 @@ def _write_to_stream(self, response): # the stream is indeed compressed, this will delegate the checksum # object to the decoder and return a _DoNothingHash here. local_checksum_object = _add_decoder(response.raw, checksum_object) - body_iter = response.iter_content( - chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False - ) - for chunk in body_iter: - self._stream.write(chunk) - self._bytes_downloaded += len(chunk) - local_checksum_object.update(chunk) + + if self._single_shot_download: + # This is useful for smaller files, or when the user wants to + # download the entire file in one go. + content = response.content + self._stream.write(content) + self._bytes_downloaded += len(content) + local_checksum_object.update(content) + else: + body_iter = response.iter_content( + chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False + ) + for chunk in body_iter: + self._stream.write(chunk) + self._bytes_downloaded += len(chunk) + local_checksum_object.update(chunk) # Don't validate the checksum for partial responses. if ( @@ -345,14 +354,22 @@ def _write_to_stream(self, response): checksum_object = self._checksum_object with response: - body_iter = response.raw.stream( - _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False - ) - for chunk in body_iter: - self._stream.write(chunk) - self._bytes_downloaded += len(chunk) - checksum_object.update(chunk) - response._content_consumed = True + if self._single_shot_download: + # This is useful for smaller files, or when the user wants to + # download the entire file in one go. + content = response.raw.read() + self._stream.write(content) + self._bytes_downloaded += len(content) + checksum_object.update(content) + else: + body_iter = response.raw.stream( + _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False + ) + for chunk in body_iter: + self._stream.write(chunk) + self._bytes_downloaded += len(chunk) + checksum_object.update(chunk) + response._content_consumed = True # Don't validate the checksum for partial responses. if ( diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 0d0e8ee80..c5e0c092e 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -987,6 +987,7 @@ def _do_download( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Perform a download without any error handling. @@ -1047,6 +1048,10 @@ def _do_download( See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for information on retry types and how to configure them. + + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. """ extra_attributes = { @@ -1054,6 +1059,7 @@ def _do_download( "download.chunk_size": f"{self.chunk_size}", "download.raw_download": raw_download, "upload.checksum": f"{checksum}", + "download.single_shot_download": single_shot_download, } args = {"timeout": timeout} @@ -1073,6 +1079,7 @@ def _do_download( end=end, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) with create_trace_span( name=f"Storage.{download_class}/consume", @@ -1127,6 +1134,7 @@ def download_to_file( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of this blob into a file-like object. @@ -1222,6 +1230,10 @@ def download_to_file( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + :raises: :class:`google.cloud.exceptions.NotFound` """ with create_trace_span(name="Storage.Blob.downloadToFile"): @@ -1240,6 +1252,7 @@ def download_to_file( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) def _handle_filename_and_download(self, filename, *args, **kwargs): @@ -1285,6 +1298,7 @@ def download_to_filename( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of this blob into a named file. @@ -1370,6 +1384,10 @@ def download_to_filename( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + :raises: :class:`google.cloud.exceptions.NotFound` """ with create_trace_span(name="Storage.Blob.downloadToFilename"): @@ -1388,6 +1406,7 @@ def download_to_filename( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) def download_as_bytes( @@ -1405,6 +1424,7 @@ def download_as_bytes( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of this blob as a bytes object. @@ -1484,6 +1504,10 @@ def download_as_bytes( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + :rtype: bytes :returns: The data stored in this blob. @@ -1507,6 +1531,7 @@ def download_as_bytes( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) return string_buffer.getvalue() @@ -1524,6 +1549,7 @@ def download_as_string( if_metageneration_not_match=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, + single_shot_download=False, ): """(Deprecated) Download the contents of this blob as a bytes object. @@ -1594,6 +1620,10 @@ def download_as_string( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + :rtype: bytes :returns: The data stored in this blob. @@ -1616,6 +1646,7 @@ def download_as_string( if_metageneration_not_match=if_metageneration_not_match, timeout=timeout, retry=retry, + single_shot_download=single_shot_download, ) def download_as_text( @@ -1633,6 +1664,7 @@ def download_as_text( if_metageneration_not_match=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of this blob as text (*not* bytes). @@ -1705,6 +1737,10 @@ def download_as_text( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + :rtype: text :returns: The data stored in this blob, decoded to text. """ @@ -1722,6 +1758,7 @@ def download_as_text( if_metageneration_not_match=if_metageneration_not_match, timeout=timeout, retry=retry, + single_shot_download=single_shot_download, ) if encoding is not None: @@ -4019,6 +4056,7 @@ def open( For downloads only, the following additional arguments are supported: - ``raw_download`` + - ``single_shot_download`` For uploads only, the following additional arguments are supported: @@ -4209,6 +4247,7 @@ def _prep_and_do_download( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, command=None, ): """Download the contents of a blob object into a file-like object. @@ -4294,6 +4333,10 @@ def _prep_and_do_download( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + :type command: str :param command: (Optional) Information about which interface for download was used, @@ -4349,6 +4392,7 @@ def _prep_and_do_download( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) except InvalidResponse as exc: _raise_from_invalid_response(exc) diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index ba94b26fc..2f56d8719 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -1143,6 +1143,7 @@ def download_blob_to_file( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of a blob object or blob URI into a file-like object. @@ -1216,6 +1217,9 @@ def download_blob_to_file( See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for information on retry types and how to configure them. + + single_shot_download (bool): + (Optional) If true, download the object in a single request. """ with create_trace_span(name="Storage.Client.downloadBlobToFile"): if not isinstance(blob_or_uri, Blob): @@ -1236,6 +1240,7 @@ def download_blob_to_file( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) def list_blobs( diff --git a/google/cloud/storage/fileio.py b/google/cloud/storage/fileio.py index 2b4754648..cc2ebacb5 100644 --- a/google/cloud/storage/fileio.py +++ b/google/cloud/storage/fileio.py @@ -35,6 +35,7 @@ "timeout", "retry", "raw_download", + "single_shot_download", } # Valid keyword arguments for upload methods. @@ -99,8 +100,9 @@ class BlobReader(io.BufferedIOBase): - ``if_metageneration_not_match`` - ``timeout`` - ``raw_download`` + - ``single_shot_download`` - Note that download_kwargs (excluding ``raw_download``) are also applied to blob.reload(), + Note that download_kwargs (excluding ``raw_download`` and ``single_shot_download``) are also applied to blob.reload(), if a reload is needed during seek(). """ @@ -177,7 +179,7 @@ def seek(self, pos, whence=0): if self._blob.size is None: reload_kwargs = { - k: v for k, v in self._download_kwargs.items() if k != "raw_download" + k: v for k, v in self._download_kwargs.items() if (k != "raw_download" and k != "single_shot_download") } self._blob.reload(**reload_kwargs) From e45b18b3860e8ff928c62639dcb7c5e4da0a834c Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Fri, 20 Jun 2025 10:45:50 +0000 Subject: [PATCH 02/16] Adding tests in test_blob file --- google/cloud/storage/_media/_download.py | 2 +- .../cloud/storage/_media/requests/download.py | 8 ++-- tests/unit/test_blob.py | 48 ++++++++++++++++--- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/google/cloud/storage/_media/_download.py b/google/cloud/storage/_media/_download.py index 8fbaabd5a..4043388a0 100644 --- a/google/cloud/storage/_media/_download.py +++ b/google/cloud/storage/_media/_download.py @@ -179,11 +179,11 @@ def __init__( self.checksum = ( "crc32c" if _helpers._is_crc32c_available_and_fast() else "md5" ) + self.single_shot_download = single_shot_download self._bytes_downloaded = 0 self._expected_checksum = None self._checksum_object = None self._object_generation = None - self._single_shot_download = single_shot_download def _prepare_request(self): """Prepare the contents of an HTTP request. diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index bff902d73..173301bc3 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -133,10 +133,10 @@ def _write_to_stream(self, response): # object to the decoder and return a _DoNothingHash here. local_checksum_object = _add_decoder(response.raw, checksum_object) - if self._single_shot_download: + if self.single_shot_download: # This is useful for smaller files, or when the user wants to # download the entire file in one go. - content = response.content + content = response.raw.read() self._stream.write(content) self._bytes_downloaded += len(content) local_checksum_object.update(content) @@ -354,7 +354,7 @@ def _write_to_stream(self, response): checksum_object = self._checksum_object with response: - if self._single_shot_download: + if self.single_shot_download: # This is useful for smaller files, or when the user wants to # download the entire file in one go. content = response.raw.read() @@ -369,7 +369,7 @@ def _write_to_stream(self, response): self._stream.write(chunk) self._bytes_downloaded += len(chunk) checksum_object.update(chunk) - response._content_consumed = True + response._content_consumed = True # Don't validate the checksum for partial responses. if ( diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 06ba62220..a742df64e 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -1282,6 +1282,7 @@ def _do_download_helper_wo_chunks( end=3, checksum="auto", retry=retry, + single_shot_download=False, ) else: patched.assert_called_once_with( @@ -1292,6 +1293,7 @@ def _do_download_helper_wo_chunks( end=None, checksum="auto", retry=retry, + single_shot_download=False, ) patched.return_value.consume.assert_called_once_with( @@ -1499,6 +1501,7 @@ def test_download_to_file_with_failure(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_to_file_wo_media_link(self): @@ -1530,6 +1533,7 @@ def test_download_to_file_wo_media_link(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_to_file_w_etag_match(self): @@ -1557,6 +1561,7 @@ def test_download_to_file_w_etag_match(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_to_file_w_generation_match(self): @@ -1584,10 +1589,11 @@ def test_download_to_file_w_generation_match(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def _download_to_file_helper( - self, use_chunks, raw_download, timeout=None, **extra_kwargs + self, use_chunks, raw_download, timeout=None, single_shot_download=False, **extra_kwargs ): blob_name = "blob-name" client = self._make_client() @@ -1612,9 +1618,9 @@ def _download_to_file_helper( with mock.patch.object(blob, "_prep_and_do_download"): if raw_download: - blob.download_to_file(file_obj, raw_download=True, **extra_kwargs) + blob.download_to_file(file_obj, raw_download=True, single_shot_download=single_shot_download, **extra_kwargs) else: - blob.download_to_file(file_obj, **extra_kwargs) + blob.download_to_file(file_obj, single_shot_download=single_shot_download, **extra_kwargs) expected_retry = extra_kwargs.get("retry", DEFAULT_RETRY) blob._prep_and_do_download.assert_called_once_with( @@ -1632,6 +1638,7 @@ def _download_to_file_helper( timeout=expected_timeout, checksum="auto", retry=expected_retry, + single_shot_download=single_shot_download, ) def test_download_to_file_wo_chunks_wo_raw(self): @@ -1643,6 +1650,18 @@ def test_download_to_file_wo_chunks_no_retry(self): def test_download_to_file_w_chunks_wo_raw(self): self._download_to_file_helper(use_chunks=True, raw_download=False) + def test_download_to_file_wo_single_shot_download_wo_raw(self): + self._download_to_file_helper(use_chunks=False, raw_download=False, single_shot_download=False) + + def test_download_to_file_w_single_shot_download_wo_raw(self): + self._download_to_file_helper(use_chunks=False, raw_download=False, single_shot_download=True) + + def test_download_to_file_wo_single_shot_download_w_raw(self): + self._download_to_file_helper(use_chunks=False, raw_download=True, single_shot_download=False) + + def test_download_to_file_w_single_shot_download_w_raw(self): + self._download_to_file_helper(use_chunks=False, raw_download=True, single_shot_download=True) + def test_download_to_file_wo_chunks_w_raw(self): self._download_to_file_helper(use_chunks=False, raw_download=True) @@ -1711,6 +1730,7 @@ def _download_to_filename_helper( timeout=expected_timeout, checksum="auto", retry=expected_retry, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, temp.name) @@ -1767,6 +1787,7 @@ def test_download_to_filename_w_etag_match(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, temp.name) @@ -1800,6 +1821,7 @@ def test_download_to_filename_w_generation_match(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, temp.name) @@ -1842,6 +1864,7 @@ def test_download_to_filename_corrupted(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, filename) @@ -1884,11 +1907,12 @@ def test_download_to_filename_notfound(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, filename) - def _download_as_bytes_helper(self, raw_download, timeout=None, **extra_kwargs): + def _download_as_bytes_helper(self, raw_download, timeout=None, single_shot_download=False, **extra_kwargs): blob_name = "blob-name" client = self._make_client() bucket = _Bucket(client) @@ -1898,12 +1922,12 @@ def _download_as_bytes_helper(self, raw_download, timeout=None, **extra_kwargs): if timeout is None: expected_timeout = self._get_default_timeout() fetched = blob.download_as_bytes( - raw_download=raw_download, **extra_kwargs + raw_download=raw_download, single_shot_download=single_shot_download, **extra_kwargs ) else: expected_timeout = timeout fetched = blob.download_as_bytes( - raw_download=raw_download, timeout=timeout, **extra_kwargs + raw_download=raw_download, timeout=timeout, single_shot_download=single_shot_download, **extra_kwargs ) self.assertEqual(fetched, b"") @@ -1924,6 +1948,7 @@ def _download_as_bytes_helper(self, raw_download, timeout=None, **extra_kwargs): timeout=expected_timeout, checksum="auto", retry=expected_retry, + single_shot_download=single_shot_download, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertIsInstance(stream, io.BytesIO) @@ -1959,6 +1984,7 @@ def test_download_as_bytes_w_etag_match(self): timeout=self._get_default_timeout(), checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_as_bytes_w_generation_match(self): @@ -1989,6 +2015,7 @@ def test_download_as_bytes_w_generation_match(self): timeout=self._get_default_timeout(), checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_as_bytes_wo_raw(self): @@ -2003,6 +2030,12 @@ def test_download_as_bytes_w_raw(self): def test_download_as_byte_w_custom_timeout(self): self._download_as_bytes_helper(raw_download=False, timeout=9.58) + def test_download_as_bytes_wo_single_shot_download(self): + self._download_as_bytes_helper(raw_download=False, retry=None, single_shot_download=False) + + def test_download_as_bytes_w_single_shot_download(self): + self._download_as_bytes_helper(raw_download=False, retry=None, single_shot_download=True) + def _download_as_text_helper( self, raw_download, @@ -2100,6 +2133,7 @@ def _download_as_text_helper( if_metageneration_match=if_metageneration_match, if_metageneration_not_match=if_metageneration_not_match, retry=expected_retry, + single_shot_download=False, ) def test_download_as_text_wo_raw(self): @@ -2226,6 +2260,7 @@ def test_download_as_string(self, mock_warn): timeout=self._get_default_timeout(), checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) mock_warn.assert_any_call( @@ -2264,6 +2299,7 @@ def test_download_as_string_no_retry(self, mock_warn): timeout=self._get_default_timeout(), checksum="auto", retry=None, + single_shot_download=False, ) mock_warn.assert_any_call( From 1c48b6c77d977fe8fae6cbebefcf695bf03515e1 Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Fri, 20 Jun 2025 13:22:22 +0000 Subject: [PATCH 03/16] Adding system tests in the test_blob file --- tests/system/test_blob.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/system/test_blob.py b/tests/system/test_blob.py index 00f218534..2db8c960f 100644 --- a/tests/system/test_blob.py +++ b/tests/system/test_blob.py @@ -1149,3 +1149,23 @@ def test_object_retention_lock(storage_client, buckets_to_delete, blobs_to_delet blob.retention.retain_until_time = None blob.patch(override_unlocked_retention=True) assert blob.retention.mode is None + + +def test_blob_download_as_bytes_single_shot_download( + shared_bucket, blobs_to_delete, file_data, service_account +): + blob_name = f"download-single-shot-{uuid.uuid4().hex}" + info = file_data["simple"] + payload = None + with open(info["path"], "rb") as f: + payload = f.read() + + blob = shared_bucket.blob(blob_name) + blob.upload_from_string(payload) + blobs_to_delete.append(blob) + + result_regular_download = blob.download_as_bytes(single_shot_download=False) + assert result_regular_download == payload + + result_single_shot_download = blob.download_as_bytes(single_shot_download=True) + assert result_single_shot_download == payload From 2a04a4f1628b10f8fe2353ae5aeddc0cbc19c44f Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Mon, 23 Jun 2025 06:26:22 +0000 Subject: [PATCH 04/16] fixing kokoro build --- tests/unit/test_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index b671cc092..a8b1e6ee4 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -2032,6 +2032,7 @@ def _download_blob_to_file_helper( checksum="auto", timeout=_DEFAULT_TIMEOUT, retry=expected_retry, + single_shot_download=False, ) def test_download_blob_to_file_wo_chunks_wo_raw(self): From 40dd69653cd5a9bf88823bf1db8bb494efa03814 Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Mon, 23 Jun 2025 06:55:16 +0000 Subject: [PATCH 05/16] fixing kokoro build --- tests/unit/test_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index a8b1e6ee4..db8094a95 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1866,6 +1866,7 @@ def test_download_blob_to_file_with_failure(self): checksum="auto", timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_blob_to_file_with_uri(self): @@ -1905,6 +1906,7 @@ def test_download_blob_to_file_with_uri(self): checksum="auto", timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_blob_to_file_with_invalid_uri(self): From 2d12b6dd87d9b455eb0afa60bd898c4a5f5483d1 Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Mon, 23 Jun 2025 07:25:51 +0000 Subject: [PATCH 06/16] Add tests in test_download file --- .../system/requests/test_download.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/resumable_media/system/requests/test_download.py b/tests/resumable_media/system/requests/test_download.py index 04c7246f6..15c4405de 100644 --- a/tests/resumable_media/system/requests/test_download.py +++ b/tests/resumable_media/system/requests/test_download.py @@ -286,6 +286,21 @@ def test_download_full(self, add_files, authorized_transport, checksum): assert self._read_response_content(response) == actual_contents check_tombstoned(download, authorized_transport) + @pytest.mark.parametrize("checksum", ["auto", "md5", "crc32c", None]) + def test_single_shot_download_full(self, add_files, authorized_transport, checksum): + for info in ALL_FILES: + actual_contents = self._get_contents(info) + blob_name = get_blob_name(info) + + # Create the actual download object. + media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) + download = self._make_one(media_url, checksum=checksum, single_shot_download=True) + # Consume the resource with single_shot_download enabled. + response = download.consume(authorized_transport) + assert response.status_code == http.client.OK + assert self._read_response_content(response) == actual_contents + check_tombstoned(download, authorized_transport) + def test_download_to_stream(self, add_files, authorized_transport): for info in ALL_FILES: actual_contents = self._get_contents(info) From 37e59d13568a4ae55682e9b8205dbaf3de71bc5e Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Mon, 23 Jun 2025 09:23:05 +0000 Subject: [PATCH 07/16] fixing code coverage --- .../unit/requests/test_download.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/resumable_media/unit/requests/test_download.py b/tests/resumable_media/unit/requests/test_download.py index 568d3238c..4c6f23a94 100644 --- a/tests/resumable_media/unit/requests/test_download.py +++ b/tests/resumable_media/unit/requests/test_download.py @@ -213,6 +213,23 @@ def test__write_to_stream_incomplete_read(self, checksum): in error.args[0] ) + @pytest.mark.parametrize("checksum", ["auto", "md5", "crc32c", None]) + def test__write_to_stream_single_shot_download(self, checksum): + stream = io.BytesIO() + download = download_mod.Download(EXAMPLE_URL, stream=stream, checksum=checksum, single_shot_download=True) + + chunk1 = b"all at once!" + response = _mock_response(chunks=[chunk1], headers={}) + ret_val = download._write_to_stream(response) + + assert ret_val is None + assert stream.getvalue() == chunk1 + assert download._bytes_downloaded == len(chunk1) + + response.__enter__.assert_called_once_with() + response.__exit__.assert_called_once_with(None, None, None) + response.raw.read.assert_called_once_with() + def _consume_helper( self, stream=None, @@ -692,6 +709,22 @@ def test__write_to_stream_incomplete_read(self, checksum): in error.args[0] ) + def test__write_to_stream_single_shot_download(self): + stream = io.BytesIO() + download = download_mod.RawDownload(EXAMPLE_URL, stream=stream, single_shot_download=True) + + chunk1 = b"all at once, raw!" + response = _mock_raw_response(chunks=[chunk1], headers={}) + ret_val = download._write_to_stream(response) + + assert ret_val is None + assert stream.getvalue() == chunk1 + assert download._bytes_downloaded == len(chunk1) + + response.__enter__.assert_called_once_with() + response.__exit__.assert_called_once_with(None, None, None) + response.raw.read.assert_called_once_with() + def _consume_helper( self, stream=None, @@ -1333,6 +1366,7 @@ def _mock_response(status_code=http.client.OK, chunks=None, headers=None): response.__enter__.return_value = response response.__exit__.return_value = None response.iter_content.return_value = iter(chunks) + response.raw.read = mock.Mock(return_value=b"".join(chunks)) return response else: return mock.Mock( @@ -1348,6 +1382,7 @@ def _mock_raw_response(status_code=http.client.OK, chunks=(), headers=None): mock_raw = mock.Mock(headers=headers, spec=["stream"]) mock_raw.stream.return_value = iter(chunks) + mock_raw.read = mock.Mock(return_value=b"".join(chunks)) response = mock.MagicMock( headers=headers, status_code=int(status_code), From 0ce88e7b58c60d41bb8c0e7eb161e61c7710ea50 Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Mon, 23 Jun 2025 14:01:46 +0000 Subject: [PATCH 08/16] File Formatting --- .../cloud/storage/_media/requests/download.py | 3 +- google/cloud/storage/fileio.py | 4 +- .../system/requests/test_download.py | 26 ++++++++- .../unit/requests/test_download.py | 8 ++- tests/unit/test_blob.py | 55 +++++++++++++++---- 5 files changed, 79 insertions(+), 17 deletions(-) diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index 173301bc3..e842c5405 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -142,7 +142,8 @@ def _write_to_stream(self, response): local_checksum_object.update(content) else: body_iter = response.iter_content( - chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False + chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, + decode_unicode=False, ) for chunk in body_iter: self._stream.write(chunk) diff --git a/google/cloud/storage/fileio.py b/google/cloud/storage/fileio.py index 130e3d9d0..7c30f39be 100644 --- a/google/cloud/storage/fileio.py +++ b/google/cloud/storage/fileio.py @@ -179,7 +179,9 @@ def seek(self, pos, whence=0): if self._blob.size is None: reload_kwargs = { - k: v for k, v in self._download_kwargs.items() if (k != "raw_download" and k != "single_shot_download") + k: v + for k, v in self._download_kwargs.items() + if (k != "raw_download" and k != "single_shot_download") } self._blob.reload(**reload_kwargs) diff --git a/tests/resumable_media/system/requests/test_download.py b/tests/resumable_media/system/requests/test_download.py index 15c4405de..a0253ce54 100644 --- a/tests/resumable_media/system/requests/test_download.py +++ b/tests/resumable_media/system/requests/test_download.py @@ -294,7 +294,9 @@ def test_single_shot_download_full(self, add_files, authorized_transport, checks # Create the actual download object. media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) - download = self._make_one(media_url, checksum=checksum, single_shot_download=True) + download = self._make_one( + media_url, checksum=checksum, single_shot_download=True + ) # Consume the resource with single_shot_download enabled. response = download.consume(authorized_transport) assert response.status_code == http.client.OK @@ -321,6 +323,28 @@ def test_download_to_stream(self, add_files, authorized_transport): assert stream.getvalue() == actual_contents check_tombstoned(download, authorized_transport) + def test_single_shot_download_to_stream(self, add_files, authorized_transport): + for info in ALL_FILES: + actual_contents = self._get_contents(info) + blob_name = get_blob_name(info) + + # Create the actual download object. + media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) + stream = io.BytesIO() + download = self._make_one( + media_url, stream=stream, single_shot_download=True + ) + # Consume the resource with single_shot_download enabled. + response = download.consume(authorized_transport) + assert response.status_code == http.client.OK + with pytest.raises(RuntimeError) as exc_info: + getattr(response, "content") + assert exc_info.value.args == (NO_BODY_ERR,) + assert response._content is False + assert response._content_consumed is True + assert stream.getvalue() == actual_contents + check_tombstoned(download, authorized_transport) + def test_download_gzip_w_stored_content_headers( self, add_files, authorized_transport ): diff --git a/tests/resumable_media/unit/requests/test_download.py b/tests/resumable_media/unit/requests/test_download.py index 4c6f23a94..78e652f97 100644 --- a/tests/resumable_media/unit/requests/test_download.py +++ b/tests/resumable_media/unit/requests/test_download.py @@ -216,7 +216,9 @@ def test__write_to_stream_incomplete_read(self, checksum): @pytest.mark.parametrize("checksum", ["auto", "md5", "crc32c", None]) def test__write_to_stream_single_shot_download(self, checksum): stream = io.BytesIO() - download = download_mod.Download(EXAMPLE_URL, stream=stream, checksum=checksum, single_shot_download=True) + download = download_mod.Download( + EXAMPLE_URL, stream=stream, checksum=checksum, single_shot_download=True + ) chunk1 = b"all at once!" response = _mock_response(chunks=[chunk1], headers={}) @@ -711,7 +713,9 @@ def test__write_to_stream_incomplete_read(self, checksum): def test__write_to_stream_single_shot_download(self): stream = io.BytesIO() - download = download_mod.RawDownload(EXAMPLE_URL, stream=stream, single_shot_download=True) + download = download_mod.RawDownload( + EXAMPLE_URL, stream=stream, single_shot_download=True + ) chunk1 = b"all at once, raw!" response = _mock_raw_response(chunks=[chunk1], headers={}) diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index a742df64e..937bebaf5 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -1593,7 +1593,12 @@ def test_download_to_file_w_generation_match(self): ) def _download_to_file_helper( - self, use_chunks, raw_download, timeout=None, single_shot_download=False, **extra_kwargs + self, + use_chunks, + raw_download, + timeout=None, + single_shot_download=False, + **extra_kwargs, ): blob_name = "blob-name" client = self._make_client() @@ -1618,9 +1623,16 @@ def _download_to_file_helper( with mock.patch.object(blob, "_prep_and_do_download"): if raw_download: - blob.download_to_file(file_obj, raw_download=True, single_shot_download=single_shot_download, **extra_kwargs) + blob.download_to_file( + file_obj, + raw_download=True, + single_shot_download=single_shot_download, + **extra_kwargs, + ) else: - blob.download_to_file(file_obj, single_shot_download=single_shot_download, **extra_kwargs) + blob.download_to_file( + file_obj, single_shot_download=single_shot_download, **extra_kwargs + ) expected_retry = extra_kwargs.get("retry", DEFAULT_RETRY) blob._prep_and_do_download.assert_called_once_with( @@ -1651,16 +1663,24 @@ def test_download_to_file_w_chunks_wo_raw(self): self._download_to_file_helper(use_chunks=True, raw_download=False) def test_download_to_file_wo_single_shot_download_wo_raw(self): - self._download_to_file_helper(use_chunks=False, raw_download=False, single_shot_download=False) + self._download_to_file_helper( + use_chunks=False, raw_download=False, single_shot_download=False + ) def test_download_to_file_w_single_shot_download_wo_raw(self): - self._download_to_file_helper(use_chunks=False, raw_download=False, single_shot_download=True) + self._download_to_file_helper( + use_chunks=False, raw_download=False, single_shot_download=True + ) def test_download_to_file_wo_single_shot_download_w_raw(self): - self._download_to_file_helper(use_chunks=False, raw_download=True, single_shot_download=False) + self._download_to_file_helper( + use_chunks=False, raw_download=True, single_shot_download=False + ) def test_download_to_file_w_single_shot_download_w_raw(self): - self._download_to_file_helper(use_chunks=False, raw_download=True, single_shot_download=True) + self._download_to_file_helper( + use_chunks=False, raw_download=True, single_shot_download=True + ) def test_download_to_file_wo_chunks_w_raw(self): self._download_to_file_helper(use_chunks=False, raw_download=True) @@ -1912,7 +1932,9 @@ def test_download_to_filename_notfound(self): stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, filename) - def _download_as_bytes_helper(self, raw_download, timeout=None, single_shot_download=False, **extra_kwargs): + def _download_as_bytes_helper( + self, raw_download, timeout=None, single_shot_download=False, **extra_kwargs + ): blob_name = "blob-name" client = self._make_client() bucket = _Bucket(client) @@ -1922,12 +1944,17 @@ def _download_as_bytes_helper(self, raw_download, timeout=None, single_shot_down if timeout is None: expected_timeout = self._get_default_timeout() fetched = blob.download_as_bytes( - raw_download=raw_download, single_shot_download=single_shot_download, **extra_kwargs + raw_download=raw_download, + single_shot_download=single_shot_download, + **extra_kwargs, ) else: expected_timeout = timeout fetched = blob.download_as_bytes( - raw_download=raw_download, timeout=timeout, single_shot_download=single_shot_download, **extra_kwargs + raw_download=raw_download, + timeout=timeout, + single_shot_download=single_shot_download, + **extra_kwargs, ) self.assertEqual(fetched, b"") @@ -2031,10 +2058,14 @@ def test_download_as_byte_w_custom_timeout(self): self._download_as_bytes_helper(raw_download=False, timeout=9.58) def test_download_as_bytes_wo_single_shot_download(self): - self._download_as_bytes_helper(raw_download=False, retry=None, single_shot_download=False) + self._download_as_bytes_helper( + raw_download=False, retry=None, single_shot_download=False + ) def test_download_as_bytes_w_single_shot_download(self): - self._download_as_bytes_helper(raw_download=False, retry=None, single_shot_download=True) + self._download_as_bytes_helper( + raw_download=False, retry=None, single_shot_download=True + ) def _download_as_text_helper( self, From b7b6b6bc1db2093e4be33e1b7623cd81853753b6 Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Mon, 23 Jun 2025 14:57:45 +0000 Subject: [PATCH 09/16] fix kokoro --- google/cloud/storage/_media/requests/download.py | 1 + 1 file changed, 1 insertion(+) diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index e842c5405..b8a4cc24c 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -140,6 +140,7 @@ def _write_to_stream(self, response): self._stream.write(content) self._bytes_downloaded += len(content) local_checksum_object.update(content) + response._content_consumed = True else: body_iter = response.iter_content( chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, From 2ba5b566b2f5d7f689b0f3998823ec9c27e3c7c7 Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Mon, 23 Jun 2025 17:04:40 +0000 Subject: [PATCH 10/16] minor fix --- google/cloud/storage/_media/requests/download.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index b8a4cc24c..b34dddd33 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -139,7 +139,10 @@ def _write_to_stream(self, response): content = response.raw.read() self._stream.write(content) self._bytes_downloaded += len(content) - local_checksum_object.update(content) + if isinstance(local_checksum_object, _helpers._DoNothingHash): + checksum_object.update(content) + else: + local_checksum_object.update(content) response._content_consumed = True else: body_iter = response.iter_content( From a4ad641df96998f53f71749503cb4d0ca7ad16af Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Tue, 24 Jun 2025 05:40:22 +0000 Subject: [PATCH 11/16] minor fix --- .../cloud/storage/_media/requests/download.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index b34dddd33..560dc7ca2 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -136,13 +136,10 @@ def _write_to_stream(self, response): if self.single_shot_download: # This is useful for smaller files, or when the user wants to # download the entire file in one go. - content = response.raw.read() + content = response.raw.read(decode_content=True) self._stream.write(content) self._bytes_downloaded += len(content) - if isinstance(local_checksum_object, _helpers._DoNothingHash): - checksum_object.update(content) - else: - local_checksum_object.update(content) + local_checksum_object.update(content) response._content_consumed = True else: body_iter = response.iter_content( @@ -699,6 +696,23 @@ def _add_decoder(response_raw, checksum): return checksum +# def _get_decoded_content(raw_content): +# """Get the decoded content from a raw response. + +# Args: +# raw_content (bytes): The raw bytes from the response. + +# Returns: +# bytes: The decoded bytes from ``raw_content``. +# """ +# # If the content is gzip-encoded, decode it. +# if raw_content.headers.get("content-encoding", "").lower() == "gzip": +# return raw_content.read(decode_content=True) +# else: +# return raw_content.read() # type: ignore + + + class _GzipDecoder(urllib3.response.GzipDecoder): """Custom subclass of ``urllib3`` decoder for ``gzip``-ed bytes. From f5d467de51403218fe6074a41ace247a615b58e3 Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Tue, 24 Jun 2025 06:42:15 +0000 Subject: [PATCH 12/16] minor fix --- tests/resumable_media/unit/requests/test_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/resumable_media/unit/requests/test_download.py b/tests/resumable_media/unit/requests/test_download.py index 78e652f97..b17fbb905 100644 --- a/tests/resumable_media/unit/requests/test_download.py +++ b/tests/resumable_media/unit/requests/test_download.py @@ -230,7 +230,7 @@ def test__write_to_stream_single_shot_download(self, checksum): response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) - response.raw.read.assert_called_once_with() + response.raw.read.assert_called_once_with(decode_content=True) def _consume_helper( self, @@ -1370,7 +1370,7 @@ def _mock_response(status_code=http.client.OK, chunks=None, headers=None): response.__enter__.return_value = response response.__exit__.return_value = None response.iter_content.return_value = iter(chunks) - response.raw.read = mock.Mock(return_value=b"".join(chunks)) + response.raw.read = mock.Mock(side_effect=lambda *args, **kwargs: b"".join(chunks)) return response else: return mock.Mock( From 7e8f213ecb6fe7ff172762ca329300e526a04660 Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Tue, 24 Jun 2025 07:08:09 +0000 Subject: [PATCH 13/16] remove commented code --- .../cloud/storage/_media/requests/download.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index 560dc7ca2..2e7c22604 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -696,23 +696,6 @@ def _add_decoder(response_raw, checksum): return checksum -# def _get_decoded_content(raw_content): -# """Get the decoded content from a raw response. - -# Args: -# raw_content (bytes): The raw bytes from the response. - -# Returns: -# bytes: The decoded bytes from ``raw_content``. -# """ -# # If the content is gzip-encoded, decode it. -# if raw_content.headers.get("content-encoding", "").lower() == "gzip": -# return raw_content.read(decode_content=True) -# else: -# return raw_content.read() # type: ignore - - - class _GzipDecoder(urllib3.response.GzipDecoder): """Custom subclass of ``urllib3`` decoder for ``gzip``-ed bytes. From be281fa2648b85897d38b6befdd9e68734aac13d Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Tue, 24 Jun 2025 11:16:13 +0000 Subject: [PATCH 14/16] resolving comments --- google/cloud/storage/_media/requests/download.py | 8 ++++---- google/cloud/storage/blob.py | 14 ++++++++++++++ tests/system/test_blob.py | 1 - 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index 2e7c22604..b8e2758e1 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -133,9 +133,9 @@ def _write_to_stream(self, response): # object to the decoder and return a _DoNothingHash here. local_checksum_object = _add_decoder(response.raw, checksum_object) + # This is useful for smaller files, or when the user wants to + # download the entire file in one go. if self.single_shot_download: - # This is useful for smaller files, or when the user wants to - # download the entire file in one go. content = response.raw.read(decode_content=True) self._stream.write(content) self._bytes_downloaded += len(content) @@ -356,9 +356,9 @@ def _write_to_stream(self, response): checksum_object = self._checksum_object with response: + # This is useful for smaller files, or when the user wants to + # download the entire file in one go. if self.single_shot_download: - # This is useful for smaller files, or when the user wants to - # download the entire file in one go. content = response.raw.read() self._stream.write(content) self._bytes_downloaded += len(content) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index c5e0c092e..998164ca4 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -1052,6 +1052,8 @@ def _do_download( :type single_shot_download: bool :param single_shot_download: (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your user use case. """ extra_attributes = { @@ -1233,6 +1235,8 @@ def download_to_file( :type single_shot_download: bool :param single_shot_download: (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your user use case. :raises: :class:`google.cloud.exceptions.NotFound` """ @@ -1387,6 +1391,8 @@ def download_to_filename( :type single_shot_download: bool :param single_shot_download: (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your user use case. :raises: :class:`google.cloud.exceptions.NotFound` """ @@ -1507,6 +1513,8 @@ def download_as_bytes( :type single_shot_download: bool :param single_shot_download: (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your user use case. :rtype: bytes :returns: The data stored in this blob. @@ -1623,6 +1631,8 @@ def download_as_string( :type single_shot_download: bool :param single_shot_download: (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your user use case. :rtype: bytes :returns: The data stored in this blob. @@ -1740,6 +1750,8 @@ def download_as_text( :type single_shot_download: bool :param single_shot_download: (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your user use case. :rtype: text :returns: The data stored in this blob, decoded to text. @@ -4336,6 +4348,8 @@ def _prep_and_do_download( :type single_shot_download: bool :param single_shot_download: (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your user use case. :type command: str :param command: diff --git a/tests/system/test_blob.py b/tests/system/test_blob.py index 2db8c960f..8b50322ba 100644 --- a/tests/system/test_blob.py +++ b/tests/system/test_blob.py @@ -1156,7 +1156,6 @@ def test_blob_download_as_bytes_single_shot_download( ): blob_name = f"download-single-shot-{uuid.uuid4().hex}" info = file_data["simple"] - payload = None with open(info["path"], "rb") as f: payload = f.read() From 4df2f87823fce3ff0ce530d8d9569433cf35e149 Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Tue, 24 Jun 2025 12:01:31 +0000 Subject: [PATCH 15/16] resolving comments --- google/cloud/storage/blob.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 998164ca4..8f058db2b 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -1053,7 +1053,7 @@ def _do_download( :param single_shot_download: (Optional) If true, download the object in a single request. Caution: Enabling this will increase the memory overload for your application. - Please enable this as per your user use case. + Please enable this as per your use case. """ extra_attributes = { @@ -1236,7 +1236,7 @@ def download_to_file( :param single_shot_download: (Optional) If true, download the object in a single request. Caution: Enabling this will increase the memory overload for your application. - Please enable this as per your user use case. + Please enable this as per your use case. :raises: :class:`google.cloud.exceptions.NotFound` """ @@ -1392,7 +1392,7 @@ def download_to_filename( :param single_shot_download: (Optional) If true, download the object in a single request. Caution: Enabling this will increase the memory overload for your application. - Please enable this as per your user use case. + Please enable this as per your use case. :raises: :class:`google.cloud.exceptions.NotFound` """ @@ -1514,7 +1514,7 @@ def download_as_bytes( :param single_shot_download: (Optional) If true, download the object in a single request. Caution: Enabling this will increase the memory overload for your application. - Please enable this as per your user use case. + Please enable this as per your use case. :rtype: bytes :returns: The data stored in this blob. @@ -1632,7 +1632,7 @@ def download_as_string( :param single_shot_download: (Optional) If true, download the object in a single request. Caution: Enabling this will increase the memory overload for your application. - Please enable this as per your user use case. + Please enable this as per your use case. :rtype: bytes :returns: The data stored in this blob. @@ -1751,7 +1751,7 @@ def download_as_text( :param single_shot_download: (Optional) If true, download the object in a single request. Caution: Enabling this will increase the memory overload for your application. - Please enable this as per your user use case. + Please enable this as per your use case. :rtype: text :returns: The data stored in this blob, decoded to text. @@ -4349,7 +4349,7 @@ def _prep_and_do_download( :param single_shot_download: (Optional) If true, download the object in a single request. Caution: Enabling this will increase the memory overload for your application. - Please enable this as per your user use case. + Please enable this as per your use case. :type command: str :param command: From d314b59dea233da8a130971d83572c621e433efc Mon Sep 17 00:00:00 2001 From: Shubham Kaushal Date: Fri, 27 Jun 2025 05:25:32 +0000 Subject: [PATCH 16/16] resolving comments --- google/cloud/storage/_media/_download.py | 5 ++++- google/cloud/storage/blob.py | 3 +++ tests/resumable_media/system/requests/test_download.py | 5 +++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/google/cloud/storage/_media/_download.py b/google/cloud/storage/_media/_download.py index 4043388a0..422b98041 100644 --- a/google/cloud/storage/_media/_download.py +++ b/google/cloud/storage/_media/_download.py @@ -140,7 +140,7 @@ class Download(DownloadBase): ``start`` to the end of the media. headers (Optional[Mapping[str, str]]): Extra headers that should be sent with the request, e.g. headers for encrypted data. - checksum Optional([str]): The type of checksum to compute to verify + checksum (Optional[str]): The type of checksum to compute to verify the integrity of the object. The response headers must contain a checksum of the requested type. If the headers lack an appropriate checksum (for instance in the case of transcoded or @@ -157,6 +157,9 @@ class Download(DownloadBase): See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for information on retry types and how to configure them. + single_shot_download (Optional[bool]): If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your use case. """ diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 8f058db2b..0eb94fd47 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -1081,6 +1081,9 @@ def _do_download( end=end, checksum=checksum, retry=retry, + # NOTE: single_shot_download is only supported in Download and RawDownload + # classes, i.e., when chunk_size is set to None (the default value). It is + # not supported for chunked downloads. single_shot_download=single_shot_download, ) with create_trace_span( diff --git a/tests/resumable_media/system/requests/test_download.py b/tests/resumable_media/system/requests/test_download.py index a0253ce54..84c44c94c 100644 --- a/tests/resumable_media/system/requests/test_download.py +++ b/tests/resumable_media/system/requests/test_download.py @@ -323,7 +323,8 @@ def test_download_to_stream(self, add_files, authorized_transport): assert stream.getvalue() == actual_contents check_tombstoned(download, authorized_transport) - def test_single_shot_download_to_stream(self, add_files, authorized_transport): + @pytest.mark.parametrize("checksum", ["auto", "md5", "crc32c", None]) + def test_single_shot_download_to_stream(self, add_files, authorized_transport, checksum): for info in ALL_FILES: actual_contents = self._get_contents(info) blob_name = get_blob_name(info) @@ -332,7 +333,7 @@ def test_single_shot_download_to_stream(self, add_files, authorized_transport): media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) stream = io.BytesIO() download = self._make_one( - media_url, stream=stream, single_shot_download=True + media_url, checksum=checksum, stream=stream, single_shot_download=True ) # Consume the resource with single_shot_download enabled. response = download.consume(authorized_transport)