feat: send entire object checksum in the final api call of resumable upload (#1654)

feat: send entire object checksum in the final api call of resumable
upload
fixes b/461994245
---------
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: chandra-siri

google/cloud/storage/_media/_upload.py

@@ -688,6 +688,13 @@ def _prepare_request(self):
 _CONTENT_TYPE_HEADER: self._content_type,
 _helpers.CONTENT_RANGE_HEADER: content_range,
  }
+if (start_byte+len(payload) ==self._total_bytes) and (
+self._checksum_objectisnotNone
+ ):
+local_checksum=_helpers.prepare_checksum_digest(
+self._checksum_object.digest()
+ )
+headers["x-goog-hash"] =f"{self._checksum_type}={local_checksum}"
 return_PUT, self.resumable_url, payload, headers
 
 def_update_checksum(self, start_byte, payload):

tests/resumable_media/system/requests/test_upload.py

@@ -27,7 +27,6 @@
 importgoogle.cloud.storage._media.requestsasresumable_requests
 fromgoogle.cloud.storage._mediaimport_helpers
 from .. importutils
-fromgoogle.cloud.storage._mediaimport_upload
 fromgoogle.cloud.storage.exceptionsimportInvalidResponse
 fromgoogle.cloud.storage.exceptionsimportDataCorruption
 
@@ -372,29 +371,6 @@ def test_resumable_upload_with_headers(
 _resumable_upload_helper(authorized_transport, img_stream, cleanup, headers=headers)
 
 
-@pytest.mark.parametrize("checksum", ["md5", "crc32c"])
-deftest_resumable_upload_with_bad_checksum(
-authorized_transport, img_stream, bucket, cleanup, checksum
-):
-fake_checksum_object=_helpers._get_checksum_object(checksum)
-fake_checksum_object.update(b"bad data")
-fake_prepared_checksum_digest=_helpers.prepare_checksum_digest(
-fake_checksum_object.digest()
- )
-withmock.patch.object(
-_helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest
- ):
-withpytest.raises(DataCorruption) asexc_info:
-_resumable_upload_helper(
-authorized_transport, img_stream, cleanup, checksum=checksum
- )
-expected_checksums= {"md5": "1bsd83IYNug8hd+V1ING3Q==", "crc32c": "YQGPxA=="}
-expected_message=_upload._UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
-checksum.upper(), fake_prepared_checksum_digest, expected_checksums[checksum]
- )
-assertexc_info.value.args[0] ==expected_message
-
-
 deftest_resumable_upload_bad_chunk_size(authorized_transport, img_stream):
 blob_name=os.path.basename(img_stream.name)
 # Create the actual upload object.

tests/unit/test_blob.py

@@ -3049,15 +3049,22 @@ def test__initiate_resumable_upload_with_client_custom_headers(self):
 self._initiate_resumable_helper(client=client)
 
 def_make_resumable_transport(
-self, headers1, headers2, headers3, total_bytes, data_corruption=False
+self,
+headers1,
+headers2,
+headers3,
+total_bytes,
+data_corruption=False,
+md5_checksum_value=None,
+crc32c_checksum_value=None,
  ):
 fake_transport=mock.Mock(spec=["request"])
 
 fake_response1=self._mock_requests_response(http.client.OK, headers1)
 fake_response2=self._mock_requests_response(
 http.client.PERMANENT_REDIRECT, headers2
  )
-json_body=f'{{"size": "{total_bytes:d}"}}'
+json_body=json.dumps({"size": str(total_bytes), "md5Hash": md5_checksum_value, "crc32c": crc32c_checksum_value})
 ifdata_corruption:
 fake_response3=DataCorruption(None)
 else:
@@ -3151,6 +3158,9 @@ def _do_resumable_upload_call2(
 if_metageneration_match=None,
 if_metageneration_not_match=None,
 timeout=None,
+checksum=None,
+crc32c_checksum_value=None,
+md5_checksum_value=None,
  ):
 # Third mock transport.request() does sends last chunk.
 content_range=f"bytes {blob.chunk_size:d}-{total_bytes-1:d}/{total_bytes:d}"
@@ -3161,6 +3171,11 @@ def _do_resumable_upload_call2(
 "content-type": content_type,
 "content-range": content_range,
  }
+ifchecksum=="crc32c":
+expected_headers["x-goog-hash"] =f"crc32c={crc32c_checksum_value}"
+elifchecksum=="md5":
+expected_headers["x-goog-hash"] =f"md5={md5_checksum_value}"
+
 payload=data[blob.chunk_size :]
 returnmock.call(
 "PUT",
@@ -3181,12 +3196,17 @@ def _do_resumable_helper(
 timeout=None,
 data_corruption=False,
 retry=None,
+checksum=None, # None is also a valid value, when user decides to disable checksum validation.
  ):
 CHUNK_SIZE=256*1024
 USER_AGENT="testing 1.2.3"
 content_type="text/html"
 # Data to be uploaded.
 data=b"<html>"+ (b"A"*CHUNK_SIZE) +b"</html>"
+
+# Data calcuated offline and entered here. (Unit test best practice).
+crc32c_checksum_value="mQ30hg=="
+md5_checksum_value="wajHeg1f2Q2u9afI6fjPOw=="
 total_bytes=len(data)
 ifuse_size:
 size=total_bytes
@@ -3213,6 +3233,8 @@ def _do_resumable_helper(
 headers3,
 total_bytes,
 data_corruption=data_corruption,
+md5_checksum_value=md5_checksum_value,
+crc32c_checksum_value=crc32c_checksum_value,
  )
 
 # Create some mock arguments and call the method under test.
@@ -3247,7 +3269,7 @@ def _do_resumable_helper(
 if_generation_not_match,
 if_metageneration_match,
 if_metageneration_not_match,
-checksum=None,
+checksum=checksum,
 retry=retry,
 **timeout_kwarg,
  )
@@ -3296,6 +3318,9 @@ def _do_resumable_helper(
 if_metageneration_match=if_metageneration_match,
 if_metageneration_not_match=if_metageneration_not_match,
 timeout=expected_timeout,
+checksum=checksum,
+crc32c_checksum_value=crc32c_checksum_value,
+md5_checksum_value=md5_checksum_value,
  )
 self.assertEqual(transport.request.mock_calls, [call0, call1, call2])
 
@@ -3308,6 +3333,12 @@ def test__do_resumable_upload_no_size(self):
 deftest__do_resumable_upload_with_size(self):
 self._do_resumable_helper(use_size=True)
 
+deftest__do_resumable_upload_with_size_with_crc32c_checksum(self):
+self._do_resumable_helper(use_size=True, checksum="crc32c")
+
+deftest__do_resumable_upload_with_size_with_md5_checksum(self):
+self._do_resumable_helper(use_size=True, checksum="md5")
+
 deftest__do_resumable_upload_with_retry(self):
 self._do_resumable_helper(retry=DEFAULT_RETRY)