Feat: Add "auto" checksum option and make default

README.rst

@@ -72,6 +72,17 @@ setup.py file. Applications which do not import directly from
 `google-resumable-media` can safely disregard this dependency. This backwards
 compatibility feature will be removed in a future major version update.
 
+Checksum Defaults
+~~~~~~~~~~~~~~~~~
+
+In Python Storage 3.0, uploads and downloads now have a default of "auto" where
+applicable. "Auto" will use crc32c checksums, except for unusual cases where the
+fast (C extension) crc32c implementation is not available, in which case it will
+use md5 instead. Before Python Storage 3.0, the default was md5 for most
+downloads and None for most uploads. Note that ranged downloads ("start" or
+"end" set) still do not support any checksumming, and some features in
+`transfer_manager.py` still support crc32c only.
+
 Miscellaneous
 ~~~~~~~~~~~~~
 

google/cloud/storage/_media/_download.py

@@ -130,16 +130,28 @@ class Download(DownloadBase):
  appropriate checksum (for instance in the case of transcoded or
  ranged downloads where the remote service does not know the
  correct checksum) an INFO-level log will be emitted. Supported
- values are "md5", "crc32c" and None.
+ values are "md5", "crc32c", "auto" and None. The default is "auto",
+ which will try to detect if the C extension for crc32c is installed
+ and fall back to md5 otherwise.
  """
 
  def __init__(
- self, media_url, stream=None, start=None, end=None, headers=None, checksum="md5"
+ self,
+ media_url,
+ stream=None,
+ start=None,
+ end=None,
+ headers=None,
+ checksum="auto",
  ):
  super(Download, self).__init__(
  media_url, stream=stream, start=start, end=end, headers=headers
  )
  self.checksum = checksum
+ if self.checksum == "auto":
+ self.checksum = (
+ "crc32c" if _helpers._is_crc32c_available_and_fast() else "md5"
+ )
  self._bytes_downloaded = 0
  self._expected_checksum = None
  self._checksum_object = None

google/cloud/storage/_media/_helpers.py

@@ -20,7 +20,6 @@
 import hashlib
 import logging
 import random
-import warnings
 
 from urllib.parse import parse_qs
 from urllib.parse import urlencode
@@ -142,43 +141,6 @@ def calculate_retry_wait(base_wait, max_sleep, multiplier=2.0):
  return new_base_wait, new_base_wait + 0.001 * jitter_ms
 
 
-def _get_crc32c_object():
- """Get crc32c object
- Attempt to use the Google-CRC32c package. If it isn't available, try
- to use CRCMod. CRCMod might be using a 'slow' varietal. If so, warn...
- """
- try:
- import google_crc32c # type: ignore
-
- crc_obj = google_crc32c.Checksum()
- except ImportError:
- try:
- import crcmod # type: ignore
-
- crc_obj = crcmod.predefined.Crc("crc-32c")
- _is_fast_crcmod()
-
- except ImportError:
- raise ImportError("Failed to import either `google-crc32c` or `crcmod`")
-
- return crc_obj
-
-
-def _is_fast_crcmod():
- # Determine if this is using the slow form of crcmod.
- nested_crcmod = __import__(
- "crcmod.crcmod",
- globals(),
- locals(),
- ["_usingExtension"],
- 0,
- )
- fast_crc = getattr(nested_crcmod, "_usingExtension", False)
- if not fast_crc:
- warnings.warn(_SLOW_CRC32C_WARNING, RuntimeWarning, stacklevel=2)
- return fast_crc
-
-
 def _get_metadata_key(checksum_type):
  if checksum_type == "md5":
  return "md5Hash"
@@ -231,10 +193,7 @@ def _get_expected_checksum(response, get_headers, media_url, checksum_type):
  _LOGGER.info(msg)
  checksum_object = _DoNothingHash()
  else:
- if checksum_type == "md5":
- checksum_object = hashlib.md5()
- else:
- checksum_object = _get_crc32c_object()
+ checksum_object = _get_checksum_object(checksum_type)
  else:
  expected_checksum = None
  checksum_object = _DoNothingHash()
@@ -331,13 +290,33 @@ def _get_checksum_object(checksum_type):
  if checksum_type == "md5":
  return hashlib.md5()
  elif checksum_type == "crc32c":
- return _get_crc32c_object()
+ # In order to support platforms that don't have google_crc32c
+ # support, only perform the import on demand.
+ import google_crc32c
+
+ return google_crc32c.Checksum()
  elif checksum_type is None:
  return None
  else:
  raise ValueError("checksum must be ``'md5'``, ``'crc32c'`` or ``None``")
 
 
+def _is_crc32c_available_and_fast():
+ """Return True if the google_crc32c C extension is installed.
+
+ Return False if either the package is not installed, or if only the
+ pure-Python version is installed.
+ """
+ try:
+ import google_crc32c
+
+ if google_crc32c.implementation == "c":
+ return True
+ except Exception:
+ pass
+ return False
+
+
 def _parse_generation_header(response, get_headers):
  """Parses the generation header from an ``X-Goog-Generation`` value.
 

google/cloud/storage/_media/_upload.py

@@ -249,19 +249,25 @@ class MultipartUpload(UploadBase):
  upload_url (str): The URL where the content will be uploaded.
  headers (Optional[Mapping[str, str]]): Extra headers that should
  be sent with the request, e.g. headers for encrypted data.
- checksum (Optional([str])): The type of checksum to compute to verify
+ checksum Optional([str]): The type of checksum to compute to verify
  the integrity of the object. The request metadata will be amended
  to include the computed value. Using this option will override a
- manually-set checksum value. Supported values are "md5", "crc32c"
- and None. The default is None.
+ manually-set checksum value. Supported values are "md5",
+ "crc32c", "auto", and None. The default is "auto", which will try
+ to detect if the C extension for crc32c is installed and fall back
+ to md5 otherwise.
 
  Attributes:
  upload_url (str): The URL where the content will be uploaded.
  """
 
- def __init__(self, upload_url, headers=None, checksum=None):
+ def __init__(self, upload_url, headers=None, checksum="auto"):
  super(MultipartUpload, self).__init__(upload_url, headers=headers)
  self._checksum_type = checksum
+ if self._checksum_type == "auto":
+ self._checksum_type = (
+ "crc32c" if _helpers._is_crc32c_available_and_fast() else "md5"
+ )
 
  def _prepare_request(self, data, metadata, content_type):
  """Prepare the contents of an HTTP request.
@@ -355,13 +361,15 @@ class ResumableUpload(UploadBase):
  chunk_size (int): The size of each chunk used to upload the resource.
  headers (Optional[Mapping[str, str]]): Extra headers that should
  be sent with every request.
- checksum (Optional([str])): The type of checksum to compute to verify
+ checksum Optional([str]): The type of checksum to compute to verify
  the integrity of the object. After the upload is complete, the
- server-computed checksum of the resulting object will be read
+ server-computed checksum of the resulting object will be checked
  and google.cloud.storage.exceptions.DataCorruption will be raised on
  a mismatch. The corrupted file will not be deleted from the remote
- host automatically. Supported values are "md5", "crc32c" and None.
- The default is None.
+ host automatically. Supported values are "md5", "crc32c", "auto",
+ and None. The default is "auto", which will try to detect if the C
+ extension for crc32c is installed and fall back to md5 otherwise.
+
 
  Attributes:
  upload_url (str): The URL where the content will be uploaded.
@@ -371,7 +379,7 @@ class ResumableUpload(UploadBase):
  :data:`.UPLOAD_CHUNK_SIZE`.
  """
 
- def __init__(self, upload_url, chunk_size, checksum=None, headers=None):
+ def __init__(self, upload_url, chunk_size, checksum="auto", headers=None):
  super(ResumableUpload, self).__init__(upload_url, headers=headers)
  if chunk_size % UPLOAD_CHUNK_SIZE != 0:
  raise ValueError(
@@ -383,6 +391,10 @@ def __init__(self, upload_url, chunk_size, checksum=None, headers=None):
  self._bytes_uploaded = 0
  self._bytes_checksummed = 0
  self._checksum_type = checksum
+ if self._checksum_type == "auto":
+ self._checksum_type = (
+ "crc32c" if _helpers._is_crc32c_available_and_fast() else "md5"
+ )
  self._checksum_object = None
  self._total_bytes = None
  self._resumable_url = None
@@ -1185,9 +1197,10 @@ class XMLMPUPart(UploadBase):
  be sent with every request.
  checksum (Optional([str])): The type of checksum to compute to verify
  the integrity of the object. The request headers will be amended
- to include the computed value. Supported values are "md5", "crc32c"
- and None. The default is None.
-
+ to include the computed value. Supported values are "md5", "crc32c",
+ "auto" and None. The default is "auto", which will try to detect if
+ the C extension for crc32c is installed and fall back to md5
+ otherwise.
  Attributes:
  upload_url (str): The URL of the object (without query parameters).
  upload_id (str): The ID of the upload from the initialization response.
@@ -1208,7 +1221,7 @@ def __init__(
  end,
  part_number,
  headers=None,
- checksum=None,
+ checksum="auto",
  ):
  super().__init__(upload_url, headers=headers)
  self._filename = filename
@@ -1218,6 +1231,10 @@ def __init__(
  self._part_number = part_number
  self._etag = None
  self._checksum_type = checksum
+ if self._checksum_type == "auto":
+ self._checksum_type = (
+ "crc32c" if _helpers._is_crc32c_available_and_fast() else "md5"
+ )
  self._checksum_object = None
 
  @property

google/cloud/storage/_media/requests/download.py

@@ -69,7 +69,9 @@ class Download(_request_helpers.RequestsMixin, _download.Download):
  appropriate checksum (for instance in the case of transcoded or
  ranged downloads where the remote service does not know the
  correct checksum) an INFO-level log will be emitted. Supported
- values are "md5", "crc32c" and None. The default is "md5".
+ values are "md5", "crc32c", "auto" and None. The default is "auto",
+ which will try to detect if the C extension for crc32c is installed
+ and fall back to md5 otherwise.
 
  Attributes:
  media_url (str): The URL containing the media to be downloaded.
@@ -263,7 +265,9 @@ class RawDownload(_request_helpers.RawRequestsMixin, _download.Download):
  appropriate checksum (for instance in the case of transcoded or
  ranged downloads where the remote service does not know the
  correct checksum) an INFO-level log will be emitted. Supported
- values are "md5", "crc32c" and None. The default is "md5".
+ values are "md5", "crc32c", "auto" and None. The default is "auto",
+ which will try to detect if the C extension for crc32c is installed
+ and fall back to md5 otherwise.
  Attributes:
  media_url (str): The URL containing the media to be downloaded.
  start (Optional[int]): The first byte in a range to be downloaded.

google/cloud/storage/_media/requests/upload.py

@@ -98,7 +98,9 @@ class MultipartUpload(_request_helpers.RequestsMixin, _upload.MultipartUpload):
  the integrity of the object. The request metadata will be amended
  to include the computed value. Using this option will override a
  manually-set checksum value. Supported values are "md5",
- "crc32c" and None. The default is None.
+ "crc32c", "auto", and None. The default is "auto", which will try
+ to detect if the C extension for crc32c is installed and fall back
+ to md5 otherwise.
 
  Attributes:
  upload_url (str): The URL where the content will be uploaded.
@@ -334,8 +336,9 @@ class ResumableUpload(_request_helpers.RequestsMixin, _upload.ResumableUpload):
  server-computed checksum of the resulting object will be checked
  and google.cloud.storage.exceptions.DataCorruption will be raised on
  a mismatch. The corrupted file will not be deleted from the remote
- host automatically. Supported values are "md5", "crc32c" and None.
- The default is None.
+ host automatically. Supported values are "md5", "crc32c", "auto",
+ and None. The default is "auto", which will try to detect if the C
+ extension for crc32c is installed and fall back to md5 otherwise.
 
  Attributes:
  upload_url (str): The URL where the content will be uploaded.