added support for generation-based query and delete of blobs.

gcloud/storage/_helpers.py

@@ -22,7 +22,7 @@
 
 
 class _PropertyMixin(object):
- """Abstract mixin for cloud storage classes with associated propertties.
+ """Abstract mixin for cloud storage classes with associated properties.
 
  Non-abstract subclasses should implement:
  - client

gcloud/storage/blob.py

@@ -67,12 +67,13 @@ class Blob(_PropertyMixin):
  _CHUNK_SIZE_MULTIPLE = 256 * 1024
  """Number (256 KB, in bytes) that must divide the chunk size."""
 
- def __init__(self, name, bucket, chunk_size=None):
+ def __init__(self, name, bucket, chunk_size=None, generation=None):
  super(Blob, self).__init__(name=name)
 
  self.chunk_size = chunk_size # Check that setter accepts value.
  self.bucket = bucket
  self._acl = ObjectACL(self)
+ self.generation = generation
 
  @property
  def chunk_size(self):
@@ -138,6 +139,22 @@ def path(self):
 
  return self.path_helper(self.bucket.path, self.name)
 
+ @property
+ def path_with_params(self):
+ """Getter property for the URL path to this Blob, with version.
+
+ :rtype: tuple of ``path`` (a string) and ``params`` (a dictionary)
+ :returns: the URL path to this blob and a dictionary with the
+ generation that can be used in query_params for
+ connection.api_request
+ """
+
+ params = {}
+ if self.generation is not None:
+ params = {'generation': self.generation}
+
+ return (self.path, params)
+
  @property
  def client(self):
  """The client bound to this blob."""
@@ -253,6 +270,9 @@ def exists(self, client=None):
  # We only need the status code (200 or not) so we seek to
  # minimize the returned payload.
  query_params = {'fields': 'name'}
+ if self.generation is not None:
+ query_params['generation'] = self.generation
+
  # We intentionally pass `_target_object=None` since fields=name
  # would limit the local properties.
  client.connection.api_request(method='GET', path=self.path,
@@ -278,7 +298,8 @@ def delete(self, client=None):
  (propagated from
  :meth:`gcloud.storage.bucket.Bucket.delete_blob`).
  """
- return self.bucket.delete_blob(self.name, client=client)
+ return self.bucket.delete_blob(self.name, client=client,
+ generation=self.generation)
 
  def download_to_file(self, file_obj, encryption_key=None, client=None):
  """Download the contents of this blob into a file-like object.
@@ -743,6 +764,19 @@ def generation(self):
  if generation is not None:
  return int(generation)
 
+ @generation.setter
+ def generation(self, value):
+ """Set the generation for this blob.
+
+ See: https://cloud.google.com/storage/docs/json_api/v1/objects
+
+ :type value: integer or ``NoneType``
+ :param value: the generation value for this blob. Setting this
+ value is useful when trying to retrieve specific
+ versions of a blob.
+ """
+ self._patch_property('generation', value)
+
  @property
  def id(self):
  """Retrieve the ID for the object.

gcloud/storage/bucket.py

@@ -103,7 +103,7 @@ def client(self):
  """The client bound to this bucket."""
  return self._client
 
- def blob(self, blob_name, chunk_size=None):
+ def blob(self, blob_name, chunk_size=None, generation=None):
  """Factory constructor for blob object.
 
  .. note::
@@ -118,10 +118,16 @@ def blob(self, blob_name, chunk_size=None):
  (1 MB). This must be a multiple of 256 KB per the
  API specification.
 
+ :type generation: integer
+ :param generation: The desired generation of the blob object. This
+ parameter affects generation based queries on
+ buckets that support versioning.
+
  :rtype: :class:`gcloud.storage.blob.Blob`
  :returns: The blob object created.
  """
- return Blob(name=blob_name, bucket=self, chunk_size=chunk_size)
+ return Blob(name=blob_name, bucket=self, chunk_size=chunk_size,
+ generation=generation)
 
  def exists(self, client=None):
  """Determines whether or not this bucket exists.
@@ -205,7 +211,7 @@ def path(self):
 
  return self.path_helper(self.name)
 
- def get_blob(self, blob_name, client=None):
+ def get_blob(self, blob_name, client=None, generation=None):
  """Get a blob object by name.
 
  This will return None if the blob doesn't exist::
@@ -217,6 +223,10 @@ def get_blob(self, blob_name, client=None):
  <Blob: my-bucket, /path/to/blob.txt>
  >>> print bucket.get_blob('/does-not-exist.txt')
  None
+ >>> print bucket.get_blob(
+ ... '/path/to/versioned_blob.txt',
+ ... generation=generation_id)
+ <Blob: my-bucket, /path/to/versioned_blob.txt>
 
  :type blob_name: string
  :param blob_name: The name of the blob to retrieve.
@@ -225,14 +235,20 @@ def get_blob(self, blob_name, client=None):
  :param client: Optional. The client to use. If not passed, falls back
  to the ``client`` stored on the current bucket.
 
+ :type generation: int
+ :param generation: Optional. The generation id to retrieve in a bucket
+ that supports versioning.
+
  :rtype: :class:`gcloud.storage.blob.Blob` or None
  :returns: The blob object if it exists, otherwise None.
  """
  client = self._require_client(client)
- blob = Blob(bucket=self, name=blob_name)
+ blob = Blob(bucket=self, name=blob_name, generation=generation)
+ blob_path, query_params = blob.path_with_params
  try:
  response = client.connection.api_request(
- method='GET', path=blob.path, _target_object=blob)
+ method='GET', path=blob_path,
+ query_params=query_params, _target_object=blob)
  # NOTE: We assume response.get('name') matches `blob_name`.
  blob._set_properties(response)
  # NOTE: This will not fail immediately in a batch. However, when
@@ -360,7 +376,7 @@ def delete(self, force=False, client=None):
  client.connection.api_request(method='DELETE', path=self.path,
  _target_object=None)
 
- def delete_blob(self, blob_name, client=None):
+ def delete_blob(self, blob_name, client=None, generation=None):
  """Deletes a blob from the current bucket.
 
  If the blob isn't found (backend 404), raises a
@@ -387,6 +403,10 @@ def delete_blob(self, blob_name, client=None):
  :param client: Optional. The client to use. If not passed, falls back
  to the ``client`` stored on the current bucket.
 
+ :type generation: int
+ :param generation: Optional. The generation of this object to delete.
+ Only works on buckets with versioning enabled.
+
  :raises: :class:`gcloud.exceptions.NotFound` (to suppress
  the exception, call ``delete_blobs``, passing a no-op
  ``on_error`` callback, e.g.::
@@ -395,10 +415,15 @@ def delete_blob(self, blob_name, client=None):
  """
  client = self._require_client(client)
  blob_path = Blob.path_helper(self.path, blob_name)
+ query_params = {}
+ if generation is not None:
+ query_params = {'generation': generation}
+
  # We intentionally pass `_target_object=None` since a DELETE
  # request has no response value (whether in a standard request or
  # in a batch request).
  client.connection.api_request(method='DELETE', path=blob_path,
+ query_params=query_params,
  _target_object=None)
 
  def delete_blobs(self, blobs, on_error=None, client=None):
@@ -426,15 +451,21 @@ def delete_blobs(self, blobs, on_error=None, client=None):
  blob_name = blob
  if not isinstance(blob_name, six.string_types):
  blob_name = blob.name
- self.delete_blob(blob_name, client=client)
+
+ generation = None
+ if hasattr(blob, 'generation'):
+ generation = blob.generation
+
+ self.delete_blob(blob_name, client=client,
+ generation=generation)
  except NotFound:
  if on_error is not None:
  on_error(blob)
  else:
  raise
 
  def copy_blob(self, blob, destination_bucket, new_name=None,
- client=None):
+ client=None, versions=False):
  """Copy the given blob to the given bucket, optionally with a new name.
 
  :type blob: :class:`gcloud.storage.blob.Blob`
@@ -451,18 +482,38 @@ def copy_blob(self, blob, destination_bucket, new_name=None,
  :param client: Optional. The client to use. If not passed, falls back
  to the ``client`` stored on the current bucket.
 
+ :type versions: boolean
+ :param versions: Optional. Copy each version.
+
  :rtype: :class:`gcloud.storage.blob.Blob`
- :returns: The new Blob.
+ :returns: The new Blob if versions is ``False``, or will return
+ a list of new blob versions, and their old blob version
+ counterparts.
  """
  client = self._require_client(client)
  if new_name is None:
  new_name = blob.name
- new_blob = Blob(bucket=destination_bucket, name=new_name)
- api_path = blob.path + '/copyTo' + new_blob.path
- copy_result = client.connection.api_request(
- method='POST', path=api_path, _target_object=new_blob)
- new_blob._set_properties(copy_result)
- return new_blob
+
+ tmp_blob = Blob(bucket=destination_bucket, name=new_name)
+ api_path = blob.path + '/copyTo' + tmp_blob.path
+ del tmp_blob
+
+ # TODO(tsinha): Support multi-page results from list_blobs
+ old_blobs = list(self.list_blobs(prefix=blob.name, versions=versions))
+ new_blobs = []
+ for old_blob in old_blobs:
+ new_blob = Blob(bucket=destination_bucket, name=new_name)
+ copy_result = client.connection.api_request(
+ method='POST', path=api_path,
+ query_params={'sourceGeneration': old_blob.generation},
+ _target_object=new_blob)
+ new_blob._set_properties(copy_result)
+ new_blobs.append(new_blob)
+
+ if versions:
+ return (new_blobs, old_blobs)
+ else:
+ return new_blobs[0]
 
  def rename_blob(self, blob, new_name, client=None):
  """Rename the given blob using copy and delete operations.
@@ -487,11 +538,19 @@ def rename_blob(self, blob, new_name, client=None):
  to the ``client`` stored on the current bucket.
 
  :rtype: :class:`Blob`
- :returns: The newly-renamed blob.
- """
- new_blob = self.copy_blob(blob, self, new_name, client=client)
- blob.delete(client=client)
- return new_blob
+ :returns: The newly-renamed blob if bucket versioning is off (or
+ there is only one version), otherwise will return blobs
+ for each newly-renamed version.
+ """
+ new_blobs, old_blobs = self.copy_blob(blob, self, new_name,
+ client=client, versions=True)
+ for old_blob in old_blobs:
+ old_blob.delete(client=client)
+
+ if len(new_blobs) == 1:
+ return new_blobs[0]
+ else:
+ return new_blobs
 
  @property
  def cors(self):

gcloud/storage/test_blob.py

@@ -278,6 +278,38 @@ def test_exists_hit(self):
  bucket._blobs[BLOB_NAME] = 1
  self.assertTrue(blob.exists())
 
+ def test_exists_hit_w_generation(self):
+ from six.moves.http_client import OK
+ BLOB_NAME = 'blob-name'
+ GENERATION = 999
+ found_response = {'status': OK}
+ connection = _Connection(found_response)
+ client = _Client(connection)
+ bucket = _Bucket(client)
+ blob = self._makeOne(BLOB_NAME, bucket=bucket,
+ properties={'generation': GENERATION})
+ bucket._blobs[BLOB_NAME] = 1
+ self.assertTrue(blob.exists())
+ kw, = connection._requested
+ self.assertEqual(kw['method'], 'GET')
+ self.assertEqual(kw['query_params']['generation'], GENERATION)
+
+ def test_exists_hit_w_none_generation(self):
+ from six.moves.http_client import OK
+ BLOB_NAME = 'blob-name'
+ GENERATION = None
+ found_response = {'status': OK}
+ connection = _Connection(found_response)
+ client = _Client(connection)
+ bucket = _Bucket(client)
+ blob = self._makeOne(BLOB_NAME, bucket=bucket,
+ properties={'generation': GENERATION})
+ bucket._blobs[BLOB_NAME] = 1
+ self.assertTrue(blob.exists())
+ kw, = connection._requested
+ self.assertEqual(kw['method'], 'GET')
+ self.assertNotIn('generation', kw['query_params'])
+
  def test_delete(self):
  from six.moves.http_client import NOT_FOUND
  BLOB_NAME = 'blob-name'
@@ -291,6 +323,42 @@ def test_delete(self):
  self.assertFalse(blob.exists())
  self.assertEqual(bucket._deleted, [(BLOB_NAME, None)])
 
+ def test_delete_w_generation(self):
+ from six.moves.http_client import NOT_FOUND
+ BLOB_NAME = 'blob-name'
+ GENERATION = 999
+ not_found_response = {'status': NOT_FOUND}
+ connection = _Connection(not_found_response)
+ client = _Client(connection)
+ bucket = _Bucket(client)
+ blob = self._makeOne(BLOB_NAME, bucket=bucket,
+ properties={'generation': GENERATION})
+ bucket._blobs[BLOB_NAME] = 1
+ blob.delete()
+ self.assertFalse(blob.exists())
+ self.assertEqual(bucket._deleted, [(BLOB_NAME, None)])
+ kw, = connection._requested
+ self.assertEqual(kw['method'], 'GET')
+ self.assertEqual(kw['query_params']['generation'], GENERATION)
+
+ def test_delete_w_none_generation(self):
+ from six.moves.http_client import NOT_FOUND
+ BLOB_NAME = 'blob-name'
+ GENERATION = None
+ not_found_response = {'status': NOT_FOUND}
+ connection = _Connection(not_found_response)
+ client = _Client(connection)
+ bucket = _Bucket(client)
+ blob = self._makeOne(BLOB_NAME, bucket=bucket,
+ properties={'generation': GENERATION})
+ bucket._blobs[BLOB_NAME] = 1
+ blob.delete()
+ self.assertFalse(blob.exists())
+ self.assertEqual(bucket._deleted, [(BLOB_NAME, None)])
+ kw, = connection._requested
+ self.assertEqual(kw['method'], 'GET')
+ self.assertNotIn('generation', kw['query_params'])
+
  def test_download_to_file_wo_media_link(self):
  from six.moves.http_client import OK
  from six.moves.http_client import PARTIAL_CONTENT
@@ -1372,7 +1440,8 @@ def __init__(self, client=None):
  self._copied = []
  self._deleted = []
 
- def delete_blob(self, blob_name, client=None):
+ def delete_blob(self, blob_name, client=None,
+ generation=None): # pylint: disable=W0613
  del self._blobs[blob_name]
  self._deleted.append((blob_name, client))