#260: break datastore cycles via factories

gcloud/datastore/__init__.py

@@ -7,8 +7,10 @@
 ... 'long-email@googleapis.com',
 ... '/path/to/private.key')
 >>> # Then do other things...
->>> query = dataset.query().kind('EntityKind')
->>> entity = dataset.entity('EntityKind')
+>>> from gcloud.datastore.entity import Entity
+>>> from gcloud.datastore.query import Query
+>>> query = Query(dataset, 'EntityKind')
+>>> entity = Entity(dataset, 'EntityKind')
 
 The main concepts with this API are:
 
@@ -32,6 +34,14 @@
  which represents a lookup or search over the rows in the datastore.
 """
 
+# import submodules which register factories.
+import gcloud.datastore.connection
+import gcloud.datastore.dataset
+import gcloud.datastore.entity
+import gcloud.datastore.key
+import gcloud.datastore.query
+import gcloud.datastore.transaction
+
 __version__ = '0.1.2'
 
 SCOPE = ('https://www.googleapis.com/auth/datastore ',
@@ -80,7 +90,9 @@ def get_dataset(dataset_id, client_email, private_key_path):
  >>> from gcloud import datastore
  >>> dataset = datastore.get_dataset('dataset-id', email, key_path)
  >>> # Now you can do things with the dataset.
- >>> dataset.query().kind('TestKind').fetch()
+ >>> from gcloud.datastore.query import Query
+ >>> query = Query(dataset, 'TestKind')
+ >>> query.fetch()
  [...]
 
  :type dataset_id: string

gcloud/datastore/_helpers.py

@@ -8,12 +8,53 @@
 from google.protobuf.internal.type_checkers import Int64ValueChecker
 import pytz
 
-from gcloud.datastore.entity import Entity
-from gcloud.datastore.key import Key
 
 INT_VALUE_CHECKER = Int64ValueChecker()
 
 
+class DuplicateFactory(Exception):
+ """Conflicting factory registration."""
+
+
+class InvalidFactory(Exception):
+ """Unknown factory invocation."""
+
+
+class _FactoryRegistry(object):
+ """Single registry for named factories.
+
+ This registry provides an API for modules to instantiate objects from
+ other modules without needing to import them directly, allowing us
+ to break circular import chains.
+ """
+
+ _MARKER = object()
+
+ def __init__(self):
+ self._registered = {}
+
+ def register(self, name, factory):
+ """Register a factory by name."""
+ if self._registered.get(name) not in (None, factory):
+ raise DuplicateFactory(name)
+ self._registered[name] = factory
+
+ def get(self, name, default=_MARKER):
+ """Look up a factory by name."""
+ found = self._registered.get(name, default)
+ if found is self._MARKER:
+ raise InvalidFactory(name)
+ return found
+
+ def invoke(self, name, *args, **kw):
+ """Look up and call a factory by name."""
+ return self.get(name)(*args, **kw)
+
+
+_FACTORIES = _FactoryRegistry() # singleton
+del _FactoryRegistry
+
+
 def _get_protobuf_attribute_and_value(val):
  """Given a value, return the protobuf attribute name and proper value.
 
@@ -47,6 +88,8 @@ def _get_protobuf_attribute_and_value(val):
 
  :returns: A tuple of the attribute name and proper value type.
  """
+ key_class = _FACTORIES.get('Key')
+ entity_class = _FACTORIES.get('Entity')
 
  if isinstance(val, datetime.datetime):
  name = 'timestamp_microseconds'
@@ -58,8 +101,6 @@ def _get_protobuf_attribute_and_value(val):
  val = val.astimezone(pytz.utc)
  # Convert the datetime to a microsecond timestamp.
  value = long(calendar.timegm(val.timetuple()) * 1e6) + val.microsecond
- elif isinstance(val, Key):
- name, value = 'key', val.to_protobuf()
  elif isinstance(val, bool):
  name, value = 'boolean', val
  elif isinstance(val, float):
@@ -71,10 +112,12 @@ def _get_protobuf_attribute_and_value(val):
  name, value = 'string', val
  elif isinstance(val, (bytes, str)):
  name, value = 'blob', val
- elif isinstance(val, Entity):
- name, value = 'entity', val
  elif isinstance(val, list):
  name, value = 'list', val
+ elif key_class and isinstance(val, key_class):
+ name, value = 'key', val.to_protobuf()
+ elif entity_class and isinstance(val, entity_class):
+ name, value = 'entity', val
  else:
  raise ValueError("Unknown protobuf attr type %s" % type(val))
 
@@ -105,7 +148,7 @@ def _get_value_from_value_pb(value_pb):
  result = naive.replace(tzinfo=pytz.utc)
 
  elif value_pb.HasField('key_value'):
- result = Key.from_protobuf(value_pb.key_value)
+ result = _FACTORIES.invoke('Key_from_protobuf', value_pb.key_value)
 
  elif value_pb.HasField('boolean_value'):
  result = value_pb.boolean_value
@@ -123,7 +166,8 @@ def _get_value_from_value_pb(value_pb):
  result = value_pb.blob_value
 
  elif value_pb.HasField('entity_value'):
- result = Entity.from_protobuf(value_pb.entity_value)
+ result = _FACTORIES.invoke(
+ 'Entity_from_protobuf', value_pb.entity_value)
 
  elif value_pb.list_value:
  result = [_get_value_from_value_pb(x) for x in value_pb.list_value]

gcloud/datastore/connection.py

@@ -210,9 +210,10 @@ def run_query(self, dataset_id, query_pb, namespace=None):
  uses this method to fetch data:
 
  >>> from gcloud import datastore
+ >>> from gcloud.datastore.query import Query
  >>> connection = datastore.get_connection(email, key_path)
  >>> dataset = connection.dataset('dataset-id')
- >>> query = dataset.query().kind('MyKind').filter('property =', 'val')
+ >>> query = Query(dataset, 'MyKind').filter('property =', 'val')
 
  Using the `fetch`` method...
 

gcloud/datastore/dataset.py

@@ -1,5 +1,7 @@
 """Create / interact with gcloud datastore datasets."""
 
+from gcloud.datastore import _helpers
+
 
 class Dataset(object):
  """A dataset in the Cloud Datastore.
@@ -60,34 +62,6 @@ def id(self):
 
  return self._id
 
- def query(self, *args, **kwargs):
- """Create a query bound to this dataset.
-
- :param args: positional arguments, passed through to the Query
-
- :param kw: keyword arguments, passed through to the Query
-
- :rtype: :class:`gcloud.datastore.query.Query`
- :returns: a new Query instance, bound to this dataset.
- """
- # This import is here to avoid circular references.
- from gcloud.datastore.query import Query
- kwargs['dataset'] = self
- return Query(*args, **kwargs)
-
- def entity(self, kind):
- """Create an entity bound to this dataset.
-
- :type kind: string
- :param kind: the "kind" of the new entity.
-
- :rtype: :class:`gcloud.datastore.entity.Entity`
- :returns: a new Entity instance, bound to this dataset.
- """
- # This import is here to avoid circular references.
- from gcloud.datastore.entity import Entity
- return Entity(dataset=self, kind=kind)
-
  def transaction(self, *args, **kwargs):
  """Create a transaction bound to this dataset.
 
@@ -98,10 +72,8 @@ def transaction(self, *args, **kwargs):
  :rtype: :class:`gcloud.datastore.transaction.Transaction`
  :returns: a new Transaction instance, bound to this dataset.
  """
- # This import is here to avoid circular references.
- from gcloud.datastore.transaction import Transaction
  kwargs['dataset'] = self
- return Transaction(*args, **kwargs)
+ return _helpers._FACTORIES.invoke('Transaction', *args, **kwargs)
 
  def get_entity(self, key):
  """Retrieves entity from the dataset, along with its attributes.
@@ -125,15 +97,16 @@ def get_entities(self, keys):
  :rtype: list of :class:`gcloud.datastore.entity.Entity`
  :return: The requested entities.
  """
- # This import is here to avoid circular references.
- from gcloud.datastore.entity import Entity
-
  entity_pbs = self.connection().lookup(
  dataset_id=self.id(),
  key_pbs=[k.to_protobuf() for k in keys]
  )
 
  entities = []
  for entity_pb in entity_pbs:
- entities.append(Entity.from_protobuf(entity_pb, dataset=self))
+ entities.append(_helpers._FACTORIES.invoke(
+ 'Entity_from_protobuf', entity_pb, dataset=self))
  return entities
+
+
+_helpers._FACTORIES.register('Dataset', Dataset)

gcloud/datastore/demo/demo.py

@@ -6,10 +6,13 @@
 
 # Let's start by importing the demo module and getting a dataset:
 from gcloud.datastore import demo
+from gcloud.datastore.entity import Entity
+from gcloud.datastore.query import Query
+
 dataset = demo.get_dataset()
 
 # Let's create a new entity of type "Thing" and name it 'Toy':
-toy = dataset.entity('Thing')
+toy = Entity(dataset, 'Thing')
 toy.update({'name': 'Toy'})
 
 # Now let's save it to our datastore:
@@ -26,7 +29,7 @@
 
 # Now let's try a more advanced query.
 # We'll start by look at all Thing entities:
-query = dataset.query().kind('Thing')
+query = Query(dataset, 'Thing')
 
 # Let's look at the first two.
 print query.limit(2).fetch()
@@ -42,13 +45,13 @@
 # (Check the official docs for explanations of what's happening here.)
 with dataset.transaction():
  print 'Creating and savng an entity...'
- thing = dataset.entity('Thing')
+ thing = Entity(dataset, 'Thing')
  thing.key(thing.key().name('foo'))
  thing['age'] = 10
  thing.save()
 
  print 'Creating and saving another entity...'
- thing2 = dataset.entity('Thing')
+ thing2 = Entity(dataset, 'Thing')
  thing2.key(thing2.key().name('bar'))
  thing2['age'] = 15
  thing2.save()
@@ -60,7 +63,7 @@
 
 # To rollback a transaction, just call .rollback()
 with dataset.transaction() as t:
- thing = dataset.entity('Thing')
+ thing = Entity(dataset, 'Thing')
  thing.key(thing.key().name('another'))
  thing.save()
  t.rollback()
@@ -72,7 +75,7 @@
 # Remember, a key won't be complete until the transaction is commited.
 # That is, while inside the transaction block, thing.key() will be incomplete.
 with dataset.transaction():
- thing = dataset.entity('Thing')
+ thing = Entity(dataset, 'Thing')
  thing.save()
  print thing.key() # This will be partial
 

gcloud/datastore/entity.py

@@ -16,7 +16,7 @@
 """
 
 from gcloud.datastore import datastore_v1_pb2 as datastore_pb
-from gcloud.datastore.key import Key
+from gcloud.datastore import _helpers
 
 
 class NoKey(RuntimeError):
@@ -39,15 +39,10 @@ class Entity(dict):
  This means you could take an existing entity and change the key
  to duplicate the object.
 
-This can be used on its own, however it is likely easier to use
- the shortcut methods provided by :class:`gcloud.datastore.dataset.Dataset`
+Entities can be constructed directly, or obtained via
+ the "lookup" methods provided by :class:`gcloud.datastore.dataset.Dataset`
  such as:
 
- - :func:`gcloud.datastore.dataset.Dataset.entity` to create a new entity.
-
- >>> dataset.entity('MyEntityKind')
- <Entity[{'kind': 'MyEntityKind'}] {}>
-
  - :func:`gcloud.datastore.dataset.Dataset.get_entity`
  to retrieve an existing entity.
 
@@ -74,7 +69,7 @@ def __init__(self, dataset=None, kind=None):
  super(Entity, self).__init__()
  self._dataset = dataset
  if kind:
- self._key = Key().kind(kind)
+ self._key = _helpers._FACTORIES.invoke('Key').kind(kind)
  else:
  self._key = None
 
@@ -159,11 +154,7 @@ def from_protobuf(cls, pb, dataset=None):
  :returns: The :class:`Entity` derived from the
  :class:`gcloud.datastore.datastore_v1_pb2.Entity`.
  """
-
- # This is here to avoid circular imports.
- from gcloud.datastore import _helpers
-
- key = Key.from_protobuf(pb.key)
+ key = _helpers._FACTORIES.invoke('Key_from_protobuf', pb.key)
  entity = cls.from_key(key, dataset)
 
  for property_pb in pb.property:
@@ -248,7 +239,8 @@ def save(self):
  transaction.add_auto_id_entity(self)
 
  if isinstance(key_pb, datastore_pb.Key):
- updated_key = Key.from_protobuf(key_pb)
+ updated_key = _helpers._FACTORIES.invoke(
+ 'Key_from_protobuf', key_pb)
  # Update the path (which may have been altered).
  self._key = key.path(updated_key.path())
 
@@ -275,3 +267,7 @@ def __repr__(self):
  super(Entity, self).__repr__())
  else:
  return '<Entity %s>' % (super(Entity, self).__repr__())
+
+
+_helpers._FACTORIES.register('Entity', Entity)
+_helpers._FACTORIES.register('Entity_from_protobuf', Entity.from_protobuf)

gcloud/datastore/key.py

@@ -4,6 +4,7 @@
 from itertools import izip
 
 from gcloud.datastore import datastore_v1_pb2 as datastore_pb
+from gcloud.datastore import _helpers
 
 
 class Key(object):
@@ -251,3 +252,8 @@ def parent(self):
 
  def __repr__(self):
  return '<Key%s>' % self.path()
+
+
+_helpers._FACTORIES.register('Key', Key)
+_helpers._FACTORIES.register('Key_from_protobuf', Key.from_protobuf)
+_helpers._FACTORIES.register('Key_from_path', Key.from_path)