Adding bigtable-emulator tox environment.

system_tests/bigtable.py

@@ -14,6 +14,7 @@
 
 import datetime
 import operator
+import os
 
 import unittest
 
@@ -28,9 +29,11 @@
 from google.cloud.bigtable.row_filters import RowFilterUnion
 from google.cloud.bigtable.row_data import Cell
 from google.cloud.bigtable.row_data import PartialRowData
+from google.cloud.environment_vars import BIGTABLE_EMULATOR
 
 from retry import RetryErrors
 from retry import RetryResult
+from system_test_utils import EmulatorCreds
 from system_test_utils import unique_resource_id
 
 
@@ -59,6 +62,7 @@ class Config(object):
  """
  CLIENT = None
  INSTANCE = None
+ IN_EMULATOR = False
 
 
 def _wait_until_complete(operation, max_attempts=5):
@@ -91,29 +95,43 @@ def _retry_on_unavailable(exc):
 def setUpModule():
  from google.cloud.exceptions import GrpcRendezvous
 
- Config.CLIENT = Client(admin=True)
+ Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None
+
+ if Config.IN_EMULATOR:
+ credentials = EmulatorCreds()
+ Config.CLIENT = Client(admin=True, credentials=credentials)
+ else:
+ Config.CLIENT = Client(admin=True)
+
  Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, LOCATION_ID)
- retry = RetryErrors(GrpcRendezvous, error_predicate=_retry_on_unavailable)
- instances, failed_locations = retry(Config.CLIENT.list_instances)()
 
- if len(failed_locations) != 0:
- raise ValueError('List instances failed in module set up.')
+ if not Config.IN_EMULATOR:
+ retry = RetryErrors(GrpcRendezvous,
+ error_predicate=_retry_on_unavailable)
+ instances, failed_locations = retry(Config.CLIENT.list_instances)()
 
- EXISTING_INSTANCES[:] = instances
+ if len(failed_locations) != 0:
+ raise ValueError('List instances failed in module set up.')
 
- # After listing, create the test instance.
- created_op = Config.INSTANCE.create()
- if not _wait_until_complete(created_op):
- raise RuntimeError('Instance creation exceed 5 seconds.')
+ EXISTING_INSTANCES[:] = instances
+
+ # After listing, create the test instance.
+ created_op = Config.INSTANCE.create()
+ if not _wait_until_complete(created_op):
+ raise RuntimeError('Instance creation exceed 5 seconds.')
 
 
 def tearDownModule():
- Config.INSTANCE.delete()
+ if not Config.IN_EMULATOR:
+ Config.INSTANCE.delete()
 
 
 class TestInstanceAdminAPI(unittest.TestCase):
 
  def setUp(self):
+ if Config.IN_EMULATOR is not None:
+ self.skipTest(
+ 'Instance Admin API not supported in Bigtable emulator')
  self.instances_to_delete = []
 
  def tearDown(self):
@@ -292,6 +310,13 @@ def tearDownClass(cls):
  # Will also delete any data contained in the table.
  cls._table.delete()
 
+ def _maybe_emulator_skip(self, message):
+ # NOTE: This method is necessary because ``Config.IN_EMULATOR``
+ # is set at runtime rather than import time, which means we
+ # can't use the @unittest.skipIf decorator.
+ if Config.IN_EMULATOR is not None:
+ self.skipTest(message)
+
  def setUp(self):
  self.rows_to_delete = []
 
@@ -401,6 +426,7 @@ def test_read_rows(self):
  self.assertEqual(rows_data.rows, expected_rows)
 
  def test_read_with_label_applied(self):
+ self._maybe_emulator_skip('Labels not supported by Bigtable emulator')
  row = self._table.row(ROW_KEY)
  self.rows_to_delete.append(row)
 

system_tests/pubsub.py

@@ -45,16 +45,18 @@ class Config(object):
  global state.
  """
  CLIENT = None
+ IN_EMULATOR = False
 
 
 def setUpModule():
- if os.getenv(PUBSUB_EMULATOR) is None:
- Config.CLIENT = client.Client()
- else:
+ Config.IN_EMULATOR = os.getenv(PUBSUB_EMULATOR) is not None
+ if Config.IN_EMULATOR:
  credentials = EmulatorCreds()
  http = httplib2.Http() # Un-authorized.
  Config.CLIENT = client.Client(credentials=credentials,
  http=http)
+ else:
+ Config.CLIENT = client.Client()
 
 
 class TestPubsub(unittest.TestCase):
@@ -203,11 +205,10 @@ def suction(self):
  self.assertEqual(message2.attributes['extra'], EXTRA_2)
 
  def _maybe_emulator_skip(self):
- # NOTE: We check at run-time rather than using the @unittest.skipIf
- # decorator. This matches the philosophy behind using
- # setUpModule to determine the environment at run-time
- # rather than at import time.
- if os.getenv(PUBSUB_EMULATOR) is not None:
+ # NOTE: This method is necessary because ``Config.IN_EMULATOR``
+ # is set at runtime rather than import time, which means we
+ # can't use the @unittest.skipIf decorator.
+ if Config.IN_EMULATOR:
  self.skipTest('IAM not supported by Pub/Sub emulator')
 
  def test_topic_iam_policy(self):

system_tests/run_emulator.py

@@ -25,21 +25,27 @@
 
 import psutil
 
+from google.cloud.environment_vars import BIGTABLE_EMULATOR
 from google.cloud.environment_vars import GCD_DATASET
 from google.cloud.environment_vars import GCD_HOST
 from google.cloud.environment_vars import PUBSUB_EMULATOR
 from run_system_test import run_module_tests
 
 
+BIGTABLE = 'bigtable'
+DATASTORE = 'datastore'
+PUBSUB = 'pubsub'
 PACKAGE_INFO = {
- 'datastore': (GCD_DATASET, GCD_HOST),
- 'pubsub': (PUBSUB_EMULATOR,)
+ BIGTABLE: (BIGTABLE_EMULATOR,),
+ DATASTORE: (GCD_DATASET, GCD_HOST),
+ PUBSUB: (PUBSUB_EMULATOR,),
 }
 EXTRA = {
-'datastore': ('--no-legacy',),
+DATASTORE: ('--no-legacy',),
 }
 _DS_READY_LINE = '[datastore] Dev App Server is now running.\n'
 _PS_READY_LINE_PREFIX = '[pubsub] INFO: Server started, listening on '
+_BT_READY_LINE_PREFIX = '[bigtable] Cloud Bigtable emulator running on '
 
 
 def get_parser():
@@ -51,8 +57,8 @@ def get_parser():
  parser = argparse.ArgumentParser(
  description='Run google-cloud system tests against local emulator.')
  parser.add_argument('--package', dest='package',
- choices=('datastore', 'pubsub'),
- default='datastore', help='Package to be tested.')
+ choices=sorted(PACKAGE_INFO.keys()),
+ default=DATASTORE, help='Package to be tested.')
  return parser
 
 
@@ -95,16 +101,18 @@ def datastore_wait_ready(popen):
  emulator_ready = popen.stderr.readline() == _DS_READY_LINE
 
 
-def pubsub_wait_ready(popen):
- """Wait until the pubsub emulator is ready to use.
+def wait_ready_prefix(popen, prefix):
+ """Wait until the a process encounters a line with matching prefix.
 
  :type popen: :class:`subprocess.Popen`
  :param popen: An open subprocess to interact with.
+
+ :type prefix: str
+ :param prefix: The prefix to match
  """
  emulator_ready = False
  while not emulator_ready:
- emulator_ready = popen.stderr.readline().startswith(
- _PS_READY_LINE_PREFIX)
+ emulator_ready = popen.stderr.readline().startswith(prefix)
 
 
 def wait_ready(package, popen):
@@ -117,14 +125,16 @@ def wait_ready(package, popen):
  :param popen: An open subprocess to interact with.
 
  :raises: :class:`KeyError` if the ``package`` is not among
- ``datastore``, ``pubsub``.
+ ``datastore``, ``pubsub`` or ``bigtable``.
  """
- if package == 'datastore':
+ if package == DATASTORE:
  datastore_wait_ready(popen)
- elif package == 'pubsub':
- pubsub_wait_ready(popen)
+ elif package == PUBSUB:
+ wait_ready_prefix(popen, _PS_READY_LINE_PREFIX)
+ elif package == BIGTABLE:
+ wait_ready_prefix(popen, _BT_READY_LINE_PREFIX)
  else:
- raise KeyError('')
+ raise KeyError('Package not supported', package)
 
 
 def cleanup(pid):

tox.ini

@@ -112,23 +112,34 @@ commands =
  python {toxinidir}/system_tests/attempt_system_tests.py {posargs}
 passenv = {[testenv:system-tests]passenv}
 
-[testenv:datastore-emulator]
-basepython =
- python2.7
-commands =
- python {toxinidir}/system_tests/run_emulator.py --package=datastore
+[emulator]
+deps =
+ {[testenv]deps}
+ psutil
 setenv =
  GOOGLE_CLOUD_NO_PRINT=true
 passenv =
  GOOGLE_CLOUD_DISABLE_GRPC
-deps =
- {[testenv]deps}
- psutil
+emulatorcmd =
+ python {toxinidir}/system_tests/run_emulator.py
+
+[testenv:datastore-emulator]
+commands =
+ {[emulator]emulatorcmd} --package=datastore
+setenv = {[emulator]setenv}
+passenv = {[emulator]passenv}
+deps = {[emulator]deps}
 
 [testenv:pubsub-emulator]
-basepython =
- python2.7
 commands =
- python {toxinidir}/system_tests/run_emulator.py --package=pubsub
-passenv = GOOGLE_CLOUD_*
-deps = {[testenv:datastore-emulator]deps}
+ {[emulator]emulatorcmd} --package=pubsub
+setenv = {[emulator]setenv}
+passenv = {[emulator]passenv}
+deps = {[emulator]deps}
+
+[testenv:bigtable-emulator]
+commands =
+ {[emulator]emulatorcmd} --package=bigtable
+setenv = {[emulator]setenv}
+passenv = {[emulator]passenv}
+deps = {[emulator]deps}