'test_cluster_create' systest flakes with 'TimeoutError'

[tseaver]

From this failed systest build:

_____________________________ test_cluster_create ______________________________

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f4d5c321d30>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7f4d5ef43ee0>
sleep_generator = <generator object exponential_sleep_generator at 0x7f4d5c1b1660>
deadline = 30, on_error = None

 def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
 """Call a function and retry if it fails.

 This is the lowest-level retry helper. Generally, you'll use the
 higher-level retry helper :class:`Retry`.

 Args:
 target(Callable): The function to call and retry. This must be a
 nullary function - apply arguments with `functools.partial`.
 predicate (Callable[Exception]): A callable used to determine if an
 exception raised by the target should be considered retryable.
 It should return True to retry or False otherwise.
 sleep_generator (Iterable[float]): An infinite iterator that determines
 how long to sleep between retries.
 deadline (float): How long to keep retrying the target. The last sleep
 period is shortened as necessary, so that the last retry runs at
 ``deadline`` (and not considerably beyond it).
 on_error (Callable[Exception]): A function to call while processing a
 retryable exception. Any error raised by this function will *not*
 be caught.

 Returns:
 Any: the return value of the target function.

 Raises:
 google.api_core.RetryError: If the deadline is exceeded while retrying.
 ValueError: If the sleep generator stops yielding values.
 Exception: If the target raises a method that isn't retryable.
 """
 if deadline is not None:
 deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
 seconds=deadline
 )
 else:
 deadline_datetime = None

 last_exc = None

 for sleep in sleep_generator:
 try:
> return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:190:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <google.api_core.operation.Operation object at 0x7f4d5c321d30>
retry = <google.api_core.retry.Retry object at 0x7f4d5f0bad60>

 def _done_or_raise(self, retry=DEFAULT_RETRY):
 """Check if the future is done and raise if it's not."""
 kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}

 if not self.done(**kwargs):
> raise _OperationNotComplete()
E google.api_core.future.polling._OperationNotComplete

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:89: _OperationNotComplete

The above exception was the direct cause of the following exception:

self = <google.api_core.operation.Operation object at 0x7f4d5c321d30>
timeout = 30, retry = <google.api_core.retry.Retry object at 0x7f4d5f0bad60>

 def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
 """Poll and wait for the Future to be resolved.

 Args:
 timeout (int):
 How long (in seconds) to wait for the operation to complete.
 If None, wait indefinitely.
 """
 if self._result_set:
 return

 retry_ = self._retry.with_deadline(timeout)

 try:
 kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
> retry_(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

args = (), kwargs = {}
target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f4d5c321d30>>)
sleep_generator = <generator object exponential_sleep_generator at 0x7f4d5c1b1660>

 @functools.wraps(func)
 def retry_wrapped_func(*args, **kwargs):
 """A wrapper that calls target function with retry."""
 target = functools.partial(func, *args, **kwargs)
 sleep_generator = exponential_sleep_generator(
 self._initial, self._maximum, multiplier=self._multiplier
 )
> return retry_target(
 target,
 self._predicate,
 sleep_generator,
 self._deadline,
 on_error=on_error,
 )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:283:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f4d5c321d30>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7f4d5ef43ee0>
sleep_generator = <generator object exponential_sleep_generator at 0x7f4d5c1b1660>
deadline = 30, on_error = None

 def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
 """Call a function and retry if it fails.

 This is the lowest-level retry helper. Generally, you'll use the
 higher-level retry helper :class:`Retry`.

 Args:
 target(Callable): The function to call and retry. This must be a
 nullary function - apply arguments with `functools.partial`.
 predicate (Callable[Exception]): A callable used to determine if an
 exception raised by the target should be considered retryable.
 It should return True to retry or False otherwise.
 sleep_generator (Iterable[float]): An infinite iterator that determines
 how long to sleep between retries.
 deadline (float): How long to keep retrying the target. The last sleep
 period is shortened as necessary, so that the last retry runs at
 ``deadline`` (and not considerably beyond it).
 on_error (Callable[Exception]): A function to call while processing a
 retryable exception. Any error raised by this function will *not*
 be caught.

 Returns:
 Any: the return value of the target function.

 Raises:
 google.api_core.RetryError: If the deadline is exceeded while retrying.
 ValueError: If the sleep generator stops yielding values.
 Exception: If the target raises a method that isn't retryable.
 """
 if deadline is not None:
 deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
 seconds=deadline
 )
 else:
 deadline_datetime = None

 last_exc = None

 for sleep in sleep_generator:
 try:
 return target()

 # pylint: disable=broad-except
 # This function explicitly must deal with broad exceptions.
 except Exception as exc:
 if not predicate(exc):
 raise
 last_exc = exc
 if on_error is not None:
 on_error(exc)

 now = datetime_helpers.utcnow()

 if deadline_datetime is not None:
 if deadline_datetime <= now:
> raise exceptions.RetryError(
 "Deadline of {:.1f}s exceeded while calling {}".format(
 deadline, target
 ),
 last_exc,
 ) from last_exc
E google.api_core.exceptions.RetryError: Deadline of 30.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f4d5c321d30>>), last exception:

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:205: RetryError

During handling of the above exception, another exception occurred:

admin_instance_populated = <google.cloud.bigtable.instance.Instance object at 0x7f4d5eef10a0>
admin_instance_id = 'g-c-p-1634667466967', skip_on_emulator = None

 def test_cluster_create(
 admin_instance_populated, admin_instance_id, skip_on_emulator,
 ):
 alt_cluster_id = f"{admin_instance_id}-c2"
 alt_location_id = "us-central1-f"
 serve_nodes = 2

 cluster_2 = admin_instance_populated.cluster(
 alt_cluster_id,
 location_id=alt_location_id,
 serve_nodes=serve_nodes,
 default_storage_type=(enums.StorageType.SSD),
 )
 operation = cluster_2.create()
> operation.result(timeout=30) # Ensure the operation completes.

tests/system/test_instance_admin.py:576:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:132: in result
 self._blocking_poll(timeout=timeout, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <google.api_core.operation.Operation object at 0x7f4d5c321d30>
timeout = 30, retry = <google.api_core.retry.Retry object at 0x7f4d5f0bad60>

 def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
 """Poll and wait for the Future to be resolved.

 Args:
 timeout (int):
 How long (in seconds) to wait for the operation to complete.
 If None, wait indefinitely.
 """
 if self._result_set:
 return

 retry_ = self._retry.with_deadline(timeout)

 try:
 kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
 retry_(self._done_or_raise)(**kwargs)
 except exceptions.RetryError:
> raise concurrent.futures.TimeoutError(
 "Operation did not complete within the designated " "timeout."
 )
E concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:112: TimeoutError