feat: add support for use_auth_w_custom_endpoint

google/cloud/storage/_helpers.py

@@ -33,17 +33,15 @@
 STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
 """Environment variable defining host for Storage emulator."""
 
-_DEFAULT_STORAGE_HOST = os.getenv(
- "API_ENDPOINT_OVERRIDE", "https://storage.googleapis.com"
-)
+_BASE_STORAGE_URI = "https://storage.googleapis.com"
+"""Base request endpoint URI for JSON API."""
+
+_DEFAULT_STORAGE_HOST = os.getenv("API_ENDPOINT_OVERRIDE", _BASE_STORAGE_URI)
 """Default storage host for JSON API."""
 
 _API_VERSION = os.getenv("API_VERSION_OVERRIDE", "v1")
 """API version of the default storage host"""
 
-_BASE_STORAGE_URI = "storage.googleapis.com"
-"""Base request endpoint URI for JSON API."""
-
 # etag match parameters in snake case and equivalent header
 _ETAG_MATCH_PARAMETERS = (
  ("if_etag_match", "If-Match"),

google/cloud/storage/client.py

@@ -96,6 +96,12 @@ class Client(ClientWithProject):
  :type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict`
  :param client_options: (Optional) Client options used to set user options on the client.
  API Endpoint should be set through client_options.
+
+ :type use_auth_w_custom_endpoint: bool
+ :param use_auth_w_custom_endpoint:
+ (Optional) Whether authentication is required under custom endpoints.
+ If false, uses AnonymousCredentials and bypasses authentication.
+ Defaults to True. Note this is only used when a custom endpoint is set in conjunction.
  """
 
  SCOPE = (
@@ -112,6 +118,7 @@ def __init__(
  _http=None,
  client_info=None,
  client_options=None,
+ use_auth_w_custom_endpoint=True,
  ):
  self._base_connection = None
 
@@ -132,7 +139,7 @@ def __init__(
  # then mTLS logic will be applied to decide which endpoint will be used.
  storage_host = _get_storage_host()
  kw_args["api_endpoint"] = (
- storage_host if storage_host != _DEFAULT_STORAGE_HOST else None
+ storage_host if storage_host != _BASE_STORAGE_URI else None
  )
 
  if client_options:
@@ -144,19 +151,23 @@ def __init__(
  api_endpoint = client_options.api_endpoint
  kw_args["api_endpoint"] = api_endpoint
 
- # Use anonymous credentials and no project when
- # STORAGE_EMULATOR_HOST or a non-default api_endpoint is set.
- if (
- kw_args["api_endpoint"] is not None
- and _BASE_STORAGE_URI not in kw_args["api_endpoint"]
- ):
- if credentials is None:
- credentials = AnonymousCredentials()
- if project is None:
- project = _get_environ_project()
- if project is None:
- no_project = True
- project = "<none>"
+ # If a custom endpoint is set, the client checks for credentials
+ # or finds the default credentials based on the current environment.
+ # Authentication may be bypassed under certain conditions:
+ # (1) STORAGE_EMULATOR_HOST is set (for backwards compatibility), OR
+ # (2) use_auth_w_custom_endpoint is set to False.
+ if kw_args["api_endpoint"] is not None:
+ if (
+ kw_args["api_endpoint"] == storage_host
+ or not use_auth_w_custom_endpoint
+ ):
+ if credentials is None:
+ credentials = AnonymousCredentials()
+ if project is None:
+ project = _get_environ_project()
+ if project is None:
+ no_project = True
+ project = "<none>"
 
  super(Client, self).__init__(
  project=project,
@@ -897,7 +908,7 @@ def create_bucket(
  project = self.project
 
  # Use no project if STORAGE_EMULATOR_HOST is set
- if _BASE_STORAGE_URI not in _get_storage_host():
+ if _get_storage_host() != _DEFAULT_STORAGE_HOST:
  if project is None:
  project = _get_environ_project()
  if project is None:
@@ -1327,7 +1338,7 @@ def list_buckets(
  project = self.project
 
  # Use no project if STORAGE_EMULATOR_HOST is set
- if _BASE_STORAGE_URI not in _get_storage_host():
+ if _get_storage_host() != _DEFAULT_STORAGE_HOST:
  if project is None:
  project = _get_environ_project()
  if project is None:

tests/unit/test_client.py

@@ -28,9 +28,10 @@
 from google.auth.credentials import AnonymousCredentials
 from google.oauth2.service_account import Credentials
 
+from google.cloud.storage import _helpers
 from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
 from google.cloud.storage._helpers import _get_default_headers
-from google.cloud.storage import _helpers
+from google.cloud.storage._http import Connection
 from google.cloud.storage.retry import DEFAULT_RETRY
 from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED
 from tests.unit.test__helpers import GCCL_INVOCATION_TEST_CONST
@@ -119,7 +120,6 @@ def _make_one(self, *args, **kw):
 
  def test_ctor_connection_type(self):
  from google.cloud._http import ClientInfo
- from google.cloud.storage._http import Connection
 
  PROJECT = "PROJECT"
  credentials = _make_credentials()
@@ -179,8 +179,6 @@ def test_ctor_w_client_options_object(self):
  )
 
  def test_ctor_wo_project(self):
- from google.cloud.storage._http import Connection
-
  PROJECT = "PROJECT"
  credentials = _make_credentials(project=PROJECT)
 
@@ -193,8 +191,6 @@ def test_ctor_wo_project(self):
  self.assertEqual(list(client._batch_stack), [])
 
  def test_ctor_w_project_explicit_none(self):
- from google.cloud.storage._http import Connection
-
  credentials = _make_credentials()
 
  client = self._make_one(project=None, credentials=credentials)
@@ -207,7 +203,6 @@ def test_ctor_w_project_explicit_none(self):
 
  def test_ctor_w_client_info(self):
  from google.cloud._http import ClientInfo
- from google.cloud.storage._http import Connection
 
  credentials = _make_credentials()
  client_info = ClientInfo()
@@ -239,8 +234,40 @@ def test_ctor_mtls(self):
  self.assertEqual(client._connection.ALLOW_AUTO_SWITCH_TO_MTLS_URL, False)
  self.assertEqual(client._connection.API_BASE_URL, "http://foo")
 
+ def test_ctor_w_custom_endpoint_use_auth(self):
+ custom_endpoint = "storage-example.p.googleapis.com"
+ client = self._make_one(client_options={"api_endpoint": custom_endpoint})
+ self.assertEqual(client._connection.API_BASE_URL, custom_endpoint)
+ self.assertIsNotNone(client.project)
+ self.assertIsInstance(client._connection, Connection)
+ self.assertIsNotNone(client._connection.credentials)
+ self.assertNotIsInstance(client._connection.credentials, AnonymousCredentials)
+
+ def test_ctor_w_custom_endpoint_bypass_auth(self):
+ custom_endpoint = "storage-example.p.googleapis.com"
+ client = self._make_one(
+ client_options={"api_endpoint": custom_endpoint},
+ use_auth_w_custom_endpoint=False,
+ )
+ self.assertEqual(client._connection.API_BASE_URL, custom_endpoint)
+ self.assertEqual(client.project, None)
+ self.assertIsInstance(client._connection, Connection)
+ self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
+
+ def test_ctor_w_custom_endpoint_w_credentials(self):
+ PROJECT = "PROJECT"
+ custom_endpoint = "storage-example.p.googleapis.com"
+ credentials = _make_credentials(project=PROJECT)
+ client = self._make_one(
+ credentials=credentials, client_options={"api_endpoint": custom_endpoint}
+ )
+ self.assertEqual(client._connection.API_BASE_URL, custom_endpoint)
+ self.assertEqual(client.project, PROJECT)
+ self.assertIsInstance(client._connection, Connection)
+ self.assertIs(client._connection.credentials, credentials)
+
  def test_ctor_w_emulator_wo_project(self):
- # avoids authentication if STORAGE_EMULATOR_ENV_VAR is set
+ # bypasses authentication if STORAGE_EMULATOR_ENV_VAR is set
  host = "http://localhost:8080"
  environ = {STORAGE_EMULATOR_ENV_VAR: host}
  with mock.patch("os.environ", environ):
@@ -250,16 +277,8 @@ def test_ctor_w_emulator_wo_project(self):
  self.assertEqual(client._connection.API_BASE_URL, host)
  self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
 
- # avoids authentication if storage emulator is set through api_endpoint
- client = self._make_one(
- client_options={"api_endpoint": "http://localhost:8080"}
- )
- self.assertIsNone(client.project)
- self.assertEqual(client._connection.API_BASE_URL, host)
- self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
-
  def test_ctor_w_emulator_w_environ_project(self):
- # avoids authentication and infers the project from the environment
+ # bypasses authentication and infers the project from the environment
  host = "http://localhost:8080"
  environ_project = "environ-project"
  environ = {
@@ -289,9 +308,17 @@ def test_ctor_w_emulator_w_project_arg(self):
  self.assertEqual(client._connection.API_BASE_URL, host)
  self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
 
- def test_create_anonymous_client(self):
- from google.cloud.storage._http import Connection
+ def test_ctor_w_emulator_w_credentials(self):
+ host = "http://localhost:8080"
+ environ = {STORAGE_EMULATOR_ENV_VAR: host}
+ credentials = _make_credentials()
+ with mock.patch("os.environ", environ):
+ client = self._make_one(credentials=credentials)
 
+ self.assertEqual(client._connection.API_BASE_URL, host)
+ self.assertIs(client._connection.credentials, credentials)
+
+ def test_create_anonymous_client(self):
  klass = self._get_target_class()
  client = klass.create_anonymous_client()