feat: add cloud.region, request_tag and transaction_tag in span attributes (#1449)

This change enhances observability by introducing these new features:
1. **Cloud Region Attribute**: The `cloud.region` attribute is now added
to all OpenTelemetry spans generated by the Spanner client. This
provides better geographical context for traces, aiding in performance
analysis and debugging across different regions.
2. **Transaction Tag**: The `transaction_tag` set on a `Transaction`
object is now correctly propagated and included in the `Commit` request.
This allows for better end-to-end traceability of transactions.
3. **Request Tag**: This introduces support for `request_tag` on
individual Spanner operations like `read`, `execute_sql`, and
`execute_update`. When a `request_tag` is provided in the
`request_options`, it is now added as a `spanner.request_tag` attribute
to the corresponding OpenTelemetry span. This allows for more granular
tracing and debugging of specific requests within a transaction or a
snapshot.
---------
Co-authored-by: surbhigarg92 <surbhigarg.92@gmail.com>

Author: sinhasubham

google/cloud/spanner_v1/_helpers.py

@@ -20,6 +20,7 @@
 importtime
 importbase64
 importthreading
+importlogging
 
 fromgoogle.protobuf.struct_pb2importListValue
 fromgoogle.protobuf.struct_pb2importValue
@@ -29,16 +30,27 @@
 fromgoogle.api_coreimportdatetime_helpers
 fromgoogle.api_core.exceptionsimportAborted
 fromgoogle.cloud._helpersimport_date_from_iso8601_date
-fromgoogle.cloud.spanner_v1importTypeCode
-fromgoogle.cloud.spanner_v1importExecuteSqlRequest
-fromgoogle.cloud.spanner_v1importJsonObject, Interval
-fromgoogle.cloud.spanner_v1importTransactionOptions
+fromgoogle.cloud.spanner_v1.typesimportExecuteSqlRequest
+fromgoogle.cloud.spanner_v1.typesimportTransactionOptions
+fromgoogle.cloud.spanner_v1.data_typesimportJsonObject, Interval
 fromgoogle.cloud.spanner_v1.request_id_headerimportwith_request_id
+fromgoogle.cloud.spanner_v1.typesimportTypeCode
+
 fromgoogle.rpc.error_details_pb2importRetryInfo
 
 try:
 fromopentelemetry.propagateimportinject
 fromopentelemetry.propagators.textmapimportSetter
+fromopentelemetry.semconv.resourceimportResourceAttributes
+fromopentelemetry.resourcedetectorimportgcp_resource_detector
+fromopentelemetry.resourcedetector.gcp_resource_detectorimport (
+GoogleCloudResourceDetector,
+ )
+
+# Overwrite the requests timeout for the detector.
+# This is necessary as the client will wait the full timeout if the
+# code is not run in a GCP environment, with the location endpoints available.
+gcp_resource_detector._TIMEOUT_SEC=0.2
 
 HAS_OPENTELEMETRY_INSTALLED=True
 exceptImportError:
@@ -55,6 +67,12 @@
 +"numeric has a whole component with precision {}"
 )
 
+GOOGLE_CLOUD_REGION_GLOBAL="global"
+
+log=logging.getLogger(__name__)
+
+_cloud_region: str=None
+
 
 ifHAS_OPENTELEMETRY_INSTALLED:
 
@@ -79,6 +97,33 @@ def set(self, carrier: List[Tuple[str, str]], key: str, value: str) -> None:
 carrier.append((key, value))
 
 
+def_get_cloud_region() ->str:
+"""Get the location of the resource, caching the result.
+
+ Returns:
+ str: The location of the resource. If OpenTelemetry is not installed, returns a global region.
+ """
+global_cloud_region
+if_cloud_regionisnotNone:
+return_cloud_region
+
+try:
+detector=GoogleCloudResourceDetector()
+resources=detector.detect()
+ifResourceAttributes.CLOUD_REGIONinresources.attributes:
+_cloud_region=resources.attributes[ResourceAttributes.CLOUD_REGION]
+else:
+_cloud_region=GOOGLE_CLOUD_REGION_GLOBAL
+exceptExceptionase:
+log.warning(
+"Failed to detect GCP resource location for Spanner metrics, defaulting to 'global'. Error: %s",
+e,
+ )
+_cloud_region=GOOGLE_CLOUD_REGION_GLOBAL
+
+return_cloud_region
+
+
 def_try_to_coerce_bytes(bytestring):
 """Try to coerce a byte string into the right thing based on Python
  version and whether or not it is base64 encoded.

google/cloud/spanner_v1/_opentelemetry_tracing.py

@@ -21,6 +21,7 @@
 fromgoogle.cloud.spanner_v1importSpannerClient
 fromgoogle.cloud.spanner_v1importgapic_version
 fromgoogle.cloud.spanner_v1._helpersimport (
+_get_cloud_region,
 _metadata_with_span_context,
 )
 
@@ -75,6 +76,7 @@ def trace_call(
 enable_end_to_end_tracing=False
 
 db_name=""
+cloud_region=None
 ifsessionandgetattr(session, "_database", None):
 db_name=session._database.name
 
@@ -88,6 +90,7 @@ def trace_call(
  )
 db_name=observability_options.get("db_name", db_name)
 
+cloud_region=_get_cloud_region()
 tracer=get_tracer(tracer_provider)
 
 # Set base attributes that we know for every trace created
@@ -97,6 +100,7 @@ def trace_call(
 "db.instance": db_name,
 "net.host.name": SpannerClient.DEFAULT_ENDPOINT,
 OTEL_SCOPE_NAME: TRACER_NAME,
+"cloud.region": cloud_region,
 OTEL_SCOPE_VERSION: TRACER_VERSION,
 # Standard GCP attributes for OTel, attributes are used for internal purpose and are subjected to change
 "gcp.client.service": "spanner",
@@ -107,6 +111,11 @@ def trace_call(
 ifextra_attributes:
 attributes.update(extra_attributes)
 
+if"request_options"inattributes:
+request_options=attributes.pop("request_options")
+ifrequest_optionsandrequest_options.request_tag:
+attributes["request.tag"] =request_options.request_tag
+
 ifextended_tracing_globally_disabled:
 enable_extended_tracing=False
 

google/cloud/spanner_v1/database.py

@@ -1025,8 +1025,14 @@ def run_in_transaction(self, func, *args, **kw):
  reraises any non-ABORT exceptions raised by ``func``.
  """
 observability_options=getattr(self, "observability_options", None)
+transaction_tag=kw.get("transaction_tag")
+extra_attributes= {}
+iftransaction_tag:
+extra_attributes["transaction.tag"] =transaction_tag
+
 withtrace_call(
 "CloudSpanner.Database.run_in_transaction",
+extra_attributes=extra_attributes,
 observability_options=observability_options,
  ), MetricsCapture():
 # Sanity check: Is there a transaction already running?

google/cloud/spanner_v1/metrics/spanner_metrics_tracer_factory.py

@@ -18,20 +18,9 @@
 from .metrics_tracer_factoryimportMetricsTracerFactory
 importos
 importlogging
-from .constantsimport (
-SPANNER_SERVICE_NAME,
-GOOGLE_CLOUD_REGION_KEY,
-GOOGLE_CLOUD_REGION_GLOBAL,
-)
+from .constantsimportSPANNER_SERVICE_NAME
 
 try:
-fromopentelemetry.resourcedetectorimportgcp_resource_detector
-
-# Overwrite the requests timeout for the detector.
-# This is necessary as the client will wait the full timeout if the
-# code is not run in a GCP environment, with the location endpoints available.
-gcp_resource_detector._TIMEOUT_SEC=0.2
-
 importmmh3
 
 logging.getLogger("opentelemetry.resourcedetector.gcp_resource_detector").setLevel(
@@ -44,6 +33,7 @@
 
 from .metrics_tracerimportMetricsTracer
 fromgoogle.cloud.spanner_v1import__version__
+fromgoogle.cloud.spanner_v1._helpersimport_get_cloud_region
 fromuuidimportuuid4
 
 log=logging.getLogger(__name__)
@@ -86,7 +76,7 @@ def __new__(
 cls._metrics_tracer_factory.set_client_hash(
 cls._generate_client_hash(client_uid)
  )
-cls._metrics_tracer_factory.set_location(cls._get_location())
+cls._metrics_tracer_factory.set_location(_get_cloud_region())
 cls._metrics_tracer_factory.gfe_enabled=gfe_enabled
 
 ifcls._metrics_tracer_factory.enabled!=enabled:
@@ -153,28 +143,3 @@ def _generate_client_hash(client_uid: str) -> str:
 
 # Return as 6 digit zero padded hex string
 returnf"{sig_figs:06x}"
-
-@staticmethod
-def_get_location() ->str:
-"""Get the location of the resource.
-
- In case of any error during detection, this method will log a warning
- and default to the "global" location.
-
- Returns:
- str: The location of the resource. If OpenTelemetry is not installed, returns a global region.
- """
-ifnotHAS_OPENTELEMETRY_INSTALLED:
-returnGOOGLE_CLOUD_REGION_GLOBAL
-try:
-detector=gcp_resource_detector.GoogleCloudResourceDetector()
-resources=detector.detect()
-
-ifGOOGLE_CLOUD_REGION_KEYinresources.attributes:
-returnresources.attributes[GOOGLE_CLOUD_REGION_KEY]
-exceptExceptionase:
-log.warning(
-"Failed to detect GCP resource location for Spanner metrics, defaulting to 'global'. Error: %s",
-e,
- )
-returnGOOGLE_CLOUD_REGION_GLOBAL

google/cloud/spanner_v1/session.py

@@ -532,9 +532,14 @@ def run_in_transaction(self, func, *args, **kw):
 database=self._database
 log_commit_stats=database.log_commit_stats
 
+extra_attributes= {}
+iftransaction_tag:
+extra_attributes["transaction.tag"] =transaction_tag
+
 withtrace_call(
 "CloudSpanner.Session.run_in_transaction",
 self,
+extra_attributes=extra_attributes,
 observability_options=getattr(database, "observability_options", None),
  ) asspan, MetricsCapture():
 attempts: int=0

google/cloud/spanner_v1/snapshot.py

@@ -409,7 +409,11 @@ def read(
 method=streaming_read_method,
 request=read_request,
 metadata=metadata,
-trace_attributes={"table_id": table, "columns": columns},
+trace_attributes={
+"table_id": table,
+"columns": columns,
+"request_options": request_options,
+ },
 column_info=column_info,
 lazy_decode=lazy_decode,
  )
@@ -601,7 +605,7 @@ def execute_sql(
 method=execute_streaming_sql_method,
 request=execute_sql_request,
 metadata=metadata,
-trace_attributes={"db.statement": sql},
+trace_attributes={"db.statement": sql, "request_options": request_options},
 column_info=column_info,
 lazy_decode=lazy_decode,
  )

google/cloud/spanner_v1/transaction.py

@@ -479,7 +479,10 @@ def execute_update(
 request_options=RequestOptions(request_options)
 request_options.transaction_tag=self.transaction_tag
 
-trace_attributes= {"db.statement": dml}
+trace_attributes= {
+"db.statement": dml,
+"request_options": request_options,
+ }
 
 # If this request begins the transaction, we need to lock
 # the transaction until the transaction ID is updated.
@@ -629,7 +632,8 @@ def batch_update(
 
 trace_attributes= {
 # Get just the queries from the DML statement batch
-"db.statement": ";".join([statement.sqlforstatementinparsed])
+"db.statement": ";".join([statement.sqlforstatementinparsed]),
+"request_options": request_options,
  }
 
 # If this request begins the transaction, we need to lock

tests/system/test_session_api.py

@@ -30,6 +30,7 @@
 fromgoogle.cloud.spanner_admin_database_v1importDatabaseDialect
 fromgoogle.cloud._helpersimportUTC
 
+fromgoogle.cloud.spanner_v1._helpersimport_get_cloud_region
 fromgoogle.cloud.spanner_v1._helpersimportAtomicCounter
 fromgoogle.cloud.spanner_v1.data_typesimportJsonObject
 fromgoogle.cloud.spanner_v1.database_sessions_managerimportTransactionType
@@ -356,6 +357,7 @@ def _make_attributes(db_instance, **kwargs):
 "db.url": "spanner.googleapis.com",
 "net.host.name": "spanner.googleapis.com",
 "db.instance": db_instance,
+"cloud.region": _get_cloud_region(),
 "gcp.client.service": "spanner",
 "gcp.client.version": ot_helpers.LIB_VERSION,
 "gcp.client.repo": "googleapis/python-spanner",

tests/unit/test__helpers.py

@@ -16,7 +16,11 @@
 importunittest
 importmock
 
-fromgoogle.cloud.spanner_v1importTransactionOptions
+fromopentelemetry.sdk.resourcesimportResource
+fromopentelemetry.semconv.resourceimportResourceAttributes
+
+
+fromgoogle.cloud.spanner_v1importTransactionOptions, _helpers
 
 
 classTest_merge_query_options(unittest.TestCase):
@@ -89,6 +93,48 @@ def test_base_object_merge_dict(self):
 self.assertEqual(result, expected)
 
 
+classTest_get_cloud_region(unittest.TestCase):
+defsetUp(self):
+_helpers._cloud_region=None
+
+def_callFUT(self, *args, **kw):
+fromgoogle.cloud.spanner_v1._helpersimport_get_cloud_region
+
+return_get_cloud_region(*args, **kw)
+
+@mock.patch("google.cloud.spanner_v1._helpers.GoogleCloudResourceDetector.detect")
+deftest_get_location_with_region(self, mock_detect):
+"""Test that _get_cloud_region returns the region when detected."""
+mock_resource=Resource.create(
+ {ResourceAttributes.CLOUD_REGION: "us-central1"}
+ )
+mock_detect.return_value=mock_resource
+
+location=self._callFUT()
+self.assertEqual(location, "us-central1")
+
+@mock.patch("google.cloud.spanner_v1._helpers.GoogleCloudResourceDetector.detect")
+deftest_get_location_without_region(self, mock_detect):
+"""Test that _get_cloud_region returns 'global' when no region is detected."""
+mock_resource=Resource.create({}) # No region attribute
+mock_detect.return_value=mock_resource
+
+location=self._callFUT()
+self.assertEqual(location, "global")
+
+@mock.patch("google.cloud.spanner_v1._helpers.GoogleCloudResourceDetector.detect")
+deftest_get_location_with_exception(self, mock_detect):
+"""Test that _get_cloud_region returns 'global' and logs a warning on exception."""
+mock_detect.side_effect=Exception("detector failed")
+
+withself.assertLogs(
+"google.cloud.spanner_v1._helpers", level="WARNING"
+ ) aslog:
+location=self._callFUT()
+self.assertEqual(location, "global")
+self.assertIn("Failed to detect GCP resource location", log.output[0])
+
+
 classTest_make_value_pb(unittest.TestCase):
 def_callFUT(self, *args, **kw):
 fromgoogle.cloud.spanner_v1._helpersimport_make_value_pb