feat: Implement custom telemetry logging in SDK#4721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
knikure merged 1 commit into from
Jun 12, 2024
Merged

feat: Implement custom telemetry logging in SDK #4721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -95,6 +95,13 @@ SageMaker Python SDK is tested on:
- Python 3.10
- Python 3.11

Telemetry
~~~~~~~~~~~~~~~

The ``sagemaker`` library has telemetry enabled to help us better understand user needs, diagnose issues, and deliver new features. This telemetry tracks the usage of various SageMaker functions.

If you prefer to opt out of telemetry, you can easily do so by setting the ``TelemetryOptOut`` parameter to ``true`` in the SDK defaults configuration. For detailed instructions, please visit `Configuring and using defaults with the SageMaker Python SDK <https://sagemaker.readthedocs.io/en/stable/overview.html#configuring-and-using-defaults-with-the-sagemaker-python-sdk>`__.

AWS Permissions
~~~~~~~~~~~~~~~

Expand Down
9 changes: 9 additions & 0 deletions src/sagemaker/local/local_session.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,8 @@
_LocalPipeline,
)
from sagemaker.session import Session
from sagemaker.telemetry.telemetry_logging import _telemetry_emitter
from sagemaker.telemetry.constants import Feature
from sagemaker.utils import (
get_config_value,
_module_import_error,
Expand DownExpand Up@@ -83,6 +85,7 @@ def __init__(self, sagemaker_session=None):
"""
self.sagemaker_session = sagemaker_session or LocalSession()

@_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_processing_job")
def create_processing_job(
self,
ProcessingJobName,
Expand DownExpand Up@@ -165,6 +168,7 @@ def describe_processing_job(self, ProcessingJobName):
raise ClientError(error_response, "describe_processing_job")
return LocalSagemakerClient._processing_jobs[ProcessingJobName].describe()

@_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_training_job")
def create_training_job(
self,
TrainingJobName,
Expand DownExpand Up@@ -235,6 +239,7 @@ def describe_training_job(self, TrainingJobName):
raise ClientError(error_response, "describe_training_job")
return LocalSagemakerClient._training_jobs[TrainingJobName].describe()

@_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_transform_job")
def create_transform_job(
self,
TransformJobName,
Expand DownExpand Up@@ -280,6 +285,7 @@ def describe_transform_job(self, TransformJobName):
raise ClientError(error_response, "describe_transform_job")
return LocalSagemakerClient._transform_jobs[TransformJobName].describe()

@_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_model")
def create_model(
self, ModelName, PrimaryContainer, *args, **kwargs
): # pylint: disable=unused-argument
Expand DownExpand Up@@ -329,6 +335,7 @@ def describe_endpoint_config(self, EndpointConfigName):
raise ClientError(error_response, "describe_endpoint_config")
return LocalSagemakerClient._endpoint_configs[EndpointConfigName].describe()

@_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint_config")
def create_endpoint_config(self, EndpointConfigName, ProductionVariants, Tags=None):
"""Create the endpoint configuration.

Expand DownExpand Up@@ -360,6 +367,7 @@ def describe_endpoint(self, EndpointName):
raise ClientError(error_response, "describe_endpoint")
return LocalSagemakerClient._endpoints[EndpointName].describe()

@_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint")
def create_endpoint(self, EndpointName, EndpointConfigName, Tags=None):
"""Create the endpoint.

Expand DownExpand Up@@ -428,6 +436,7 @@ def delete_model(self, ModelName):
if ModelName in LocalSagemakerClient._models:
del LocalSagemakerClient._models[ModelName]

@_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_pipeline")
def create_pipeline(
self, pipeline, pipeline_description, **kwargs # pylint: disable=unused-argument
):
Expand Down
16 changes: 16 additions & 0 deletions src/sagemaker/telemetry/__init__.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Placeholder docstring"""
from __future__ import absolute_import

from .telemetry_logging import _telemetry_emitter # noqa: F401
41 changes: 41 additions & 0 deletions src/sagemaker/telemetry/constants.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Constants used in SageMaker Python SDK telemetry."""

from __future__ import absolute_import
from enum import Enum

# Default AWS region used by SageMaker
DEFAULT_AWS_REGION = "us-west-2"


class Feature(Enum):
"""Enumeration of feature names used in telemetry."""

SDK_DEFAULTS = 1
LOCAL_MODE = 2

def __str__(self): # pylint: disable=E0307
"""Return the feature name."""
return self.name


class Status(Enum):
"""Enumeration of status values used in telemetry."""

SUCCESS = 1
FAILURE = 0

def __str__(self): # pylint: disable=E0307
"""Return the status name."""
return self.name
229 changes: 229 additions & 0 deletions src/sagemaker/telemetry/telemetry_logging.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Telemetry module for SageMaker Python SDK to collect usage data and metrics."""
from __future__ import absolute_import
import logging
import platform
import sys
from time import perf_counter
from typing import List

from sagemaker.utils import resolve_value_from_config
from sagemaker.config.config_schema import TELEMETRY_OPT_OUT_PATH
from sagemaker.telemetry.constants import (
Feature,
Status,
DEFAULT_AWS_REGION,
)
from sagemaker.user_agent import SDK_VERSION, process_studio_metadata_file

logger = logging.getLogger(__name__)

OS_NAME = platform.system() or "UnresolvedOS"
OS_VERSION = platform.release() or "UnresolvedOSVersion"
OS_NAME_VERSION = "{}/{}".format(OS_NAME, OS_VERSION)
PYTHON_VERSION = "{}.{}.{}".format(
sys.version_info.major, sys.version_info.minor, sys.version_info.micro
)

TELEMETRY_OPT_OUT_MESSAGING = (
"SageMaker Python SDK will collect telemetry to help us better understand our user's needs, "
"diagnose issues, and deliver additional features.\n"
"To opt out of telemetry, please disable via TelemetryOptOut parameter in SDK defaults config. "
"For more information, refer to https://sagemaker.readthedocs.io/en/stable/overview.html"
"#configuring-and-using-defaults-with-the-sagemaker-python-sdk."
)

FEATURE_TO_CODE = {
str(Feature.SDK_DEFAULTS): 1,
str(Feature.LOCAL_MODE): 2,
}

STATUS_TO_CODE = {
str(Status.SUCCESS): 1,
str(Status.FAILURE): 0,
}


def _telemetry_emitter(feature: str, func_name: str):
"""Decorator to emit telemetry logs for SageMaker Python SDK functions"""

def decorator(func):
def wrapper(self, *args, **kwargs):
Comment thread
knikure marked this conversation as resolved.
logger.info(TELEMETRY_OPT_OUT_MESSAGING)
response = None
caught_ex = None
studio_app_type = process_studio_metadata_file()

# Check if telemetry is opted out
telemetry_opt_out_flag = resolve_value_from_config(
direct_input=None,
config_path=TELEMETRY_OPT_OUT_PATH,
default_value=False,
sagemaker_session=self.sagemaker_session,
)
logger.debug("TelemetryOptOut flag is set to: %s", telemetry_opt_out_flag)

# Construct the feature list to track feature combinations
feature_list: List[int] = [FEATURE_TO_CODE[str(feature)]]
if self.sagemaker_session:
if self.sagemaker_session.sagemaker_config and feature != Feature.SDK_DEFAULTS:
feature_list.append(FEATURE_TO_CODE[str(Feature.SDK_DEFAULTS)])

if self.sagemaker_session.local_mode and feature != Feature.LOCAL_MODE:
feature_list.append(FEATURE_TO_CODE[str(Feature.LOCAL_MODE)])

# Construct the extra info to track platform and environment usage metadata
extra = (
f"{func_name}"
f"&x-sdkVersion={SDK_VERSION}"
f"&x-env={PYTHON_VERSION}"
f"&x-sys={OS_NAME_VERSION}"
f"&x-platform={studio_app_type}"
)

# Add endpoint ARN to the extra info if available
if self.sagemaker_session and self.sagemaker_session.endpoint_arn:
extra += f"&x-endpointArn={self.sagemaker_session.endpoint_arn}"

start_timer = perf_counter()
try:
# Call the original function
response = func(self, *args, **kwargs)
stop_timer = perf_counter()
elapsed = stop_timer - start_timer
extra += f"&x-latency={round(elapsed, 2)}"
if not telemetry_opt_out_flag:
_send_telemetry_request(
STATUS_TO_CODE[str(Status.SUCCESS)],
feature_list,
self.sagemaker_session,
None,
None,
extra,
)
except Exception as e: # pylint: disable=W0703
stop_timer = perf_counter()
elapsed = stop_timer - start_timer
extra += f"&x-latency={round(elapsed, 2)}"
if not telemetry_opt_out_flag:
_send_telemetry_request(
STATUS_TO_CODE[str(Status.FAILURE)],
feature_list,
self.sagemaker_session,
str(e),
e.__class__.__name__,
extra,
)
caught_ex = e
finally:
if caught_ex:
raise caught_ex
return response # pylint: disable=W0150

return wrapper

return decorator


from sagemaker.session import Session # noqa: E402 pylint: disable=C0413


def _send_telemetry_request(
status: int,
feature_list: List[int],
session: Session,
failure_reason: str = None,
failure_type: str = None,
extra_info: str = None,
) -> None:
"""Make GET request to an empty object in S3 bucket"""
try:
accountId = _get_accountId(session)
region = _get_region_or_default(session)
url = _construct_url(
accountId,
region,
str(status),
str(
",".join(map(str, feature_list))
), # Remove brackets and quotes to cut down on length
failure_reason,
failure_type,
extra_info,
)
# Send the telemetry request
logger.debug("Sending telemetry request to [%s]", url)
_requests_helper(url, 2)
logger.debug("SageMaker Python SDK telemetry successfully emitted!")
except Exception: # pylint: disable=W0703
logger.debug("SageMaker Python SDK telemetry not emitted!!")


def _construct_url(
accountId: str,
region: str,
status: str,
feature: str,
failure_reason: str,
failure_type: str,
extra_info: str,
) -> str:
"""Construct the URL for the telemetry request"""

base_url = (
f"https://sm-pysdk-t-{region}.s3.{region}.amazonaws.com/telemetry?"
f"x-accountId={accountId}"
f"&x-status={status}"
f"&x-feature={feature}"
)
logger.debug("Failure reason: %s", failure_reason)
if failure_reason:
base_url += f"&x-failureReason={failure_reason}"
base_url += f"&x-failureType={failure_type}"
if extra_info:
base_url += f"&x-extra={extra_info}"
return base_url


import requests # noqa: E402 pylint: disable=C0413,C0411


def _requests_helper(url, timeout):
"""Make a GET request to the given URL"""

response = None
try:
response = requests.get(url, timeout)
except requests.exceptions.RequestException as e:
logger.exception("Request exception: %s", str(e))
return response


def _get_accountId(session):
"""Return the account ID from the boto session"""

try:
sts = session.boto_session.client("sts")
return sts.get_caller_identity()["Account"]
except Exception: # pylint: disable=W0703
return None


def _get_region_or_default(session):
"""Return the region name from the boto session or default to us-west-2"""

try:
return session.boto_session.region_name
except Exception: # pylint: disable=W0703
return DEFAULT_AWS_REGION
4 changes: 3 additions & 1 deletion tests/integ/sagemaker/conftest.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -102,7 +102,9 @@
"channels:\n"
" - defaults\n"
"dependencies:\n"
" - scipy=1.10.1\n"
" - requests=2.32.3\n"
" - charset-normalizer=3.3.2\n"
" - scipy=1.13.1\n"
" - pip:\n"
" - /sagemaker-{sagemaker_version}.tar.gz\n"
"prefix: /opt/conda/bin/conda\n"
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
feat: Implement custom telemetry logging in SDK#4721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

knikure
merged 1 commit into

from

Jun 12, 2024
 Merged


feat: Implement custom telemetry logging in SDK
#4721
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
README.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -95,6 +95,13 @@ SageMaker Python SDK is tested on:
- Python 3.10
- Python 3.11

Telemetry
~~~~~~~~~~~~~~~

The ``sagemaker`` library has telemetry enabled to help us better understand user needs, diagnose issues, and deliver new features. This telemetry tracks the usage of various SageMaker functions.

If you prefer to opt out of telemetry, you can easily do so by setting the ``TelemetryOptOut`` parameter to ``true`` in the SDK defaults configuration. For detailed instructions, please visit `Configuring and using defaults with the SageMaker Python SDK <https://sagemaker.readthedocs.io/en/stable/overview.html#configuring-and-using-defaults-with-the-sagemaker-python-sdk>`__.

AWS Permissions
~~~~~~~~~~~~~~~

Expand Down

9 changes: 9 additions & 0 deletions
src/sagemaker/local/local_session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,8 @@
 _LocalPipeline,
)
from sagemaker.session import Session
from sagemaker.telemetry.telemetry_logging import _telemetry_emitter
from sagemaker.telemetry.constants import Feature
from sagemaker.utils import (
 get_config_value,
 _module_import_error,
Expand DownExpand Up@@ -83,6 +85,7 @@ def __init__(self, sagemaker_session=None):
 """
 self.sagemaker_session = sagemaker_session or LocalSession()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_processing_job")
 def create_processing_job(
 self,
 ProcessingJobName,
Expand DownExpand Up@@ -165,6 +168,7 @@ def describe_processing_job(self, ProcessingJobName):
 raise ClientError(error_response, "describe_processing_job")
 return LocalSagemakerClient._processing_jobs[ProcessingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_training_job")
 def create_training_job(
 self,
 TrainingJobName,
Expand DownExpand Up@@ -235,6 +239,7 @@ def describe_training_job(self, TrainingJobName):
 raise ClientError(error_response, "describe_training_job")
 return LocalSagemakerClient._training_jobs[TrainingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_transform_job")
 def create_transform_job(
 self,
 TransformJobName,
Expand DownExpand Up@@ -280,6 +285,7 @@ def describe_transform_job(self, TransformJobName):
 raise ClientError(error_response, "describe_transform_job")
 return LocalSagemakerClient._transform_jobs[TransformJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_model")
 def create_model(
 self, ModelName, PrimaryContainer, *args, **kwargs
 ): # pylint: disable=unused-argument
Expand DownExpand Up@@ -329,6 +335,7 @@ def describe_endpoint_config(self, EndpointConfigName):
 raise ClientError(error_response, "describe_endpoint_config")
 return LocalSagemakerClient._endpoint_configs[EndpointConfigName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint_config")
 def create_endpoint_config(self, EndpointConfigName, ProductionVariants, Tags=None):
 """Create the endpoint configuration.

Expand DownExpand Up@@ -360,6 +367,7 @@ def describe_endpoint(self, EndpointName):
 raise ClientError(error_response, "describe_endpoint")
 return LocalSagemakerClient._endpoints[EndpointName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint")
 def create_endpoint(self, EndpointName, EndpointConfigName, Tags=None):
 """Create the endpoint.

Expand DownExpand Up@@ -428,6 +436,7 @@ def delete_model(self, ModelName):
 if ModelName in LocalSagemakerClient._models:
 del LocalSagemakerClient._models[ModelName]

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_pipeline")
 def create_pipeline(
 self, pipeline, pipeline_description, **kwargs # pylint: disable=unused-argument
 ):
Expand Down

16 changes: 16 additions & 0 deletions
src/sagemaker/telemetry/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Placeholder docstring"""
from __future__ import absolute_import

from .telemetry_logging import _telemetry_emitter # noqa: F401

41 changes: 41 additions & 0 deletions
src/sagemaker/telemetry/constants.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Constants used in SageMaker Python SDK telemetry."""

from __future__ import absolute_import
from enum import Enum

# Default AWS region used by SageMaker
DEFAULT_AWS_REGION = "us-west-2"


class Feature(Enum):
 """Enumeration of feature names used in telemetry."""

 SDK_DEFAULTS = 1
 LOCAL_MODE = 2

 def __str__(self): # pylint: disable=E0307
 """Return the feature name."""
 return self.name


class Status(Enum):
 """Enumeration of status values used in telemetry."""

 SUCCESS = 1
 FAILURE = 0

 def __str__(self): # pylint: disable=E0307
 """Return the status name."""
 return self.name

229 changes: 229 additions & 0 deletions
src/sagemaker/telemetry/telemetry_logging.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Telemetry module for SageMaker Python SDK to collect usage data and metrics."""
from __future__ import absolute_import
import logging
import platform
import sys
from time import perf_counter
from typing import List

from sagemaker.utils import resolve_value_from_config
from sagemaker.config.config_schema import TELEMETRY_OPT_OUT_PATH
from sagemaker.telemetry.constants import (
 Feature,
 Status,
 DEFAULT_AWS_REGION,
)
from sagemaker.user_agent import SDK_VERSION, process_studio_metadata_file

logger = logging.getLogger(__name__)

OS_NAME = platform.system() or "UnresolvedOS"
OS_VERSION = platform.release() or "UnresolvedOSVersion"
OS_NAME_VERSION = "{}/{}".format(OS_NAME, OS_VERSION)
PYTHON_VERSION = "{}.{}.{}".format(
 sys.version_info.major, sys.version_info.minor, sys.version_info.micro
)

TELEMETRY_OPT_OUT_MESSAGING = (
 "SageMaker Python SDK will collect telemetry to help us better understand our user's needs, "
 "diagnose issues, and deliver additional features.\n"
 "To opt out of telemetry, please disable via TelemetryOptOut parameter in SDK defaults config. "
 "For more information, refer to https://sagemaker.readthedocs.io/en/stable/overview.html"
 "#configuring-and-using-defaults-with-the-sagemaker-python-sdk."
)

FEATURE_TO_CODE = {
 str(Feature.SDK_DEFAULTS): 1,
 str(Feature.LOCAL_MODE): 2,
}

STATUS_TO_CODE = {
 str(Status.SUCCESS): 1,
 str(Status.FAILURE): 0,
}


def _telemetry_emitter(feature: str, func_name: str):
 """Decorator to emit telemetry logs for SageMaker Python SDK functions"""

 def decorator(func):
 def wrapper(self, *args, **kwargs):
Comment thread
knikure marked this conversation as resolved.

 logger.info(TELEMETRY_OPT_OUT_MESSAGING)
 response = None
 caught_ex = None
 studio_app_type = process_studio_metadata_file()

 # Check if telemetry is opted out
 telemetry_opt_out_flag = resolve_value_from_config(
 direct_input=None,
 config_path=TELEMETRY_OPT_OUT_PATH,
 default_value=False,
 sagemaker_session=self.sagemaker_session,
 )
 logger.debug("TelemetryOptOut flag is set to: %s", telemetry_opt_out_flag)

 # Construct the feature list to track feature combinations
 feature_list: List[int] = [FEATURE_TO_CODE[str(feature)]]
 if self.sagemaker_session:
 if self.sagemaker_session.sagemaker_config and feature != Feature.SDK_DEFAULTS:
 feature_list.append(FEATURE_TO_CODE[str(Feature.SDK_DEFAULTS)])

 if self.sagemaker_session.local_mode and feature != Feature.LOCAL_MODE:
 feature_list.append(FEATURE_TO_CODE[str(Feature.LOCAL_MODE)])

 # Construct the extra info to track platform and environment usage metadata
 extra = (
 f"{func_name}"
 f"&x-sdkVersion={SDK_VERSION}"
 f"&x-env={PYTHON_VERSION}"
 f"&x-sys={OS_NAME_VERSION}"
 f"&x-platform={studio_app_type}"
 )

 # Add endpoint ARN to the extra info if available
 if self.sagemaker_session and self.sagemaker_session.endpoint_arn:
 extra += f"&x-endpointArn={self.sagemaker_session.endpoint_arn}"

 start_timer = perf_counter()
 try:
 # Call the original function
 response = func(self, *args, **kwargs)
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.SUCCESS)],
 feature_list,
 self.sagemaker_session,
 None,
 None,
 extra,
 )
 except Exception as e: # pylint: disable=W0703
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.FAILURE)],
 feature_list,
 self.sagemaker_session,
 str(e),
 e.__class__.__name__,
 extra,
 )
 caught_ex = e
 finally:
 if caught_ex:
 raise caught_ex
 return response # pylint: disable=W0150

 return wrapper

 return decorator


from sagemaker.session import Session # noqa: E402 pylint: disable=C0413


def _send_telemetry_request(
 status: int,
 feature_list: List[int],
 session: Session,
 failure_reason: str = None,
 failure_type: str = None,
 extra_info: str = None,
) -> None:
 """Make GET request to an empty object in S3 bucket"""
 try:
 accountId = _get_accountId(session)
 region = _get_region_or_default(session)
 url = _construct_url(
 accountId,
 region,
 str(status),
 str(
 ",".join(map(str, feature_list))
 ), # Remove brackets and quotes to cut down on length
 failure_reason,
 failure_type,
 extra_info,
 )
 # Send the telemetry request
 logger.debug("Sending telemetry request to [%s]", url)
 _requests_helper(url, 2)
 logger.debug("SageMaker Python SDK telemetry successfully emitted!")
 except Exception: # pylint: disable=W0703
 logger.debug("SageMaker Python SDK telemetry not emitted!!")


def _construct_url(
 accountId: str,
 region: str,
 status: str,
 feature: str,
 failure_reason: str,
 failure_type: str,
 extra_info: str,
) -> str:
 """Construct the URL for the telemetry request"""

 base_url = (
 f"https://sm-pysdk-t-{region}.s3.{region}.amazonaws.com/telemetry?"
 f"x-accountId={accountId}"
 f"&x-status={status}"
 f"&x-feature={feature}"
 )
 logger.debug("Failure reason: %s", failure_reason)
 if failure_reason:
 base_url += f"&x-failureReason={failure_reason}"
 base_url += f"&x-failureType={failure_type}"
 if extra_info:
 base_url += f"&x-extra={extra_info}"
 return base_url


import requests # noqa: E402 pylint: disable=C0413,C0411


def _requests_helper(url, timeout):
 """Make a GET request to the given URL"""

 response = None
 try:
 response = requests.get(url, timeout)
 except requests.exceptions.RequestException as e:
 logger.exception("Request exception: %s", str(e))
 return response


def _get_accountId(session):
 """Return the account ID from the boto session"""

 try:
 sts = session.boto_session.client("sts")
 return sts.get_caller_identity()["Account"]
 except Exception: # pylint: disable=W0703
 return None


def _get_region_or_default(session):
 """Return the region name from the boto session or default to us-west-2"""

 try:
 return session.boto_session.region_name
 except Exception: # pylint: disable=W0703
 return DEFAULT_AWS_REGION

4 changes: 3 additions & 1 deletion
tests/integ/sagemaker/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -102,7 +102,9 @@
 "channels:\n"
 " - defaults\n"
 "dependencies:\n"
 " - scipy=1.10.1\n"
 " - requests=2.32.3\n"
 " - charset-normalizer=3.3.2\n"
 " - scipy=1.13.1\n"
 " - pip:\n"
 " - /sagemaker-{sagemaker_version}.tar.gz\n"
 "prefix: /opt/conda/bin/conda\n"
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat: Implement custom telemetry logging in SDK#4721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

knikure
merged 1 commit into

from

Jun 12, 2024
 Merged


feat: Implement custom telemetry logging in SDK
#4721
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
README.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -95,6 +95,13 @@ SageMaker Python SDK is tested on:
- Python 3.10
- Python 3.11

Telemetry
~~~~~~~~~~~~~~~

The ``sagemaker`` library has telemetry enabled to help us better understand user needs, diagnose issues, and deliver new features. This telemetry tracks the usage of various SageMaker functions.

If you prefer to opt out of telemetry, you can easily do so by setting the ``TelemetryOptOut`` parameter to ``true`` in the SDK defaults configuration. For detailed instructions, please visit `Configuring and using defaults with the SageMaker Python SDK <https://sagemaker.readthedocs.io/en/stable/overview.html#configuring-and-using-defaults-with-the-sagemaker-python-sdk>`__.

AWS Permissions
~~~~~~~~~~~~~~~

Expand Down

9 changes: 9 additions & 0 deletions
src/sagemaker/local/local_session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,8 @@
 _LocalPipeline,
)
from sagemaker.session import Session
from sagemaker.telemetry.telemetry_logging import _telemetry_emitter
from sagemaker.telemetry.constants import Feature
from sagemaker.utils import (
 get_config_value,
 _module_import_error,
Expand DownExpand Up@@ -83,6 +85,7 @@ def __init__(self, sagemaker_session=None):
 """
 self.sagemaker_session = sagemaker_session or LocalSession()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_processing_job")
 def create_processing_job(
 self,
 ProcessingJobName,
Expand DownExpand Up@@ -165,6 +168,7 @@ def describe_processing_job(self, ProcessingJobName):
 raise ClientError(error_response, "describe_processing_job")
 return LocalSagemakerClient._processing_jobs[ProcessingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_training_job")
 def create_training_job(
 self,
 TrainingJobName,
Expand DownExpand Up@@ -235,6 +239,7 @@ def describe_training_job(self, TrainingJobName):
 raise ClientError(error_response, "describe_training_job")
 return LocalSagemakerClient._training_jobs[TrainingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_transform_job")
 def create_transform_job(
 self,
 TransformJobName,
Expand DownExpand Up@@ -280,6 +285,7 @@ def describe_transform_job(self, TransformJobName):
 raise ClientError(error_response, "describe_transform_job")
 return LocalSagemakerClient._transform_jobs[TransformJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_model")
 def create_model(
 self, ModelName, PrimaryContainer, *args, **kwargs
 ): # pylint: disable=unused-argument
Expand DownExpand Up@@ -329,6 +335,7 @@ def describe_endpoint_config(self, EndpointConfigName):
 raise ClientError(error_response, "describe_endpoint_config")
 return LocalSagemakerClient._endpoint_configs[EndpointConfigName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint_config")
 def create_endpoint_config(self, EndpointConfigName, ProductionVariants, Tags=None):
 """Create the endpoint configuration.

Expand DownExpand Up@@ -360,6 +367,7 @@ def describe_endpoint(self, EndpointName):
 raise ClientError(error_response, "describe_endpoint")
 return LocalSagemakerClient._endpoints[EndpointName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint")
 def create_endpoint(self, EndpointName, EndpointConfigName, Tags=None):
 """Create the endpoint.

Expand DownExpand Up@@ -428,6 +436,7 @@ def delete_model(self, ModelName):
 if ModelName in LocalSagemakerClient._models:
 del LocalSagemakerClient._models[ModelName]

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_pipeline")
 def create_pipeline(
 self, pipeline, pipeline_description, **kwargs # pylint: disable=unused-argument
 ):
Expand Down

16 changes: 16 additions & 0 deletions
src/sagemaker/telemetry/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Placeholder docstring"""
from __future__ import absolute_import

from .telemetry_logging import _telemetry_emitter # noqa: F401

41 changes: 41 additions & 0 deletions
src/sagemaker/telemetry/constants.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Constants used in SageMaker Python SDK telemetry."""

from __future__ import absolute_import
from enum import Enum

# Default AWS region used by SageMaker
DEFAULT_AWS_REGION = "us-west-2"


class Feature(Enum):
 """Enumeration of feature names used in telemetry."""

 SDK_DEFAULTS = 1
 LOCAL_MODE = 2

 def __str__(self): # pylint: disable=E0307
 """Return the feature name."""
 return self.name


class Status(Enum):
 """Enumeration of status values used in telemetry."""

 SUCCESS = 1
 FAILURE = 0

 def __str__(self): # pylint: disable=E0307
 """Return the status name."""
 return self.name

229 changes: 229 additions & 0 deletions
src/sagemaker/telemetry/telemetry_logging.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Telemetry module for SageMaker Python SDK to collect usage data and metrics."""
from __future__ import absolute_import
import logging
import platform
import sys
from time import perf_counter
from typing import List

from sagemaker.utils import resolve_value_from_config
from sagemaker.config.config_schema import TELEMETRY_OPT_OUT_PATH
from sagemaker.telemetry.constants import (
 Feature,
 Status,
 DEFAULT_AWS_REGION,
)
from sagemaker.user_agent import SDK_VERSION, process_studio_metadata_file

logger = logging.getLogger(__name__)

OS_NAME = platform.system() or "UnresolvedOS"
OS_VERSION = platform.release() or "UnresolvedOSVersion"
OS_NAME_VERSION = "{}/{}".format(OS_NAME, OS_VERSION)
PYTHON_VERSION = "{}.{}.{}".format(
 sys.version_info.major, sys.version_info.minor, sys.version_info.micro
)

TELEMETRY_OPT_OUT_MESSAGING = (
 "SageMaker Python SDK will collect telemetry to help us better understand our user's needs, "
 "diagnose issues, and deliver additional features.\n"
 "To opt out of telemetry, please disable via TelemetryOptOut parameter in SDK defaults config. "
 "For more information, refer to https://sagemaker.readthedocs.io/en/stable/overview.html"
 "#configuring-and-using-defaults-with-the-sagemaker-python-sdk."
)

FEATURE_TO_CODE = {
 str(Feature.SDK_DEFAULTS): 1,
 str(Feature.LOCAL_MODE): 2,
}

STATUS_TO_CODE = {
 str(Status.SUCCESS): 1,
 str(Status.FAILURE): 0,
}


def _telemetry_emitter(feature: str, func_name: str):
 """Decorator to emit telemetry logs for SageMaker Python SDK functions"""

 def decorator(func):
 def wrapper(self, *args, **kwargs):
Comment thread
knikure marked this conversation as resolved.

 logger.info(TELEMETRY_OPT_OUT_MESSAGING)
 response = None
 caught_ex = None
 studio_app_type = process_studio_metadata_file()

 # Check if telemetry is opted out
 telemetry_opt_out_flag = resolve_value_from_config(
 direct_input=None,
 config_path=TELEMETRY_OPT_OUT_PATH,
 default_value=False,
 sagemaker_session=self.sagemaker_session,
 )
 logger.debug("TelemetryOptOut flag is set to: %s", telemetry_opt_out_flag)

 # Construct the feature list to track feature combinations
 feature_list: List[int] = [FEATURE_TO_CODE[str(feature)]]
 if self.sagemaker_session:
 if self.sagemaker_session.sagemaker_config and feature != Feature.SDK_DEFAULTS:
 feature_list.append(FEATURE_TO_CODE[str(Feature.SDK_DEFAULTS)])

 if self.sagemaker_session.local_mode and feature != Feature.LOCAL_MODE:
 feature_list.append(FEATURE_TO_CODE[str(Feature.LOCAL_MODE)])

 # Construct the extra info to track platform and environment usage metadata
 extra = (
 f"{func_name}"
 f"&x-sdkVersion={SDK_VERSION}"
 f"&x-env={PYTHON_VERSION}"
 f"&x-sys={OS_NAME_VERSION}"
 f"&x-platform={studio_app_type}"
 )

 # Add endpoint ARN to the extra info if available
 if self.sagemaker_session and self.sagemaker_session.endpoint_arn:
 extra += f"&x-endpointArn={self.sagemaker_session.endpoint_arn}"

 start_timer = perf_counter()
 try:
 # Call the original function
 response = func(self, *args, **kwargs)
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.SUCCESS)],
 feature_list,
 self.sagemaker_session,
 None,
 None,
 extra,
 )
 except Exception as e: # pylint: disable=W0703
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.FAILURE)],
 feature_list,
 self.sagemaker_session,
 str(e),
 e.__class__.__name__,
 extra,
 )
 caught_ex = e
 finally:
 if caught_ex:
 raise caught_ex
 return response # pylint: disable=W0150

 return wrapper

 return decorator


from sagemaker.session import Session # noqa: E402 pylint: disable=C0413


def _send_telemetry_request(
 status: int,
 feature_list: List[int],
 session: Session,
 failure_reason: str = None,
 failure_type: str = None,
 extra_info: str = None,
) -> None:
 """Make GET request to an empty object in S3 bucket"""
 try:
 accountId = _get_accountId(session)
 region = _get_region_or_default(session)
 url = _construct_url(
 accountId,
 region,
 str(status),
 str(
 ",".join(map(str, feature_list))
 ), # Remove brackets and quotes to cut down on length
 failure_reason,
 failure_type,
 extra_info,
 )
 # Send the telemetry request
 logger.debug("Sending telemetry request to [%s]", url)
 _requests_helper(url, 2)
 logger.debug("SageMaker Python SDK telemetry successfully emitted!")
 except Exception: # pylint: disable=W0703
 logger.debug("SageMaker Python SDK telemetry not emitted!!")


def _construct_url(
 accountId: str,
 region: str,
 status: str,
 feature: str,
 failure_reason: str,
 failure_type: str,
 extra_info: str,
) -> str:
 """Construct the URL for the telemetry request"""

 base_url = (
 f"https://sm-pysdk-t-{region}.s3.{region}.amazonaws.com/telemetry?"
 f"x-accountId={accountId}"
 f"&x-status={status}"
 f"&x-feature={feature}"
 )
 logger.debug("Failure reason: %s", failure_reason)
 if failure_reason:
 base_url += f"&x-failureReason={failure_reason}"
 base_url += f"&x-failureType={failure_type}"
 if extra_info:
 base_url += f"&x-extra={extra_info}"
 return base_url


import requests # noqa: E402 pylint: disable=C0413,C0411


def _requests_helper(url, timeout):
 """Make a GET request to the given URL"""

 response = None
 try:
 response = requests.get(url, timeout)
 except requests.exceptions.RequestException as e:
 logger.exception("Request exception: %s", str(e))
 return response


def _get_accountId(session):
 """Return the account ID from the boto session"""

 try:
 sts = session.boto_session.client("sts")
 return sts.get_caller_identity()["Account"]
 except Exception: # pylint: disable=W0703
 return None


def _get_region_or_default(session):
 """Return the region name from the boto session or default to us-west-2"""

 try:
 return session.boto_session.region_name
 except Exception: # pylint: disable=W0703
 return DEFAULT_AWS_REGION

4 changes: 3 additions & 1 deletion
tests/integ/sagemaker/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -102,7 +102,9 @@
 "channels:\n"
 " - defaults\n"
 "dependencies:\n"
 " - scipy=1.10.1\n"
 " - requests=2.32.3\n"
 " - charset-normalizer=3.3.2\n"
 " - scipy=1.13.1\n"
 " - pip:\n"
 " - /sagemaker-{sagemaker_version}.tar.gz\n"
 "prefix: /opt/conda/bin/conda\n"
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat: Implement custom telemetry logging in SDK#4721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

knikure
merged 1 commit into

from

Jun 12, 2024
 Merged


feat: Implement custom telemetry logging in SDK
#4721
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
README.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -95,6 +95,13 @@ SageMaker Python SDK is tested on:
- Python 3.10
- Python 3.11

Telemetry
~~~~~~~~~~~~~~~

The ``sagemaker`` library has telemetry enabled to help us better understand user needs, diagnose issues, and deliver new features. This telemetry tracks the usage of various SageMaker functions.

If you prefer to opt out of telemetry, you can easily do so by setting the ``TelemetryOptOut`` parameter to ``true`` in the SDK defaults configuration. For detailed instructions, please visit `Configuring and using defaults with the SageMaker Python SDK <https://sagemaker.readthedocs.io/en/stable/overview.html#configuring-and-using-defaults-with-the-sagemaker-python-sdk>`__.

AWS Permissions
~~~~~~~~~~~~~~~

Expand Down

9 changes: 9 additions & 0 deletions
src/sagemaker/local/local_session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,8 @@
 _LocalPipeline,
)
from sagemaker.session import Session
from sagemaker.telemetry.telemetry_logging import _telemetry_emitter
from sagemaker.telemetry.constants import Feature
from sagemaker.utils import (
 get_config_value,
 _module_import_error,
Expand DownExpand Up@@ -83,6 +85,7 @@ def __init__(self, sagemaker_session=None):
 """
 self.sagemaker_session = sagemaker_session or LocalSession()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_processing_job")
 def create_processing_job(
 self,
 ProcessingJobName,
Expand DownExpand Up@@ -165,6 +168,7 @@ def describe_processing_job(self, ProcessingJobName):
 raise ClientError(error_response, "describe_processing_job")
 return LocalSagemakerClient._processing_jobs[ProcessingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_training_job")
 def create_training_job(
 self,
 TrainingJobName,
Expand DownExpand Up@@ -235,6 +239,7 @@ def describe_training_job(self, TrainingJobName):
 raise ClientError(error_response, "describe_training_job")
 return LocalSagemakerClient._training_jobs[TrainingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_transform_job")
 def create_transform_job(
 self,
 TransformJobName,
Expand DownExpand Up@@ -280,6 +285,7 @@ def describe_transform_job(self, TransformJobName):
 raise ClientError(error_response, "describe_transform_job")
 return LocalSagemakerClient._transform_jobs[TransformJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_model")
 def create_model(
 self, ModelName, PrimaryContainer, *args, **kwargs
 ): # pylint: disable=unused-argument
Expand DownExpand Up@@ -329,6 +335,7 @@ def describe_endpoint_config(self, EndpointConfigName):
 raise ClientError(error_response, "describe_endpoint_config")
 return LocalSagemakerClient._endpoint_configs[EndpointConfigName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint_config")
 def create_endpoint_config(self, EndpointConfigName, ProductionVariants, Tags=None):
 """Create the endpoint configuration.

Expand DownExpand Up@@ -360,6 +367,7 @@ def describe_endpoint(self, EndpointName):
 raise ClientError(error_response, "describe_endpoint")
 return LocalSagemakerClient._endpoints[EndpointName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint")
 def create_endpoint(self, EndpointName, EndpointConfigName, Tags=None):
 """Create the endpoint.

Expand DownExpand Up@@ -428,6 +436,7 @@ def delete_model(self, ModelName):
 if ModelName in LocalSagemakerClient._models:
 del LocalSagemakerClient._models[ModelName]

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_pipeline")
 def create_pipeline(
 self, pipeline, pipeline_description, **kwargs # pylint: disable=unused-argument
 ):
Expand Down

16 changes: 16 additions & 0 deletions
src/sagemaker/telemetry/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Placeholder docstring"""
from __future__ import absolute_import

from .telemetry_logging import _telemetry_emitter # noqa: F401

41 changes: 41 additions & 0 deletions
src/sagemaker/telemetry/constants.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Constants used in SageMaker Python SDK telemetry."""

from __future__ import absolute_import
from enum import Enum

# Default AWS region used by SageMaker
DEFAULT_AWS_REGION = "us-west-2"


class Feature(Enum):
 """Enumeration of feature names used in telemetry."""

 SDK_DEFAULTS = 1
 LOCAL_MODE = 2

 def __str__(self): # pylint: disable=E0307
 """Return the feature name."""
 return self.name


class Status(Enum):
 """Enumeration of status values used in telemetry."""

 SUCCESS = 1
 FAILURE = 0

 def __str__(self): # pylint: disable=E0307
 """Return the status name."""
 return self.name

229 changes: 229 additions & 0 deletions
src/sagemaker/telemetry/telemetry_logging.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Telemetry module for SageMaker Python SDK to collect usage data and metrics."""
from __future__ import absolute_import
import logging
import platform
import sys
from time import perf_counter
from typing import List

from sagemaker.utils import resolve_value_from_config
from sagemaker.config.config_schema import TELEMETRY_OPT_OUT_PATH
from sagemaker.telemetry.constants import (
 Feature,
 Status,
 DEFAULT_AWS_REGION,
)
from sagemaker.user_agent import SDK_VERSION, process_studio_metadata_file

logger = logging.getLogger(__name__)

OS_NAME = platform.system() or "UnresolvedOS"
OS_VERSION = platform.release() or "UnresolvedOSVersion"
OS_NAME_VERSION = "{}/{}".format(OS_NAME, OS_VERSION)
PYTHON_VERSION = "{}.{}.{}".format(
 sys.version_info.major, sys.version_info.minor, sys.version_info.micro
)

TELEMETRY_OPT_OUT_MESSAGING = (
 "SageMaker Python SDK will collect telemetry to help us better understand our user's needs, "
 "diagnose issues, and deliver additional features.\n"
 "To opt out of telemetry, please disable via TelemetryOptOut parameter in SDK defaults config. "
 "For more information, refer to https://sagemaker.readthedocs.io/en/stable/overview.html"
 "#configuring-and-using-defaults-with-the-sagemaker-python-sdk."
)

FEATURE_TO_CODE = {
 str(Feature.SDK_DEFAULTS): 1,
 str(Feature.LOCAL_MODE): 2,
}

STATUS_TO_CODE = {
 str(Status.SUCCESS): 1,
 str(Status.FAILURE): 0,
}


def _telemetry_emitter(feature: str, func_name: str):
 """Decorator to emit telemetry logs for SageMaker Python SDK functions"""

 def decorator(func):
 def wrapper(self, *args, **kwargs):
Comment thread
knikure marked this conversation as resolved.

 logger.info(TELEMETRY_OPT_OUT_MESSAGING)
 response = None
 caught_ex = None
 studio_app_type = process_studio_metadata_file()

 # Check if telemetry is opted out
 telemetry_opt_out_flag = resolve_value_from_config(
 direct_input=None,
 config_path=TELEMETRY_OPT_OUT_PATH,
 default_value=False,
 sagemaker_session=self.sagemaker_session,
 )
 logger.debug("TelemetryOptOut flag is set to: %s", telemetry_opt_out_flag)

 # Construct the feature list to track feature combinations
 feature_list: List[int] = [FEATURE_TO_CODE[str(feature)]]
 if self.sagemaker_session:
 if self.sagemaker_session.sagemaker_config and feature != Feature.SDK_DEFAULTS:
 feature_list.append(FEATURE_TO_CODE[str(Feature.SDK_DEFAULTS)])

 if self.sagemaker_session.local_mode and feature != Feature.LOCAL_MODE:
 feature_list.append(FEATURE_TO_CODE[str(Feature.LOCAL_MODE)])

 # Construct the extra info to track platform and environment usage metadata
 extra = (
 f"{func_name}"
 f"&x-sdkVersion={SDK_VERSION}"
 f"&x-env={PYTHON_VERSION}"
 f"&x-sys={OS_NAME_VERSION}"
 f"&x-platform={studio_app_type}"
 )

 # Add endpoint ARN to the extra info if available
 if self.sagemaker_session and self.sagemaker_session.endpoint_arn:
 extra += f"&x-endpointArn={self.sagemaker_session.endpoint_arn}"

 start_timer = perf_counter()
 try:
 # Call the original function
 response = func(self, *args, **kwargs)
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.SUCCESS)],
 feature_list,
 self.sagemaker_session,
 None,
 None,
 extra,
 )
 except Exception as e: # pylint: disable=W0703
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.FAILURE)],
 feature_list,
 self.sagemaker_session,
 str(e),
 e.__class__.__name__,
 extra,
 )
 caught_ex = e
 finally:
 if caught_ex:
 raise caught_ex
 return response # pylint: disable=W0150

 return wrapper

 return decorator


from sagemaker.session import Session # noqa: E402 pylint: disable=C0413


def _send_telemetry_request(
 status: int,
 feature_list: List[int],
 session: Session,
 failure_reason: str = None,
 failure_type: str = None,
 extra_info: str = None,
) -> None:
 """Make GET request to an empty object in S3 bucket"""
 try:
 accountId = _get_accountId(session)
 region = _get_region_or_default(session)
 url = _construct_url(
 accountId,
 region,
 str(status),
 str(
 ",".join(map(str, feature_list))
 ), # Remove brackets and quotes to cut down on length
 failure_reason,
 failure_type,
 extra_info,
 )
 # Send the telemetry request
 logger.debug("Sending telemetry request to [%s]", url)
 _requests_helper(url, 2)
 logger.debug("SageMaker Python SDK telemetry successfully emitted!")
 except Exception: # pylint: disable=W0703
 logger.debug("SageMaker Python SDK telemetry not emitted!!")


def _construct_url(
 accountId: str,
 region: str,
 status: str,
 feature: str,
 failure_reason: str,
 failure_type: str,
 extra_info: str,
) -> str:
 """Construct the URL for the telemetry request"""

 base_url = (
 f"https://sm-pysdk-t-{region}.s3.{region}.amazonaws.com/telemetry?"
 f"x-accountId={accountId}"
 f"&x-status={status}"
 f"&x-feature={feature}"
 )
 logger.debug("Failure reason: %s", failure_reason)
 if failure_reason:
 base_url += f"&x-failureReason={failure_reason}"
 base_url += f"&x-failureType={failure_type}"
 if extra_info:
 base_url += f"&x-extra={extra_info}"
 return base_url


import requests # noqa: E402 pylint: disable=C0413,C0411


def _requests_helper(url, timeout):
 """Make a GET request to the given URL"""

 response = None
 try:
 response = requests.get(url, timeout)
 except requests.exceptions.RequestException as e:
 logger.exception("Request exception: %s", str(e))
 return response


def _get_accountId(session):
 """Return the account ID from the boto session"""

 try:
 sts = session.boto_session.client("sts")
 return sts.get_caller_identity()["Account"]
 except Exception: # pylint: disable=W0703
 return None


def _get_region_or_default(session):
 """Return the region name from the boto session or default to us-west-2"""

 try:
 return session.boto_session.region_name
 except Exception: # pylint: disable=W0703
 return DEFAULT_AWS_REGION

4 changes: 3 additions & 1 deletion
tests/integ/sagemaker/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -102,7 +102,9 @@
 "channels:\n"
 " - defaults\n"
 "dependencies:\n"
 " - scipy=1.10.1\n"
 " - requests=2.32.3\n"
 " - charset-normalizer=3.3.2\n"
 " - scipy=1.13.1\n"
 " - pip:\n"
 " - /sagemaker-{sagemaker_version}.tar.gz\n"
 "prefix: /opt/conda/bin/conda\n"
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
feat: Implement custom telemetry logging in SDK#4721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

knikure
merged 1 commit into

from

Jun 12, 2024
 Merged


feat: Implement custom telemetry logging in SDK
#4721
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
README.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -95,6 +95,13 @@ SageMaker Python SDK is tested on:
- Python 3.10
- Python 3.11

Telemetry
~~~~~~~~~~~~~~~

The ``sagemaker`` library has telemetry enabled to help us better understand user needs, diagnose issues, and deliver new features. This telemetry tracks the usage of various SageMaker functions.

If you prefer to opt out of telemetry, you can easily do so by setting the ``TelemetryOptOut`` parameter to ``true`` in the SDK defaults configuration. For detailed instructions, please visit `Configuring and using defaults with the SageMaker Python SDK <https://sagemaker.readthedocs.io/en/stable/overview.html#configuring-and-using-defaults-with-the-sagemaker-python-sdk>`__.

AWS Permissions
~~~~~~~~~~~~~~~

Expand Down

9 changes: 9 additions & 0 deletions
src/sagemaker/local/local_session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,8 @@
 _LocalPipeline,
)
from sagemaker.session import Session
from sagemaker.telemetry.telemetry_logging import _telemetry_emitter
from sagemaker.telemetry.constants import Feature
from sagemaker.utils import (
 get_config_value,
 _module_import_error,
Expand DownExpand Up@@ -83,6 +85,7 @@ def __init__(self, sagemaker_session=None):
 """
 self.sagemaker_session = sagemaker_session or LocalSession()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_processing_job")
 def create_processing_job(
 self,
 ProcessingJobName,
Expand DownExpand Up@@ -165,6 +168,7 @@ def describe_processing_job(self, ProcessingJobName):
 raise ClientError(error_response, "describe_processing_job")
 return LocalSagemakerClient._processing_jobs[ProcessingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_training_job")
 def create_training_job(
 self,
 TrainingJobName,
Expand DownExpand Up@@ -235,6 +239,7 @@ def describe_training_job(self, TrainingJobName):
 raise ClientError(error_response, "describe_training_job")
 return LocalSagemakerClient._training_jobs[TrainingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_transform_job")
 def create_transform_job(
 self,
 TransformJobName,
Expand DownExpand Up@@ -280,6 +285,7 @@ def describe_transform_job(self, TransformJobName):
 raise ClientError(error_response, "describe_transform_job")
 return LocalSagemakerClient._transform_jobs[TransformJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_model")
 def create_model(
 self, ModelName, PrimaryContainer, *args, **kwargs
 ): # pylint: disable=unused-argument
Expand DownExpand Up@@ -329,6 +335,7 @@ def describe_endpoint_config(self, EndpointConfigName):
 raise ClientError(error_response, "describe_endpoint_config")
 return LocalSagemakerClient._endpoint_configs[EndpointConfigName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint_config")
 def create_endpoint_config(self, EndpointConfigName, ProductionVariants, Tags=None):
 """Create the endpoint configuration.

Expand DownExpand Up@@ -360,6 +367,7 @@ def describe_endpoint(self, EndpointName):
 raise ClientError(error_response, "describe_endpoint")
 return LocalSagemakerClient._endpoints[EndpointName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint")
 def create_endpoint(self, EndpointName, EndpointConfigName, Tags=None):
 """Create the endpoint.

Expand DownExpand Up@@ -428,6 +436,7 @@ def delete_model(self, ModelName):
 if ModelName in LocalSagemakerClient._models:
 del LocalSagemakerClient._models[ModelName]

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_pipeline")
 def create_pipeline(
 self, pipeline, pipeline_description, **kwargs # pylint: disable=unused-argument
 ):
Expand Down

16 changes: 16 additions & 0 deletions
src/sagemaker/telemetry/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Placeholder docstring"""
from __future__ import absolute_import

from .telemetry_logging import _telemetry_emitter # noqa: F401

41 changes: 41 additions & 0 deletions
src/sagemaker/telemetry/constants.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Constants used in SageMaker Python SDK telemetry."""

from __future__ import absolute_import
from enum import Enum

# Default AWS region used by SageMaker
DEFAULT_AWS_REGION = "us-west-2"


class Feature(Enum):
 """Enumeration of feature names used in telemetry."""

 SDK_DEFAULTS = 1
 LOCAL_MODE = 2

 def __str__(self): # pylint: disable=E0307
 """Return the feature name."""
 return self.name


class Status(Enum):
 """Enumeration of status values used in telemetry."""

 SUCCESS = 1
 FAILURE = 0

 def __str__(self): # pylint: disable=E0307
 """Return the status name."""
 return self.name

229 changes: 229 additions & 0 deletions
src/sagemaker/telemetry/telemetry_logging.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Telemetry module for SageMaker Python SDK to collect usage data and metrics."""
from __future__ import absolute_import
import logging
import platform
import sys
from time import perf_counter
from typing import List

from sagemaker.utils import resolve_value_from_config
from sagemaker.config.config_schema import TELEMETRY_OPT_OUT_PATH
from sagemaker.telemetry.constants import (
 Feature,
 Status,
 DEFAULT_AWS_REGION,
)
from sagemaker.user_agent import SDK_VERSION, process_studio_metadata_file

logger = logging.getLogger(__name__)

OS_NAME = platform.system() or "UnresolvedOS"
OS_VERSION = platform.release() or "UnresolvedOSVersion"
OS_NAME_VERSION = "{}/{}".format(OS_NAME, OS_VERSION)
PYTHON_VERSION = "{}.{}.{}".format(
 sys.version_info.major, sys.version_info.minor, sys.version_info.micro
)

TELEMETRY_OPT_OUT_MESSAGING = (
 "SageMaker Python SDK will collect telemetry to help us better understand our user's needs, "
 "diagnose issues, and deliver additional features.\n"
 "To opt out of telemetry, please disable via TelemetryOptOut parameter in SDK defaults config. "
 "For more information, refer to https://sagemaker.readthedocs.io/en/stable/overview.html"
 "#configuring-and-using-defaults-with-the-sagemaker-python-sdk."
)

FEATURE_TO_CODE = {
 str(Feature.SDK_DEFAULTS): 1,
 str(Feature.LOCAL_MODE): 2,
}

STATUS_TO_CODE = {
 str(Status.SUCCESS): 1,
 str(Status.FAILURE): 0,
}


def _telemetry_emitter(feature: str, func_name: str):
 """Decorator to emit telemetry logs for SageMaker Python SDK functions"""

 def decorator(func):
 def wrapper(self, *args, **kwargs):
Comment thread
knikure marked this conversation as resolved.

 logger.info(TELEMETRY_OPT_OUT_MESSAGING)
 response = None
 caught_ex = None
 studio_app_type = process_studio_metadata_file()

 # Check if telemetry is opted out
 telemetry_opt_out_flag = resolve_value_from_config(
 direct_input=None,
 config_path=TELEMETRY_OPT_OUT_PATH,
 default_value=False,
 sagemaker_session=self.sagemaker_session,
 )
 logger.debug("TelemetryOptOut flag is set to: %s", telemetry_opt_out_flag)

 # Construct the feature list to track feature combinations
 feature_list: List[int] = [FEATURE_TO_CODE[str(feature)]]
 if self.sagemaker_session:
 if self.sagemaker_session.sagemaker_config and feature != Feature.SDK_DEFAULTS:
 feature_list.append(FEATURE_TO_CODE[str(Feature.SDK_DEFAULTS)])

 if self.sagemaker_session.local_mode and feature != Feature.LOCAL_MODE:
 feature_list.append(FEATURE_TO_CODE[str(Feature.LOCAL_MODE)])

 # Construct the extra info to track platform and environment usage metadata
 extra = (
 f"{func_name}"
 f"&x-sdkVersion={SDK_VERSION}"
 f"&x-env={PYTHON_VERSION}"
 f"&x-sys={OS_NAME_VERSION}"
 f"&x-platform={studio_app_type}"
 )

 # Add endpoint ARN to the extra info if available
 if self.sagemaker_session and self.sagemaker_session.endpoint_arn:
 extra += f"&x-endpointArn={self.sagemaker_session.endpoint_arn}"

 start_timer = perf_counter()
 try:
 # Call the original function
 response = func(self, *args, **kwargs)
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.SUCCESS)],
 feature_list,
 self.sagemaker_session,
 None,
 None,
 extra,
 )
 except Exception as e: # pylint: disable=W0703
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.FAILURE)],
 feature_list,
 self.sagemaker_session,
 str(e),
 e.__class__.__name__,
 extra,
 )
 caught_ex = e
 finally:
 if caught_ex:
 raise caught_ex
 return response # pylint: disable=W0150

 return wrapper

 return decorator


from sagemaker.session import Session # noqa: E402 pylint: disable=C0413


def _send_telemetry_request(
 status: int,
 feature_list: List[int],
 session: Session,
 failure_reason: str = None,
 failure_type: str = None,
 extra_info: str = None,
) -> None:
 """Make GET request to an empty object in S3 bucket"""
 try:
 accountId = _get_accountId(session)
 region = _get_region_or_default(session)
 url = _construct_url(
 accountId,
 region,
 str(status),
 str(
 ",".join(map(str, feature_list))
 ), # Remove brackets and quotes to cut down on length
 failure_reason,
 failure_type,
 extra_info,
 )
 # Send the telemetry request
 logger.debug("Sending telemetry request to [%s]", url)
 _requests_helper(url, 2)
 logger.debug("SageMaker Python SDK telemetry successfully emitted!")
 except Exception: # pylint: disable=W0703
 logger.debug("SageMaker Python SDK telemetry not emitted!!")


def _construct_url(
 accountId: str,
 region: str,
 status: str,
 feature: str,
 failure_reason: str,
 failure_type: str,
 extra_info: str,
) -> str:
 """Construct the URL for the telemetry request"""

 base_url = (
 f"https://sm-pysdk-t-{region}.s3.{region}.amazonaws.com/telemetry?"
 f"x-accountId={accountId}"
 f"&x-status={status}"
 f"&x-feature={feature}"
 )
 logger.debug("Failure reason: %s", failure_reason)
 if failure_reason:
 base_url += f"&x-failureReason={failure_reason}"
 base_url += f"&x-failureType={failure_type}"
 if extra_info:
 base_url += f"&x-extra={extra_info}"
 return base_url


import requests # noqa: E402 pylint: disable=C0413,C0411


def _requests_helper(url, timeout):
 """Make a GET request to the given URL"""

 response = None
 try:
 response = requests.get(url, timeout)
 except requests.exceptions.RequestException as e:
 logger.exception("Request exception: %s", str(e))
 return response


def _get_accountId(session):
 """Return the account ID from the boto session"""

 try:
 sts = session.boto_session.client("sts")
 return sts.get_caller_identity()["Account"]
 except Exception: # pylint: disable=W0703
 return None


def _get_region_or_default(session):
 """Return the region name from the boto session or default to us-west-2"""

 try:
 return session.boto_session.region_name
 except Exception: # pylint: disable=W0703
 return DEFAULT_AWS_REGION

4 changes: 3 additions & 1 deletion
tests/integ/sagemaker/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -102,7 +102,9 @@
 "channels:\n"
 " - defaults\n"
 "dependencies:\n"
 " - scipy=1.10.1\n"
 " - requests=2.32.3\n"
 " - charset-normalizer=3.3.2\n"
 " - scipy=1.13.1\n"
 " - pip:\n"
 " - /sagemaker-{sagemaker_version}.tar.gz\n"
 "prefix: /opt/conda/bin/conda\n"
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat: Implement custom telemetry logging in SDK#4721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

knikure
merged 1 commit into

from

Jun 12, 2024
 Merged


feat: Implement custom telemetry logging in SDK
#4721
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
README.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -95,6 +95,13 @@ SageMaker Python SDK is tested on:
- Python 3.10
- Python 3.11

Telemetry
~~~~~~~~~~~~~~~

The ``sagemaker`` library has telemetry enabled to help us better understand user needs, diagnose issues, and deliver new features. This telemetry tracks the usage of various SageMaker functions.

If you prefer to opt out of telemetry, you can easily do so by setting the ``TelemetryOptOut`` parameter to ``true`` in the SDK defaults configuration. For detailed instructions, please visit `Configuring and using defaults with the SageMaker Python SDK <https://sagemaker.readthedocs.io/en/stable/overview.html#configuring-and-using-defaults-with-the-sagemaker-python-sdk>`__.

AWS Permissions
~~~~~~~~~~~~~~~

Expand Down

9 changes: 9 additions & 0 deletions
src/sagemaker/local/local_session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,8 @@
 _LocalPipeline,
)
from sagemaker.session import Session
from sagemaker.telemetry.telemetry_logging import _telemetry_emitter
from sagemaker.telemetry.constants import Feature
from sagemaker.utils import (
 get_config_value,
 _module_import_error,
Expand DownExpand Up@@ -83,6 +85,7 @@ def __init__(self, sagemaker_session=None):
 """
 self.sagemaker_session = sagemaker_session or LocalSession()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_processing_job")
 def create_processing_job(
 self,
 ProcessingJobName,
Expand DownExpand Up@@ -165,6 +168,7 @@ def describe_processing_job(self, ProcessingJobName):
 raise ClientError(error_response, "describe_processing_job")
 return LocalSagemakerClient._processing_jobs[ProcessingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_training_job")
 def create_training_job(
 self,
 TrainingJobName,
Expand DownExpand Up@@ -235,6 +239,7 @@ def describe_training_job(self, TrainingJobName):
 raise ClientError(error_response, "describe_training_job")
 return LocalSagemakerClient._training_jobs[TrainingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_transform_job")
 def create_transform_job(
 self,
 TransformJobName,
Expand DownExpand Up@@ -280,6 +285,7 @@ def describe_transform_job(self, TransformJobName):
 raise ClientError(error_response, "describe_transform_job")
 return LocalSagemakerClient._transform_jobs[TransformJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_model")
 def create_model(
 self, ModelName, PrimaryContainer, *args, **kwargs
 ): # pylint: disable=unused-argument
Expand DownExpand Up@@ -329,6 +335,7 @@ def describe_endpoint_config(self, EndpointConfigName):
 raise ClientError(error_response, "describe_endpoint_config")
 return LocalSagemakerClient._endpoint_configs[EndpointConfigName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint_config")
 def create_endpoint_config(self, EndpointConfigName, ProductionVariants, Tags=None):
 """Create the endpoint configuration.

Expand DownExpand Up@@ -360,6 +367,7 @@ def describe_endpoint(self, EndpointName):
 raise ClientError(error_response, "describe_endpoint")
 return LocalSagemakerClient._endpoints[EndpointName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint")
 def create_endpoint(self, EndpointName, EndpointConfigName, Tags=None):
 """Create the endpoint.

Expand DownExpand Up@@ -428,6 +436,7 @@ def delete_model(self, ModelName):
 if ModelName in LocalSagemakerClient._models:
 del LocalSagemakerClient._models[ModelName]

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_pipeline")
 def create_pipeline(
 self, pipeline, pipeline_description, **kwargs # pylint: disable=unused-argument
 ):
Expand Down

16 changes: 16 additions & 0 deletions
src/sagemaker/telemetry/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Placeholder docstring"""
from __future__ import absolute_import

from .telemetry_logging import _telemetry_emitter # noqa: F401

41 changes: 41 additions & 0 deletions
src/sagemaker/telemetry/constants.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Constants used in SageMaker Python SDK telemetry."""

from __future__ import absolute_import
from enum import Enum

# Default AWS region used by SageMaker
DEFAULT_AWS_REGION = "us-west-2"


class Feature(Enum):
 """Enumeration of feature names used in telemetry."""

 SDK_DEFAULTS = 1
 LOCAL_MODE = 2

 def __str__(self): # pylint: disable=E0307
 """Return the feature name."""
 return self.name


class Status(Enum):
 """Enumeration of status values used in telemetry."""

 SUCCESS = 1
 FAILURE = 0

 def __str__(self): # pylint: disable=E0307
 """Return the status name."""
 return self.name

229 changes: 229 additions & 0 deletions
src/sagemaker/telemetry/telemetry_logging.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Telemetry module for SageMaker Python SDK to collect usage data and metrics."""
from __future__ import absolute_import
import logging
import platform
import sys
from time import perf_counter
from typing import List

from sagemaker.utils import resolve_value_from_config
from sagemaker.config.config_schema import TELEMETRY_OPT_OUT_PATH
from sagemaker.telemetry.constants import (
 Feature,
 Status,
 DEFAULT_AWS_REGION,
)
from sagemaker.user_agent import SDK_VERSION, process_studio_metadata_file

logger = logging.getLogger(__name__)

OS_NAME = platform.system() or "UnresolvedOS"
OS_VERSION = platform.release() or "UnresolvedOSVersion"
OS_NAME_VERSION = "{}/{}".format(OS_NAME, OS_VERSION)
PYTHON_VERSION = "{}.{}.{}".format(
 sys.version_info.major, sys.version_info.minor, sys.version_info.micro
)

TELEMETRY_OPT_OUT_MESSAGING = (
 "SageMaker Python SDK will collect telemetry to help us better understand our user's needs, "
 "diagnose issues, and deliver additional features.\n"
 "To opt out of telemetry, please disable via TelemetryOptOut parameter in SDK defaults config. "
 "For more information, refer to https://sagemaker.readthedocs.io/en/stable/overview.html"
 "#configuring-and-using-defaults-with-the-sagemaker-python-sdk."
)

FEATURE_TO_CODE = {
 str(Feature.SDK_DEFAULTS): 1,
 str(Feature.LOCAL_MODE): 2,
}

STATUS_TO_CODE = {
 str(Status.SUCCESS): 1,
 str(Status.FAILURE): 0,
}


def _telemetry_emitter(feature: str, func_name: str):
 """Decorator to emit telemetry logs for SageMaker Python SDK functions"""

 def decorator(func):
 def wrapper(self, *args, **kwargs):
Comment thread
knikure marked this conversation as resolved.

 logger.info(TELEMETRY_OPT_OUT_MESSAGING)
 response = None
 caught_ex = None
 studio_app_type = process_studio_metadata_file()

 # Check if telemetry is opted out
 telemetry_opt_out_flag = resolve_value_from_config(
 direct_input=None,
 config_path=TELEMETRY_OPT_OUT_PATH,
 default_value=False,
 sagemaker_session=self.sagemaker_session,
 )
 logger.debug("TelemetryOptOut flag is set to: %s", telemetry_opt_out_flag)

 # Construct the feature list to track feature combinations
 feature_list: List[int] = [FEATURE_TO_CODE[str(feature)]]
 if self.sagemaker_session:
 if self.sagemaker_session.sagemaker_config and feature != Feature.SDK_DEFAULTS:
 feature_list.append(FEATURE_TO_CODE[str(Feature.SDK_DEFAULTS)])

 if self.sagemaker_session.local_mode and feature != Feature.LOCAL_MODE:
 feature_list.append(FEATURE_TO_CODE[str(Feature.LOCAL_MODE)])

 # Construct the extra info to track platform and environment usage metadata
 extra = (
 f"{func_name}"
 f"&x-sdkVersion={SDK_VERSION}"
 f"&x-env={PYTHON_VERSION}"
 f"&x-sys={OS_NAME_VERSION}"
 f"&x-platform={studio_app_type}"
 )

 # Add endpoint ARN to the extra info if available
 if self.sagemaker_session and self.sagemaker_session.endpoint_arn:
 extra += f"&x-endpointArn={self.sagemaker_session.endpoint_arn}"

 start_timer = perf_counter()
 try:
 # Call the original function
 response = func(self, *args, **kwargs)
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.SUCCESS)],
 feature_list,
 self.sagemaker_session,
 None,
 None,
 extra,
 )
 except Exception as e: # pylint: disable=W0703
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.FAILURE)],
 feature_list,
 self.sagemaker_session,
 str(e),
 e.__class__.__name__,
 extra,
 )
 caught_ex = e
 finally:
 if caught_ex:
 raise caught_ex
 return response # pylint: disable=W0150

 return wrapper

 return decorator


from sagemaker.session import Session # noqa: E402 pylint: disable=C0413


def _send_telemetry_request(
 status: int,
 feature_list: List[int],
 session: Session,
 failure_reason: str = None,
 failure_type: str = None,
 extra_info: str = None,
) -> None:
 """Make GET request to an empty object in S3 bucket"""
 try:
 accountId = _get_accountId(session)
 region = _get_region_or_default(session)
 url = _construct_url(
 accountId,
 region,
 str(status),
 str(
 ",".join(map(str, feature_list))
 ), # Remove brackets and quotes to cut down on length
 failure_reason,
 failure_type,
 extra_info,
 )
 # Send the telemetry request
 logger.debug("Sending telemetry request to [%s]", url)
 _requests_helper(url, 2)
 logger.debug("SageMaker Python SDK telemetry successfully emitted!")
 except Exception: # pylint: disable=W0703
 logger.debug("SageMaker Python SDK telemetry not emitted!!")


def _construct_url(
 accountId: str,
 region: str,
 status: str,
 feature: str,
 failure_reason: str,
 failure_type: str,
 extra_info: str,
) -> str:
 """Construct the URL for the telemetry request"""

 base_url = (
 f"https://sm-pysdk-t-{region}.s3.{region}.amazonaws.com/telemetry?"
 f"x-accountId={accountId}"
 f"&x-status={status}"
 f"&x-feature={feature}"
 )
 logger.debug("Failure reason: %s", failure_reason)
 if failure_reason:
 base_url += f"&x-failureReason={failure_reason}"
 base_url += f"&x-failureType={failure_type}"
 if extra_info:
 base_url += f"&x-extra={extra_info}"
 return base_url


import requests # noqa: E402 pylint: disable=C0413,C0411


def _requests_helper(url, timeout):
 """Make a GET request to the given URL"""

 response = None
 try:
 response = requests.get(url, timeout)
 except requests.exceptions.RequestException as e:
 logger.exception("Request exception: %s", str(e))
 return response


def _get_accountId(session):
 """Return the account ID from the boto session"""

 try:
 sts = session.boto_session.client("sts")
 return sts.get_caller_identity()["Account"]
 except Exception: # pylint: disable=W0703
 return None


def _get_region_or_default(session):
 """Return the region name from the boto session or default to us-west-2"""

 try:
 return session.boto_session.region_name
 except Exception: # pylint: disable=W0703
 return DEFAULT_AWS_REGION

4 changes: 3 additions & 1 deletion
tests/integ/sagemaker/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -102,7 +102,9 @@
 "channels:\n"
 " - defaults\n"
 "dependencies:\n"
 " - scipy=1.10.1\n"
 " - requests=2.32.3\n"
 " - charset-normalizer=3.3.2\n"
 " - scipy=1.13.1\n"
 " - pip:\n"
 " - /sagemaker-{sagemaker_version}.tar.gz\n"
 "prefix: /opt/conda/bin/conda\n"
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat: Implement custom telemetry logging in SDK#4721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

knikure
merged 1 commit into

from

Jun 12, 2024
 Merged


feat: Implement custom telemetry logging in SDK
#4721
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
README.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -95,6 +95,13 @@ SageMaker Python SDK is tested on:
- Python 3.10
- Python 3.11

Telemetry
~~~~~~~~~~~~~~~

The ``sagemaker`` library has telemetry enabled to help us better understand user needs, diagnose issues, and deliver new features. This telemetry tracks the usage of various SageMaker functions.

If you prefer to opt out of telemetry, you can easily do so by setting the ``TelemetryOptOut`` parameter to ``true`` in the SDK defaults configuration. For detailed instructions, please visit `Configuring and using defaults with the SageMaker Python SDK <https://sagemaker.readthedocs.io/en/stable/overview.html#configuring-and-using-defaults-with-the-sagemaker-python-sdk>`__.

AWS Permissions
~~~~~~~~~~~~~~~

Expand Down

9 changes: 9 additions & 0 deletions
src/sagemaker/local/local_session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,8 @@
 _LocalPipeline,
)
from sagemaker.session import Session
from sagemaker.telemetry.telemetry_logging import _telemetry_emitter
from sagemaker.telemetry.constants import Feature
from sagemaker.utils import (
 get_config_value,
 _module_import_error,
Expand DownExpand Up@@ -83,6 +85,7 @@ def __init__(self, sagemaker_session=None):
 """
 self.sagemaker_session = sagemaker_session or LocalSession()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_processing_job")
 def create_processing_job(
 self,
 ProcessingJobName,
Expand DownExpand Up@@ -165,6 +168,7 @@ def describe_processing_job(self, ProcessingJobName):
 raise ClientError(error_response, "describe_processing_job")
 return LocalSagemakerClient._processing_jobs[ProcessingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_training_job")
 def create_training_job(
 self,
 TrainingJobName,
Expand DownExpand Up@@ -235,6 +239,7 @@ def describe_training_job(self, TrainingJobName):
 raise ClientError(error_response, "describe_training_job")
 return LocalSagemakerClient._training_jobs[TrainingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_transform_job")
 def create_transform_job(
 self,
 TransformJobName,
Expand DownExpand Up@@ -280,6 +285,7 @@ def describe_transform_job(self, TransformJobName):
 raise ClientError(error_response, "describe_transform_job")
 return LocalSagemakerClient._transform_jobs[TransformJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_model")
 def create_model(
 self, ModelName, PrimaryContainer, *args, **kwargs
 ): # pylint: disable=unused-argument
Expand DownExpand Up@@ -329,6 +335,7 @@ def describe_endpoint_config(self, EndpointConfigName):
 raise ClientError(error_response, "describe_endpoint_config")
 return LocalSagemakerClient._endpoint_configs[EndpointConfigName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint_config")
 def create_endpoint_config(self, EndpointConfigName, ProductionVariants, Tags=None):
 """Create the endpoint configuration.

Expand DownExpand Up@@ -360,6 +367,7 @@ def describe_endpoint(self, EndpointName):
 raise ClientError(error_response, "describe_endpoint")
 return LocalSagemakerClient._endpoints[EndpointName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint")
 def create_endpoint(self, EndpointName, EndpointConfigName, Tags=None):
 """Create the endpoint.

Expand DownExpand Up@@ -428,6 +436,7 @@ def delete_model(self, ModelName):
 if ModelName in LocalSagemakerClient._models:
 del LocalSagemakerClient._models[ModelName]

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_pipeline")
 def create_pipeline(
 self, pipeline, pipeline_description, **kwargs # pylint: disable=unused-argument
 ):
Expand Down

16 changes: 16 additions & 0 deletions
src/sagemaker/telemetry/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Placeholder docstring"""
from __future__ import absolute_import

from .telemetry_logging import _telemetry_emitter # noqa: F401

41 changes: 41 additions & 0 deletions
src/sagemaker/telemetry/constants.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Constants used in SageMaker Python SDK telemetry."""

from __future__ import absolute_import
from enum import Enum

# Default AWS region used by SageMaker
DEFAULT_AWS_REGION = "us-west-2"


class Feature(Enum):
 """Enumeration of feature names used in telemetry."""

 SDK_DEFAULTS = 1
 LOCAL_MODE = 2

 def __str__(self): # pylint: disable=E0307
 """Return the feature name."""
 return self.name


class Status(Enum):
 """Enumeration of status values used in telemetry."""

 SUCCESS = 1
 FAILURE = 0

 def __str__(self): # pylint: disable=E0307
 """Return the status name."""
 return self.name

229 changes: 229 additions & 0 deletions
src/sagemaker/telemetry/telemetry_logging.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Telemetry module for SageMaker Python SDK to collect usage data and metrics."""
from __future__ import absolute_import
import logging
import platform
import sys
from time import perf_counter
from typing import List

from sagemaker.utils import resolve_value_from_config
from sagemaker.config.config_schema import TELEMETRY_OPT_OUT_PATH
from sagemaker.telemetry.constants import (
 Feature,
 Status,
 DEFAULT_AWS_REGION,
)
from sagemaker.user_agent import SDK_VERSION, process_studio_metadata_file

logger = logging.getLogger(__name__)

OS_NAME = platform.system() or "UnresolvedOS"
OS_VERSION = platform.release() or "UnresolvedOSVersion"
OS_NAME_VERSION = "{}/{}".format(OS_NAME, OS_VERSION)
PYTHON_VERSION = "{}.{}.{}".format(
 sys.version_info.major, sys.version_info.minor, sys.version_info.micro
)

TELEMETRY_OPT_OUT_MESSAGING = (
 "SageMaker Python SDK will collect telemetry to help us better understand our user's needs, "
 "diagnose issues, and deliver additional features.\n"
 "To opt out of telemetry, please disable via TelemetryOptOut parameter in SDK defaults config. "
 "For more information, refer to https://sagemaker.readthedocs.io/en/stable/overview.html"
 "#configuring-and-using-defaults-with-the-sagemaker-python-sdk."
)

FEATURE_TO_CODE = {
 str(Feature.SDK_DEFAULTS): 1,
 str(Feature.LOCAL_MODE): 2,
}

STATUS_TO_CODE = {
 str(Status.SUCCESS): 1,
 str(Status.FAILURE): 0,
}


def _telemetry_emitter(feature: str, func_name: str):
 """Decorator to emit telemetry logs for SageMaker Python SDK functions"""

 def decorator(func):
 def wrapper(self, *args, **kwargs):
Comment thread
knikure marked this conversation as resolved.

 logger.info(TELEMETRY_OPT_OUT_MESSAGING)
 response = None
 caught_ex = None
 studio_app_type = process_studio_metadata_file()

 # Check if telemetry is opted out
 telemetry_opt_out_flag = resolve_value_from_config(
 direct_input=None,
 config_path=TELEMETRY_OPT_OUT_PATH,
 default_value=False,
 sagemaker_session=self.sagemaker_session,
 )
 logger.debug("TelemetryOptOut flag is set to: %s", telemetry_opt_out_flag)

 # Construct the feature list to track feature combinations
 feature_list: List[int] = [FEATURE_TO_CODE[str(feature)]]
 if self.sagemaker_session:
 if self.sagemaker_session.sagemaker_config and feature != Feature.SDK_DEFAULTS:
 feature_list.append(FEATURE_TO_CODE[str(Feature.SDK_DEFAULTS)])

 if self.sagemaker_session.local_mode and feature != Feature.LOCAL_MODE:
 feature_list.append(FEATURE_TO_CODE[str(Feature.LOCAL_MODE)])

 # Construct the extra info to track platform and environment usage metadata
 extra = (
 f"{func_name}"
 f"&x-sdkVersion={SDK_VERSION}"
 f"&x-env={PYTHON_VERSION}"
 f"&x-sys={OS_NAME_VERSION}"
 f"&x-platform={studio_app_type}"
 )

 # Add endpoint ARN to the extra info if available
 if self.sagemaker_session and self.sagemaker_session.endpoint_arn:
 extra += f"&x-endpointArn={self.sagemaker_session.endpoint_arn}"

 start_timer = perf_counter()
 try:
 # Call the original function
 response = func(self, *args, **kwargs)
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.SUCCESS)],
 feature_list,
 self.sagemaker_session,
 None,
 None,
 extra,
 )
 except Exception as e: # pylint: disable=W0703
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.FAILURE)],
 feature_list,
 self.sagemaker_session,
 str(e),
 e.__class__.__name__,
 extra,
 )
 caught_ex = e
 finally:
 if caught_ex:
 raise caught_ex
 return response # pylint: disable=W0150

 return wrapper

 return decorator


from sagemaker.session import Session # noqa: E402 pylint: disable=C0413


def _send_telemetry_request(
 status: int,
 feature_list: List[int],
 session: Session,
 failure_reason: str = None,
 failure_type: str = None,
 extra_info: str = None,
) -> None:
 """Make GET request to an empty object in S3 bucket"""
 try:
 accountId = _get_accountId(session)
 region = _get_region_or_default(session)
 url = _construct_url(
 accountId,
 region,
 str(status),
 str(
 ",".join(map(str, feature_list))
 ), # Remove brackets and quotes to cut down on length
 failure_reason,
 failure_type,
 extra_info,
 )
 # Send the telemetry request
 logger.debug("Sending telemetry request to [%s]", url)
 _requests_helper(url, 2)
 logger.debug("SageMaker Python SDK telemetry successfully emitted!")
 except Exception: # pylint: disable=W0703
 logger.debug("SageMaker Python SDK telemetry not emitted!!")


def _construct_url(
 accountId: str,
 region: str,
 status: str,
 feature: str,
 failure_reason: str,
 failure_type: str,
 extra_info: str,
) -> str:
 """Construct the URL for the telemetry request"""

 base_url = (
 f"https://sm-pysdk-t-{region}.s3.{region}.amazonaws.com/telemetry?"
 f"x-accountId={accountId}"
 f"&x-status={status}"
 f"&x-feature={feature}"
 )
 logger.debug("Failure reason: %s", failure_reason)
 if failure_reason:
 base_url += f"&x-failureReason={failure_reason}"
 base_url += f"&x-failureType={failure_type}"
 if extra_info:
 base_url += f"&x-extra={extra_info}"
 return base_url


import requests # noqa: E402 pylint: disable=C0413,C0411


def _requests_helper(url, timeout):
 """Make a GET request to the given URL"""

 response = None
 try:
 response = requests.get(url, timeout)
 except requests.exceptions.RequestException as e:
 logger.exception("Request exception: %s", str(e))
 return response


def _get_accountId(session):
 """Return the account ID from the boto session"""

 try:
 sts = session.boto_session.client("sts")
 return sts.get_caller_identity()["Account"]
 except Exception: # pylint: disable=W0703
 return None


def _get_region_or_default(session):
 """Return the region name from the boto session or default to us-west-2"""

 try:
 return session.boto_session.region_name
 except Exception: # pylint: disable=W0703
 return DEFAULT_AWS_REGION

4 changes: 3 additions & 1 deletion
tests/integ/sagemaker/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -102,7 +102,9 @@
 "channels:\n"
 " - defaults\n"
 "dependencies:\n"
 " - scipy=1.10.1\n"
 " - requests=2.32.3\n"
 " - charset-normalizer=3.3.2\n"
 " - scipy=1.13.1\n"
 " - pip:\n"
 " - /sagemaker-{sagemaker_version}.tar.gz\n"
 "prefix: /opt/conda/bin/conda\n"
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
feat: Implement custom telemetry logging in SDK#4721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

knikure
merged 1 commit into

from

Jun 12, 2024
 Merged


feat: Implement custom telemetry logging in SDK
#4721
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
README.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -95,6 +95,13 @@ SageMaker Python SDK is tested on:
- Python 3.10
- Python 3.11

Telemetry
~~~~~~~~~~~~~~~

The ``sagemaker`` library has telemetry enabled to help us better understand user needs, diagnose issues, and deliver new features. This telemetry tracks the usage of various SageMaker functions.

If you prefer to opt out of telemetry, you can easily do so by setting the ``TelemetryOptOut`` parameter to ``true`` in the SDK defaults configuration. For detailed instructions, please visit `Configuring and using defaults with the SageMaker Python SDK <https://sagemaker.readthedocs.io/en/stable/overview.html#configuring-and-using-defaults-with-the-sagemaker-python-sdk>`__.

AWS Permissions
~~~~~~~~~~~~~~~

Expand Down

9 changes: 9 additions & 0 deletions
src/sagemaker/local/local_session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,8 @@
 _LocalPipeline,
)
from sagemaker.session import Session
from sagemaker.telemetry.telemetry_logging import _telemetry_emitter
from sagemaker.telemetry.constants import Feature
from sagemaker.utils import (
 get_config_value,
 _module_import_error,
Expand DownExpand Up@@ -83,6 +85,7 @@ def __init__(self, sagemaker_session=None):
 """
 self.sagemaker_session = sagemaker_session or LocalSession()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_processing_job")
 def create_processing_job(
 self,
 ProcessingJobName,
Expand DownExpand Up@@ -165,6 +168,7 @@ def describe_processing_job(self, ProcessingJobName):
 raise ClientError(error_response, "describe_processing_job")
 return LocalSagemakerClient._processing_jobs[ProcessingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_training_job")
 def create_training_job(
 self,
 TrainingJobName,
Expand DownExpand Up@@ -235,6 +239,7 @@ def describe_training_job(self, TrainingJobName):
 raise ClientError(error_response, "describe_training_job")
 return LocalSagemakerClient._training_jobs[TrainingJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_transform_job")
 def create_transform_job(
 self,
 TransformJobName,
Expand DownExpand Up@@ -280,6 +285,7 @@ def describe_transform_job(self, TransformJobName):
 raise ClientError(error_response, "describe_transform_job")
 return LocalSagemakerClient._transform_jobs[TransformJobName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_model")
 def create_model(
 self, ModelName, PrimaryContainer, *args, **kwargs
 ): # pylint: disable=unused-argument
Expand DownExpand Up@@ -329,6 +335,7 @@ def describe_endpoint_config(self, EndpointConfigName):
 raise ClientError(error_response, "describe_endpoint_config")
 return LocalSagemakerClient._endpoint_configs[EndpointConfigName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint_config")
 def create_endpoint_config(self, EndpointConfigName, ProductionVariants, Tags=None):
 """Create the endpoint configuration.

Expand DownExpand Up@@ -360,6 +367,7 @@ def describe_endpoint(self, EndpointName):
 raise ClientError(error_response, "describe_endpoint")
 return LocalSagemakerClient._endpoints[EndpointName].describe()

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_endpoint")
 def create_endpoint(self, EndpointName, EndpointConfigName, Tags=None):
 """Create the endpoint.

Expand DownExpand Up@@ -428,6 +436,7 @@ def delete_model(self, ModelName):
 if ModelName in LocalSagemakerClient._models:
 del LocalSagemakerClient._models[ModelName]

 @_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_pipeline")
 def create_pipeline(
 self, pipeline, pipeline_description, **kwargs # pylint: disable=unused-argument
 ):
Expand Down

16 changes: 16 additions & 0 deletions
src/sagemaker/telemetry/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Placeholder docstring"""
from __future__ import absolute_import

from .telemetry_logging import _telemetry_emitter # noqa: F401

41 changes: 41 additions & 0 deletions
src/sagemaker/telemetry/constants.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Constants used in SageMaker Python SDK telemetry."""

from __future__ import absolute_import
from enum import Enum

# Default AWS region used by SageMaker
DEFAULT_AWS_REGION = "us-west-2"


class Feature(Enum):
 """Enumeration of feature names used in telemetry."""

 SDK_DEFAULTS = 1
 LOCAL_MODE = 2

 def __str__(self): # pylint: disable=E0307
 """Return the feature name."""
 return self.name


class Status(Enum):
 """Enumeration of status values used in telemetry."""

 SUCCESS = 1
 FAILURE = 0

 def __str__(self): # pylint: disable=E0307
 """Return the status name."""
 return self.name

229 changes: 229 additions & 0 deletions
src/sagemaker/telemetry/telemetry_logging.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Telemetry module for SageMaker Python SDK to collect usage data and metrics."""
from __future__ import absolute_import
import logging
import platform
import sys
from time import perf_counter
from typing import List

from sagemaker.utils import resolve_value_from_config
from sagemaker.config.config_schema import TELEMETRY_OPT_OUT_PATH
from sagemaker.telemetry.constants import (
 Feature,
 Status,
 DEFAULT_AWS_REGION,
)
from sagemaker.user_agent import SDK_VERSION, process_studio_metadata_file

logger = logging.getLogger(__name__)

OS_NAME = platform.system() or "UnresolvedOS"
OS_VERSION = platform.release() or "UnresolvedOSVersion"
OS_NAME_VERSION = "{}/{}".format(OS_NAME, OS_VERSION)
PYTHON_VERSION = "{}.{}.{}".format(
 sys.version_info.major, sys.version_info.minor, sys.version_info.micro
)

TELEMETRY_OPT_OUT_MESSAGING = (
 "SageMaker Python SDK will collect telemetry to help us better understand our user's needs, "
 "diagnose issues, and deliver additional features.\n"
 "To opt out of telemetry, please disable via TelemetryOptOut parameter in SDK defaults config. "
 "For more information, refer to https://sagemaker.readthedocs.io/en/stable/overview.html"
 "#configuring-and-using-defaults-with-the-sagemaker-python-sdk."
)

FEATURE_TO_CODE = {
 str(Feature.SDK_DEFAULTS): 1,
 str(Feature.LOCAL_MODE): 2,
}

STATUS_TO_CODE = {
 str(Status.SUCCESS): 1,
 str(Status.FAILURE): 0,
}


def _telemetry_emitter(feature: str, func_name: str):
 """Decorator to emit telemetry logs for SageMaker Python SDK functions"""

 def decorator(func):
 def wrapper(self, *args, **kwargs):
Comment thread
knikure marked this conversation as resolved.

 logger.info(TELEMETRY_OPT_OUT_MESSAGING)
 response = None
 caught_ex = None
 studio_app_type = process_studio_metadata_file()

 # Check if telemetry is opted out
 telemetry_opt_out_flag = resolve_value_from_config(
 direct_input=None,
 config_path=TELEMETRY_OPT_OUT_PATH,
 default_value=False,
 sagemaker_session=self.sagemaker_session,
 )
 logger.debug("TelemetryOptOut flag is set to: %s", telemetry_opt_out_flag)

 # Construct the feature list to track feature combinations
 feature_list: List[int] = [FEATURE_TO_CODE[str(feature)]]
 if self.sagemaker_session:
 if self.sagemaker_session.sagemaker_config and feature != Feature.SDK_DEFAULTS:
 feature_list.append(FEATURE_TO_CODE[str(Feature.SDK_DEFAULTS)])

 if self.sagemaker_session.local_mode and feature != Feature.LOCAL_MODE:
 feature_list.append(FEATURE_TO_CODE[str(Feature.LOCAL_MODE)])

 # Construct the extra info to track platform and environment usage metadata
 extra = (
 f"{func_name}"
 f"&x-sdkVersion={SDK_VERSION}"
 f"&x-env={PYTHON_VERSION}"
 f"&x-sys={OS_NAME_VERSION}"
 f"&x-platform={studio_app_type}"
 )

 # Add endpoint ARN to the extra info if available
 if self.sagemaker_session and self.sagemaker_session.endpoint_arn:
 extra += f"&x-endpointArn={self.sagemaker_session.endpoint_arn}"

 start_timer = perf_counter()
 try:
 # Call the original function
 response = func(self, *args, **kwargs)
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.SUCCESS)],
 feature_list,
 self.sagemaker_session,
 None,
 None,
 extra,
 )
 except Exception as e: # pylint: disable=W0703
 stop_timer = perf_counter()
 elapsed = stop_timer - start_timer
 extra += f"&x-latency={round(elapsed, 2)}"
 if not telemetry_opt_out_flag:
 _send_telemetry_request(
 STATUS_TO_CODE[str(Status.FAILURE)],
 feature_list,
 self.sagemaker_session,
 str(e),
 e.__class__.__name__,
 extra,
 )
 caught_ex = e
 finally:
 if caught_ex:
 raise caught_ex
 return response # pylint: disable=W0150

 return wrapper

 return decorator


from sagemaker.session import Session # noqa: E402 pylint: disable=C0413


def _send_telemetry_request(
 status: int,
 feature_list: List[int],
 session: Session,
 failure_reason: str = None,
 failure_type: str = None,
 extra_info: str = None,
) -> None:
 """Make GET request to an empty object in S3 bucket"""
 try:
 accountId = _get_accountId(session)
 region = _get_region_or_default(session)
 url = _construct_url(
 accountId,
 region,
 str(status),
 str(
 ",".join(map(str, feature_list))
 ), # Remove brackets and quotes to cut down on length
 failure_reason,
 failure_type,
 extra_info,
 )
 # Send the telemetry request
 logger.debug("Sending telemetry request to [%s]", url)
 _requests_helper(url, 2)
 logger.debug("SageMaker Python SDK telemetry successfully emitted!")
 except Exception: # pylint: disable=W0703
 logger.debug("SageMaker Python SDK telemetry not emitted!!")


def _construct_url(
 accountId: str,
 region: str,
 status: str,
 feature: str,
 failure_reason: str,
 failure_type: str,
 extra_info: str,
) -> str:
 """Construct the URL for the telemetry request"""

 base_url = (
 f"https://sm-pysdk-t-{region}.s3.{region}.amazonaws.com/telemetry?"
 f"x-accountId={accountId}"
 f"&x-status={status}"
 f"&x-feature={feature}"
 )
 logger.debug("Failure reason: %s", failure_reason)
 if failure_reason:
 base_url += f"&x-failureReason={failure_reason}"
 base_url += f"&x-failureType={failure_type}"
 if extra_info:
 base_url += f"&x-extra={extra_info}"
 return base_url


import requests # noqa: E402 pylint: disable=C0413,C0411


def _requests_helper(url, timeout):
 """Make a GET request to the given URL"""

 response = None
 try:
 response = requests.get(url, timeout)
 except requests.exceptions.RequestException as e:
 logger.exception("Request exception: %s", str(e))
 return response


def _get_accountId(session):
 """Return the account ID from the boto session"""

 try:
 sts = session.boto_session.client("sts")
 return sts.get_caller_identity()["Account"]
 except Exception: # pylint: disable=W0703
 return None


def _get_region_or_default(session):
 """Return the region name from the boto session or default to us-west-2"""

 try:
 return session.boto_session.region_name
 except Exception: # pylint: disable=W0703
 return DEFAULT_AWS_REGION

4 changes: 3 additions & 1 deletion
tests/integ/sagemaker/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -102,7 +102,9 @@
 "channels:\n"
 " - defaults\n"
 "dependencies:\n"
 " - scipy=1.10.1\n"
 " - requests=2.32.3\n"
 " - charset-normalizer=3.3.2\n"
 " - scipy=1.13.1\n"
 " - pip:\n"
 " - /sagemaker-{sagemaker_version}.tar.gz\n"
 "prefix: /opt/conda/bin/conda\n"
Expand Down
Loading