change: separate out methods to get train arguments#1850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
metrizable merged 1 commit into from
Aug 24, 2020
Merged

change: separate out methods to get train arguments #1850

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions src/sagemaker/estimator.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1037,11 +1037,31 @@ def start_new(cls, estimator, inputs, experiment_config):
:meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.


Returns:
sagemaker.estimator._TrainingJob: Constructed object that captures
all information about the started training job.
"""
train_args = cls._get_train_args(estimator, inputs, experiment_config)
estimator.sagemaker_session.train(**train_args)

return cls(estimator.sagemaker_session, estimator._current_job_name)

@classmethod
def _get_train_args(cls, estimator, inputs, experiment_config):
"""Constructs a dict of arguments for an Amazon SageMaker training job from the estimator.

Args:
estimator (sagemaker.estimator.EstimatorBase): Estimator object
created by the user.
inputs (str): Parameters used when called
:meth:`~sagemaker.estimator.EstimatorBase.fit`.
experiment_config (dict[str, str]): Experiment management configuration used when called
:meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.

Returns:
Dict: dict for `sagemaker.session.Session.train` method
"""

local_mode = estimator.sagemaker_session.local_mode
model_uri = estimator.model_uri
Expand DownExpand Up@@ -1102,9 +1122,7 @@ def start_new(cls, estimator, inputs, experiment_config):
if estimator.enable_sagemaker_metrics is not None:
train_args["enable_sagemaker_metrics"] = estimator.enable_sagemaker_metrics

estimator.sagemaker_session.train(**train_args)

return cls(estimator.sagemaker_session, estimator._current_job_name)
return train_args

@classmethod
def _add_spot_checkpoint_args(cls, local_mode, estimator, train_args):
Expand Down
130 changes: 127 additions & 3 deletions src/sagemaker/session.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -511,7 +511,133 @@ def train( # noqa: C901
Returns:
str: ARN of the training job, if it is created.
"""
train_request = self._get_train_request(
input_mode=input_mode,
input_config=input_config,
role=role,
job_name=job_name,
output_config=output_config,
resource_config=resource_config,
vpc_config=vpc_config,
hyperparameters=hyperparameters,
stop_condition=stop_condition,
tags=tags,
metric_definitions=metric_definitions,
enable_network_isolation=enable_network_isolation,
image_uri=image_uri,
algorithm_arn=algorithm_arn,
encrypt_inter_container_traffic=encrypt_inter_container_traffic,
use_spot_instances=use_spot_instances,
checkpoint_s3_uri=checkpoint_s3_uri,
checkpoint_local_path=checkpoint_local_path,
experiment_config=experiment_config,
debugger_rule_configs=debugger_rule_configs,
debugger_hook_config=debugger_hook_config,
tensorboard_output_config=tensorboard_output_config,
enable_sagemaker_metrics=enable_sagemaker_metrics,
)
LOGGER.info("Creating training-job with name: %s", job_name)
LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
self.sagemaker_client.create_training_job(**train_request)

def _get_train_request( # noqa: C901
self,
input_mode,
input_config,
role,
job_name,
output_config,
resource_config,
vpc_config,
hyperparameters,
stop_condition,
tags,
metric_definitions,
enable_network_isolation=False,
image_uri=None,
algorithm_arn=None,
encrypt_inter_container_traffic=False,
use_spot_instances=False,
checkpoint_s3_uri=None,
checkpoint_local_path=None,
experiment_config=None,
debugger_rule_configs=None,
debugger_hook_config=None,
tensorboard_output_config=None,
enable_sagemaker_metrics=None,
):
"""Constructs a request compatible for creating an Amazon SageMaker training job.

Args:
input_mode (str): The input mode that the algorithm supports. Valid modes:
* 'File' - Amazon SageMaker copies the training dataset from the S3 location to
a directory in the Docker container.
* 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
Unix-named pipe.

input_config (list): A list of Channel objects. Each channel is a named input source.
Please refer to the format details described:
https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training
jobs and APIs that create Amazon SageMaker endpoints use this role to access
training data and model artifacts. You must grant sufficient permissions to this
role.
job_name (str): Name of the training job being created.
output_config (dict): The S3 URI where you want to store the training results and
optional KMS key ID.
resource_config (dict): Contains values for ResourceConfig:
* instance_count (int): Number of EC2 instances to use for training.
The key in resource_config is 'InstanceCount'.
* instance_type (str): Type of EC2 instance to use for training, for example,
'ml.c4.xlarge'. The key in resource_config is 'InstanceType'.

vpc_config (dict): Contains values for VpcConfig:
* subnets (list[str]): List of subnet ids.
The key in vpc_config is 'Subnets'.
* security_group_ids (list[str]): List of security group ids.
The key in vpc_config is 'SecurityGroupIds'.

hyperparameters (dict): Hyperparameters for model training. The hyperparameters are
made accessible as a dict[str, str] to the training code on SageMaker. For
convenience, this accepts other types for keys and values, but ``str()`` will be
called to convert them before training.
stop_condition (dict): Defines when training shall finish. Contains entries that can
be understood by the service like ``MaxRuntimeInSeconds``.
tags (list[dict]): List of tags for labeling a training job. For more, see
https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
metric_definitions (list[dict]): A list of dictionaries that defines the metric(s)
used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
the name of the metric, and 'Regex' for the regular expression used to extract the
metric from the logs.
enable_network_isolation (bool): Whether to request for the training job to run with
network isolation or not.
image_uri (str): Docker image containing training code.
algorithm_arn (str): Algorithm Arn from Marketplace.
encrypt_inter_container_traffic (bool): Specifies whether traffic between training
containers is encrypted for the training job (default: ``False``).
use_spot_instances (bool): whether to use spot instances for training.
checkpoint_s3_uri (str): The S3 URI in which to persist checkpoints
that the algorithm persists (if any) during training. (default:
``None``).
checkpoint_local_path (str): The local path that the algorithm
writes its checkpoints to. SageMaker will persist all files
under this path to `checkpoint_s3_uri` continually during
training. On job startup the reverse happens - data from the
s3 location is downloaded to this path before the algorithm is
started. If the path is unset then SageMaker assumes the
checkpoints will be provided under `/opt/ml/checkpoints/`.
(default: ``None``).
experiment_config (dict): Experiment management configuration. Dictionary contains
three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
(default: ``None``)
enable_sagemaker_metrics (bool): enable SageMaker Metrics Time
Series. For more information see:
https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
(default: ``None``).

Returns:
Dict: a training request dictionary
"""
train_request = {
"AlgorithmSpecification": {"TrainingInputMode": input_mode},
"OutputDataConfig": output_config,
Expand DownExpand Up@@ -583,9 +709,7 @@ def train( # noqa: C901
if tensorboard_output_config is not None:
train_request["TensorBoardOutputConfig"] = tensorboard_output_config

LOGGER.info("Creating training-job with name: %s", job_name)
LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
self.sagemaker_client.create_training_job(**train_request)
return train_request

def process(
self,
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
change: separate out methods to get train arguments#1850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

metrizable
merged 1 commit into

from

Aug 24, 2020
 Merged


change: separate out methods to get train arguments
#1850
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 22 additions & 4 deletions
src/sagemaker/estimator.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1037,11 +1037,31 @@ def start_new(cls, estimator, inputs, experiment_config):
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.


 Returns:
 sagemaker.estimator._TrainingJob: Constructed object that captures
 all information about the started training job.
 """
 train_args = cls._get_train_args(estimator, inputs, experiment_config)
 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)

 @classmethod
 def _get_train_args(cls, estimator, inputs, experiment_config):
 """Constructs a dict of arguments for an Amazon SageMaker training job from the estimator.

 Args:
 estimator (sagemaker.estimator.EstimatorBase): Estimator object
 created by the user.
 inputs (str): Parameters used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`.
 experiment_config (dict[str, str]): Experiment management configuration used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.

 Returns:
 Dict: dict for `sagemaker.session.Session.train` method
 """

 local_mode = estimator.sagemaker_session.local_mode
 model_uri = estimator.model_uri
Expand DownExpand Up@@ -1102,9 +1122,7 @@ def start_new(cls, estimator, inputs, experiment_config):
 if estimator.enable_sagemaker_metrics is not None:
 train_args["enable_sagemaker_metrics"] = estimator.enable_sagemaker_metrics

 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)
 return train_args

 @classmethod
 def _add_spot_checkpoint_args(cls, local_mode, estimator, train_args):
Expand Down

130 changes: 127 additions & 3 deletions
src/sagemaker/session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -511,7 +511,133 @@ def train( # noqa: C901
 Returns:
 str: ARN of the training job, if it is created.
 """
 train_request = self._get_train_request(
 input_mode=input_mode,
 input_config=input_config,
 role=role,
 job_name=job_name,
 output_config=output_config,
 resource_config=resource_config,
 vpc_config=vpc_config,
 hyperparameters=hyperparameters,
 stop_condition=stop_condition,
 tags=tags,
 metric_definitions=metric_definitions,
 enable_network_isolation=enable_network_isolation,
 image_uri=image_uri,
 algorithm_arn=algorithm_arn,
 encrypt_inter_container_traffic=encrypt_inter_container_traffic,
 use_spot_instances=use_spot_instances,
 checkpoint_s3_uri=checkpoint_s3_uri,
 checkpoint_local_path=checkpoint_local_path,
 experiment_config=experiment_config,
 debugger_rule_configs=debugger_rule_configs,
 debugger_hook_config=debugger_hook_config,
 tensorboard_output_config=tensorboard_output_config,
 enable_sagemaker_metrics=enable_sagemaker_metrics,
 )
 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)

 def _get_train_request( # noqa: C901
 self,
 input_mode,
 input_config,
 role,
 job_name,
 output_config,
 resource_config,
 vpc_config,
 hyperparameters,
 stop_condition,
 tags,
 metric_definitions,
 enable_network_isolation=False,
 image_uri=None,
 algorithm_arn=None,
 encrypt_inter_container_traffic=False,
 use_spot_instances=False,
 checkpoint_s3_uri=None,
 checkpoint_local_path=None,
 experiment_config=None,
 debugger_rule_configs=None,
 debugger_hook_config=None,
 tensorboard_output_config=None,
 enable_sagemaker_metrics=None,
 ):
 """Constructs a request compatible for creating an Amazon SageMaker training job.

 Args:
 input_mode (str): The input mode that the algorithm supports. Valid modes:
 * 'File' - Amazon SageMaker copies the training dataset from the S3 location to
 a directory in the Docker container.
 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
 Unix-named pipe.

 input_config (list): A list of Channel objects. Each channel is a named input source.
 Please refer to the format details described:
 https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
 role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training
 jobs and APIs that create Amazon SageMaker endpoints use this role to access
 training data and model artifacts. You must grant sufficient permissions to this
 role.
 job_name (str): Name of the training job being created.
 output_config (dict): The S3 URI where you want to store the training results and
 optional KMS key ID.
 resource_config (dict): Contains values for ResourceConfig:
 * instance_count (int): Number of EC2 instances to use for training.
 The key in resource_config is 'InstanceCount'.
 * instance_type (str): Type of EC2 instance to use for training, for example,
 'ml.c4.xlarge'. The key in resource_config is 'InstanceType'.

 vpc_config (dict): Contains values for VpcConfig:
 * subnets (list[str]): List of subnet ids.
 The key in vpc_config is 'Subnets'.
 * security_group_ids (list[str]): List of security group ids.
 The key in vpc_config is 'SecurityGroupIds'.

 hyperparameters (dict): Hyperparameters for model training. The hyperparameters are
 made accessible as a dict[str, str] to the training code on SageMaker. For
 convenience, this accepts other types for keys and values, but ``str()`` will be
 called to convert them before training.
 stop_condition (dict): Defines when training shall finish. Contains entries that can
 be understood by the service like ``MaxRuntimeInSeconds``.
 tags (list[dict]): List of tags for labeling a training job. For more, see
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
 metric_definitions (list[dict]): A list of dictionaries that defines the metric(s)
 used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
 the name of the metric, and 'Regex' for the regular expression used to extract the
 metric from the logs.
 enable_network_isolation (bool): Whether to request for the training job to run with
 network isolation or not.
 image_uri (str): Docker image containing training code.
 algorithm_arn (str): Algorithm Arn from Marketplace.
 encrypt_inter_container_traffic (bool): Specifies whether traffic between training
 containers is encrypted for the training job (default: ``False``).
 use_spot_instances (bool): whether to use spot instances for training.
 checkpoint_s3_uri (str): The S3 URI in which to persist checkpoints
 that the algorithm persists (if any) during training. (default:
 ``None``).
 checkpoint_local_path (str): The local path that the algorithm
 writes its checkpoints to. SageMaker will persist all files
 under this path to `checkpoint_s3_uri` continually during
 training. On job startup the reverse happens - data from the
 s3 location is downloaded to this path before the algorithm is
 started. If the path is unset then SageMaker assumes the
 checkpoints will be provided under `/opt/ml/checkpoints/`.
 (default: ``None``).
 experiment_config (dict): Experiment management configuration. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
 (default: ``None``)
 enable_sagemaker_metrics (bool): enable SageMaker Metrics Time
 Series. For more information see:
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
 (default: ``None``).

 Returns:
 Dict: a training request dictionary
 """
 train_request = {
 "AlgorithmSpecification": {"TrainingInputMode": input_mode},
 "OutputDataConfig": output_config,
Expand DownExpand Up@@ -583,9 +709,7 @@ def train( # noqa: C901
 if tensorboard_output_config is not None:
 train_request["TensorBoardOutputConfig"] = tensorboard_output_config

 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)
 return train_request

 def process(
 self,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
change: separate out methods to get train arguments#1850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

metrizable
merged 1 commit into

from

Aug 24, 2020
 Merged


change: separate out methods to get train arguments
#1850
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 22 additions & 4 deletions
src/sagemaker/estimator.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1037,11 +1037,31 @@ def start_new(cls, estimator, inputs, experiment_config):
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.


 Returns:
 sagemaker.estimator._TrainingJob: Constructed object that captures
 all information about the started training job.
 """
 train_args = cls._get_train_args(estimator, inputs, experiment_config)
 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)

 @classmethod
 def _get_train_args(cls, estimator, inputs, experiment_config):
 """Constructs a dict of arguments for an Amazon SageMaker training job from the estimator.

 Args:
 estimator (sagemaker.estimator.EstimatorBase): Estimator object
 created by the user.
 inputs (str): Parameters used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`.
 experiment_config (dict[str, str]): Experiment management configuration used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.

 Returns:
 Dict: dict for `sagemaker.session.Session.train` method
 """

 local_mode = estimator.sagemaker_session.local_mode
 model_uri = estimator.model_uri
Expand DownExpand Up@@ -1102,9 +1122,7 @@ def start_new(cls, estimator, inputs, experiment_config):
 if estimator.enable_sagemaker_metrics is not None:
 train_args["enable_sagemaker_metrics"] = estimator.enable_sagemaker_metrics

 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)
 return train_args

 @classmethod
 def _add_spot_checkpoint_args(cls, local_mode, estimator, train_args):
Expand Down

130 changes: 127 additions & 3 deletions
src/sagemaker/session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -511,7 +511,133 @@ def train( # noqa: C901
 Returns:
 str: ARN of the training job, if it is created.
 """
 train_request = self._get_train_request(
 input_mode=input_mode,
 input_config=input_config,
 role=role,
 job_name=job_name,
 output_config=output_config,
 resource_config=resource_config,
 vpc_config=vpc_config,
 hyperparameters=hyperparameters,
 stop_condition=stop_condition,
 tags=tags,
 metric_definitions=metric_definitions,
 enable_network_isolation=enable_network_isolation,
 image_uri=image_uri,
 algorithm_arn=algorithm_arn,
 encrypt_inter_container_traffic=encrypt_inter_container_traffic,
 use_spot_instances=use_spot_instances,
 checkpoint_s3_uri=checkpoint_s3_uri,
 checkpoint_local_path=checkpoint_local_path,
 experiment_config=experiment_config,
 debugger_rule_configs=debugger_rule_configs,
 debugger_hook_config=debugger_hook_config,
 tensorboard_output_config=tensorboard_output_config,
 enable_sagemaker_metrics=enable_sagemaker_metrics,
 )
 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)

 def _get_train_request( # noqa: C901
 self,
 input_mode,
 input_config,
 role,
 job_name,
 output_config,
 resource_config,
 vpc_config,
 hyperparameters,
 stop_condition,
 tags,
 metric_definitions,
 enable_network_isolation=False,
 image_uri=None,
 algorithm_arn=None,
 encrypt_inter_container_traffic=False,
 use_spot_instances=False,
 checkpoint_s3_uri=None,
 checkpoint_local_path=None,
 experiment_config=None,
 debugger_rule_configs=None,
 debugger_hook_config=None,
 tensorboard_output_config=None,
 enable_sagemaker_metrics=None,
 ):
 """Constructs a request compatible for creating an Amazon SageMaker training job.

 Args:
 input_mode (str): The input mode that the algorithm supports. Valid modes:
 * 'File' - Amazon SageMaker copies the training dataset from the S3 location to
 a directory in the Docker container.
 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
 Unix-named pipe.

 input_config (list): A list of Channel objects. Each channel is a named input source.
 Please refer to the format details described:
 https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
 role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training
 jobs and APIs that create Amazon SageMaker endpoints use this role to access
 training data and model artifacts. You must grant sufficient permissions to this
 role.
 job_name (str): Name of the training job being created.
 output_config (dict): The S3 URI where you want to store the training results and
 optional KMS key ID.
 resource_config (dict): Contains values for ResourceConfig:
 * instance_count (int): Number of EC2 instances to use for training.
 The key in resource_config is 'InstanceCount'.
 * instance_type (str): Type of EC2 instance to use for training, for example,
 'ml.c4.xlarge'. The key in resource_config is 'InstanceType'.

 vpc_config (dict): Contains values for VpcConfig:
 * subnets (list[str]): List of subnet ids.
 The key in vpc_config is 'Subnets'.
 * security_group_ids (list[str]): List of security group ids.
 The key in vpc_config is 'SecurityGroupIds'.

 hyperparameters (dict): Hyperparameters for model training. The hyperparameters are
 made accessible as a dict[str, str] to the training code on SageMaker. For
 convenience, this accepts other types for keys and values, but ``str()`` will be
 called to convert them before training.
 stop_condition (dict): Defines when training shall finish. Contains entries that can
 be understood by the service like ``MaxRuntimeInSeconds``.
 tags (list[dict]): List of tags for labeling a training job. For more, see
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
 metric_definitions (list[dict]): A list of dictionaries that defines the metric(s)
 used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
 the name of the metric, and 'Regex' for the regular expression used to extract the
 metric from the logs.
 enable_network_isolation (bool): Whether to request for the training job to run with
 network isolation or not.
 image_uri (str): Docker image containing training code.
 algorithm_arn (str): Algorithm Arn from Marketplace.
 encrypt_inter_container_traffic (bool): Specifies whether traffic between training
 containers is encrypted for the training job (default: ``False``).
 use_spot_instances (bool): whether to use spot instances for training.
 checkpoint_s3_uri (str): The S3 URI in which to persist checkpoints
 that the algorithm persists (if any) during training. (default:
 ``None``).
 checkpoint_local_path (str): The local path that the algorithm
 writes its checkpoints to. SageMaker will persist all files
 under this path to `checkpoint_s3_uri` continually during
 training. On job startup the reverse happens - data from the
 s3 location is downloaded to this path before the algorithm is
 started. If the path is unset then SageMaker assumes the
 checkpoints will be provided under `/opt/ml/checkpoints/`.
 (default: ``None``).
 experiment_config (dict): Experiment management configuration. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
 (default: ``None``)
 enable_sagemaker_metrics (bool): enable SageMaker Metrics Time
 Series. For more information see:
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
 (default: ``None``).

 Returns:
 Dict: a training request dictionary
 """
 train_request = {
 "AlgorithmSpecification": {"TrainingInputMode": input_mode},
 "OutputDataConfig": output_config,
Expand DownExpand Up@@ -583,9 +709,7 @@ def train( # noqa: C901
 if tensorboard_output_config is not None:
 train_request["TensorBoardOutputConfig"] = tensorboard_output_config

 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)
 return train_request

 def process(
 self,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
change: separate out methods to get train arguments#1850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

metrizable
merged 1 commit into

from

Aug 24, 2020
 Merged


change: separate out methods to get train arguments
#1850
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 22 additions & 4 deletions
src/sagemaker/estimator.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1037,11 +1037,31 @@ def start_new(cls, estimator, inputs, experiment_config):
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.


 Returns:
 sagemaker.estimator._TrainingJob: Constructed object that captures
 all information about the started training job.
 """
 train_args = cls._get_train_args(estimator, inputs, experiment_config)
 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)

 @classmethod
 def _get_train_args(cls, estimator, inputs, experiment_config):
 """Constructs a dict of arguments for an Amazon SageMaker training job from the estimator.

 Args:
 estimator (sagemaker.estimator.EstimatorBase): Estimator object
 created by the user.
 inputs (str): Parameters used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`.
 experiment_config (dict[str, str]): Experiment management configuration used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.

 Returns:
 Dict: dict for `sagemaker.session.Session.train` method
 """

 local_mode = estimator.sagemaker_session.local_mode
 model_uri = estimator.model_uri
Expand DownExpand Up@@ -1102,9 +1122,7 @@ def start_new(cls, estimator, inputs, experiment_config):
 if estimator.enable_sagemaker_metrics is not None:
 train_args["enable_sagemaker_metrics"] = estimator.enable_sagemaker_metrics

 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)
 return train_args

 @classmethod
 def _add_spot_checkpoint_args(cls, local_mode, estimator, train_args):
Expand Down

130 changes: 127 additions & 3 deletions
src/sagemaker/session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -511,7 +511,133 @@ def train( # noqa: C901
 Returns:
 str: ARN of the training job, if it is created.
 """
 train_request = self._get_train_request(
 input_mode=input_mode,
 input_config=input_config,
 role=role,
 job_name=job_name,
 output_config=output_config,
 resource_config=resource_config,
 vpc_config=vpc_config,
 hyperparameters=hyperparameters,
 stop_condition=stop_condition,
 tags=tags,
 metric_definitions=metric_definitions,
 enable_network_isolation=enable_network_isolation,
 image_uri=image_uri,
 algorithm_arn=algorithm_arn,
 encrypt_inter_container_traffic=encrypt_inter_container_traffic,
 use_spot_instances=use_spot_instances,
 checkpoint_s3_uri=checkpoint_s3_uri,
 checkpoint_local_path=checkpoint_local_path,
 experiment_config=experiment_config,
 debugger_rule_configs=debugger_rule_configs,
 debugger_hook_config=debugger_hook_config,
 tensorboard_output_config=tensorboard_output_config,
 enable_sagemaker_metrics=enable_sagemaker_metrics,
 )
 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)

 def _get_train_request( # noqa: C901
 self,
 input_mode,
 input_config,
 role,
 job_name,
 output_config,
 resource_config,
 vpc_config,
 hyperparameters,
 stop_condition,
 tags,
 metric_definitions,
 enable_network_isolation=False,
 image_uri=None,
 algorithm_arn=None,
 encrypt_inter_container_traffic=False,
 use_spot_instances=False,
 checkpoint_s3_uri=None,
 checkpoint_local_path=None,
 experiment_config=None,
 debugger_rule_configs=None,
 debugger_hook_config=None,
 tensorboard_output_config=None,
 enable_sagemaker_metrics=None,
 ):
 """Constructs a request compatible for creating an Amazon SageMaker training job.

 Args:
 input_mode (str): The input mode that the algorithm supports. Valid modes:
 * 'File' - Amazon SageMaker copies the training dataset from the S3 location to
 a directory in the Docker container.
 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
 Unix-named pipe.

 input_config (list): A list of Channel objects. Each channel is a named input source.
 Please refer to the format details described:
 https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
 role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training
 jobs and APIs that create Amazon SageMaker endpoints use this role to access
 training data and model artifacts. You must grant sufficient permissions to this
 role.
 job_name (str): Name of the training job being created.
 output_config (dict): The S3 URI where you want to store the training results and
 optional KMS key ID.
 resource_config (dict): Contains values for ResourceConfig:
 * instance_count (int): Number of EC2 instances to use for training.
 The key in resource_config is 'InstanceCount'.
 * instance_type (str): Type of EC2 instance to use for training, for example,
 'ml.c4.xlarge'. The key in resource_config is 'InstanceType'.

 vpc_config (dict): Contains values for VpcConfig:
 * subnets (list[str]): List of subnet ids.
 The key in vpc_config is 'Subnets'.
 * security_group_ids (list[str]): List of security group ids.
 The key in vpc_config is 'SecurityGroupIds'.

 hyperparameters (dict): Hyperparameters for model training. The hyperparameters are
 made accessible as a dict[str, str] to the training code on SageMaker. For
 convenience, this accepts other types for keys and values, but ``str()`` will be
 called to convert them before training.
 stop_condition (dict): Defines when training shall finish. Contains entries that can
 be understood by the service like ``MaxRuntimeInSeconds``.
 tags (list[dict]): List of tags for labeling a training job. For more, see
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
 metric_definitions (list[dict]): A list of dictionaries that defines the metric(s)
 used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
 the name of the metric, and 'Regex' for the regular expression used to extract the
 metric from the logs.
 enable_network_isolation (bool): Whether to request for the training job to run with
 network isolation or not.
 image_uri (str): Docker image containing training code.
 algorithm_arn (str): Algorithm Arn from Marketplace.
 encrypt_inter_container_traffic (bool): Specifies whether traffic between training
 containers is encrypted for the training job (default: ``False``).
 use_spot_instances (bool): whether to use spot instances for training.
 checkpoint_s3_uri (str): The S3 URI in which to persist checkpoints
 that the algorithm persists (if any) during training. (default:
 ``None``).
 checkpoint_local_path (str): The local path that the algorithm
 writes its checkpoints to. SageMaker will persist all files
 under this path to `checkpoint_s3_uri` continually during
 training. On job startup the reverse happens - data from the
 s3 location is downloaded to this path before the algorithm is
 started. If the path is unset then SageMaker assumes the
 checkpoints will be provided under `/opt/ml/checkpoints/`.
 (default: ``None``).
 experiment_config (dict): Experiment management configuration. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
 (default: ``None``)
 enable_sagemaker_metrics (bool): enable SageMaker Metrics Time
 Series. For more information see:
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
 (default: ``None``).

 Returns:
 Dict: a training request dictionary
 """
 train_request = {
 "AlgorithmSpecification": {"TrainingInputMode": input_mode},
 "OutputDataConfig": output_config,
Expand DownExpand Up@@ -583,9 +709,7 @@ def train( # noqa: C901
 if tensorboard_output_config is not None:
 train_request["TensorBoardOutputConfig"] = tensorboard_output_config

 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)
 return train_request

 def process(
 self,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
change: separate out methods to get train arguments#1850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

metrizable
merged 1 commit into

from

Aug 24, 2020
 Merged


change: separate out methods to get train arguments
#1850
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 22 additions & 4 deletions
src/sagemaker/estimator.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1037,11 +1037,31 @@ def start_new(cls, estimator, inputs, experiment_config):
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.


 Returns:
 sagemaker.estimator._TrainingJob: Constructed object that captures
 all information about the started training job.
 """
 train_args = cls._get_train_args(estimator, inputs, experiment_config)
 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)

 @classmethod
 def _get_train_args(cls, estimator, inputs, experiment_config):
 """Constructs a dict of arguments for an Amazon SageMaker training job from the estimator.

 Args:
 estimator (sagemaker.estimator.EstimatorBase): Estimator object
 created by the user.
 inputs (str): Parameters used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`.
 experiment_config (dict[str, str]): Experiment management configuration used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.

 Returns:
 Dict: dict for `sagemaker.session.Session.train` method
 """

 local_mode = estimator.sagemaker_session.local_mode
 model_uri = estimator.model_uri
Expand DownExpand Up@@ -1102,9 +1122,7 @@ def start_new(cls, estimator, inputs, experiment_config):
 if estimator.enable_sagemaker_metrics is not None:
 train_args["enable_sagemaker_metrics"] = estimator.enable_sagemaker_metrics

 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)
 return train_args

 @classmethod
 def _add_spot_checkpoint_args(cls, local_mode, estimator, train_args):
Expand Down

130 changes: 127 additions & 3 deletions
src/sagemaker/session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -511,7 +511,133 @@ def train( # noqa: C901
 Returns:
 str: ARN of the training job, if it is created.
 """
 train_request = self._get_train_request(
 input_mode=input_mode,
 input_config=input_config,
 role=role,
 job_name=job_name,
 output_config=output_config,
 resource_config=resource_config,
 vpc_config=vpc_config,
 hyperparameters=hyperparameters,
 stop_condition=stop_condition,
 tags=tags,
 metric_definitions=metric_definitions,
 enable_network_isolation=enable_network_isolation,
 image_uri=image_uri,
 algorithm_arn=algorithm_arn,
 encrypt_inter_container_traffic=encrypt_inter_container_traffic,
 use_spot_instances=use_spot_instances,
 checkpoint_s3_uri=checkpoint_s3_uri,
 checkpoint_local_path=checkpoint_local_path,
 experiment_config=experiment_config,
 debugger_rule_configs=debugger_rule_configs,
 debugger_hook_config=debugger_hook_config,
 tensorboard_output_config=tensorboard_output_config,
 enable_sagemaker_metrics=enable_sagemaker_metrics,
 )
 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)

 def _get_train_request( # noqa: C901
 self,
 input_mode,
 input_config,
 role,
 job_name,
 output_config,
 resource_config,
 vpc_config,
 hyperparameters,
 stop_condition,
 tags,
 metric_definitions,
 enable_network_isolation=False,
 image_uri=None,
 algorithm_arn=None,
 encrypt_inter_container_traffic=False,
 use_spot_instances=False,
 checkpoint_s3_uri=None,
 checkpoint_local_path=None,
 experiment_config=None,
 debugger_rule_configs=None,
 debugger_hook_config=None,
 tensorboard_output_config=None,
 enable_sagemaker_metrics=None,
 ):
 """Constructs a request compatible for creating an Amazon SageMaker training job.

 Args:
 input_mode (str): The input mode that the algorithm supports. Valid modes:
 * 'File' - Amazon SageMaker copies the training dataset from the S3 location to
 a directory in the Docker container.
 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
 Unix-named pipe.

 input_config (list): A list of Channel objects. Each channel is a named input source.
 Please refer to the format details described:
 https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
 role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training
 jobs and APIs that create Amazon SageMaker endpoints use this role to access
 training data and model artifacts. You must grant sufficient permissions to this
 role.
 job_name (str): Name of the training job being created.
 output_config (dict): The S3 URI where you want to store the training results and
 optional KMS key ID.
 resource_config (dict): Contains values for ResourceConfig:
 * instance_count (int): Number of EC2 instances to use for training.
 The key in resource_config is 'InstanceCount'.
 * instance_type (str): Type of EC2 instance to use for training, for example,
 'ml.c4.xlarge'. The key in resource_config is 'InstanceType'.

 vpc_config (dict): Contains values for VpcConfig:
 * subnets (list[str]): List of subnet ids.
 The key in vpc_config is 'Subnets'.
 * security_group_ids (list[str]): List of security group ids.
 The key in vpc_config is 'SecurityGroupIds'.

 hyperparameters (dict): Hyperparameters for model training. The hyperparameters are
 made accessible as a dict[str, str] to the training code on SageMaker. For
 convenience, this accepts other types for keys and values, but ``str()`` will be
 called to convert them before training.
 stop_condition (dict): Defines when training shall finish. Contains entries that can
 be understood by the service like ``MaxRuntimeInSeconds``.
 tags (list[dict]): List of tags for labeling a training job. For more, see
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
 metric_definitions (list[dict]): A list of dictionaries that defines the metric(s)
 used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
 the name of the metric, and 'Regex' for the regular expression used to extract the
 metric from the logs.
 enable_network_isolation (bool): Whether to request for the training job to run with
 network isolation or not.
 image_uri (str): Docker image containing training code.
 algorithm_arn (str): Algorithm Arn from Marketplace.
 encrypt_inter_container_traffic (bool): Specifies whether traffic between training
 containers is encrypted for the training job (default: ``False``).
 use_spot_instances (bool): whether to use spot instances for training.
 checkpoint_s3_uri (str): The S3 URI in which to persist checkpoints
 that the algorithm persists (if any) during training. (default:
 ``None``).
 checkpoint_local_path (str): The local path that the algorithm
 writes its checkpoints to. SageMaker will persist all files
 under this path to `checkpoint_s3_uri` continually during
 training. On job startup the reverse happens - data from the
 s3 location is downloaded to this path before the algorithm is
 started. If the path is unset then SageMaker assumes the
 checkpoints will be provided under `/opt/ml/checkpoints/`.
 (default: ``None``).
 experiment_config (dict): Experiment management configuration. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
 (default: ``None``)
 enable_sagemaker_metrics (bool): enable SageMaker Metrics Time
 Series. For more information see:
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
 (default: ``None``).

 Returns:
 Dict: a training request dictionary
 """
 train_request = {
 "AlgorithmSpecification": {"TrainingInputMode": input_mode},
 "OutputDataConfig": output_config,
Expand DownExpand Up@@ -583,9 +709,7 @@ def train( # noqa: C901
 if tensorboard_output_config is not None:
 train_request["TensorBoardOutputConfig"] = tensorboard_output_config

 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)
 return train_request

 def process(
 self,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
change: separate out methods to get train arguments#1850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

metrizable
merged 1 commit into

from

Aug 24, 2020
 Merged


change: separate out methods to get train arguments
#1850
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 22 additions & 4 deletions
src/sagemaker/estimator.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1037,11 +1037,31 @@ def start_new(cls, estimator, inputs, experiment_config):
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.


 Returns:
 sagemaker.estimator._TrainingJob: Constructed object that captures
 all information about the started training job.
 """
 train_args = cls._get_train_args(estimator, inputs, experiment_config)
 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)

 @classmethod
 def _get_train_args(cls, estimator, inputs, experiment_config):
 """Constructs a dict of arguments for an Amazon SageMaker training job from the estimator.

 Args:
 estimator (sagemaker.estimator.EstimatorBase): Estimator object
 created by the user.
 inputs (str): Parameters used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`.
 experiment_config (dict[str, str]): Experiment management configuration used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.

 Returns:
 Dict: dict for `sagemaker.session.Session.train` method
 """

 local_mode = estimator.sagemaker_session.local_mode
 model_uri = estimator.model_uri
Expand DownExpand Up@@ -1102,9 +1122,7 @@ def start_new(cls, estimator, inputs, experiment_config):
 if estimator.enable_sagemaker_metrics is not None:
 train_args["enable_sagemaker_metrics"] = estimator.enable_sagemaker_metrics

 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)
 return train_args

 @classmethod
 def _add_spot_checkpoint_args(cls, local_mode, estimator, train_args):
Expand Down

130 changes: 127 additions & 3 deletions
src/sagemaker/session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -511,7 +511,133 @@ def train( # noqa: C901
 Returns:
 str: ARN of the training job, if it is created.
 """
 train_request = self._get_train_request(
 input_mode=input_mode,
 input_config=input_config,
 role=role,
 job_name=job_name,
 output_config=output_config,
 resource_config=resource_config,
 vpc_config=vpc_config,
 hyperparameters=hyperparameters,
 stop_condition=stop_condition,
 tags=tags,
 metric_definitions=metric_definitions,
 enable_network_isolation=enable_network_isolation,
 image_uri=image_uri,
 algorithm_arn=algorithm_arn,
 encrypt_inter_container_traffic=encrypt_inter_container_traffic,
 use_spot_instances=use_spot_instances,
 checkpoint_s3_uri=checkpoint_s3_uri,
 checkpoint_local_path=checkpoint_local_path,
 experiment_config=experiment_config,
 debugger_rule_configs=debugger_rule_configs,
 debugger_hook_config=debugger_hook_config,
 tensorboard_output_config=tensorboard_output_config,
 enable_sagemaker_metrics=enable_sagemaker_metrics,
 )
 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)

 def _get_train_request( # noqa: C901
 self,
 input_mode,
 input_config,
 role,
 job_name,
 output_config,
 resource_config,
 vpc_config,
 hyperparameters,
 stop_condition,
 tags,
 metric_definitions,
 enable_network_isolation=False,
 image_uri=None,
 algorithm_arn=None,
 encrypt_inter_container_traffic=False,
 use_spot_instances=False,
 checkpoint_s3_uri=None,
 checkpoint_local_path=None,
 experiment_config=None,
 debugger_rule_configs=None,
 debugger_hook_config=None,
 tensorboard_output_config=None,
 enable_sagemaker_metrics=None,
 ):
 """Constructs a request compatible for creating an Amazon SageMaker training job.

 Args:
 input_mode (str): The input mode that the algorithm supports. Valid modes:
 * 'File' - Amazon SageMaker copies the training dataset from the S3 location to
 a directory in the Docker container.
 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
 Unix-named pipe.

 input_config (list): A list of Channel objects. Each channel is a named input source.
 Please refer to the format details described:
 https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
 role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training
 jobs and APIs that create Amazon SageMaker endpoints use this role to access
 training data and model artifacts. You must grant sufficient permissions to this
 role.
 job_name (str): Name of the training job being created.
 output_config (dict): The S3 URI where you want to store the training results and
 optional KMS key ID.
 resource_config (dict): Contains values for ResourceConfig:
 * instance_count (int): Number of EC2 instances to use for training.
 The key in resource_config is 'InstanceCount'.
 * instance_type (str): Type of EC2 instance to use for training, for example,
 'ml.c4.xlarge'. The key in resource_config is 'InstanceType'.

 vpc_config (dict): Contains values for VpcConfig:
 * subnets (list[str]): List of subnet ids.
 The key in vpc_config is 'Subnets'.
 * security_group_ids (list[str]): List of security group ids.
 The key in vpc_config is 'SecurityGroupIds'.

 hyperparameters (dict): Hyperparameters for model training. The hyperparameters are
 made accessible as a dict[str, str] to the training code on SageMaker. For
 convenience, this accepts other types for keys and values, but ``str()`` will be
 called to convert them before training.
 stop_condition (dict): Defines when training shall finish. Contains entries that can
 be understood by the service like ``MaxRuntimeInSeconds``.
 tags (list[dict]): List of tags for labeling a training job. For more, see
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
 metric_definitions (list[dict]): A list of dictionaries that defines the metric(s)
 used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
 the name of the metric, and 'Regex' for the regular expression used to extract the
 metric from the logs.
 enable_network_isolation (bool): Whether to request for the training job to run with
 network isolation or not.
 image_uri (str): Docker image containing training code.
 algorithm_arn (str): Algorithm Arn from Marketplace.
 encrypt_inter_container_traffic (bool): Specifies whether traffic between training
 containers is encrypted for the training job (default: ``False``).
 use_spot_instances (bool): whether to use spot instances for training.
 checkpoint_s3_uri (str): The S3 URI in which to persist checkpoints
 that the algorithm persists (if any) during training. (default:
 ``None``).
 checkpoint_local_path (str): The local path that the algorithm
 writes its checkpoints to. SageMaker will persist all files
 under this path to `checkpoint_s3_uri` continually during
 training. On job startup the reverse happens - data from the
 s3 location is downloaded to this path before the algorithm is
 started. If the path is unset then SageMaker assumes the
 checkpoints will be provided under `/opt/ml/checkpoints/`.
 (default: ``None``).
 experiment_config (dict): Experiment management configuration. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
 (default: ``None``)
 enable_sagemaker_metrics (bool): enable SageMaker Metrics Time
 Series. For more information see:
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
 (default: ``None``).

 Returns:
 Dict: a training request dictionary
 """
 train_request = {
 "AlgorithmSpecification": {"TrainingInputMode": input_mode},
 "OutputDataConfig": output_config,
Expand DownExpand Up@@ -583,9 +709,7 @@ def train( # noqa: C901
 if tensorboard_output_config is not None:
 train_request["TensorBoardOutputConfig"] = tensorboard_output_config

 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)
 return train_request

 def process(
 self,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
change: separate out methods to get train arguments#1850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

metrizable
merged 1 commit into

from

Aug 24, 2020
 Merged


change: separate out methods to get train arguments
#1850
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 22 additions & 4 deletions
src/sagemaker/estimator.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1037,11 +1037,31 @@ def start_new(cls, estimator, inputs, experiment_config):
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.


 Returns:
 sagemaker.estimator._TrainingJob: Constructed object that captures
 all information about the started training job.
 """
 train_args = cls._get_train_args(estimator, inputs, experiment_config)
 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)

 @classmethod
 def _get_train_args(cls, estimator, inputs, experiment_config):
 """Constructs a dict of arguments for an Amazon SageMaker training job from the estimator.

 Args:
 estimator (sagemaker.estimator.EstimatorBase): Estimator object
 created by the user.
 inputs (str): Parameters used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`.
 experiment_config (dict[str, str]): Experiment management configuration used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.

 Returns:
 Dict: dict for `sagemaker.session.Session.train` method
 """

 local_mode = estimator.sagemaker_session.local_mode
 model_uri = estimator.model_uri
Expand DownExpand Up@@ -1102,9 +1122,7 @@ def start_new(cls, estimator, inputs, experiment_config):
 if estimator.enable_sagemaker_metrics is not None:
 train_args["enable_sagemaker_metrics"] = estimator.enable_sagemaker_metrics

 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)
 return train_args

 @classmethod
 def _add_spot_checkpoint_args(cls, local_mode, estimator, train_args):
Expand Down

130 changes: 127 additions & 3 deletions
src/sagemaker/session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -511,7 +511,133 @@ def train( # noqa: C901
 Returns:
 str: ARN of the training job, if it is created.
 """
 train_request = self._get_train_request(
 input_mode=input_mode,
 input_config=input_config,
 role=role,
 job_name=job_name,
 output_config=output_config,
 resource_config=resource_config,
 vpc_config=vpc_config,
 hyperparameters=hyperparameters,
 stop_condition=stop_condition,
 tags=tags,
 metric_definitions=metric_definitions,
 enable_network_isolation=enable_network_isolation,
 image_uri=image_uri,
 algorithm_arn=algorithm_arn,
 encrypt_inter_container_traffic=encrypt_inter_container_traffic,
 use_spot_instances=use_spot_instances,
 checkpoint_s3_uri=checkpoint_s3_uri,
 checkpoint_local_path=checkpoint_local_path,
 experiment_config=experiment_config,
 debugger_rule_configs=debugger_rule_configs,
 debugger_hook_config=debugger_hook_config,
 tensorboard_output_config=tensorboard_output_config,
 enable_sagemaker_metrics=enable_sagemaker_metrics,
 )
 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)

 def _get_train_request( # noqa: C901
 self,
 input_mode,
 input_config,
 role,
 job_name,
 output_config,
 resource_config,
 vpc_config,
 hyperparameters,
 stop_condition,
 tags,
 metric_definitions,
 enable_network_isolation=False,
 image_uri=None,
 algorithm_arn=None,
 encrypt_inter_container_traffic=False,
 use_spot_instances=False,
 checkpoint_s3_uri=None,
 checkpoint_local_path=None,
 experiment_config=None,
 debugger_rule_configs=None,
 debugger_hook_config=None,
 tensorboard_output_config=None,
 enable_sagemaker_metrics=None,
 ):
 """Constructs a request compatible for creating an Amazon SageMaker training job.

 Args:
 input_mode (str): The input mode that the algorithm supports. Valid modes:
 * 'File' - Amazon SageMaker copies the training dataset from the S3 location to
 a directory in the Docker container.
 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
 Unix-named pipe.

 input_config (list): A list of Channel objects. Each channel is a named input source.
 Please refer to the format details described:
 https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
 role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training
 jobs and APIs that create Amazon SageMaker endpoints use this role to access
 training data and model artifacts. You must grant sufficient permissions to this
 role.
 job_name (str): Name of the training job being created.
 output_config (dict): The S3 URI where you want to store the training results and
 optional KMS key ID.
 resource_config (dict): Contains values for ResourceConfig:
 * instance_count (int): Number of EC2 instances to use for training.
 The key in resource_config is 'InstanceCount'.
 * instance_type (str): Type of EC2 instance to use for training, for example,
 'ml.c4.xlarge'. The key in resource_config is 'InstanceType'.

 vpc_config (dict): Contains values for VpcConfig:
 * subnets (list[str]): List of subnet ids.
 The key in vpc_config is 'Subnets'.
 * security_group_ids (list[str]): List of security group ids.
 The key in vpc_config is 'SecurityGroupIds'.

 hyperparameters (dict): Hyperparameters for model training. The hyperparameters are
 made accessible as a dict[str, str] to the training code on SageMaker. For
 convenience, this accepts other types for keys and values, but ``str()`` will be
 called to convert them before training.
 stop_condition (dict): Defines when training shall finish. Contains entries that can
 be understood by the service like ``MaxRuntimeInSeconds``.
 tags (list[dict]): List of tags for labeling a training job. For more, see
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
 metric_definitions (list[dict]): A list of dictionaries that defines the metric(s)
 used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
 the name of the metric, and 'Regex' for the regular expression used to extract the
 metric from the logs.
 enable_network_isolation (bool): Whether to request for the training job to run with
 network isolation or not.
 image_uri (str): Docker image containing training code.
 algorithm_arn (str): Algorithm Arn from Marketplace.
 encrypt_inter_container_traffic (bool): Specifies whether traffic between training
 containers is encrypted for the training job (default: ``False``).
 use_spot_instances (bool): whether to use spot instances for training.
 checkpoint_s3_uri (str): The S3 URI in which to persist checkpoints
 that the algorithm persists (if any) during training. (default:
 ``None``).
 checkpoint_local_path (str): The local path that the algorithm
 writes its checkpoints to. SageMaker will persist all files
 under this path to `checkpoint_s3_uri` continually during
 training. On job startup the reverse happens - data from the
 s3 location is downloaded to this path before the algorithm is
 started. If the path is unset then SageMaker assumes the
 checkpoints will be provided under `/opt/ml/checkpoints/`.
 (default: ``None``).
 experiment_config (dict): Experiment management configuration. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
 (default: ``None``)
 enable_sagemaker_metrics (bool): enable SageMaker Metrics Time
 Series. For more information see:
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
 (default: ``None``).

 Returns:
 Dict: a training request dictionary
 """
 train_request = {
 "AlgorithmSpecification": {"TrainingInputMode": input_mode},
 "OutputDataConfig": output_config,
Expand DownExpand Up@@ -583,9 +709,7 @@ def train( # noqa: C901
 if tensorboard_output_config is not None:
 train_request["TensorBoardOutputConfig"] = tensorboard_output_config

 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)
 return train_request

 def process(
 self,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
change: separate out methods to get train arguments#1850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

metrizable
merged 1 commit into

from

Aug 24, 2020
 Merged


change: separate out methods to get train arguments
#1850
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 22 additions & 4 deletions
src/sagemaker/estimator.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1037,11 +1037,31 @@ def start_new(cls, estimator, inputs, experiment_config):
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.


 Returns:
 sagemaker.estimator._TrainingJob: Constructed object that captures
 all information about the started training job.
 """
 train_args = cls._get_train_args(estimator, inputs, experiment_config)
 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)

 @classmethod
 def _get_train_args(cls, estimator, inputs, experiment_config):
 """Constructs a dict of arguments for an Amazon SageMaker training job from the estimator.

 Args:
 estimator (sagemaker.estimator.EstimatorBase): Estimator object
 created by the user.
 inputs (str): Parameters used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`.
 experiment_config (dict[str, str]): Experiment management configuration used when called
 :meth:`~sagemaker.estimator.EstimatorBase.fit`. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.

 Returns:
 Dict: dict for `sagemaker.session.Session.train` method
 """

 local_mode = estimator.sagemaker_session.local_mode
 model_uri = estimator.model_uri
Expand DownExpand Up@@ -1102,9 +1122,7 @@ def start_new(cls, estimator, inputs, experiment_config):
 if estimator.enable_sagemaker_metrics is not None:
 train_args["enable_sagemaker_metrics"] = estimator.enable_sagemaker_metrics

 estimator.sagemaker_session.train(**train_args)

 return cls(estimator.sagemaker_session, estimator._current_job_name)
 return train_args

 @classmethod
 def _add_spot_checkpoint_args(cls, local_mode, estimator, train_args):
Expand Down

130 changes: 127 additions & 3 deletions
src/sagemaker/session.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -511,7 +511,133 @@ def train( # noqa: C901
 Returns:
 str: ARN of the training job, if it is created.
 """
 train_request = self._get_train_request(
 input_mode=input_mode,
 input_config=input_config,
 role=role,
 job_name=job_name,
 output_config=output_config,
 resource_config=resource_config,
 vpc_config=vpc_config,
 hyperparameters=hyperparameters,
 stop_condition=stop_condition,
 tags=tags,
 metric_definitions=metric_definitions,
 enable_network_isolation=enable_network_isolation,
 image_uri=image_uri,
 algorithm_arn=algorithm_arn,
 encrypt_inter_container_traffic=encrypt_inter_container_traffic,
 use_spot_instances=use_spot_instances,
 checkpoint_s3_uri=checkpoint_s3_uri,
 checkpoint_local_path=checkpoint_local_path,
 experiment_config=experiment_config,
 debugger_rule_configs=debugger_rule_configs,
 debugger_hook_config=debugger_hook_config,
 tensorboard_output_config=tensorboard_output_config,
 enable_sagemaker_metrics=enable_sagemaker_metrics,
 )
 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)

 def _get_train_request( # noqa: C901
 self,
 input_mode,
 input_config,
 role,
 job_name,
 output_config,
 resource_config,
 vpc_config,
 hyperparameters,
 stop_condition,
 tags,
 metric_definitions,
 enable_network_isolation=False,
 image_uri=None,
 algorithm_arn=None,
 encrypt_inter_container_traffic=False,
 use_spot_instances=False,
 checkpoint_s3_uri=None,
 checkpoint_local_path=None,
 experiment_config=None,
 debugger_rule_configs=None,
 debugger_hook_config=None,
 tensorboard_output_config=None,
 enable_sagemaker_metrics=None,
 ):
 """Constructs a request compatible for creating an Amazon SageMaker training job.

 Args:
 input_mode (str): The input mode that the algorithm supports. Valid modes:
 * 'File' - Amazon SageMaker copies the training dataset from the S3 location to
 a directory in the Docker container.
 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
 Unix-named pipe.

 input_config (list): A list of Channel objects. Each channel is a named input source.
 Please refer to the format details described:
 https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
 role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training
 jobs and APIs that create Amazon SageMaker endpoints use this role to access
 training data and model artifacts. You must grant sufficient permissions to this
 role.
 job_name (str): Name of the training job being created.
 output_config (dict): The S3 URI where you want to store the training results and
 optional KMS key ID.
 resource_config (dict): Contains values for ResourceConfig:
 * instance_count (int): Number of EC2 instances to use for training.
 The key in resource_config is 'InstanceCount'.
 * instance_type (str): Type of EC2 instance to use for training, for example,
 'ml.c4.xlarge'. The key in resource_config is 'InstanceType'.

 vpc_config (dict): Contains values for VpcConfig:
 * subnets (list[str]): List of subnet ids.
 The key in vpc_config is 'Subnets'.
 * security_group_ids (list[str]): List of security group ids.
 The key in vpc_config is 'SecurityGroupIds'.

 hyperparameters (dict): Hyperparameters for model training. The hyperparameters are
 made accessible as a dict[str, str] to the training code on SageMaker. For
 convenience, this accepts other types for keys and values, but ``str()`` will be
 called to convert them before training.
 stop_condition (dict): Defines when training shall finish. Contains entries that can
 be understood by the service like ``MaxRuntimeInSeconds``.
 tags (list[dict]): List of tags for labeling a training job. For more, see
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
 metric_definitions (list[dict]): A list of dictionaries that defines the metric(s)
 used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
 the name of the metric, and 'Regex' for the regular expression used to extract the
 metric from the logs.
 enable_network_isolation (bool): Whether to request for the training job to run with
 network isolation or not.
 image_uri (str): Docker image containing training code.
 algorithm_arn (str): Algorithm Arn from Marketplace.
 encrypt_inter_container_traffic (bool): Specifies whether traffic between training
 containers is encrypted for the training job (default: ``False``).
 use_spot_instances (bool): whether to use spot instances for training.
 checkpoint_s3_uri (str): The S3 URI in which to persist checkpoints
 that the algorithm persists (if any) during training. (default:
 ``None``).
 checkpoint_local_path (str): The local path that the algorithm
 writes its checkpoints to. SageMaker will persist all files
 under this path to `checkpoint_s3_uri` continually during
 training. On job startup the reverse happens - data from the
 s3 location is downloaded to this path before the algorithm is
 started. If the path is unset then SageMaker assumes the
 checkpoints will be provided under `/opt/ml/checkpoints/`.
 (default: ``None``).
 experiment_config (dict): Experiment management configuration. Dictionary contains
 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
 (default: ``None``)
 enable_sagemaker_metrics (bool): enable SageMaker Metrics Time
 Series. For more information see:
 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
 (default: ``None``).

 Returns:
 Dict: a training request dictionary
 """
 train_request = {
 "AlgorithmSpecification": {"TrainingInputMode": input_mode},
 "OutputDataConfig": output_config,
Expand DownExpand Up@@ -583,9 +709,7 @@ def train( # noqa: C901
 if tensorboard_output_config is not None:
 train_request["TensorBoardOutputConfig"] = tensorboard_output_config

 LOGGER.info("Creating training-job with name: %s", job_name)
 LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
 self.sagemaker_client.create_training_job(**train_request)
 return train_request

 def process(
 self,
Expand Down