Benchmark feature v2#4618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
liujiaorr merged 17 commits into from
Apr 25, 2024
Merged

Benchmark feature v2 #4618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
63147a9
Add funtionalities to get and set deployment config
Apr 24, 2024
6520a2b
Resolve PR comments
Apr 24, 2024
a7fced7
ModelBuilder-JS
Apr 24, 2024
dd84775
Add Unit tests
Apr 25, 2024
faba3c8
Refactoring
Apr 25, 2024
dc2f5b7
Testing with Notebook
Apr 25, 2024
b4e983a
Test backward compatibility
Apr 25, 2024
469f17a
Merge branch 'master-benchmark-feature' into benchmark-feature-v2
makungaj1 Apr 25, 2024
6f30cbf
Remove Accelerated column if all not enabled
Apr 25, 2024
5724ce1
Fix docstring
Apr 25, 2024
4a61ee6
Resolved PR Review comments
Apr 25, 2024
d88ef47
Docstring
Apr 25, 2024
aaba1cb
increase code coverage
Apr 25, 2024
c20c1f3
Testing fix with Notebook
Apr 25, 2024
10bd9db
Merge remote-tracking branch 'upstream/master-benchmark-feature' into…
Apr 25, 2024
aee4169
Only fetch instance rate metrics if not present
Apr 25, 2024
76b8112
Increase code coverage
Apr 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions src/sagemaker/jumpstart/model.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
validate_model_id_and_get_type,
verify_model_region_and_return_specs,
get_jumpstart_configs,
extract_metrics_from_deployment_configs,
get_metrics_from_deployment_configs,
)
from sagemaker.jumpstart.constants import JUMPSTART_LOGGER
from sagemaker.jumpstart.enums import JumpStartModelType
Expand DownExpand Up@@ -868,7 +868,7 @@ def _get_benchmarks_data(self, config_name: str) -> Dict[str, List[str]]:
Returns:
Dict[str, List[str]]: Deployment config benchmark data.
"""
return extract_metrics_from_deployment_configs(
return get_metrics_from_deployment_configs(
self._deployment_configs,
config_name,
)
Expand DownExpand Up@@ -905,20 +905,29 @@ def _convert_to_deployment_config_metadata(
"default_inference_instance_type"
)

instance_rate = get_instance_rate_per_hour(
instance_type=default_inference_instance_type, region=self.region
)

benchmark_metrics = (
metadata_config.benchmark_metrics.get(default_inference_instance_type)
if metadata_config.benchmark_metrics is not None
else None
)
if instance_rate is not None:
if benchmark_metrics is not None:
benchmark_metrics.append(JumpStartBenchmarkStat(instance_rate))

should_fetch_instance_rate_metric = True
if benchmark_metrics is not None:
for benchmark_metric in benchmark_metrics:
if benchmark_metric.name.lower() == "instance rate":
should_fetch_instance_rate_metric = False
break

if should_fetch_instance_rate_metric:
instance_rate = get_instance_rate_per_hour(
instance_type=default_inference_instance_type, region=self.region
)
instance_rate_metric = JumpStartBenchmarkStat(instance_rate)

if benchmark_metrics is None:
benchmark_metrics = [instance_rate_metric]
else:
benchmark_metrics = [JumpStartBenchmarkStat(instance_rate)]
benchmark_metrics.append(instance_rate_metric)

init_kwargs = get_init_kwargs(
model_id=self.model_id,
Expand Down
2 changes: 1 addition & 1 deletion src/sagemaker/jumpstart/utils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1030,7 +1030,7 @@ def get_jumpstart_configs(
)


def extract_metrics_from_deployment_configs(
def get_metrics_from_deployment_configs(
deployment_configs: List[Dict[str, Any]], config_name: str
) -> Dict[str, List[str]]:
"""Extracts metrics from deployment configs.
Expand Down
5 changes: 3 additions & 2 deletions tests/unit/sagemaker/jumpstart/model/test_model.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@
get_base_spec_with_prototype_configs,
get_mock_init_kwargs,
get_base_deployment_configs,
get_base_spec_with_prototype_configs_with_missing_benchmarks,
)
import boto3

Expand DownExpand Up@@ -1790,7 +1791,7 @@ def test_model_retrieve_deployment_config(

mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
mock_verify_model_region_and_return_specs.side_effect = (
lambda *args, **kwargs: get_base_spec_with_prototype_configs()
lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
)
mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
"name": "Instance Rate",
Expand DownExpand Up@@ -1838,7 +1839,7 @@ def test_model_display_benchmark_metrics(

mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
mock_verify_model_region_and_return_specs.side_effect = (
lambda *args, **kwargs: get_base_spec_with_prototype_configs()
lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
)
mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
"name": "Instance Rate",
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/sagemaker/jumpstart/test_utils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1810,6 +1810,6 @@ def test_get_jumpstart_benchmark_stats_training(
],
)
def test_extract_metrics_from_deployment_configs(config_name, configs, expected):
data = utils.extract_metrics_from_deployment_configs(configs, config_name)
data = utils.get_metrics_from_deployment_configs(configs, config_name)

assert data == expected
20 changes: 20 additions & 0 deletions tests/unit/sagemaker/jumpstart/utils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -226,6 +226,26 @@ def get_base_spec_with_prototype_configs(
return JumpStartModelSpecs(spec)


def get_base_spec_with_prototype_configs_with_missing_benchmarks(
region: str = None,
model_id: str = None,
version: str = None,
s3_client: boto3.client = None,
model_type: JumpStartModelType = JumpStartModelType.OPEN_WEIGHTS,
) -> JumpStartModelSpecs:
spec = copy.deepcopy(BASE_SPEC)
copy_inference_configs = copy.deepcopy(INFERENCE_CONFIGS)
copy_inference_configs["inference_configs"]["neuron-inference"]["benchmark_metrics"] = None

inference_configs = {**INFERENCE_CONFIGS, **INFERENCE_CONFIG_RANKINGS}
training_configs = {**TRAINING_CONFIGS, **TRAINING_CONFIG_RANKINGS}

spec.update(inference_configs)
spec.update(training_configs)

return JumpStartModelSpecs(spec)


def get_prototype_spec_with_configs(
region: str = None,
model_id: str = None,
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Benchmark feature v2#4618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

liujiaorr
merged 17 commits into

from

Apr 25, 2024
 Merged


Benchmark feature v2
#4618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
63147a9

Add funtionalities to get and set deployment config

Apr 24, 2024
6520a2b

Resolve PR comments

Apr 24, 2024
a7fced7

ModelBuilder-JS

Apr 24, 2024
dd84775

Add Unit tests

Apr 25, 2024
faba3c8

Refactoring

Apr 25, 2024
dc2f5b7

Testing with Notebook

Apr 25, 2024
b4e983a

Test backward compatibility

Apr 25, 2024
469f17a

Merge branch 'master-benchmark-feature' into benchmark-feature-v2


makungaj1 Apr 25, 2024
6f30cbf

Remove Accelerated column if all not enabled

Apr 25, 2024
5724ce1

Fix docstring

Apr 25, 2024
4a61ee6

Resolved PR Review comments

Apr 25, 2024
d88ef47

Docstring

Apr 25, 2024
aaba1cb

increase code coverage

Apr 25, 2024
c20c1f3

Testing fix with Notebook

Apr 25, 2024
10bd9db

Merge remote-tracking branch 'upstream/master-benchmark-feature' into…

Apr 25, 2024
aee4169

Only fetch instance rate metrics if not present

Apr 25, 2024
76b8112

Increase code coverage

Apr 25, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 19 additions & 10 deletions
src/sagemaker/jumpstart/model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 validate_model_id_and_get_type,
 verify_model_region_and_return_specs,
 get_jumpstart_configs,
extract_metrics_from_deployment_configs,
get_metrics_from_deployment_configs,
)
from sagemaker.jumpstart.constants import JUMPSTART_LOGGER
from sagemaker.jumpstart.enums import JumpStartModelType
Expand DownExpand Up@@ -868,7 +868,7 @@ def _get_benchmarks_data(self, config_name: str) -> Dict[str, List[str]]:
 Returns:
 Dict[str, List[str]]: Deployment config benchmark data.
 """
 return extract_metrics_from_deployment_configs(
 return get_metrics_from_deployment_configs(
 self._deployment_configs,
 config_name,
 )
Expand DownExpand Up@@ -905,20 +905,29 @@ def _convert_to_deployment_config_metadata(
 "default_inference_instance_type"
 )

 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )

 benchmark_metrics = (
 metadata_config.benchmark_metrics.get(default_inference_instance_type)
 if metadata_config.benchmark_metrics is not None
 else None
 )
 if instance_rate is not None:
 if benchmark_metrics is not None:
 benchmark_metrics.append(JumpStartBenchmarkStat(instance_rate))

 should_fetch_instance_rate_metric = True
 if benchmark_metrics is not None:
 for benchmark_metric in benchmark_metrics:
 if benchmark_metric.name.lower() == "instance rate":
 should_fetch_instance_rate_metric = False
 break

 if should_fetch_instance_rate_metric:
 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )
 instance_rate_metric = JumpStartBenchmarkStat(instance_rate)

 if benchmark_metrics is None:
 benchmark_metrics = [instance_rate_metric]
 else:
 benchmark_metrics = [JumpStartBenchmarkStat(instance_rate)]
 benchmark_metrics.append(instance_rate_metric)

 init_kwargs = get_init_kwargs(
 model_id=self.model_id,
Expand Down

2 changes: 1 addition & 1 deletion
src/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1030,7 +1030,7 @@ def get_jumpstart_configs(
 )


def extract_metrics_from_deployment_configs(
def get_metrics_from_deployment_configs(
 deployment_configs: List[Dict[str, Any]], config_name: str
) -> Dict[str, List[str]]:
 """Extracts metrics from deployment configs.
Expand Down

5 changes: 3 additions & 2 deletions
tests/unit/sagemaker/jumpstart/model/test_model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@
 get_base_spec_with_prototype_configs,
 get_mock_init_kwargs,
 get_base_deployment_configs,
 get_base_spec_with_prototype_configs_with_missing_benchmarks,
)
import boto3

Expand DownExpand Up@@ -1790,7 +1791,7 @@ def test_model_retrieve_deployment_config(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand DownExpand Up@@ -1838,7 +1839,7 @@ def test_model_display_benchmark_metrics(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand Down

2 changes: 1 addition & 1 deletion
tests/unit/sagemaker/jumpstart/test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1810,6 +1810,6 @@ def test_get_jumpstart_benchmark_stats_training(
 ],
)
def test_extract_metrics_from_deployment_configs(config_name, configs, expected):
 data = utils.extract_metrics_from_deployment_configs(configs, config_name)
 data = utils.get_metrics_from_deployment_configs(configs, config_name)

 assert data == expected

20 changes: 20 additions & 0 deletions
tests/unit/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -226,6 +226,26 @@ def get_base_spec_with_prototype_configs(
 return JumpStartModelSpecs(spec)


def get_base_spec_with_prototype_configs_with_missing_benchmarks(
 region: str = None,
 model_id: str = None,
 version: str = None,
 s3_client: boto3.client = None,
 model_type: JumpStartModelType = JumpStartModelType.OPEN_WEIGHTS,
) -> JumpStartModelSpecs:
 spec = copy.deepcopy(BASE_SPEC)
 copy_inference_configs = copy.deepcopy(INFERENCE_CONFIGS)
 copy_inference_configs["inference_configs"]["neuron-inference"]["benchmark_metrics"] = None

 inference_configs = {**INFERENCE_CONFIGS, **INFERENCE_CONFIG_RANKINGS}
 training_configs = {**TRAINING_CONFIGS, **TRAINING_CONFIG_RANKINGS}

 spec.update(inference_configs)
 spec.update(training_configs)

 return JumpStartModelSpecs(spec)


def get_prototype_spec_with_configs(
 region: str = None,
 model_id: str = None,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Benchmark feature v2#4618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

liujiaorr
merged 17 commits into

from

Apr 25, 2024
 Merged


Benchmark feature v2
#4618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
63147a9

Add funtionalities to get and set deployment config

Apr 24, 2024
6520a2b

Resolve PR comments

Apr 24, 2024
a7fced7

ModelBuilder-JS

Apr 24, 2024
dd84775

Add Unit tests

Apr 25, 2024
faba3c8

Refactoring

Apr 25, 2024
dc2f5b7

Testing with Notebook

Apr 25, 2024
b4e983a

Test backward compatibility

Apr 25, 2024
469f17a

Merge branch 'master-benchmark-feature' into benchmark-feature-v2


makungaj1 Apr 25, 2024
6f30cbf

Remove Accelerated column if all not enabled

Apr 25, 2024
5724ce1

Fix docstring

Apr 25, 2024
4a61ee6

Resolved PR Review comments

Apr 25, 2024
d88ef47

Docstring

Apr 25, 2024
aaba1cb

increase code coverage

Apr 25, 2024
c20c1f3

Testing fix with Notebook

Apr 25, 2024
10bd9db

Merge remote-tracking branch 'upstream/master-benchmark-feature' into…

Apr 25, 2024
aee4169

Only fetch instance rate metrics if not present

Apr 25, 2024
76b8112

Increase code coverage

Apr 25, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 19 additions & 10 deletions
src/sagemaker/jumpstart/model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 validate_model_id_and_get_type,
 verify_model_region_and_return_specs,
 get_jumpstart_configs,
extract_metrics_from_deployment_configs,
get_metrics_from_deployment_configs,
)
from sagemaker.jumpstart.constants import JUMPSTART_LOGGER
from sagemaker.jumpstart.enums import JumpStartModelType
Expand DownExpand Up@@ -868,7 +868,7 @@ def _get_benchmarks_data(self, config_name: str) -> Dict[str, List[str]]:
 Returns:
 Dict[str, List[str]]: Deployment config benchmark data.
 """
 return extract_metrics_from_deployment_configs(
 return get_metrics_from_deployment_configs(
 self._deployment_configs,
 config_name,
 )
Expand DownExpand Up@@ -905,20 +905,29 @@ def _convert_to_deployment_config_metadata(
 "default_inference_instance_type"
 )

 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )

 benchmark_metrics = (
 metadata_config.benchmark_metrics.get(default_inference_instance_type)
 if metadata_config.benchmark_metrics is not None
 else None
 )
 if instance_rate is not None:
 if benchmark_metrics is not None:
 benchmark_metrics.append(JumpStartBenchmarkStat(instance_rate))

 should_fetch_instance_rate_metric = True
 if benchmark_metrics is not None:
 for benchmark_metric in benchmark_metrics:
 if benchmark_metric.name.lower() == "instance rate":
 should_fetch_instance_rate_metric = False
 break

 if should_fetch_instance_rate_metric:
 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )
 instance_rate_metric = JumpStartBenchmarkStat(instance_rate)

 if benchmark_metrics is None:
 benchmark_metrics = [instance_rate_metric]
 else:
 benchmark_metrics = [JumpStartBenchmarkStat(instance_rate)]
 benchmark_metrics.append(instance_rate_metric)

 init_kwargs = get_init_kwargs(
 model_id=self.model_id,
Expand Down

2 changes: 1 addition & 1 deletion
src/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1030,7 +1030,7 @@ def get_jumpstart_configs(
 )


def extract_metrics_from_deployment_configs(
def get_metrics_from_deployment_configs(
 deployment_configs: List[Dict[str, Any]], config_name: str
) -> Dict[str, List[str]]:
 """Extracts metrics from deployment configs.
Expand Down

5 changes: 3 additions & 2 deletions
tests/unit/sagemaker/jumpstart/model/test_model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@
 get_base_spec_with_prototype_configs,
 get_mock_init_kwargs,
 get_base_deployment_configs,
 get_base_spec_with_prototype_configs_with_missing_benchmarks,
)
import boto3

Expand DownExpand Up@@ -1790,7 +1791,7 @@ def test_model_retrieve_deployment_config(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand DownExpand Up@@ -1838,7 +1839,7 @@ def test_model_display_benchmark_metrics(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand Down

2 changes: 1 addition & 1 deletion
tests/unit/sagemaker/jumpstart/test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1810,6 +1810,6 @@ def test_get_jumpstart_benchmark_stats_training(
 ],
)
def test_extract_metrics_from_deployment_configs(config_name, configs, expected):
 data = utils.extract_metrics_from_deployment_configs(configs, config_name)
 data = utils.get_metrics_from_deployment_configs(configs, config_name)

 assert data == expected

20 changes: 20 additions & 0 deletions
tests/unit/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -226,6 +226,26 @@ def get_base_spec_with_prototype_configs(
 return JumpStartModelSpecs(spec)


def get_base_spec_with_prototype_configs_with_missing_benchmarks(
 region: str = None,
 model_id: str = None,
 version: str = None,
 s3_client: boto3.client = None,
 model_type: JumpStartModelType = JumpStartModelType.OPEN_WEIGHTS,
) -> JumpStartModelSpecs:
 spec = copy.deepcopy(BASE_SPEC)
 copy_inference_configs = copy.deepcopy(INFERENCE_CONFIGS)
 copy_inference_configs["inference_configs"]["neuron-inference"]["benchmark_metrics"] = None

 inference_configs = {**INFERENCE_CONFIGS, **INFERENCE_CONFIG_RANKINGS}
 training_configs = {**TRAINING_CONFIGS, **TRAINING_CONFIG_RANKINGS}

 spec.update(inference_configs)
 spec.update(training_configs)

 return JumpStartModelSpecs(spec)


def get_prototype_spec_with_configs(
 region: str = None,
 model_id: str = None,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Benchmark feature v2#4618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

liujiaorr
merged 17 commits into

from

Apr 25, 2024
 Merged


Benchmark feature v2
#4618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
63147a9

Add funtionalities to get and set deployment config

Apr 24, 2024
6520a2b

Resolve PR comments

Apr 24, 2024
a7fced7

ModelBuilder-JS

Apr 24, 2024
dd84775

Add Unit tests

Apr 25, 2024
faba3c8

Refactoring

Apr 25, 2024
dc2f5b7

Testing with Notebook

Apr 25, 2024
b4e983a

Test backward compatibility

Apr 25, 2024
469f17a

Merge branch 'master-benchmark-feature' into benchmark-feature-v2


makungaj1 Apr 25, 2024
6f30cbf

Remove Accelerated column if all not enabled

Apr 25, 2024
5724ce1

Fix docstring

Apr 25, 2024
4a61ee6

Resolved PR Review comments

Apr 25, 2024
d88ef47

Docstring

Apr 25, 2024
aaba1cb

increase code coverage

Apr 25, 2024
c20c1f3

Testing fix with Notebook

Apr 25, 2024
10bd9db

Merge remote-tracking branch 'upstream/master-benchmark-feature' into…

Apr 25, 2024
aee4169

Only fetch instance rate metrics if not present

Apr 25, 2024
76b8112

Increase code coverage

Apr 25, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 19 additions & 10 deletions
src/sagemaker/jumpstart/model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 validate_model_id_and_get_type,
 verify_model_region_and_return_specs,
 get_jumpstart_configs,
extract_metrics_from_deployment_configs,
get_metrics_from_deployment_configs,
)
from sagemaker.jumpstart.constants import JUMPSTART_LOGGER
from sagemaker.jumpstart.enums import JumpStartModelType
Expand DownExpand Up@@ -868,7 +868,7 @@ def _get_benchmarks_data(self, config_name: str) -> Dict[str, List[str]]:
 Returns:
 Dict[str, List[str]]: Deployment config benchmark data.
 """
 return extract_metrics_from_deployment_configs(
 return get_metrics_from_deployment_configs(
 self._deployment_configs,
 config_name,
 )
Expand DownExpand Up@@ -905,20 +905,29 @@ def _convert_to_deployment_config_metadata(
 "default_inference_instance_type"
 )

 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )

 benchmark_metrics = (
 metadata_config.benchmark_metrics.get(default_inference_instance_type)
 if metadata_config.benchmark_metrics is not None
 else None
 )
 if instance_rate is not None:
 if benchmark_metrics is not None:
 benchmark_metrics.append(JumpStartBenchmarkStat(instance_rate))

 should_fetch_instance_rate_metric = True
 if benchmark_metrics is not None:
 for benchmark_metric in benchmark_metrics:
 if benchmark_metric.name.lower() == "instance rate":
 should_fetch_instance_rate_metric = False
 break

 if should_fetch_instance_rate_metric:
 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )
 instance_rate_metric = JumpStartBenchmarkStat(instance_rate)

 if benchmark_metrics is None:
 benchmark_metrics = [instance_rate_metric]
 else:
 benchmark_metrics = [JumpStartBenchmarkStat(instance_rate)]
 benchmark_metrics.append(instance_rate_metric)

 init_kwargs = get_init_kwargs(
 model_id=self.model_id,
Expand Down

2 changes: 1 addition & 1 deletion
src/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1030,7 +1030,7 @@ def get_jumpstart_configs(
 )


def extract_metrics_from_deployment_configs(
def get_metrics_from_deployment_configs(
 deployment_configs: List[Dict[str, Any]], config_name: str
) -> Dict[str, List[str]]:
 """Extracts metrics from deployment configs.
Expand Down

5 changes: 3 additions & 2 deletions
tests/unit/sagemaker/jumpstart/model/test_model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@
 get_base_spec_with_prototype_configs,
 get_mock_init_kwargs,
 get_base_deployment_configs,
 get_base_spec_with_prototype_configs_with_missing_benchmarks,
)
import boto3

Expand DownExpand Up@@ -1790,7 +1791,7 @@ def test_model_retrieve_deployment_config(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand DownExpand Up@@ -1838,7 +1839,7 @@ def test_model_display_benchmark_metrics(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand Down

2 changes: 1 addition & 1 deletion
tests/unit/sagemaker/jumpstart/test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1810,6 +1810,6 @@ def test_get_jumpstart_benchmark_stats_training(
 ],
)
def test_extract_metrics_from_deployment_configs(config_name, configs, expected):
 data = utils.extract_metrics_from_deployment_configs(configs, config_name)
 data = utils.get_metrics_from_deployment_configs(configs, config_name)

 assert data == expected

20 changes: 20 additions & 0 deletions
tests/unit/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -226,6 +226,26 @@ def get_base_spec_with_prototype_configs(
 return JumpStartModelSpecs(spec)


def get_base_spec_with_prototype_configs_with_missing_benchmarks(
 region: str = None,
 model_id: str = None,
 version: str = None,
 s3_client: boto3.client = None,
 model_type: JumpStartModelType = JumpStartModelType.OPEN_WEIGHTS,
) -> JumpStartModelSpecs:
 spec = copy.deepcopy(BASE_SPEC)
 copy_inference_configs = copy.deepcopy(INFERENCE_CONFIGS)
 copy_inference_configs["inference_configs"]["neuron-inference"]["benchmark_metrics"] = None

 inference_configs = {**INFERENCE_CONFIGS, **INFERENCE_CONFIG_RANKINGS}
 training_configs = {**TRAINING_CONFIGS, **TRAINING_CONFIG_RANKINGS}

 spec.update(inference_configs)
 spec.update(training_configs)

 return JumpStartModelSpecs(spec)


def get_prototype_spec_with_configs(
 region: str = None,
 model_id: str = None,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Benchmark feature v2#4618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

liujiaorr
merged 17 commits into

from

Apr 25, 2024
 Merged


Benchmark feature v2
#4618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
63147a9

Add funtionalities to get and set deployment config

Apr 24, 2024
6520a2b

Resolve PR comments

Apr 24, 2024
a7fced7

ModelBuilder-JS

Apr 24, 2024
dd84775

Add Unit tests

Apr 25, 2024
faba3c8

Refactoring

Apr 25, 2024
dc2f5b7

Testing with Notebook

Apr 25, 2024
b4e983a

Test backward compatibility

Apr 25, 2024
469f17a

Merge branch 'master-benchmark-feature' into benchmark-feature-v2


makungaj1 Apr 25, 2024
6f30cbf

Remove Accelerated column if all not enabled

Apr 25, 2024
5724ce1

Fix docstring

Apr 25, 2024
4a61ee6

Resolved PR Review comments

Apr 25, 2024
d88ef47

Docstring

Apr 25, 2024
aaba1cb

increase code coverage

Apr 25, 2024
c20c1f3

Testing fix with Notebook

Apr 25, 2024
10bd9db

Merge remote-tracking branch 'upstream/master-benchmark-feature' into…

Apr 25, 2024
aee4169

Only fetch instance rate metrics if not present

Apr 25, 2024
76b8112

Increase code coverage

Apr 25, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 19 additions & 10 deletions
src/sagemaker/jumpstart/model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 validate_model_id_and_get_type,
 verify_model_region_and_return_specs,
 get_jumpstart_configs,
extract_metrics_from_deployment_configs,
get_metrics_from_deployment_configs,
)
from sagemaker.jumpstart.constants import JUMPSTART_LOGGER
from sagemaker.jumpstart.enums import JumpStartModelType
Expand DownExpand Up@@ -868,7 +868,7 @@ def _get_benchmarks_data(self, config_name: str) -> Dict[str, List[str]]:
 Returns:
 Dict[str, List[str]]: Deployment config benchmark data.
 """
 return extract_metrics_from_deployment_configs(
 return get_metrics_from_deployment_configs(
 self._deployment_configs,
 config_name,
 )
Expand DownExpand Up@@ -905,20 +905,29 @@ def _convert_to_deployment_config_metadata(
 "default_inference_instance_type"
 )

 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )

 benchmark_metrics = (
 metadata_config.benchmark_metrics.get(default_inference_instance_type)
 if metadata_config.benchmark_metrics is not None
 else None
 )
 if instance_rate is not None:
 if benchmark_metrics is not None:
 benchmark_metrics.append(JumpStartBenchmarkStat(instance_rate))

 should_fetch_instance_rate_metric = True
 if benchmark_metrics is not None:
 for benchmark_metric in benchmark_metrics:
 if benchmark_metric.name.lower() == "instance rate":
 should_fetch_instance_rate_metric = False
 break

 if should_fetch_instance_rate_metric:
 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )
 instance_rate_metric = JumpStartBenchmarkStat(instance_rate)

 if benchmark_metrics is None:
 benchmark_metrics = [instance_rate_metric]
 else:
 benchmark_metrics = [JumpStartBenchmarkStat(instance_rate)]
 benchmark_metrics.append(instance_rate_metric)

 init_kwargs = get_init_kwargs(
 model_id=self.model_id,
Expand Down

2 changes: 1 addition & 1 deletion
src/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1030,7 +1030,7 @@ def get_jumpstart_configs(
 )


def extract_metrics_from_deployment_configs(
def get_metrics_from_deployment_configs(
 deployment_configs: List[Dict[str, Any]], config_name: str
) -> Dict[str, List[str]]:
 """Extracts metrics from deployment configs.
Expand Down

5 changes: 3 additions & 2 deletions
tests/unit/sagemaker/jumpstart/model/test_model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@
 get_base_spec_with_prototype_configs,
 get_mock_init_kwargs,
 get_base_deployment_configs,
 get_base_spec_with_prototype_configs_with_missing_benchmarks,
)
import boto3

Expand DownExpand Up@@ -1790,7 +1791,7 @@ def test_model_retrieve_deployment_config(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand DownExpand Up@@ -1838,7 +1839,7 @@ def test_model_display_benchmark_metrics(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand Down

2 changes: 1 addition & 1 deletion
tests/unit/sagemaker/jumpstart/test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1810,6 +1810,6 @@ def test_get_jumpstart_benchmark_stats_training(
 ],
)
def test_extract_metrics_from_deployment_configs(config_name, configs, expected):
 data = utils.extract_metrics_from_deployment_configs(configs, config_name)
 data = utils.get_metrics_from_deployment_configs(configs, config_name)

 assert data == expected

20 changes: 20 additions & 0 deletions
tests/unit/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -226,6 +226,26 @@ def get_base_spec_with_prototype_configs(
 return JumpStartModelSpecs(spec)


def get_base_spec_with_prototype_configs_with_missing_benchmarks(
 region: str = None,
 model_id: str = None,
 version: str = None,
 s3_client: boto3.client = None,
 model_type: JumpStartModelType = JumpStartModelType.OPEN_WEIGHTS,
) -> JumpStartModelSpecs:
 spec = copy.deepcopy(BASE_SPEC)
 copy_inference_configs = copy.deepcopy(INFERENCE_CONFIGS)
 copy_inference_configs["inference_configs"]["neuron-inference"]["benchmark_metrics"] = None

 inference_configs = {**INFERENCE_CONFIGS, **INFERENCE_CONFIG_RANKINGS}
 training_configs = {**TRAINING_CONFIGS, **TRAINING_CONFIG_RANKINGS}

 spec.update(inference_configs)
 spec.update(training_configs)

 return JumpStartModelSpecs(spec)


def get_prototype_spec_with_configs(
 region: str = None,
 model_id: str = None,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Benchmark feature v2#4618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

liujiaorr
merged 17 commits into

from

Apr 25, 2024
 Merged


Benchmark feature v2
#4618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
63147a9

Add funtionalities to get and set deployment config

Apr 24, 2024
6520a2b

Resolve PR comments

Apr 24, 2024
a7fced7

ModelBuilder-JS

Apr 24, 2024
dd84775

Add Unit tests

Apr 25, 2024
faba3c8

Refactoring

Apr 25, 2024
dc2f5b7

Testing with Notebook

Apr 25, 2024
b4e983a

Test backward compatibility

Apr 25, 2024
469f17a

Merge branch 'master-benchmark-feature' into benchmark-feature-v2


makungaj1 Apr 25, 2024
6f30cbf

Remove Accelerated column if all not enabled

Apr 25, 2024
5724ce1

Fix docstring

Apr 25, 2024
4a61ee6

Resolved PR Review comments

Apr 25, 2024
d88ef47

Docstring

Apr 25, 2024
aaba1cb

increase code coverage

Apr 25, 2024
c20c1f3

Testing fix with Notebook

Apr 25, 2024
10bd9db

Merge remote-tracking branch 'upstream/master-benchmark-feature' into…

Apr 25, 2024
aee4169

Only fetch instance rate metrics if not present

Apr 25, 2024
76b8112

Increase code coverage

Apr 25, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 19 additions & 10 deletions
src/sagemaker/jumpstart/model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 validate_model_id_and_get_type,
 verify_model_region_and_return_specs,
 get_jumpstart_configs,
extract_metrics_from_deployment_configs,
get_metrics_from_deployment_configs,
)
from sagemaker.jumpstart.constants import JUMPSTART_LOGGER
from sagemaker.jumpstart.enums import JumpStartModelType
Expand DownExpand Up@@ -868,7 +868,7 @@ def _get_benchmarks_data(self, config_name: str) -> Dict[str, List[str]]:
 Returns:
 Dict[str, List[str]]: Deployment config benchmark data.
 """
 return extract_metrics_from_deployment_configs(
 return get_metrics_from_deployment_configs(
 self._deployment_configs,
 config_name,
 )
Expand DownExpand Up@@ -905,20 +905,29 @@ def _convert_to_deployment_config_metadata(
 "default_inference_instance_type"
 )

 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )

 benchmark_metrics = (
 metadata_config.benchmark_metrics.get(default_inference_instance_type)
 if metadata_config.benchmark_metrics is not None
 else None
 )
 if instance_rate is not None:
 if benchmark_metrics is not None:
 benchmark_metrics.append(JumpStartBenchmarkStat(instance_rate))

 should_fetch_instance_rate_metric = True
 if benchmark_metrics is not None:
 for benchmark_metric in benchmark_metrics:
 if benchmark_metric.name.lower() == "instance rate":
 should_fetch_instance_rate_metric = False
 break

 if should_fetch_instance_rate_metric:
 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )
 instance_rate_metric = JumpStartBenchmarkStat(instance_rate)

 if benchmark_metrics is None:
 benchmark_metrics = [instance_rate_metric]
 else:
 benchmark_metrics = [JumpStartBenchmarkStat(instance_rate)]
 benchmark_metrics.append(instance_rate_metric)

 init_kwargs = get_init_kwargs(
 model_id=self.model_id,
Expand Down

2 changes: 1 addition & 1 deletion
src/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1030,7 +1030,7 @@ def get_jumpstart_configs(
 )


def extract_metrics_from_deployment_configs(
def get_metrics_from_deployment_configs(
 deployment_configs: List[Dict[str, Any]], config_name: str
) -> Dict[str, List[str]]:
 """Extracts metrics from deployment configs.
Expand Down

5 changes: 3 additions & 2 deletions
tests/unit/sagemaker/jumpstart/model/test_model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@
 get_base_spec_with_prototype_configs,
 get_mock_init_kwargs,
 get_base_deployment_configs,
 get_base_spec_with_prototype_configs_with_missing_benchmarks,
)
import boto3

Expand DownExpand Up@@ -1790,7 +1791,7 @@ def test_model_retrieve_deployment_config(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand DownExpand Up@@ -1838,7 +1839,7 @@ def test_model_display_benchmark_metrics(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand Down

2 changes: 1 addition & 1 deletion
tests/unit/sagemaker/jumpstart/test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1810,6 +1810,6 @@ def test_get_jumpstart_benchmark_stats_training(
 ],
)
def test_extract_metrics_from_deployment_configs(config_name, configs, expected):
 data = utils.extract_metrics_from_deployment_configs(configs, config_name)
 data = utils.get_metrics_from_deployment_configs(configs, config_name)

 assert data == expected

20 changes: 20 additions & 0 deletions
tests/unit/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -226,6 +226,26 @@ def get_base_spec_with_prototype_configs(
 return JumpStartModelSpecs(spec)


def get_base_spec_with_prototype_configs_with_missing_benchmarks(
 region: str = None,
 model_id: str = None,
 version: str = None,
 s3_client: boto3.client = None,
 model_type: JumpStartModelType = JumpStartModelType.OPEN_WEIGHTS,
) -> JumpStartModelSpecs:
 spec = copy.deepcopy(BASE_SPEC)
 copy_inference_configs = copy.deepcopy(INFERENCE_CONFIGS)
 copy_inference_configs["inference_configs"]["neuron-inference"]["benchmark_metrics"] = None

 inference_configs = {**INFERENCE_CONFIGS, **INFERENCE_CONFIG_RANKINGS}
 training_configs = {**TRAINING_CONFIGS, **TRAINING_CONFIG_RANKINGS}

 spec.update(inference_configs)
 spec.update(training_configs)

 return JumpStartModelSpecs(spec)


def get_prototype_spec_with_configs(
 region: str = None,
 model_id: str = None,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Benchmark feature v2#4618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

liujiaorr
merged 17 commits into

from

Apr 25, 2024
 Merged


Benchmark feature v2
#4618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
63147a9

Add funtionalities to get and set deployment config

Apr 24, 2024
6520a2b

Resolve PR comments

Apr 24, 2024
a7fced7

ModelBuilder-JS

Apr 24, 2024
dd84775

Add Unit tests

Apr 25, 2024
faba3c8

Refactoring

Apr 25, 2024
dc2f5b7

Testing with Notebook

Apr 25, 2024
b4e983a

Test backward compatibility

Apr 25, 2024
469f17a

Merge branch 'master-benchmark-feature' into benchmark-feature-v2


makungaj1 Apr 25, 2024
6f30cbf

Remove Accelerated column if all not enabled

Apr 25, 2024
5724ce1

Fix docstring

Apr 25, 2024
4a61ee6

Resolved PR Review comments

Apr 25, 2024
d88ef47

Docstring

Apr 25, 2024
aaba1cb

increase code coverage

Apr 25, 2024
c20c1f3

Testing fix with Notebook

Apr 25, 2024
10bd9db

Merge remote-tracking branch 'upstream/master-benchmark-feature' into…

Apr 25, 2024
aee4169

Only fetch instance rate metrics if not present

Apr 25, 2024
76b8112

Increase code coverage

Apr 25, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 19 additions & 10 deletions
src/sagemaker/jumpstart/model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 validate_model_id_and_get_type,
 verify_model_region_and_return_specs,
 get_jumpstart_configs,
extract_metrics_from_deployment_configs,
get_metrics_from_deployment_configs,
)
from sagemaker.jumpstart.constants import JUMPSTART_LOGGER
from sagemaker.jumpstart.enums import JumpStartModelType
Expand DownExpand Up@@ -868,7 +868,7 @@ def _get_benchmarks_data(self, config_name: str) -> Dict[str, List[str]]:
 Returns:
 Dict[str, List[str]]: Deployment config benchmark data.
 """
 return extract_metrics_from_deployment_configs(
 return get_metrics_from_deployment_configs(
 self._deployment_configs,
 config_name,
 )
Expand DownExpand Up@@ -905,20 +905,29 @@ def _convert_to_deployment_config_metadata(
 "default_inference_instance_type"
 )

 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )

 benchmark_metrics = (
 metadata_config.benchmark_metrics.get(default_inference_instance_type)
 if metadata_config.benchmark_metrics is not None
 else None
 )
 if instance_rate is not None:
 if benchmark_metrics is not None:
 benchmark_metrics.append(JumpStartBenchmarkStat(instance_rate))

 should_fetch_instance_rate_metric = True
 if benchmark_metrics is not None:
 for benchmark_metric in benchmark_metrics:
 if benchmark_metric.name.lower() == "instance rate":
 should_fetch_instance_rate_metric = False
 break

 if should_fetch_instance_rate_metric:
 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )
 instance_rate_metric = JumpStartBenchmarkStat(instance_rate)

 if benchmark_metrics is None:
 benchmark_metrics = [instance_rate_metric]
 else:
 benchmark_metrics = [JumpStartBenchmarkStat(instance_rate)]
 benchmark_metrics.append(instance_rate_metric)

 init_kwargs = get_init_kwargs(
 model_id=self.model_id,
Expand Down

2 changes: 1 addition & 1 deletion
src/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1030,7 +1030,7 @@ def get_jumpstart_configs(
 )


def extract_metrics_from_deployment_configs(
def get_metrics_from_deployment_configs(
 deployment_configs: List[Dict[str, Any]], config_name: str
) -> Dict[str, List[str]]:
 """Extracts metrics from deployment configs.
Expand Down

5 changes: 3 additions & 2 deletions
tests/unit/sagemaker/jumpstart/model/test_model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@
 get_base_spec_with_prototype_configs,
 get_mock_init_kwargs,
 get_base_deployment_configs,
 get_base_spec_with_prototype_configs_with_missing_benchmarks,
)
import boto3

Expand DownExpand Up@@ -1790,7 +1791,7 @@ def test_model_retrieve_deployment_config(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand DownExpand Up@@ -1838,7 +1839,7 @@ def test_model_display_benchmark_metrics(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand Down

2 changes: 1 addition & 1 deletion
tests/unit/sagemaker/jumpstart/test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1810,6 +1810,6 @@ def test_get_jumpstart_benchmark_stats_training(
 ],
)
def test_extract_metrics_from_deployment_configs(config_name, configs, expected):
 data = utils.extract_metrics_from_deployment_configs(configs, config_name)
 data = utils.get_metrics_from_deployment_configs(configs, config_name)

 assert data == expected

20 changes: 20 additions & 0 deletions
tests/unit/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -226,6 +226,26 @@ def get_base_spec_with_prototype_configs(
 return JumpStartModelSpecs(spec)


def get_base_spec_with_prototype_configs_with_missing_benchmarks(
 region: str = None,
 model_id: str = None,
 version: str = None,
 s3_client: boto3.client = None,
 model_type: JumpStartModelType = JumpStartModelType.OPEN_WEIGHTS,
) -> JumpStartModelSpecs:
 spec = copy.deepcopy(BASE_SPEC)
 copy_inference_configs = copy.deepcopy(INFERENCE_CONFIGS)
 copy_inference_configs["inference_configs"]["neuron-inference"]["benchmark_metrics"] = None

 inference_configs = {**INFERENCE_CONFIGS, **INFERENCE_CONFIG_RANKINGS}
 training_configs = {**TRAINING_CONFIGS, **TRAINING_CONFIG_RANKINGS}

 spec.update(inference_configs)
 spec.update(training_configs)

 return JumpStartModelSpecs(spec)


def get_prototype_spec_with_configs(
 region: str = None,
 model_id: str = None,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Benchmark feature v2#4618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

liujiaorr
merged 17 commits into

from

Apr 25, 2024
 Merged


Benchmark feature v2
#4618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
63147a9

Add funtionalities to get and set deployment config

Apr 24, 2024
6520a2b

Resolve PR comments

Apr 24, 2024
a7fced7

ModelBuilder-JS

Apr 24, 2024
dd84775

Add Unit tests

Apr 25, 2024
faba3c8

Refactoring

Apr 25, 2024
dc2f5b7

Testing with Notebook

Apr 25, 2024
b4e983a

Test backward compatibility

Apr 25, 2024
469f17a

Merge branch 'master-benchmark-feature' into benchmark-feature-v2


makungaj1 Apr 25, 2024
6f30cbf

Remove Accelerated column if all not enabled

Apr 25, 2024
5724ce1

Fix docstring

Apr 25, 2024
4a61ee6

Resolved PR Review comments

Apr 25, 2024
d88ef47

Docstring

Apr 25, 2024
aaba1cb

increase code coverage

Apr 25, 2024
c20c1f3

Testing fix with Notebook

Apr 25, 2024
10bd9db

Merge remote-tracking branch 'upstream/master-benchmark-feature' into…

Apr 25, 2024
aee4169

Only fetch instance rate metrics if not present

Apr 25, 2024
76b8112

Increase code coverage

Apr 25, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 19 additions & 10 deletions
src/sagemaker/jumpstart/model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@
 validate_model_id_and_get_type,
 verify_model_region_and_return_specs,
 get_jumpstart_configs,
extract_metrics_from_deployment_configs,
get_metrics_from_deployment_configs,
)
from sagemaker.jumpstart.constants import JUMPSTART_LOGGER
from sagemaker.jumpstart.enums import JumpStartModelType
Expand DownExpand Up@@ -868,7 +868,7 @@ def _get_benchmarks_data(self, config_name: str) -> Dict[str, List[str]]:
 Returns:
 Dict[str, List[str]]: Deployment config benchmark data.
 """
 return extract_metrics_from_deployment_configs(
 return get_metrics_from_deployment_configs(
 self._deployment_configs,
 config_name,
 )
Expand DownExpand Up@@ -905,20 +905,29 @@ def _convert_to_deployment_config_metadata(
 "default_inference_instance_type"
 )

 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )

 benchmark_metrics = (
 metadata_config.benchmark_metrics.get(default_inference_instance_type)
 if metadata_config.benchmark_metrics is not None
 else None
 )
 if instance_rate is not None:
 if benchmark_metrics is not None:
 benchmark_metrics.append(JumpStartBenchmarkStat(instance_rate))

 should_fetch_instance_rate_metric = True
 if benchmark_metrics is not None:
 for benchmark_metric in benchmark_metrics:
 if benchmark_metric.name.lower() == "instance rate":
 should_fetch_instance_rate_metric = False
 break

 if should_fetch_instance_rate_metric:
 instance_rate = get_instance_rate_per_hour(
 instance_type=default_inference_instance_type, region=self.region
 )
 instance_rate_metric = JumpStartBenchmarkStat(instance_rate)

 if benchmark_metrics is None:
 benchmark_metrics = [instance_rate_metric]
 else:
 benchmark_metrics = [JumpStartBenchmarkStat(instance_rate)]
 benchmark_metrics.append(instance_rate_metric)

 init_kwargs = get_init_kwargs(
 model_id=self.model_id,
Expand Down

2 changes: 1 addition & 1 deletion
src/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1030,7 +1030,7 @@ def get_jumpstart_configs(
 )


def extract_metrics_from_deployment_configs(
def get_metrics_from_deployment_configs(
 deployment_configs: List[Dict[str, Any]], config_name: str
) -> Dict[str, List[str]]:
 """Extracts metrics from deployment configs.
Expand Down

5 changes: 3 additions & 2 deletions
tests/unit/sagemaker/jumpstart/model/test_model.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@
 get_base_spec_with_prototype_configs,
 get_mock_init_kwargs,
 get_base_deployment_configs,
 get_base_spec_with_prototype_configs_with_missing_benchmarks,
)
import boto3

Expand DownExpand Up@@ -1790,7 +1791,7 @@ def test_model_retrieve_deployment_config(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand DownExpand Up@@ -1838,7 +1839,7 @@ def test_model_display_benchmark_metrics(

 mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id)
 mock_verify_model_region_and_return_specs.side_effect = (
 lambda *args, **kwargs: get_base_spec_with_prototype_configs()
 lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks()
 )
 mock_get_instance_rate_per_hour.side_effect = lambda *args, **kwargs: {
 "name": "Instance Rate",
Expand Down

2 changes: 1 addition & 1 deletion
tests/unit/sagemaker/jumpstart/test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1810,6 +1810,6 @@ def test_get_jumpstart_benchmark_stats_training(
 ],
)
def test_extract_metrics_from_deployment_configs(config_name, configs, expected):
 data = utils.extract_metrics_from_deployment_configs(configs, config_name)
 data = utils.get_metrics_from_deployment_configs(configs, config_name)

 assert data == expected

20 changes: 20 additions & 0 deletions
tests/unit/sagemaker/jumpstart/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -226,6 +226,26 @@ def get_base_spec_with_prototype_configs(
 return JumpStartModelSpecs(spec)


def get_base_spec_with_prototype_configs_with_missing_benchmarks(
 region: str = None,
 model_id: str = None,
 version: str = None,
 s3_client: boto3.client = None,
 model_type: JumpStartModelType = JumpStartModelType.OPEN_WEIGHTS,
) -> JumpStartModelSpecs:
 spec = copy.deepcopy(BASE_SPEC)
 copy_inference_configs = copy.deepcopy(INFERENCE_CONFIGS)
 copy_inference_configs["inference_configs"]["neuron-inference"]["benchmark_metrics"] = None

 inference_configs = {**INFERENCE_CONFIGS, **INFERENCE_CONFIG_RANKINGS}
 training_configs = {**TRAINING_CONFIGS, **TRAINING_CONFIG_RANKINGS}

 spec.update(inference_configs)
 spec.update(training_configs)

 return JumpStartModelSpecs(spec)


def get_prototype_spec_with_configs(
 region: str = None,
 model_id: str = None,
Expand Down