datamixing recipe path fix#6073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mujtaba1747 merged 6 commits into from
Jul 22, 2026
Merged

datamixing recipe path fix #6073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fb577ce
cleanup: use a single logger in base_trainer
Jul 15, 2026
706e73e
fix: update recipe resolver unit tests
Jul 15, 2026
7e14662
Merge branch 'aws:master' into master
mujtaba1747 Jul 15, 2026
f73e729
Release 3.16.0 (2026-07-15): Bump VERSION files and internal dependen…
Jul 15, 2026
c2ace1e
Merge branch 'master' of github.com:mujtaba1747/sagemaker-python-sdk
Jul 21, 2026
cfd34e8
fix: use absolute path in data mixing recipe path construction
Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 8 additions & 13 deletions sagemaker-train/src/sagemaker/train/common_utils/data_mixing_utils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -480,16 +480,18 @@ def build_hyperpod_datamix_recipe_from_context(
4. Inject customer_data_percent and nova_data_percentages into data_mixing.sources
5. Validate non-zero categories exist in template's nova_data section
6. Write final YAML to HyperPod CLI recipes directory
7. Return (relative_recipe_path, image_uri)
7. Return (recipe_path, image_uri)

Args:
context: The HyperPodTemplateContext from resolve_hyperpod_datamix_context.
validated_config: A DataMixingConfig validated via validate_data_mixing_categories.

Returns:
Tuple of (relative_recipe_path, image_uri). relative_recipe_path is relative
to the HyperPod CLI recipes_collection/recipes directory with .yaml extension
removed. image_uri is the container image URI from the context (or None).
Tuple of (recipe_path, image_uri). recipe_path is the absolute filesystem
path (including the .yaml extension) of the generated recipe written under
the HyperPod CLI recipes_collection/recipes directory; it is consumed by the
recipe resolver as a user recipe file. image_uri is the container image URI
from the context (or None).

Raises:
RuntimeError: If hyperpod_cli is not installed.
Expand DownExpand Up@@ -652,18 +654,11 @@ def _apply_overrides(recipe: dict, overrides: dict) -> dict:
with open(recipe_path, "w") as f:
f.write(recipe_output)

relative_path = (
recipe_path.split(HYPERPOD_RECIPE_PATH, 1)[1]
.lstrip("/").lstrip("\\")
.removesuffix(".yaml")
)

logger.info(
"Generated HyperPod datamix recipe at '%s' (relative: '%s') from context '%s'.",
"Generated HyperPod datamix recipe at '%s' from context '%s'.",
recipe_path,
relative_path,
context.recipe_name,
)

return relative_path, context.image_uri
return recipe_path, context.image_uri

0 ...n/tests/integ/train/test_nova_hyperpod.py → ...sts/integ/train/test_nova_sft_hyperpod.py
View file
Open in desktop
File renamed without changes.
40 changes: 35 additions & 5 deletions sagemaker-train/tests/integ/train/test_sft_data_mixing_hyperpod.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,7 @@
import pytest
from sagemaker.train.sft_trainer import SFTTrainer
from sagemaker.train.common import TrainingType
from sagemaker.train.base_trainer import BaseTrainer
from sagemaker.train.data_mixing_config import DataMixingConfig
from sagemaker.core.training.configs import HyperPodCompute

Expand All@@ -49,8 +50,8 @@
REGION = "us-east-1"
DATA_PREFIX = "test-sft-data-mixing-hyperpod-integ"
NUM_TRAINING_SAMPLES = 300
HYPERPOD_CLUSTER_NAME = "riv-rig"
HYPERPOD_INSTANCE_TYPE = "ml.p5.48xlarge"
HYPERPOD_CLUSTER_NAME = "pysdk-hp-integ-tests"
HYPERPOD_INSTANCE_TYPE = "ml.g6.12xlarge"


def _generate_training_data() -> str:
Expand DownExpand Up@@ -125,10 +126,10 @@ def training_resources(sagemaker_session_us_east_1):
@pytest.mark.gpu_intensive
@pytest.mark.us_east_1
def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1, training_resources):
"""Test SFT trainer with Nova Lite 2 model and data mixing on HyperPod.
"""Test SFT trainer with Nova Micro model and data mixing on HyperPod.

This end-to-end test submits a real HyperPod training job with DataMixingConfig
for Nova Lite 2. The SDK resolves the datamix recipe from SageMaker Hub, validates
for Nova Micro. The SDK resolves the datamix recipe from SageMaker Hub, validates
categories, and includes the serialized config in the HyperPod override parameters.
"""
unique_id = f"{int(time.time())}-{random.randint(1000, 9999)}"
Expand DownExpand Up@@ -184,4 +185,33 @@ def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1
assert get_job_result.returncode == 0, (
f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}"
)
logger.info(f"Verified job '{job_name}' exists on the cluster.")
logger.info(f"Verified job '{job_name}' exists on the cluster using hp-cli.")

# Poll for job completion by checking for the manifest in S3.
# The manifest is written under {output_s3_path}/{job_name}/manifest.json
# once training finishes, so its presence confirms end-to-end completion.

output_s3_path = training_resources["s3_output_path"]
max_wait_time = 21600 # 6 hour timeout (HyperPod jobs can take longer)
poll_interval = 60 # Check every 60 seconds
start_time = time.time()
checkpoint_path = None

while time.time() - start_time < max_wait_time:
checkpoint_path = BaseTrainer._resolve_checkpoint_from_manifest(
job_name=job_name,
output_s3_path=output_s3_path,
sagemaker_session=sagemaker_session_us_east_1,
)
if checkpoint_path:
logger.info(f"Checkpoint resolved: {checkpoint_path}")
break

elapsed = int(time.time() - start_time)
logger.info(f"Waiting for manifest... ({elapsed}s elapsed)")
time.sleep(poll_interval)

assert checkpoint_path is not None, (
f"Job {job_name} did not produce a manifest within {max_wait_time}s"
)
logger.info(f"Training complete. Checkpoint: {checkpoint_path}")
22 changes: 15 additions & 7 deletions sagemaker-train/tests/unit/train/common_utils/test_data_mixing_utils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
"""Unit tests for data mixing utility functions."""
from __future__ import absolute_import

import os

import pytest

from sagemaker.train.data_mixing_config import DataMixingConfig
Expand DownExpand Up@@ -823,8 +825,12 @@ def capture_write(path, mode="r", **kwargs):
assert parsed["data_mixing"]["sources"]["nova_data"]["en-entertainment"] == 20
assert parsed["data_mixing"]["sources"]["nova_data"]["en-scientific"] == 10

def test_return_value_is_relative_path_and_image_uri(self):
"""Return value should be (relative_path_without_extension, image_uri)."""
def test_return_value_is_absolute_recipe_path_and_image_uri(self):
"""Return value should be (absolute_recipe_path_with_extension, image_uri).

The path must be a loadable filesystem path (absolute, .yaml extension
intact) because the recipe resolver opens it as a user recipe file.
"""
from unittest.mock import patch, MagicMock, mock_open

from sagemaker.train.common_utils.data_mixing_utils import (
Expand All@@ -843,12 +849,14 @@ def test_return_value_is_relative_path_and_image_uri(self):
with patch("builtins.open", m_open):
result = build_hyperpod_datamix_recipe_from_context(context, config)

relative_path, image_uri = result
recipe_path, image_uri = result

# relative_path should not end with .yaml
assert not relative_path.endswith(".yaml")
# relative_path should contain fine-tuning/nova
assert "fine-tuning/nova" in relative_path
# recipe_path must be an absolute filesystem path
assert os.path.isabs(recipe_path)
# recipe_path must retain the .yaml extension so it can be opened
assert recipe_path.endswith(".yaml")
# recipe_path should live under the HyperPod CLI recipes fine-tuning/nova dir
assert "fine-tuning/nova" in recipe_path
# image_uri should match the context
assert image_uri == self.IMAGE_URI

Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
datamixing recipe path fix#6073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

mujtaba1747
merged 6 commits into

from

Jul 22, 2026
 Merged


datamixing recipe path fix
#6073
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
fb577ce

cleanup: use a single logger in base_trainer

Jul 15, 2026
706e73e

fix: update recipe resolver unit tests

Jul 15, 2026
7e14662

Merge branch 'aws:master' into master


mujtaba1747 Jul 15, 2026
f73e729

Release 3.16.0 (2026-07-15): Bump VERSION files and internal dependen…

Jul 15, 2026
c2ace1e

Merge branch 'master' of github.com:mujtaba1747/sagemaker-python-sdk

Jul 21, 2026
cfd34e8

fix: use absolute path in data mixing recipe path construction

Jul 21, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 8 additions & 13 deletions
sagemaker-train/src/sagemaker/train/common_utils/data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -480,16 +480,18 @@ def build_hyperpod_datamix_recipe_from_context(
 4. Inject customer_data_percent and nova_data_percentages into data_mixing.sources
 5. Validate non-zero categories exist in template's nova_data section
 6. Write final YAML to HyperPod CLI recipes directory
 7. Return (relative_recipe_path, image_uri)
 7. Return (recipe_path, image_uri)

 Args:
 context: The HyperPodTemplateContext from resolve_hyperpod_datamix_context.
 validated_config: A DataMixingConfig validated via validate_data_mixing_categories.

 Returns:
 Tuple of (relative_recipe_path, image_uri). relative_recipe_path is relative
 to the HyperPod CLI recipes_collection/recipes directory with .yaml extension
 removed. image_uri is the container image URI from the context (or None).
 Tuple of (recipe_path, image_uri). recipe_path is the absolute filesystem
 path (including the .yaml extension) of the generated recipe written under
 the HyperPod CLI recipes_collection/recipes directory; it is consumed by the
 recipe resolver as a user recipe file. image_uri is the container image URI
 from the context (or None).

 Raises:
 RuntimeError: If hyperpod_cli is not installed.
Expand DownExpand Up@@ -652,18 +654,11 @@ def _apply_overrides(recipe: dict, overrides: dict) -> dict:
 with open(recipe_path, "w") as f:
 f.write(recipe_output)

 relative_path = (
 recipe_path.split(HYPERPOD_RECIPE_PATH, 1)[1]
 .lstrip("/").lstrip("\\")
 .removesuffix(".yaml")
 )

 logger.info(
 "Generated HyperPod datamix recipe at '%s' (relative: '%s') from context '%s'.",
 "Generated HyperPod datamix recipe at '%s' from context '%s'.",
 recipe_path,
 relative_path,
 context.recipe_name,
 )

 return relative_path, context.image_uri
 return recipe_path, context.image_uri


0
...n/tests/integ/train/test_nova_hyperpod.py → ...sts/integ/train/test_nova_sft_hyperpod.py

View file


Open in desktop


File renamed without changes.


40 changes: 35 additions & 5 deletions
sagemaker-train/tests/integ/train/test_sft_data_mixing_hyperpod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,7 @@
import pytest
from sagemaker.train.sft_trainer import SFTTrainer
from sagemaker.train.common import TrainingType
from sagemaker.train.base_trainer import BaseTrainer
from sagemaker.train.data_mixing_config import DataMixingConfig
from sagemaker.core.training.configs import HyperPodCompute

Expand All@@ -49,8 +50,8 @@
REGION = "us-east-1"
DATA_PREFIX = "test-sft-data-mixing-hyperpod-integ"
NUM_TRAINING_SAMPLES = 300
HYPERPOD_CLUSTER_NAME = "riv-rig"
HYPERPOD_INSTANCE_TYPE = "ml.p5.48xlarge"
HYPERPOD_CLUSTER_NAME = "pysdk-hp-integ-tests"
HYPERPOD_INSTANCE_TYPE = "ml.g6.12xlarge"


def _generate_training_data() -> str:
Expand DownExpand Up@@ -125,10 +126,10 @@ def training_resources(sagemaker_session_us_east_1):
@pytest.mark.gpu_intensive
@pytest.mark.us_east_1
def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1, training_resources):
 """Test SFT trainer with Nova Lite 2 model and data mixing on HyperPod.
 """Test SFT trainer with Nova Micro model and data mixing on HyperPod.

 This end-to-end test submits a real HyperPod training job with DataMixingConfig
 for Nova Lite 2. The SDK resolves the datamix recipe from SageMaker Hub, validates
 for Nova Micro. The SDK resolves the datamix recipe from SageMaker Hub, validates
 categories, and includes the serialized config in the HyperPod override parameters.
 """
 unique_id = f"{int(time.time())}-{random.randint(1000, 9999)}"
Expand DownExpand Up@@ -184,4 +185,33 @@ def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1
 assert get_job_result.returncode == 0, (
 f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}"
 )
 logger.info(f"Verified job '{job_name}' exists on the cluster.")
 logger.info(f"Verified job '{job_name}' exists on the cluster using hp-cli.")

 # Poll for job completion by checking for the manifest in S3.
 # The manifest is written under {output_s3_path}/{job_name}/manifest.json
 # once training finishes, so its presence confirms end-to-end completion.

 output_s3_path = training_resources["s3_output_path"]
 max_wait_time = 21600 # 6 hour timeout (HyperPod jobs can take longer)
 poll_interval = 60 # Check every 60 seconds
 start_time = time.time()
 checkpoint_path = None

 while time.time() - start_time < max_wait_time:
 checkpoint_path = BaseTrainer._resolve_checkpoint_from_manifest(
 job_name=job_name,
 output_s3_path=output_s3_path,
 sagemaker_session=sagemaker_session_us_east_1,
 )
 if checkpoint_path:
 logger.info(f"Checkpoint resolved: {checkpoint_path}")
 break

 elapsed = int(time.time() - start_time)
 logger.info(f"Waiting for manifest... ({elapsed}s elapsed)")
 time.sleep(poll_interval)

 assert checkpoint_path is not None, (
 f"Job {job_name} did not produce a manifest within {max_wait_time}s"
 )
 logger.info(f"Training complete. Checkpoint: {checkpoint_path}")

22 changes: 15 additions & 7 deletions
sagemaker-train/tests/unit/train/common_utils/test_data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
"""Unit tests for data mixing utility functions."""
from __future__ import absolute_import

import os

import pytest

from sagemaker.train.data_mixing_config import DataMixingConfig
Expand DownExpand Up@@ -823,8 +825,12 @@ def capture_write(path, mode="r", **kwargs):
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-entertainment"] == 20
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-scientific"] == 10

 def test_return_value_is_relative_path_and_image_uri(self):
 """Return value should be (relative_path_without_extension, image_uri)."""
 def test_return_value_is_absolute_recipe_path_and_image_uri(self):
 """Return value should be (absolute_recipe_path_with_extension, image_uri).

 The path must be a loadable filesystem path (absolute, .yaml extension
 intact) because the recipe resolver opens it as a user recipe file.
 """
 from unittest.mock import patch, MagicMock, mock_open

 from sagemaker.train.common_utils.data_mixing_utils import (
Expand All@@ -843,12 +849,14 @@ def test_return_value_is_relative_path_and_image_uri(self):
 with patch("builtins.open", m_open):
 result = build_hyperpod_datamix_recipe_from_context(context, config)

relative_path, image_uri = result
recipe_path, image_uri = result

 # relative_path should not end with .yaml
 assert not relative_path.endswith(".yaml")
 # relative_path should contain fine-tuning/nova
 assert "fine-tuning/nova" in relative_path
 # recipe_path must be an absolute filesystem path
 assert os.path.isabs(recipe_path)
 # recipe_path must retain the .yaml extension so it can be opened
 assert recipe_path.endswith(".yaml")
 # recipe_path should live under the HyperPod CLI recipes fine-tuning/nova dir
 assert "fine-tuning/nova" in recipe_path
 # image_uri should match the context
 assert image_uri == self.IMAGE_URI

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
datamixing recipe path fix#6073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

mujtaba1747
merged 6 commits into

from

Jul 22, 2026
 Merged


datamixing recipe path fix
#6073
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
fb577ce

cleanup: use a single logger in base_trainer

Jul 15, 2026
706e73e

fix: update recipe resolver unit tests

Jul 15, 2026
7e14662

Merge branch 'aws:master' into master


mujtaba1747 Jul 15, 2026
f73e729

Release 3.16.0 (2026-07-15): Bump VERSION files and internal dependen…

Jul 15, 2026
c2ace1e

Merge branch 'master' of github.com:mujtaba1747/sagemaker-python-sdk

Jul 21, 2026
cfd34e8

fix: use absolute path in data mixing recipe path construction

Jul 21, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 8 additions & 13 deletions
sagemaker-train/src/sagemaker/train/common_utils/data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -480,16 +480,18 @@ def build_hyperpod_datamix_recipe_from_context(
 4. Inject customer_data_percent and nova_data_percentages into data_mixing.sources
 5. Validate non-zero categories exist in template's nova_data section
 6. Write final YAML to HyperPod CLI recipes directory
 7. Return (relative_recipe_path, image_uri)
 7. Return (recipe_path, image_uri)

 Args:
 context: The HyperPodTemplateContext from resolve_hyperpod_datamix_context.
 validated_config: A DataMixingConfig validated via validate_data_mixing_categories.

 Returns:
 Tuple of (relative_recipe_path, image_uri). relative_recipe_path is relative
 to the HyperPod CLI recipes_collection/recipes directory with .yaml extension
 removed. image_uri is the container image URI from the context (or None).
 Tuple of (recipe_path, image_uri). recipe_path is the absolute filesystem
 path (including the .yaml extension) of the generated recipe written under
 the HyperPod CLI recipes_collection/recipes directory; it is consumed by the
 recipe resolver as a user recipe file. image_uri is the container image URI
 from the context (or None).

 Raises:
 RuntimeError: If hyperpod_cli is not installed.
Expand DownExpand Up@@ -652,18 +654,11 @@ def _apply_overrides(recipe: dict, overrides: dict) -> dict:
 with open(recipe_path, "w") as f:
 f.write(recipe_output)

 relative_path = (
 recipe_path.split(HYPERPOD_RECIPE_PATH, 1)[1]
 .lstrip("/").lstrip("\\")
 .removesuffix(".yaml")
 )

 logger.info(
 "Generated HyperPod datamix recipe at '%s' (relative: '%s') from context '%s'.",
 "Generated HyperPod datamix recipe at '%s' from context '%s'.",
 recipe_path,
 relative_path,
 context.recipe_name,
 )

 return relative_path, context.image_uri
 return recipe_path, context.image_uri


0
...n/tests/integ/train/test_nova_hyperpod.py → ...sts/integ/train/test_nova_sft_hyperpod.py

View file


Open in desktop


File renamed without changes.


40 changes: 35 additions & 5 deletions
sagemaker-train/tests/integ/train/test_sft_data_mixing_hyperpod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,7 @@
import pytest
from sagemaker.train.sft_trainer import SFTTrainer
from sagemaker.train.common import TrainingType
from sagemaker.train.base_trainer import BaseTrainer
from sagemaker.train.data_mixing_config import DataMixingConfig
from sagemaker.core.training.configs import HyperPodCompute

Expand All@@ -49,8 +50,8 @@
REGION = "us-east-1"
DATA_PREFIX = "test-sft-data-mixing-hyperpod-integ"
NUM_TRAINING_SAMPLES = 300
HYPERPOD_CLUSTER_NAME = "riv-rig"
HYPERPOD_INSTANCE_TYPE = "ml.p5.48xlarge"
HYPERPOD_CLUSTER_NAME = "pysdk-hp-integ-tests"
HYPERPOD_INSTANCE_TYPE = "ml.g6.12xlarge"


def _generate_training_data() -> str:
Expand DownExpand Up@@ -125,10 +126,10 @@ def training_resources(sagemaker_session_us_east_1):
@pytest.mark.gpu_intensive
@pytest.mark.us_east_1
def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1, training_resources):
 """Test SFT trainer with Nova Lite 2 model and data mixing on HyperPod.
 """Test SFT trainer with Nova Micro model and data mixing on HyperPod.

 This end-to-end test submits a real HyperPod training job with DataMixingConfig
 for Nova Lite 2. The SDK resolves the datamix recipe from SageMaker Hub, validates
 for Nova Micro. The SDK resolves the datamix recipe from SageMaker Hub, validates
 categories, and includes the serialized config in the HyperPod override parameters.
 """
 unique_id = f"{int(time.time())}-{random.randint(1000, 9999)}"
Expand DownExpand Up@@ -184,4 +185,33 @@ def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1
 assert get_job_result.returncode == 0, (
 f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}"
 )
 logger.info(f"Verified job '{job_name}' exists on the cluster.")
 logger.info(f"Verified job '{job_name}' exists on the cluster using hp-cli.")

 # Poll for job completion by checking for the manifest in S3.
 # The manifest is written under {output_s3_path}/{job_name}/manifest.json
 # once training finishes, so its presence confirms end-to-end completion.

 output_s3_path = training_resources["s3_output_path"]
 max_wait_time = 21600 # 6 hour timeout (HyperPod jobs can take longer)
 poll_interval = 60 # Check every 60 seconds
 start_time = time.time()
 checkpoint_path = None

 while time.time() - start_time < max_wait_time:
 checkpoint_path = BaseTrainer._resolve_checkpoint_from_manifest(
 job_name=job_name,
 output_s3_path=output_s3_path,
 sagemaker_session=sagemaker_session_us_east_1,
 )
 if checkpoint_path:
 logger.info(f"Checkpoint resolved: {checkpoint_path}")
 break

 elapsed = int(time.time() - start_time)
 logger.info(f"Waiting for manifest... ({elapsed}s elapsed)")
 time.sleep(poll_interval)

 assert checkpoint_path is not None, (
 f"Job {job_name} did not produce a manifest within {max_wait_time}s"
 )
 logger.info(f"Training complete. Checkpoint: {checkpoint_path}")

22 changes: 15 additions & 7 deletions
sagemaker-train/tests/unit/train/common_utils/test_data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
"""Unit tests for data mixing utility functions."""
from __future__ import absolute_import

import os

import pytest

from sagemaker.train.data_mixing_config import DataMixingConfig
Expand DownExpand Up@@ -823,8 +825,12 @@ def capture_write(path, mode="r", **kwargs):
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-entertainment"] == 20
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-scientific"] == 10

 def test_return_value_is_relative_path_and_image_uri(self):
 """Return value should be (relative_path_without_extension, image_uri)."""
 def test_return_value_is_absolute_recipe_path_and_image_uri(self):
 """Return value should be (absolute_recipe_path_with_extension, image_uri).

 The path must be a loadable filesystem path (absolute, .yaml extension
 intact) because the recipe resolver opens it as a user recipe file.
 """
 from unittest.mock import patch, MagicMock, mock_open

 from sagemaker.train.common_utils.data_mixing_utils import (
Expand All@@ -843,12 +849,14 @@ def test_return_value_is_relative_path_and_image_uri(self):
 with patch("builtins.open", m_open):
 result = build_hyperpod_datamix_recipe_from_context(context, config)

relative_path, image_uri = result
recipe_path, image_uri = result

 # relative_path should not end with .yaml
 assert not relative_path.endswith(".yaml")
 # relative_path should contain fine-tuning/nova
 assert "fine-tuning/nova" in relative_path
 # recipe_path must be an absolute filesystem path
 assert os.path.isabs(recipe_path)
 # recipe_path must retain the .yaml extension so it can be opened
 assert recipe_path.endswith(".yaml")
 # recipe_path should live under the HyperPod CLI recipes fine-tuning/nova dir
 assert "fine-tuning/nova" in recipe_path
 # image_uri should match the context
 assert image_uri == self.IMAGE_URI

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
datamixing recipe path fix#6073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

mujtaba1747
merged 6 commits into

from

Jul 22, 2026
 Merged


datamixing recipe path fix
#6073
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
fb577ce

cleanup: use a single logger in base_trainer

Jul 15, 2026
706e73e

fix: update recipe resolver unit tests

Jul 15, 2026
7e14662

Merge branch 'aws:master' into master


mujtaba1747 Jul 15, 2026
f73e729

Release 3.16.0 (2026-07-15): Bump VERSION files and internal dependen…

Jul 15, 2026
c2ace1e

Merge branch 'master' of github.com:mujtaba1747/sagemaker-python-sdk

Jul 21, 2026
cfd34e8

fix: use absolute path in data mixing recipe path construction

Jul 21, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 8 additions & 13 deletions
sagemaker-train/src/sagemaker/train/common_utils/data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -480,16 +480,18 @@ def build_hyperpod_datamix_recipe_from_context(
 4. Inject customer_data_percent and nova_data_percentages into data_mixing.sources
 5. Validate non-zero categories exist in template's nova_data section
 6. Write final YAML to HyperPod CLI recipes directory
 7. Return (relative_recipe_path, image_uri)
 7. Return (recipe_path, image_uri)

 Args:
 context: The HyperPodTemplateContext from resolve_hyperpod_datamix_context.
 validated_config: A DataMixingConfig validated via validate_data_mixing_categories.

 Returns:
 Tuple of (relative_recipe_path, image_uri). relative_recipe_path is relative
 to the HyperPod CLI recipes_collection/recipes directory with .yaml extension
 removed. image_uri is the container image URI from the context (or None).
 Tuple of (recipe_path, image_uri). recipe_path is the absolute filesystem
 path (including the .yaml extension) of the generated recipe written under
 the HyperPod CLI recipes_collection/recipes directory; it is consumed by the
 recipe resolver as a user recipe file. image_uri is the container image URI
 from the context (or None).

 Raises:
 RuntimeError: If hyperpod_cli is not installed.
Expand DownExpand Up@@ -652,18 +654,11 @@ def _apply_overrides(recipe: dict, overrides: dict) -> dict:
 with open(recipe_path, "w") as f:
 f.write(recipe_output)

 relative_path = (
 recipe_path.split(HYPERPOD_RECIPE_PATH, 1)[1]
 .lstrip("/").lstrip("\\")
 .removesuffix(".yaml")
 )

 logger.info(
 "Generated HyperPod datamix recipe at '%s' (relative: '%s') from context '%s'.",
 "Generated HyperPod datamix recipe at '%s' from context '%s'.",
 recipe_path,
 relative_path,
 context.recipe_name,
 )

 return relative_path, context.image_uri
 return recipe_path, context.image_uri


0
...n/tests/integ/train/test_nova_hyperpod.py → ...sts/integ/train/test_nova_sft_hyperpod.py

View file


Open in desktop


File renamed without changes.


40 changes: 35 additions & 5 deletions
sagemaker-train/tests/integ/train/test_sft_data_mixing_hyperpod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,7 @@
import pytest
from sagemaker.train.sft_trainer import SFTTrainer
from sagemaker.train.common import TrainingType
from sagemaker.train.base_trainer import BaseTrainer
from sagemaker.train.data_mixing_config import DataMixingConfig
from sagemaker.core.training.configs import HyperPodCompute

Expand All@@ -49,8 +50,8 @@
REGION = "us-east-1"
DATA_PREFIX = "test-sft-data-mixing-hyperpod-integ"
NUM_TRAINING_SAMPLES = 300
HYPERPOD_CLUSTER_NAME = "riv-rig"
HYPERPOD_INSTANCE_TYPE = "ml.p5.48xlarge"
HYPERPOD_CLUSTER_NAME = "pysdk-hp-integ-tests"
HYPERPOD_INSTANCE_TYPE = "ml.g6.12xlarge"


def _generate_training_data() -> str:
Expand DownExpand Up@@ -125,10 +126,10 @@ def training_resources(sagemaker_session_us_east_1):
@pytest.mark.gpu_intensive
@pytest.mark.us_east_1
def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1, training_resources):
 """Test SFT trainer with Nova Lite 2 model and data mixing on HyperPod.
 """Test SFT trainer with Nova Micro model and data mixing on HyperPod.

 This end-to-end test submits a real HyperPod training job with DataMixingConfig
 for Nova Lite 2. The SDK resolves the datamix recipe from SageMaker Hub, validates
 for Nova Micro. The SDK resolves the datamix recipe from SageMaker Hub, validates
 categories, and includes the serialized config in the HyperPod override parameters.
 """
 unique_id = f"{int(time.time())}-{random.randint(1000, 9999)}"
Expand DownExpand Up@@ -184,4 +185,33 @@ def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1
 assert get_job_result.returncode == 0, (
 f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}"
 )
 logger.info(f"Verified job '{job_name}' exists on the cluster.")
 logger.info(f"Verified job '{job_name}' exists on the cluster using hp-cli.")

 # Poll for job completion by checking for the manifest in S3.
 # The manifest is written under {output_s3_path}/{job_name}/manifest.json
 # once training finishes, so its presence confirms end-to-end completion.

 output_s3_path = training_resources["s3_output_path"]
 max_wait_time = 21600 # 6 hour timeout (HyperPod jobs can take longer)
 poll_interval = 60 # Check every 60 seconds
 start_time = time.time()
 checkpoint_path = None

 while time.time() - start_time < max_wait_time:
 checkpoint_path = BaseTrainer._resolve_checkpoint_from_manifest(
 job_name=job_name,
 output_s3_path=output_s3_path,
 sagemaker_session=sagemaker_session_us_east_1,
 )
 if checkpoint_path:
 logger.info(f"Checkpoint resolved: {checkpoint_path}")
 break

 elapsed = int(time.time() - start_time)
 logger.info(f"Waiting for manifest... ({elapsed}s elapsed)")
 time.sleep(poll_interval)

 assert checkpoint_path is not None, (
 f"Job {job_name} did not produce a manifest within {max_wait_time}s"
 )
 logger.info(f"Training complete. Checkpoint: {checkpoint_path}")

22 changes: 15 additions & 7 deletions
sagemaker-train/tests/unit/train/common_utils/test_data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
"""Unit tests for data mixing utility functions."""
from __future__ import absolute_import

import os

import pytest

from sagemaker.train.data_mixing_config import DataMixingConfig
Expand DownExpand Up@@ -823,8 +825,12 @@ def capture_write(path, mode="r", **kwargs):
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-entertainment"] == 20
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-scientific"] == 10

 def test_return_value_is_relative_path_and_image_uri(self):
 """Return value should be (relative_path_without_extension, image_uri)."""
 def test_return_value_is_absolute_recipe_path_and_image_uri(self):
 """Return value should be (absolute_recipe_path_with_extension, image_uri).

 The path must be a loadable filesystem path (absolute, .yaml extension
 intact) because the recipe resolver opens it as a user recipe file.
 """
 from unittest.mock import patch, MagicMock, mock_open

 from sagemaker.train.common_utils.data_mixing_utils import (
Expand All@@ -843,12 +849,14 @@ def test_return_value_is_relative_path_and_image_uri(self):
 with patch("builtins.open", m_open):
 result = build_hyperpod_datamix_recipe_from_context(context, config)

relative_path, image_uri = result
recipe_path, image_uri = result

 # relative_path should not end with .yaml
 assert not relative_path.endswith(".yaml")
 # relative_path should contain fine-tuning/nova
 assert "fine-tuning/nova" in relative_path
 # recipe_path must be an absolute filesystem path
 assert os.path.isabs(recipe_path)
 # recipe_path must retain the .yaml extension so it can be opened
 assert recipe_path.endswith(".yaml")
 # recipe_path should live under the HyperPod CLI recipes fine-tuning/nova dir
 assert "fine-tuning/nova" in recipe_path
 # image_uri should match the context
 assert image_uri == self.IMAGE_URI

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
datamixing recipe path fix#6073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

mujtaba1747
merged 6 commits into

from

Jul 22, 2026
 Merged


datamixing recipe path fix
#6073
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
fb577ce

cleanup: use a single logger in base_trainer

Jul 15, 2026
706e73e

fix: update recipe resolver unit tests

Jul 15, 2026
7e14662

Merge branch 'aws:master' into master


mujtaba1747 Jul 15, 2026
f73e729

Release 3.16.0 (2026-07-15): Bump VERSION files and internal dependen…

Jul 15, 2026
c2ace1e

Merge branch 'master' of github.com:mujtaba1747/sagemaker-python-sdk

Jul 21, 2026
cfd34e8

fix: use absolute path in data mixing recipe path construction

Jul 21, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 8 additions & 13 deletions
sagemaker-train/src/sagemaker/train/common_utils/data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -480,16 +480,18 @@ def build_hyperpod_datamix_recipe_from_context(
 4. Inject customer_data_percent and nova_data_percentages into data_mixing.sources
 5. Validate non-zero categories exist in template's nova_data section
 6. Write final YAML to HyperPod CLI recipes directory
 7. Return (relative_recipe_path, image_uri)
 7. Return (recipe_path, image_uri)

 Args:
 context: The HyperPodTemplateContext from resolve_hyperpod_datamix_context.
 validated_config: A DataMixingConfig validated via validate_data_mixing_categories.

 Returns:
 Tuple of (relative_recipe_path, image_uri). relative_recipe_path is relative
 to the HyperPod CLI recipes_collection/recipes directory with .yaml extension
 removed. image_uri is the container image URI from the context (or None).
 Tuple of (recipe_path, image_uri). recipe_path is the absolute filesystem
 path (including the .yaml extension) of the generated recipe written under
 the HyperPod CLI recipes_collection/recipes directory; it is consumed by the
 recipe resolver as a user recipe file. image_uri is the container image URI
 from the context (or None).

 Raises:
 RuntimeError: If hyperpod_cli is not installed.
Expand DownExpand Up@@ -652,18 +654,11 @@ def _apply_overrides(recipe: dict, overrides: dict) -> dict:
 with open(recipe_path, "w") as f:
 f.write(recipe_output)

 relative_path = (
 recipe_path.split(HYPERPOD_RECIPE_PATH, 1)[1]
 .lstrip("/").lstrip("\\")
 .removesuffix(".yaml")
 )

 logger.info(
 "Generated HyperPod datamix recipe at '%s' (relative: '%s') from context '%s'.",
 "Generated HyperPod datamix recipe at '%s' from context '%s'.",
 recipe_path,
 relative_path,
 context.recipe_name,
 )

 return relative_path, context.image_uri
 return recipe_path, context.image_uri


0
...n/tests/integ/train/test_nova_hyperpod.py → ...sts/integ/train/test_nova_sft_hyperpod.py

View file


Open in desktop


File renamed without changes.


40 changes: 35 additions & 5 deletions
sagemaker-train/tests/integ/train/test_sft_data_mixing_hyperpod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,7 @@
import pytest
from sagemaker.train.sft_trainer import SFTTrainer
from sagemaker.train.common import TrainingType
from sagemaker.train.base_trainer import BaseTrainer
from sagemaker.train.data_mixing_config import DataMixingConfig
from sagemaker.core.training.configs import HyperPodCompute

Expand All@@ -49,8 +50,8 @@
REGION = "us-east-1"
DATA_PREFIX = "test-sft-data-mixing-hyperpod-integ"
NUM_TRAINING_SAMPLES = 300
HYPERPOD_CLUSTER_NAME = "riv-rig"
HYPERPOD_INSTANCE_TYPE = "ml.p5.48xlarge"
HYPERPOD_CLUSTER_NAME = "pysdk-hp-integ-tests"
HYPERPOD_INSTANCE_TYPE = "ml.g6.12xlarge"


def _generate_training_data() -> str:
Expand DownExpand Up@@ -125,10 +126,10 @@ def training_resources(sagemaker_session_us_east_1):
@pytest.mark.gpu_intensive
@pytest.mark.us_east_1
def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1, training_resources):
 """Test SFT trainer with Nova Lite 2 model and data mixing on HyperPod.
 """Test SFT trainer with Nova Micro model and data mixing on HyperPod.

 This end-to-end test submits a real HyperPod training job with DataMixingConfig
 for Nova Lite 2. The SDK resolves the datamix recipe from SageMaker Hub, validates
 for Nova Micro. The SDK resolves the datamix recipe from SageMaker Hub, validates
 categories, and includes the serialized config in the HyperPod override parameters.
 """
 unique_id = f"{int(time.time())}-{random.randint(1000, 9999)}"
Expand DownExpand Up@@ -184,4 +185,33 @@ def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1
 assert get_job_result.returncode == 0, (
 f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}"
 )
 logger.info(f"Verified job '{job_name}' exists on the cluster.")
 logger.info(f"Verified job '{job_name}' exists on the cluster using hp-cli.")

 # Poll for job completion by checking for the manifest in S3.
 # The manifest is written under {output_s3_path}/{job_name}/manifest.json
 # once training finishes, so its presence confirms end-to-end completion.

 output_s3_path = training_resources["s3_output_path"]
 max_wait_time = 21600 # 6 hour timeout (HyperPod jobs can take longer)
 poll_interval = 60 # Check every 60 seconds
 start_time = time.time()
 checkpoint_path = None

 while time.time() - start_time < max_wait_time:
 checkpoint_path = BaseTrainer._resolve_checkpoint_from_manifest(
 job_name=job_name,
 output_s3_path=output_s3_path,
 sagemaker_session=sagemaker_session_us_east_1,
 )
 if checkpoint_path:
 logger.info(f"Checkpoint resolved: {checkpoint_path}")
 break

 elapsed = int(time.time() - start_time)
 logger.info(f"Waiting for manifest... ({elapsed}s elapsed)")
 time.sleep(poll_interval)

 assert checkpoint_path is not None, (
 f"Job {job_name} did not produce a manifest within {max_wait_time}s"
 )
 logger.info(f"Training complete. Checkpoint: {checkpoint_path}")

22 changes: 15 additions & 7 deletions
sagemaker-train/tests/unit/train/common_utils/test_data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
"""Unit tests for data mixing utility functions."""
from __future__ import absolute_import

import os

import pytest

from sagemaker.train.data_mixing_config import DataMixingConfig
Expand DownExpand Up@@ -823,8 +825,12 @@ def capture_write(path, mode="r", **kwargs):
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-entertainment"] == 20
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-scientific"] == 10

 def test_return_value_is_relative_path_and_image_uri(self):
 """Return value should be (relative_path_without_extension, image_uri)."""
 def test_return_value_is_absolute_recipe_path_and_image_uri(self):
 """Return value should be (absolute_recipe_path_with_extension, image_uri).

 The path must be a loadable filesystem path (absolute, .yaml extension
 intact) because the recipe resolver opens it as a user recipe file.
 """
 from unittest.mock import patch, MagicMock, mock_open

 from sagemaker.train.common_utils.data_mixing_utils import (
Expand All@@ -843,12 +849,14 @@ def test_return_value_is_relative_path_and_image_uri(self):
 with patch("builtins.open", m_open):
 result = build_hyperpod_datamix_recipe_from_context(context, config)

relative_path, image_uri = result
recipe_path, image_uri = result

 # relative_path should not end with .yaml
 assert not relative_path.endswith(".yaml")
 # relative_path should contain fine-tuning/nova
 assert "fine-tuning/nova" in relative_path
 # recipe_path must be an absolute filesystem path
 assert os.path.isabs(recipe_path)
 # recipe_path must retain the .yaml extension so it can be opened
 assert recipe_path.endswith(".yaml")
 # recipe_path should live under the HyperPod CLI recipes fine-tuning/nova dir
 assert "fine-tuning/nova" in recipe_path
 # image_uri should match the context
 assert image_uri == self.IMAGE_URI

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
datamixing recipe path fix#6073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

mujtaba1747
merged 6 commits into

from

Jul 22, 2026
 Merged


datamixing recipe path fix
#6073
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
fb577ce

cleanup: use a single logger in base_trainer

Jul 15, 2026
706e73e

fix: update recipe resolver unit tests

Jul 15, 2026
7e14662

Merge branch 'aws:master' into master


mujtaba1747 Jul 15, 2026
f73e729

Release 3.16.0 (2026-07-15): Bump VERSION files and internal dependen…

Jul 15, 2026
c2ace1e

Merge branch 'master' of github.com:mujtaba1747/sagemaker-python-sdk

Jul 21, 2026
cfd34e8

fix: use absolute path in data mixing recipe path construction

Jul 21, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 8 additions & 13 deletions
sagemaker-train/src/sagemaker/train/common_utils/data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -480,16 +480,18 @@ def build_hyperpod_datamix_recipe_from_context(
 4. Inject customer_data_percent and nova_data_percentages into data_mixing.sources
 5. Validate non-zero categories exist in template's nova_data section
 6. Write final YAML to HyperPod CLI recipes directory
 7. Return (relative_recipe_path, image_uri)
 7. Return (recipe_path, image_uri)

 Args:
 context: The HyperPodTemplateContext from resolve_hyperpod_datamix_context.
 validated_config: A DataMixingConfig validated via validate_data_mixing_categories.

 Returns:
 Tuple of (relative_recipe_path, image_uri). relative_recipe_path is relative
 to the HyperPod CLI recipes_collection/recipes directory with .yaml extension
 removed. image_uri is the container image URI from the context (or None).
 Tuple of (recipe_path, image_uri). recipe_path is the absolute filesystem
 path (including the .yaml extension) of the generated recipe written under
 the HyperPod CLI recipes_collection/recipes directory; it is consumed by the
 recipe resolver as a user recipe file. image_uri is the container image URI
 from the context (or None).

 Raises:
 RuntimeError: If hyperpod_cli is not installed.
Expand DownExpand Up@@ -652,18 +654,11 @@ def _apply_overrides(recipe: dict, overrides: dict) -> dict:
 with open(recipe_path, "w") as f:
 f.write(recipe_output)

 relative_path = (
 recipe_path.split(HYPERPOD_RECIPE_PATH, 1)[1]
 .lstrip("/").lstrip("\\")
 .removesuffix(".yaml")
 )

 logger.info(
 "Generated HyperPod datamix recipe at '%s' (relative: '%s') from context '%s'.",
 "Generated HyperPod datamix recipe at '%s' from context '%s'.",
 recipe_path,
 relative_path,
 context.recipe_name,
 )

 return relative_path, context.image_uri
 return recipe_path, context.image_uri


0
...n/tests/integ/train/test_nova_hyperpod.py → ...sts/integ/train/test_nova_sft_hyperpod.py

View file


Open in desktop


File renamed without changes.


40 changes: 35 additions & 5 deletions
sagemaker-train/tests/integ/train/test_sft_data_mixing_hyperpod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,7 @@
import pytest
from sagemaker.train.sft_trainer import SFTTrainer
from sagemaker.train.common import TrainingType
from sagemaker.train.base_trainer import BaseTrainer
from sagemaker.train.data_mixing_config import DataMixingConfig
from sagemaker.core.training.configs import HyperPodCompute

Expand All@@ -49,8 +50,8 @@
REGION = "us-east-1"
DATA_PREFIX = "test-sft-data-mixing-hyperpod-integ"
NUM_TRAINING_SAMPLES = 300
HYPERPOD_CLUSTER_NAME = "riv-rig"
HYPERPOD_INSTANCE_TYPE = "ml.p5.48xlarge"
HYPERPOD_CLUSTER_NAME = "pysdk-hp-integ-tests"
HYPERPOD_INSTANCE_TYPE = "ml.g6.12xlarge"


def _generate_training_data() -> str:
Expand DownExpand Up@@ -125,10 +126,10 @@ def training_resources(sagemaker_session_us_east_1):
@pytest.mark.gpu_intensive
@pytest.mark.us_east_1
def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1, training_resources):
 """Test SFT trainer with Nova Lite 2 model and data mixing on HyperPod.
 """Test SFT trainer with Nova Micro model and data mixing on HyperPod.

 This end-to-end test submits a real HyperPod training job with DataMixingConfig
 for Nova Lite 2. The SDK resolves the datamix recipe from SageMaker Hub, validates
 for Nova Micro. The SDK resolves the datamix recipe from SageMaker Hub, validates
 categories, and includes the serialized config in the HyperPod override parameters.
 """
 unique_id = f"{int(time.time())}-{random.randint(1000, 9999)}"
Expand DownExpand Up@@ -184,4 +185,33 @@ def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1
 assert get_job_result.returncode == 0, (
 f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}"
 )
 logger.info(f"Verified job '{job_name}' exists on the cluster.")
 logger.info(f"Verified job '{job_name}' exists on the cluster using hp-cli.")

 # Poll for job completion by checking for the manifest in S3.
 # The manifest is written under {output_s3_path}/{job_name}/manifest.json
 # once training finishes, so its presence confirms end-to-end completion.

 output_s3_path = training_resources["s3_output_path"]
 max_wait_time = 21600 # 6 hour timeout (HyperPod jobs can take longer)
 poll_interval = 60 # Check every 60 seconds
 start_time = time.time()
 checkpoint_path = None

 while time.time() - start_time < max_wait_time:
 checkpoint_path = BaseTrainer._resolve_checkpoint_from_manifest(
 job_name=job_name,
 output_s3_path=output_s3_path,
 sagemaker_session=sagemaker_session_us_east_1,
 )
 if checkpoint_path:
 logger.info(f"Checkpoint resolved: {checkpoint_path}")
 break

 elapsed = int(time.time() - start_time)
 logger.info(f"Waiting for manifest... ({elapsed}s elapsed)")
 time.sleep(poll_interval)

 assert checkpoint_path is not None, (
 f"Job {job_name} did not produce a manifest within {max_wait_time}s"
 )
 logger.info(f"Training complete. Checkpoint: {checkpoint_path}")

22 changes: 15 additions & 7 deletions
sagemaker-train/tests/unit/train/common_utils/test_data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
"""Unit tests for data mixing utility functions."""
from __future__ import absolute_import

import os

import pytest

from sagemaker.train.data_mixing_config import DataMixingConfig
Expand DownExpand Up@@ -823,8 +825,12 @@ def capture_write(path, mode="r", **kwargs):
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-entertainment"] == 20
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-scientific"] == 10

 def test_return_value_is_relative_path_and_image_uri(self):
 """Return value should be (relative_path_without_extension, image_uri)."""
 def test_return_value_is_absolute_recipe_path_and_image_uri(self):
 """Return value should be (absolute_recipe_path_with_extension, image_uri).

 The path must be a loadable filesystem path (absolute, .yaml extension
 intact) because the recipe resolver opens it as a user recipe file.
 """
 from unittest.mock import patch, MagicMock, mock_open

 from sagemaker.train.common_utils.data_mixing_utils import (
Expand All@@ -843,12 +849,14 @@ def test_return_value_is_relative_path_and_image_uri(self):
 with patch("builtins.open", m_open):
 result = build_hyperpod_datamix_recipe_from_context(context, config)

relative_path, image_uri = result
recipe_path, image_uri = result

 # relative_path should not end with .yaml
 assert not relative_path.endswith(".yaml")
 # relative_path should contain fine-tuning/nova
 assert "fine-tuning/nova" in relative_path
 # recipe_path must be an absolute filesystem path
 assert os.path.isabs(recipe_path)
 # recipe_path must retain the .yaml extension so it can be opened
 assert recipe_path.endswith(".yaml")
 # recipe_path should live under the HyperPod CLI recipes fine-tuning/nova dir
 assert "fine-tuning/nova" in recipe_path
 # image_uri should match the context
 assert image_uri == self.IMAGE_URI

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
datamixing recipe path fix#6073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

mujtaba1747
merged 6 commits into

from

Jul 22, 2026
 Merged


datamixing recipe path fix
#6073
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
fb577ce

cleanup: use a single logger in base_trainer

Jul 15, 2026
706e73e

fix: update recipe resolver unit tests

Jul 15, 2026
7e14662

Merge branch 'aws:master' into master


mujtaba1747 Jul 15, 2026
f73e729

Release 3.16.0 (2026-07-15): Bump VERSION files and internal dependen…

Jul 15, 2026
c2ace1e

Merge branch 'master' of github.com:mujtaba1747/sagemaker-python-sdk

Jul 21, 2026
cfd34e8

fix: use absolute path in data mixing recipe path construction

Jul 21, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 8 additions & 13 deletions
sagemaker-train/src/sagemaker/train/common_utils/data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -480,16 +480,18 @@ def build_hyperpod_datamix_recipe_from_context(
 4. Inject customer_data_percent and nova_data_percentages into data_mixing.sources
 5. Validate non-zero categories exist in template's nova_data section
 6. Write final YAML to HyperPod CLI recipes directory
 7. Return (relative_recipe_path, image_uri)
 7. Return (recipe_path, image_uri)

 Args:
 context: The HyperPodTemplateContext from resolve_hyperpod_datamix_context.
 validated_config: A DataMixingConfig validated via validate_data_mixing_categories.

 Returns:
 Tuple of (relative_recipe_path, image_uri). relative_recipe_path is relative
 to the HyperPod CLI recipes_collection/recipes directory with .yaml extension
 removed. image_uri is the container image URI from the context (or None).
 Tuple of (recipe_path, image_uri). recipe_path is the absolute filesystem
 path (including the .yaml extension) of the generated recipe written under
 the HyperPod CLI recipes_collection/recipes directory; it is consumed by the
 recipe resolver as a user recipe file. image_uri is the container image URI
 from the context (or None).

 Raises:
 RuntimeError: If hyperpod_cli is not installed.
Expand DownExpand Up@@ -652,18 +654,11 @@ def _apply_overrides(recipe: dict, overrides: dict) -> dict:
 with open(recipe_path, "w") as f:
 f.write(recipe_output)

 relative_path = (
 recipe_path.split(HYPERPOD_RECIPE_PATH, 1)[1]
 .lstrip("/").lstrip("\\")
 .removesuffix(".yaml")
 )

 logger.info(
 "Generated HyperPod datamix recipe at '%s' (relative: '%s') from context '%s'.",
 "Generated HyperPod datamix recipe at '%s' from context '%s'.",
 recipe_path,
 relative_path,
 context.recipe_name,
 )

 return relative_path, context.image_uri
 return recipe_path, context.image_uri


0
...n/tests/integ/train/test_nova_hyperpod.py → ...sts/integ/train/test_nova_sft_hyperpod.py

View file


Open in desktop


File renamed without changes.


40 changes: 35 additions & 5 deletions
sagemaker-train/tests/integ/train/test_sft_data_mixing_hyperpod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,7 @@
import pytest
from sagemaker.train.sft_trainer import SFTTrainer
from sagemaker.train.common import TrainingType
from sagemaker.train.base_trainer import BaseTrainer
from sagemaker.train.data_mixing_config import DataMixingConfig
from sagemaker.core.training.configs import HyperPodCompute

Expand All@@ -49,8 +50,8 @@
REGION = "us-east-1"
DATA_PREFIX = "test-sft-data-mixing-hyperpod-integ"
NUM_TRAINING_SAMPLES = 300
HYPERPOD_CLUSTER_NAME = "riv-rig"
HYPERPOD_INSTANCE_TYPE = "ml.p5.48xlarge"
HYPERPOD_CLUSTER_NAME = "pysdk-hp-integ-tests"
HYPERPOD_INSTANCE_TYPE = "ml.g6.12xlarge"


def _generate_training_data() -> str:
Expand DownExpand Up@@ -125,10 +126,10 @@ def training_resources(sagemaker_session_us_east_1):
@pytest.mark.gpu_intensive
@pytest.mark.us_east_1
def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1, training_resources):
 """Test SFT trainer with Nova Lite 2 model and data mixing on HyperPod.
 """Test SFT trainer with Nova Micro model and data mixing on HyperPod.

 This end-to-end test submits a real HyperPod training job with DataMixingConfig
 for Nova Lite 2. The SDK resolves the datamix recipe from SageMaker Hub, validates
 for Nova Micro. The SDK resolves the datamix recipe from SageMaker Hub, validates
 categories, and includes the serialized config in the HyperPod override parameters.
 """
 unique_id = f"{int(time.time())}-{random.randint(1000, 9999)}"
Expand DownExpand Up@@ -184,4 +185,33 @@ def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1
 assert get_job_result.returncode == 0, (
 f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}"
 )
 logger.info(f"Verified job '{job_name}' exists on the cluster.")
 logger.info(f"Verified job '{job_name}' exists on the cluster using hp-cli.")

 # Poll for job completion by checking for the manifest in S3.
 # The manifest is written under {output_s3_path}/{job_name}/manifest.json
 # once training finishes, so its presence confirms end-to-end completion.

 output_s3_path = training_resources["s3_output_path"]
 max_wait_time = 21600 # 6 hour timeout (HyperPod jobs can take longer)
 poll_interval = 60 # Check every 60 seconds
 start_time = time.time()
 checkpoint_path = None

 while time.time() - start_time < max_wait_time:
 checkpoint_path = BaseTrainer._resolve_checkpoint_from_manifest(
 job_name=job_name,
 output_s3_path=output_s3_path,
 sagemaker_session=sagemaker_session_us_east_1,
 )
 if checkpoint_path:
 logger.info(f"Checkpoint resolved: {checkpoint_path}")
 break

 elapsed = int(time.time() - start_time)
 logger.info(f"Waiting for manifest... ({elapsed}s elapsed)")
 time.sleep(poll_interval)

 assert checkpoint_path is not None, (
 f"Job {job_name} did not produce a manifest within {max_wait_time}s"
 )
 logger.info(f"Training complete. Checkpoint: {checkpoint_path}")

22 changes: 15 additions & 7 deletions
sagemaker-train/tests/unit/train/common_utils/test_data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
"""Unit tests for data mixing utility functions."""
from __future__ import absolute_import

import os

import pytest

from sagemaker.train.data_mixing_config import DataMixingConfig
Expand DownExpand Up@@ -823,8 +825,12 @@ def capture_write(path, mode="r", **kwargs):
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-entertainment"] == 20
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-scientific"] == 10

 def test_return_value_is_relative_path_and_image_uri(self):
 """Return value should be (relative_path_without_extension, image_uri)."""
 def test_return_value_is_absolute_recipe_path_and_image_uri(self):
 """Return value should be (absolute_recipe_path_with_extension, image_uri).

 The path must be a loadable filesystem path (absolute, .yaml extension
 intact) because the recipe resolver opens it as a user recipe file.
 """
 from unittest.mock import patch, MagicMock, mock_open

 from sagemaker.train.common_utils.data_mixing_utils import (
Expand All@@ -843,12 +849,14 @@ def test_return_value_is_relative_path_and_image_uri(self):
 with patch("builtins.open", m_open):
 result = build_hyperpod_datamix_recipe_from_context(context, config)

relative_path, image_uri = result
recipe_path, image_uri = result

 # relative_path should not end with .yaml
 assert not relative_path.endswith(".yaml")
 # relative_path should contain fine-tuning/nova
 assert "fine-tuning/nova" in relative_path
 # recipe_path must be an absolute filesystem path
 assert os.path.isabs(recipe_path)
 # recipe_path must retain the .yaml extension so it can be opened
 assert recipe_path.endswith(".yaml")
 # recipe_path should live under the HyperPod CLI recipes fine-tuning/nova dir
 assert "fine-tuning/nova" in recipe_path
 # image_uri should match the context
 assert image_uri == self.IMAGE_URI

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
datamixing recipe path fix#6073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

mujtaba1747
merged 6 commits into

from

Jul 22, 2026
 Merged


datamixing recipe path fix
#6073
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
fb577ce

cleanup: use a single logger in base_trainer

Jul 15, 2026
706e73e

fix: update recipe resolver unit tests

Jul 15, 2026
7e14662

Merge branch 'aws:master' into master


mujtaba1747 Jul 15, 2026
f73e729

Release 3.16.0 (2026-07-15): Bump VERSION files and internal dependen…

Jul 15, 2026
c2ace1e

Merge branch 'master' of github.com:mujtaba1747/sagemaker-python-sdk

Jul 21, 2026
cfd34e8

fix: use absolute path in data mixing recipe path construction

Jul 21, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 8 additions & 13 deletions
sagemaker-train/src/sagemaker/train/common_utils/data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -480,16 +480,18 @@ def build_hyperpod_datamix_recipe_from_context(
 4. Inject customer_data_percent and nova_data_percentages into data_mixing.sources
 5. Validate non-zero categories exist in template's nova_data section
 6. Write final YAML to HyperPod CLI recipes directory
 7. Return (relative_recipe_path, image_uri)
 7. Return (recipe_path, image_uri)

 Args:
 context: The HyperPodTemplateContext from resolve_hyperpod_datamix_context.
 validated_config: A DataMixingConfig validated via validate_data_mixing_categories.

 Returns:
 Tuple of (relative_recipe_path, image_uri). relative_recipe_path is relative
 to the HyperPod CLI recipes_collection/recipes directory with .yaml extension
 removed. image_uri is the container image URI from the context (or None).
 Tuple of (recipe_path, image_uri). recipe_path is the absolute filesystem
 path (including the .yaml extension) of the generated recipe written under
 the HyperPod CLI recipes_collection/recipes directory; it is consumed by the
 recipe resolver as a user recipe file. image_uri is the container image URI
 from the context (or None).

 Raises:
 RuntimeError: If hyperpod_cli is not installed.
Expand DownExpand Up@@ -652,18 +654,11 @@ def _apply_overrides(recipe: dict, overrides: dict) -> dict:
 with open(recipe_path, "w") as f:
 f.write(recipe_output)

 relative_path = (
 recipe_path.split(HYPERPOD_RECIPE_PATH, 1)[1]
 .lstrip("/").lstrip("\\")
 .removesuffix(".yaml")
 )

 logger.info(
 "Generated HyperPod datamix recipe at '%s' (relative: '%s') from context '%s'.",
 "Generated HyperPod datamix recipe at '%s' from context '%s'.",
 recipe_path,
 relative_path,
 context.recipe_name,
 )

 return relative_path, context.image_uri
 return recipe_path, context.image_uri


0
...n/tests/integ/train/test_nova_hyperpod.py → ...sts/integ/train/test_nova_sft_hyperpod.py

View file


Open in desktop


File renamed without changes.


40 changes: 35 additions & 5 deletions
sagemaker-train/tests/integ/train/test_sft_data_mixing_hyperpod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,7 @@
import pytest
from sagemaker.train.sft_trainer import SFTTrainer
from sagemaker.train.common import TrainingType
from sagemaker.train.base_trainer import BaseTrainer
from sagemaker.train.data_mixing_config import DataMixingConfig
from sagemaker.core.training.configs import HyperPodCompute

Expand All@@ -49,8 +50,8 @@
REGION = "us-east-1"
DATA_PREFIX = "test-sft-data-mixing-hyperpod-integ"
NUM_TRAINING_SAMPLES = 300
HYPERPOD_CLUSTER_NAME = "riv-rig"
HYPERPOD_INSTANCE_TYPE = "ml.p5.48xlarge"
HYPERPOD_CLUSTER_NAME = "pysdk-hp-integ-tests"
HYPERPOD_INSTANCE_TYPE = "ml.g6.12xlarge"


def _generate_training_data() -> str:
Expand DownExpand Up@@ -125,10 +126,10 @@ def training_resources(sagemaker_session_us_east_1):
@pytest.mark.gpu_intensive
@pytest.mark.us_east_1
def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1, training_resources):
 """Test SFT trainer with Nova Lite 2 model and data mixing on HyperPod.
 """Test SFT trainer with Nova Micro model and data mixing on HyperPod.

 This end-to-end test submits a real HyperPod training job with DataMixingConfig
 for Nova Lite 2. The SDK resolves the datamix recipe from SageMaker Hub, validates
 for Nova Micro. The SDK resolves the datamix recipe from SageMaker Hub, validates
 categories, and includes the serialized config in the HyperPod override parameters.
 """
 unique_id = f"{int(time.time())}-{random.randint(1000, 9999)}"
Expand DownExpand Up@@ -184,4 +185,33 @@ def test_sft_trainer_nova_micro_data_mixing_hyperpod(sagemaker_session_us_east_1
 assert get_job_result.returncode == 0, (
 f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}"
 )
 logger.info(f"Verified job '{job_name}' exists on the cluster.")
 logger.info(f"Verified job '{job_name}' exists on the cluster using hp-cli.")

 # Poll for job completion by checking for the manifest in S3.
 # The manifest is written under {output_s3_path}/{job_name}/manifest.json
 # once training finishes, so its presence confirms end-to-end completion.

 output_s3_path = training_resources["s3_output_path"]
 max_wait_time = 21600 # 6 hour timeout (HyperPod jobs can take longer)
 poll_interval = 60 # Check every 60 seconds
 start_time = time.time()
 checkpoint_path = None

 while time.time() - start_time < max_wait_time:
 checkpoint_path = BaseTrainer._resolve_checkpoint_from_manifest(
 job_name=job_name,
 output_s3_path=output_s3_path,
 sagemaker_session=sagemaker_session_us_east_1,
 )
 if checkpoint_path:
 logger.info(f"Checkpoint resolved: {checkpoint_path}")
 break

 elapsed = int(time.time() - start_time)
 logger.info(f"Waiting for manifest... ({elapsed}s elapsed)")
 time.sleep(poll_interval)

 assert checkpoint_path is not None, (
 f"Job {job_name} did not produce a manifest within {max_wait_time}s"
 )
 logger.info(f"Training complete. Checkpoint: {checkpoint_path}")

22 changes: 15 additions & 7 deletions
sagemaker-train/tests/unit/train/common_utils/test_data_mixing_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
"""Unit tests for data mixing utility functions."""
from __future__ import absolute_import

import os

import pytest

from sagemaker.train.data_mixing_config import DataMixingConfig
Expand DownExpand Up@@ -823,8 +825,12 @@ def capture_write(path, mode="r", **kwargs):
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-entertainment"] == 20
 assert parsed["data_mixing"]["sources"]["nova_data"]["en-scientific"] == 10

 def test_return_value_is_relative_path_and_image_uri(self):
 """Return value should be (relative_path_without_extension, image_uri)."""
 def test_return_value_is_absolute_recipe_path_and_image_uri(self):
 """Return value should be (absolute_recipe_path_with_extension, image_uri).

 The path must be a loadable filesystem path (absolute, .yaml extension
 intact) because the recipe resolver opens it as a user recipe file.
 """
 from unittest.mock import patch, MagicMock, mock_open

 from sagemaker.train.common_utils.data_mixing_utils import (
Expand All@@ -843,12 +849,14 @@ def test_return_value_is_relative_path_and_image_uri(self):
 with patch("builtins.open", m_open):
 result = build_hyperpod_datamix_recipe_from_context(context, config)

relative_path, image_uri = result
recipe_path, image_uri = result

 # relative_path should not end with .yaml
 assert not relative_path.endswith(".yaml")
 # relative_path should contain fine-tuning/nova
 assert "fine-tuning/nova" in relative_path
 # recipe_path must be an absolute filesystem path
 assert os.path.isabs(recipe_path)
 # recipe_path must retain the .yaml extension so it can be opened
 assert recipe_path.endswith(".yaml")
 # recipe_path should live under the HyperPod CLI recipes fine-tuning/nova dir
 assert "fine-tuning/nova" in recipe_path
 # image_uri should match the context
 assert image_uri == self.IMAGE_URI

Expand Down
Loading