fix: prevent race condition in vpc tests#863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jesterhazy merged 2 commits into from
Jun 19, 2019
Merged

fix: prevent race condition in vpc tests #863

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
429b8d2
fix: prevent race condition in vpc tests
jesterhazy Jun 19, 2019
0bdad3a
remove extra files
jesterhazy Jun 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,4 +25,4 @@ venv/
*~
.pytest_cache/
*.swp
tests/data/local_mode_lock
.docker/
21 changes: 11 additions & 10 deletions tests/integ/local_mode_utils.py → tests/integ/lock.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,24 +15,25 @@
import fcntl
import os
import time
import tempfile
from contextlib import contextmanager

import tests.integ

LOCK_PATH = os.path.join(tests.integ.DATA_DIR, 'local_mode_lock')
DEFAULT_LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_lock')


@contextmanager
def lock():
# Since Local Mode uses the same port for serving, we need a lock in order
# to allow concurrent test execution.

@laurenyulaurenyuJun 19, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be good to have an explanation of why these locks are needed somewhere

@jesterhazyjesterhazyJun 19, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably at (multiple) points of use. or would a generic "lock for tests that need locks" comment work?

@laurenyulaurenyuJun 19, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a generic comment with specific callouts for the most common reasons (e.g. local mode)?

local_mode_lock_fd = open(LOCK_PATH, 'w')
local_mode_lock = local_mode_lock_fd.fileno()
def lock(path=DEFAULT_LOCK_PATH):
"""Create a file lock to control concurrent test execution. Certain tests or
test operations need to limit concurrency to work reliably. Examples include
local mode endpoint tests and vpc creation tests.
"""
f = open(path, 'w')
fd = f.fileno()

fcntl.lockf(local_mode_lock, fcntl.LOCK_EX)
fcntl.lockf(fd, fcntl.LOCK_EX)

try:
yield
finally:
time.sleep(5)
fcntl.lockf(local_mode_lock, fcntl.LOCK_UN)
fcntl.lockf(fd, fcntl.LOCK_UN)
24 changes: 14 additions & 10 deletions tests/integ/test_local_mode.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,14 +18,18 @@
import boto3
import numpy
import pytest
import tests.integ.local_mode_utils as local_mode_utils
import tempfile

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ.timeout import timeout

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
from sagemaker.tensorflow import TensorFlow

# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_local_mode_lock')
DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data')
DEFAULT_REGION = 'us-west-2'

Expand DownExpand Up@@ -101,7 +105,7 @@ def test_tf_local_mode(tf_full_version, sagemaker_local_session):
print('job succeeded: {}'.format(estimator.latest_training_job.name))

endpoint_name = estimator.latest_training_job.name
with local_mode_utils.lock():
with lock.lock(LOCK_PATH):
try:
json_predictor = estimator.deploy(initial_instance_count=1,
instance_type='local',
Expand DownExpand Up@@ -140,7 +144,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):

endpoint_name = estimator.latest_training_job.name

with local_mode_utils.lock():
with lock.lock(LOCK_PATH):
try:
json_predictor = estimator.deploy(initial_instance_count=1,
instance_type='local',
Expand DownExpand Up@@ -178,7 +182,7 @@ def test_tf_local_data(sagemaker_local_session):
print('job succeeded: {}'.format(estimator.latest_training_job.name))

endpoint_name = estimator.latest_training_job.name
with local_mode_utils.lock():
with lock.lock(LOCK_PATH):
try:
json_predictor = estimator.deploy(initial_instance_count=1,
instance_type='local',
Expand DownExpand Up@@ -217,7 +221,7 @@ def test_tf_local_data_local_script():
print('job succeeded: {}'.format(estimator.latest_training_job.name))

endpoint_name = estimator.latest_training_job.name
with local_mode_utils.lock():
with lock.lock(LOCK_PATH):
try:
json_predictor = estimator.deploy(initial_instance_count=1,
instance_type='local',
Expand All@@ -241,7 +245,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
s3_model.sagemaker_session = sagemaker_local_session

predictor = None
with local_mode_utils.lock():
with lock.lock(LOCK_PATH):
try:
predictor = s3_model.deploy(initial_instance_count=1, instance_type='local')
data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -255,7 +259,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mxnet_model):
predictor = None

with local_mode_utils.lock():
with lock.lock(LOCK_PATH):
try:
path = 'file://%s' % (str(tmpdir))
model = mxnet_model(path)
Expand DownExpand Up@@ -285,7 +289,7 @@ def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):
mx.fit({'train': train_input, 'test': test_input})
endpoint_name = mx.latest_training_job.name

with local_mode_utils.lock():
with lock.lock(LOCK_PATH):
try:
predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -310,7 +314,7 @@ def test_mxnet_local_data_local_script(mxnet_full_version):
mx.fit({'train': train_input, 'test': test_input})
endpoint_name = mx.latest_training_job.name

with local_mode_utils.lock():
with lock.lock(LOCK_PATH):
try:
predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
Expand DownExpand Up@@ -365,7 +369,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
transformer = mx.transformer(1, 'local', assemble_with='Line', max_payload=1,
strategy='SingleRecord', output_path=output_path)

with local_mode_utils.lock():
with lock.lock(LOCK_PATH):
transformer.transform(transform_input, content_type='text/csv', split_type='Line')
transformer.wait()

Expand Down
5 changes: 3 additions & 2 deletions tests/integ/test_source_dirs.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@

import pytest

import tests.integ.local_mode_utils as local_mode_utils
import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION

from sagemaker.pytorch.estimator import PyTorch
Expand All@@ -37,7 +37,8 @@ def test_source_dirs(tmpdir, sagemaker_local_session):
sagemaker_session=sagemaker_local_session)
estimator.fit()

with local_mode_utils.lock():
# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
with lock.lock():
try:
predictor = estimator.deploy(initial_instance_count=1, instance_type='local')
predict_response = predictor.predict([7])
Expand Down
39 changes: 27 additions & 12 deletions tests/integ/vpc_test_utils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,13 @@
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import os
import tempfile

import tests.integ.lock as lock

VPC_NAME = 'sagemaker-python-sdk-test-vpc'
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_vpc_lock')


def _get_subnet_ids_by_name(ec2_client, name):
Expand DownExpand Up@@ -61,20 +67,24 @@ def _create_vpc_with_name(ec2_client, region, name):
AvailabilityZone=(region + 'b'))['Subnet']['SubnetId']
print('created subnet: {}'.format(subnet_id_b))

s3_service = [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if s.endswith('s3')][0]
s3_service = \
[s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if
s.endswith('s3')][0]
ec2_client.create_vpc_endpoint(VpcId=vpc_id, ServiceName=s3_service,
RouteTableIds=[_get_route_table_id(ec2_client, vpc_id)])
print('created s3 vpc endpoint')

security_group_id = ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
security_group_id = \
ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
print('created security group: {}'.format(security_group_id))

# multi-host vpc jobs require communication among hosts
ec2_client.authorize_security_group_ingress(GroupId=security_group_id,
IpPermissions=[{'IpProtocol': 'tcp',
'FromPort': 0,
'ToPort': 65535,
'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
'UserIdGroupPairs': [{
'GroupId': security_group_id}]}])

ec2_client.create_tags(Resources=[vpc_id, subnet_id_a, subnet_id_b, security_group_id],
Tags=[{'Key': 'Name', 'Value': name}])
Expand All@@ -83,23 +93,28 @@ def _create_vpc_with_name(ec2_client, region, name):


def get_or_create_vpc_resources(ec2_client, region, name=VPC_NAME):
if _vpc_exists(ec2_client, name):
print('using existing vpc: {}'.format(name))
return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client, name)
else:
print('creating new vpc: {}'.format(name))
return _create_vpc_with_name(ec2_client, region, name)
# use lock to prevent race condition when tests are running concurrently
with lock.lock(LOCK_PATH):
if _vpc_exists(ec2_client, name):
print('using existing vpc: {}'.format(name))
return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client,
name)
else:
print('creating new vpc: {}'.format(name))
return _create_vpc_with_name(ec2_client, region, name)


def setup_security_group_for_encryption(ec2_client, security_group_id):
sg_desc = ec2_client.describe_security_groups(GroupIds=[security_group_id])
ingress_perms = sg_desc['SecurityGroups'][0]['IpPermissions']
if len(ingress_perms) == 1:
ec2_client.\
ec2_client.\
authorize_security_group_ingress(GroupId=security_group_id,
IpPermissions=[{'IpProtocol': '50',
'UserIdGroupPairs': [{'GroupId': security_group_id}]},
'UserIdGroupPairs': [
{'GroupId': security_group_id}]},
{'IpProtocol': 'udp',
'FromPort': 500,
'ToPort': 500,
'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
'UserIdGroupPairs': [
{'GroupId': security_group_id}]}])
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
fix: prevent race condition in vpc tests#863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jesterhazy
merged 2 commits into

from

Jun 19, 2019
 Merged


fix: prevent race condition in vpc tests
#863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
429b8d2

fix: prevent race condition in vpc tests


jesterhazy Jun 19, 2019
0bdad3a

remove extra files


jesterhazy Jun 19, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.gitignore

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,4 +25,4 @@ venv/
*~
.pytest_cache/
*.swp
tests/data/local_mode_lock
.docker/

21 changes: 11 additions & 10 deletions
tests/integ/local_mode_utils.py → tests/integ/lock.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,24 +15,25 @@
import fcntl
import os
import time
import tempfile
from contextlib import contextmanager

import tests.integ

LOCK_PATH = os.path.join(tests.integ.DATA_DIR, 'local_mode_lock')
DEFAULT_LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_lock')


@contextmanager
def lock():
 # Since Local Mode uses the same port for serving, we need a lock in order
 # to allow concurrent test execution.
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be good to have an explanation of why these locks are needed somewhere
@jesterhazyjesterhazyJun 19, 2019

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably at (multiple) points of use. or would a generic "lock for tests that need locks" comment work?
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a generic comment with specific callouts for the most common reasons (e.g. local mode)?
 local_mode_lock_fd = open(LOCK_PATH, 'w')
 local_mode_lock = local_mode_lock_fd.fileno()
def lock(path=DEFAULT_LOCK_PATH):
 """Create a file lock to control concurrent test execution. Certain tests or
 test operations need to limit concurrency to work reliably. Examples include
 local mode endpoint tests and vpc creation tests.
 """
 f = open(path, 'w')
 fd = f.fileno()

 fcntl.lockf(local_mode_lock, fcntl.LOCK_EX)
 fcntl.lockf(fd, fcntl.LOCK_EX)

 try:
 yield
 finally:
 time.sleep(5)
 fcntl.lockf(local_mode_lock, fcntl.LOCK_UN)
 fcntl.lockf(fd, fcntl.LOCK_UN)

24 changes: 14 additions & 10 deletions
tests/integ/test_local_mode.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,14 +18,18 @@
import boto3
import numpy
import pytest
import tests.integ.local_mode_utils as local_mode_utils
import tempfile

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ.timeout import timeout

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
from sagemaker.tensorflow import TensorFlow

# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_local_mode_lock')
DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data')
DEFAULT_REGION = 'us-west-2'

Expand DownExpand Up@@ -101,7 +105,7 @@ def test_tf_local_mode(tf_full_version, sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -140,7 +144,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):

 endpoint_name = estimator.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -178,7 +182,7 @@ def test_tf_local_data(sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -217,7 +221,7 @@ def test_tf_local_data_local_script():
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand All@@ -241,7 +245,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
 s3_model.sagemaker_session = sagemaker_local_session

 predictor = None
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = s3_model.deploy(initial_instance_count=1, instance_type='local')
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -255,7 +259,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mxnet_model):
 predictor = None

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 path = 'file://%s' % (str(tmpdir))
 model = mxnet_model(path)
Expand DownExpand Up@@ -285,7 +289,7 @@ def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -310,7 +314,7 @@ def test_mxnet_local_data_local_script(mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand DownExpand Up@@ -365,7 +369,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
 transformer = mx.transformer(1, 'local', assemble_with='Line', max_payload=1,
 strategy='SingleRecord', output_path=output_path)

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 transformer.transform(transform_input, content_type='text/csv', split_type='Line')
 transformer.wait()

Expand Down

5 changes: 3 additions & 2 deletions
tests/integ/test_source_dirs.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@

import pytest

import tests.integ.local_mode_utils as local_mode_utils
import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION

from sagemaker.pytorch.estimator import PyTorch
Expand All@@ -37,7 +37,8 @@ def test_source_dirs(tmpdir, sagemaker_local_session):
 sagemaker_session=sagemaker_local_session)
 estimator.fit()

 with local_mode_utils.lock():
 # endpoint tests all use the same port, so we use this lock to prevent concurrent execution
 with lock.lock():
 try:
 predictor = estimator.deploy(initial_instance_count=1, instance_type='local')
 predict_response = predictor.predict([7])
Expand Down

39 changes: 27 additions & 12 deletions
tests/integ/vpc_test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,13 @@
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import os
import tempfile

import tests.integ.lock as lock

VPC_NAME = 'sagemaker-python-sdk-test-vpc'
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_vpc_lock')


def _get_subnet_ids_by_name(ec2_client, name):
Expand DownExpand Up@@ -61,20 +67,24 @@ def _create_vpc_with_name(ec2_client, region, name):
 AvailabilityZone=(region + 'b'))['Subnet']['SubnetId']
 print('created subnet: {}'.format(subnet_id_b))

 s3_service = [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if s.endswith('s3')][0]
 s3_service = \
 [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if
 s.endswith('s3')][0]
 ec2_client.create_vpc_endpoint(VpcId=vpc_id, ServiceName=s3_service,
 RouteTableIds=[_get_route_table_id(ec2_client, vpc_id)])
 print('created s3 vpc endpoint')

 security_group_id = ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 security_group_id = \
 ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 print('created security group: {}'.format(security_group_id))

 # multi-host vpc jobs require communication among hosts
 ec2_client.authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': 'tcp',
 'FromPort': 0,
 'ToPort': 65535,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [{
 'GroupId': security_group_id}]}])

 ec2_client.create_tags(Resources=[vpc_id, subnet_id_a, subnet_id_b, security_group_id],
 Tags=[{'Key': 'Name', 'Value': name}])
Expand All@@ -83,23 +93,28 @@ def _create_vpc_with_name(ec2_client, region, name):


def get_or_create_vpc_resources(ec2_client, region, name=VPC_NAME):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client, name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)
 # use lock to prevent race condition when tests are running concurrently
 with lock.lock(LOCK_PATH):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client,
 name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)


def setup_security_group_for_encryption(ec2_client, security_group_id):
 sg_desc = ec2_client.describe_security_groups(GroupIds=[security_group_id])
 ingress_perms = sg_desc['SecurityGroups'][0]['IpPermissions']
 if len(ingress_perms) == 1:
 ec2_client.\
 ec2_client.\
 authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': '50',
 'UserIdGroupPairs': [{'GroupId': security_group_id}]},
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]},
 {'IpProtocol': 'udp',
 'FromPort': 500,
 'ToPort': 500,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]}])

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: prevent race condition in vpc tests#863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jesterhazy
merged 2 commits into

from

Jun 19, 2019
 Merged


fix: prevent race condition in vpc tests
#863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
429b8d2

fix: prevent race condition in vpc tests


jesterhazy Jun 19, 2019
0bdad3a

remove extra files


jesterhazy Jun 19, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.gitignore

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,4 +25,4 @@ venv/
*~
.pytest_cache/
*.swp
tests/data/local_mode_lock
.docker/

21 changes: 11 additions & 10 deletions
tests/integ/local_mode_utils.py → tests/integ/lock.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,24 +15,25 @@
import fcntl
import os
import time
import tempfile
from contextlib import contextmanager

import tests.integ

LOCK_PATH = os.path.join(tests.integ.DATA_DIR, 'local_mode_lock')
DEFAULT_LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_lock')


@contextmanager
def lock():
 # Since Local Mode uses the same port for serving, we need a lock in order
 # to allow concurrent test execution.
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be good to have an explanation of why these locks are needed somewhere
@jesterhazyjesterhazyJun 19, 2019

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably at (multiple) points of use. or would a generic "lock for tests that need locks" comment work?
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a generic comment with specific callouts for the most common reasons (e.g. local mode)?
 local_mode_lock_fd = open(LOCK_PATH, 'w')
 local_mode_lock = local_mode_lock_fd.fileno()
def lock(path=DEFAULT_LOCK_PATH):
 """Create a file lock to control concurrent test execution. Certain tests or
 test operations need to limit concurrency to work reliably. Examples include
 local mode endpoint tests and vpc creation tests.
 """
 f = open(path, 'w')
 fd = f.fileno()

 fcntl.lockf(local_mode_lock, fcntl.LOCK_EX)
 fcntl.lockf(fd, fcntl.LOCK_EX)

 try:
 yield
 finally:
 time.sleep(5)
 fcntl.lockf(local_mode_lock, fcntl.LOCK_UN)
 fcntl.lockf(fd, fcntl.LOCK_UN)

24 changes: 14 additions & 10 deletions
tests/integ/test_local_mode.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,14 +18,18 @@
import boto3
import numpy
import pytest
import tests.integ.local_mode_utils as local_mode_utils
import tempfile

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ.timeout import timeout

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
from sagemaker.tensorflow import TensorFlow

# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_local_mode_lock')
DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data')
DEFAULT_REGION = 'us-west-2'

Expand DownExpand Up@@ -101,7 +105,7 @@ def test_tf_local_mode(tf_full_version, sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -140,7 +144,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):

 endpoint_name = estimator.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -178,7 +182,7 @@ def test_tf_local_data(sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -217,7 +221,7 @@ def test_tf_local_data_local_script():
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand All@@ -241,7 +245,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
 s3_model.sagemaker_session = sagemaker_local_session

 predictor = None
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = s3_model.deploy(initial_instance_count=1, instance_type='local')
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -255,7 +259,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mxnet_model):
 predictor = None

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 path = 'file://%s' % (str(tmpdir))
 model = mxnet_model(path)
Expand DownExpand Up@@ -285,7 +289,7 @@ def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -310,7 +314,7 @@ def test_mxnet_local_data_local_script(mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand DownExpand Up@@ -365,7 +369,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
 transformer = mx.transformer(1, 'local', assemble_with='Line', max_payload=1,
 strategy='SingleRecord', output_path=output_path)

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 transformer.transform(transform_input, content_type='text/csv', split_type='Line')
 transformer.wait()

Expand Down

5 changes: 3 additions & 2 deletions
tests/integ/test_source_dirs.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@

import pytest

import tests.integ.local_mode_utils as local_mode_utils
import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION

from sagemaker.pytorch.estimator import PyTorch
Expand All@@ -37,7 +37,8 @@ def test_source_dirs(tmpdir, sagemaker_local_session):
 sagemaker_session=sagemaker_local_session)
 estimator.fit()

 with local_mode_utils.lock():
 # endpoint tests all use the same port, so we use this lock to prevent concurrent execution
 with lock.lock():
 try:
 predictor = estimator.deploy(initial_instance_count=1, instance_type='local')
 predict_response = predictor.predict([7])
Expand Down

39 changes: 27 additions & 12 deletions
tests/integ/vpc_test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,13 @@
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import os
import tempfile

import tests.integ.lock as lock

VPC_NAME = 'sagemaker-python-sdk-test-vpc'
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_vpc_lock')


def _get_subnet_ids_by_name(ec2_client, name):
Expand DownExpand Up@@ -61,20 +67,24 @@ def _create_vpc_with_name(ec2_client, region, name):
 AvailabilityZone=(region + 'b'))['Subnet']['SubnetId']
 print('created subnet: {}'.format(subnet_id_b))

 s3_service = [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if s.endswith('s3')][0]
 s3_service = \
 [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if
 s.endswith('s3')][0]
 ec2_client.create_vpc_endpoint(VpcId=vpc_id, ServiceName=s3_service,
 RouteTableIds=[_get_route_table_id(ec2_client, vpc_id)])
 print('created s3 vpc endpoint')

 security_group_id = ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 security_group_id = \
 ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 print('created security group: {}'.format(security_group_id))

 # multi-host vpc jobs require communication among hosts
 ec2_client.authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': 'tcp',
 'FromPort': 0,
 'ToPort': 65535,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [{
 'GroupId': security_group_id}]}])

 ec2_client.create_tags(Resources=[vpc_id, subnet_id_a, subnet_id_b, security_group_id],
 Tags=[{'Key': 'Name', 'Value': name}])
Expand All@@ -83,23 +93,28 @@ def _create_vpc_with_name(ec2_client, region, name):


def get_or_create_vpc_resources(ec2_client, region, name=VPC_NAME):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client, name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)
 # use lock to prevent race condition when tests are running concurrently
 with lock.lock(LOCK_PATH):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client,
 name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)


def setup_security_group_for_encryption(ec2_client, security_group_id):
 sg_desc = ec2_client.describe_security_groups(GroupIds=[security_group_id])
 ingress_perms = sg_desc['SecurityGroups'][0]['IpPermissions']
 if len(ingress_perms) == 1:
 ec2_client.\
 ec2_client.\
 authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': '50',
 'UserIdGroupPairs': [{'GroupId': security_group_id}]},
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]},
 {'IpProtocol': 'udp',
 'FromPort': 500,
 'ToPort': 500,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]}])

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: prevent race condition in vpc tests#863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jesterhazy
merged 2 commits into

from

Jun 19, 2019
 Merged


fix: prevent race condition in vpc tests
#863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
429b8d2

fix: prevent race condition in vpc tests


jesterhazy Jun 19, 2019
0bdad3a

remove extra files


jesterhazy Jun 19, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.gitignore

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,4 +25,4 @@ venv/
*~
.pytest_cache/
*.swp
tests/data/local_mode_lock
.docker/

21 changes: 11 additions & 10 deletions
tests/integ/local_mode_utils.py → tests/integ/lock.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,24 +15,25 @@
import fcntl
import os
import time
import tempfile
from contextlib import contextmanager

import tests.integ

LOCK_PATH = os.path.join(tests.integ.DATA_DIR, 'local_mode_lock')
DEFAULT_LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_lock')


@contextmanager
def lock():
 # Since Local Mode uses the same port for serving, we need a lock in order
 # to allow concurrent test execution.
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be good to have an explanation of why these locks are needed somewhere
@jesterhazyjesterhazyJun 19, 2019

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably at (multiple) points of use. or would a generic "lock for tests that need locks" comment work?
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a generic comment with specific callouts for the most common reasons (e.g. local mode)?
 local_mode_lock_fd = open(LOCK_PATH, 'w')
 local_mode_lock = local_mode_lock_fd.fileno()
def lock(path=DEFAULT_LOCK_PATH):
 """Create a file lock to control concurrent test execution. Certain tests or
 test operations need to limit concurrency to work reliably. Examples include
 local mode endpoint tests and vpc creation tests.
 """
 f = open(path, 'w')
 fd = f.fileno()

 fcntl.lockf(local_mode_lock, fcntl.LOCK_EX)
 fcntl.lockf(fd, fcntl.LOCK_EX)

 try:
 yield
 finally:
 time.sleep(5)
 fcntl.lockf(local_mode_lock, fcntl.LOCK_UN)
 fcntl.lockf(fd, fcntl.LOCK_UN)

24 changes: 14 additions & 10 deletions
tests/integ/test_local_mode.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,14 +18,18 @@
import boto3
import numpy
import pytest
import tests.integ.local_mode_utils as local_mode_utils
import tempfile

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ.timeout import timeout

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
from sagemaker.tensorflow import TensorFlow

# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_local_mode_lock')
DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data')
DEFAULT_REGION = 'us-west-2'

Expand DownExpand Up@@ -101,7 +105,7 @@ def test_tf_local_mode(tf_full_version, sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -140,7 +144,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):

 endpoint_name = estimator.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -178,7 +182,7 @@ def test_tf_local_data(sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -217,7 +221,7 @@ def test_tf_local_data_local_script():
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand All@@ -241,7 +245,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
 s3_model.sagemaker_session = sagemaker_local_session

 predictor = None
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = s3_model.deploy(initial_instance_count=1, instance_type='local')
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -255,7 +259,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mxnet_model):
 predictor = None

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 path = 'file://%s' % (str(tmpdir))
 model = mxnet_model(path)
Expand DownExpand Up@@ -285,7 +289,7 @@ def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -310,7 +314,7 @@ def test_mxnet_local_data_local_script(mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand DownExpand Up@@ -365,7 +369,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
 transformer = mx.transformer(1, 'local', assemble_with='Line', max_payload=1,
 strategy='SingleRecord', output_path=output_path)

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 transformer.transform(transform_input, content_type='text/csv', split_type='Line')
 transformer.wait()

Expand Down

5 changes: 3 additions & 2 deletions
tests/integ/test_source_dirs.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@

import pytest

import tests.integ.local_mode_utils as local_mode_utils
import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION

from sagemaker.pytorch.estimator import PyTorch
Expand All@@ -37,7 +37,8 @@ def test_source_dirs(tmpdir, sagemaker_local_session):
 sagemaker_session=sagemaker_local_session)
 estimator.fit()

 with local_mode_utils.lock():
 # endpoint tests all use the same port, so we use this lock to prevent concurrent execution
 with lock.lock():
 try:
 predictor = estimator.deploy(initial_instance_count=1, instance_type='local')
 predict_response = predictor.predict([7])
Expand Down

39 changes: 27 additions & 12 deletions
tests/integ/vpc_test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,13 @@
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import os
import tempfile

import tests.integ.lock as lock

VPC_NAME = 'sagemaker-python-sdk-test-vpc'
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_vpc_lock')


def _get_subnet_ids_by_name(ec2_client, name):
Expand DownExpand Up@@ -61,20 +67,24 @@ def _create_vpc_with_name(ec2_client, region, name):
 AvailabilityZone=(region + 'b'))['Subnet']['SubnetId']
 print('created subnet: {}'.format(subnet_id_b))

 s3_service = [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if s.endswith('s3')][0]
 s3_service = \
 [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if
 s.endswith('s3')][0]
 ec2_client.create_vpc_endpoint(VpcId=vpc_id, ServiceName=s3_service,
 RouteTableIds=[_get_route_table_id(ec2_client, vpc_id)])
 print('created s3 vpc endpoint')

 security_group_id = ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 security_group_id = \
 ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 print('created security group: {}'.format(security_group_id))

 # multi-host vpc jobs require communication among hosts
 ec2_client.authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': 'tcp',
 'FromPort': 0,
 'ToPort': 65535,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [{
 'GroupId': security_group_id}]}])

 ec2_client.create_tags(Resources=[vpc_id, subnet_id_a, subnet_id_b, security_group_id],
 Tags=[{'Key': 'Name', 'Value': name}])
Expand All@@ -83,23 +93,28 @@ def _create_vpc_with_name(ec2_client, region, name):


def get_or_create_vpc_resources(ec2_client, region, name=VPC_NAME):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client, name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)
 # use lock to prevent race condition when tests are running concurrently
 with lock.lock(LOCK_PATH):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client,
 name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)


def setup_security_group_for_encryption(ec2_client, security_group_id):
 sg_desc = ec2_client.describe_security_groups(GroupIds=[security_group_id])
 ingress_perms = sg_desc['SecurityGroups'][0]['IpPermissions']
 if len(ingress_perms) == 1:
 ec2_client.\
 ec2_client.\
 authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': '50',
 'UserIdGroupPairs': [{'GroupId': security_group_id}]},
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]},
 {'IpProtocol': 'udp',
 'FromPort': 500,
 'ToPort': 500,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]}])

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
fix: prevent race condition in vpc tests#863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jesterhazy
merged 2 commits into

from

Jun 19, 2019
 Merged


fix: prevent race condition in vpc tests
#863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
429b8d2

fix: prevent race condition in vpc tests


jesterhazy Jun 19, 2019
0bdad3a

remove extra files


jesterhazy Jun 19, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.gitignore

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,4 +25,4 @@ venv/
*~
.pytest_cache/
*.swp
tests/data/local_mode_lock
.docker/

21 changes: 11 additions & 10 deletions
tests/integ/local_mode_utils.py → tests/integ/lock.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,24 +15,25 @@
import fcntl
import os
import time
import tempfile
from contextlib import contextmanager

import tests.integ

LOCK_PATH = os.path.join(tests.integ.DATA_DIR, 'local_mode_lock')
DEFAULT_LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_lock')


@contextmanager
def lock():
 # Since Local Mode uses the same port for serving, we need a lock in order
 # to allow concurrent test execution.
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be good to have an explanation of why these locks are needed somewhere
@jesterhazyjesterhazyJun 19, 2019

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably at (multiple) points of use. or would a generic "lock for tests that need locks" comment work?
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a generic comment with specific callouts for the most common reasons (e.g. local mode)?
 local_mode_lock_fd = open(LOCK_PATH, 'w')
 local_mode_lock = local_mode_lock_fd.fileno()
def lock(path=DEFAULT_LOCK_PATH):
 """Create a file lock to control concurrent test execution. Certain tests or
 test operations need to limit concurrency to work reliably. Examples include
 local mode endpoint tests and vpc creation tests.
 """
 f = open(path, 'w')
 fd = f.fileno()

 fcntl.lockf(local_mode_lock, fcntl.LOCK_EX)
 fcntl.lockf(fd, fcntl.LOCK_EX)

 try:
 yield
 finally:
 time.sleep(5)
 fcntl.lockf(local_mode_lock, fcntl.LOCK_UN)
 fcntl.lockf(fd, fcntl.LOCK_UN)

24 changes: 14 additions & 10 deletions
tests/integ/test_local_mode.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,14 +18,18 @@
import boto3
import numpy
import pytest
import tests.integ.local_mode_utils as local_mode_utils
import tempfile

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ.timeout import timeout

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
from sagemaker.tensorflow import TensorFlow

# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_local_mode_lock')
DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data')
DEFAULT_REGION = 'us-west-2'

Expand DownExpand Up@@ -101,7 +105,7 @@ def test_tf_local_mode(tf_full_version, sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -140,7 +144,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):

 endpoint_name = estimator.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -178,7 +182,7 @@ def test_tf_local_data(sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -217,7 +221,7 @@ def test_tf_local_data_local_script():
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand All@@ -241,7 +245,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
 s3_model.sagemaker_session = sagemaker_local_session

 predictor = None
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = s3_model.deploy(initial_instance_count=1, instance_type='local')
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -255,7 +259,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mxnet_model):
 predictor = None

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 path = 'file://%s' % (str(tmpdir))
 model = mxnet_model(path)
Expand DownExpand Up@@ -285,7 +289,7 @@ def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -310,7 +314,7 @@ def test_mxnet_local_data_local_script(mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand DownExpand Up@@ -365,7 +369,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
 transformer = mx.transformer(1, 'local', assemble_with='Line', max_payload=1,
 strategy='SingleRecord', output_path=output_path)

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 transformer.transform(transform_input, content_type='text/csv', split_type='Line')
 transformer.wait()

Expand Down

5 changes: 3 additions & 2 deletions
tests/integ/test_source_dirs.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@

import pytest

import tests.integ.local_mode_utils as local_mode_utils
import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION

from sagemaker.pytorch.estimator import PyTorch
Expand All@@ -37,7 +37,8 @@ def test_source_dirs(tmpdir, sagemaker_local_session):
 sagemaker_session=sagemaker_local_session)
 estimator.fit()

 with local_mode_utils.lock():
 # endpoint tests all use the same port, so we use this lock to prevent concurrent execution
 with lock.lock():
 try:
 predictor = estimator.deploy(initial_instance_count=1, instance_type='local')
 predict_response = predictor.predict([7])
Expand Down

39 changes: 27 additions & 12 deletions
tests/integ/vpc_test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,13 @@
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import os
import tempfile

import tests.integ.lock as lock

VPC_NAME = 'sagemaker-python-sdk-test-vpc'
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_vpc_lock')


def _get_subnet_ids_by_name(ec2_client, name):
Expand DownExpand Up@@ -61,20 +67,24 @@ def _create_vpc_with_name(ec2_client, region, name):
 AvailabilityZone=(region + 'b'))['Subnet']['SubnetId']
 print('created subnet: {}'.format(subnet_id_b))

 s3_service = [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if s.endswith('s3')][0]
 s3_service = \
 [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if
 s.endswith('s3')][0]
 ec2_client.create_vpc_endpoint(VpcId=vpc_id, ServiceName=s3_service,
 RouteTableIds=[_get_route_table_id(ec2_client, vpc_id)])
 print('created s3 vpc endpoint')

 security_group_id = ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 security_group_id = \
 ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 print('created security group: {}'.format(security_group_id))

 # multi-host vpc jobs require communication among hosts
 ec2_client.authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': 'tcp',
 'FromPort': 0,
 'ToPort': 65535,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [{
 'GroupId': security_group_id}]}])

 ec2_client.create_tags(Resources=[vpc_id, subnet_id_a, subnet_id_b, security_group_id],
 Tags=[{'Key': 'Name', 'Value': name}])
Expand All@@ -83,23 +93,28 @@ def _create_vpc_with_name(ec2_client, region, name):


def get_or_create_vpc_resources(ec2_client, region, name=VPC_NAME):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client, name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)
 # use lock to prevent race condition when tests are running concurrently
 with lock.lock(LOCK_PATH):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client,
 name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)


def setup_security_group_for_encryption(ec2_client, security_group_id):
 sg_desc = ec2_client.describe_security_groups(GroupIds=[security_group_id])
 ingress_perms = sg_desc['SecurityGroups'][0]['IpPermissions']
 if len(ingress_perms) == 1:
 ec2_client.\
 ec2_client.\
 authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': '50',
 'UserIdGroupPairs': [{'GroupId': security_group_id}]},
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]},
 {'IpProtocol': 'udp',
 'FromPort': 500,
 'ToPort': 500,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]}])

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: prevent race condition in vpc tests#863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jesterhazy
merged 2 commits into

from

Jun 19, 2019
 Merged


fix: prevent race condition in vpc tests
#863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
429b8d2

fix: prevent race condition in vpc tests


jesterhazy Jun 19, 2019
0bdad3a

remove extra files


jesterhazy Jun 19, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.gitignore

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,4 +25,4 @@ venv/
*~
.pytest_cache/
*.swp
tests/data/local_mode_lock
.docker/

21 changes: 11 additions & 10 deletions
tests/integ/local_mode_utils.py → tests/integ/lock.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,24 +15,25 @@
import fcntl
import os
import time
import tempfile
from contextlib import contextmanager

import tests.integ

LOCK_PATH = os.path.join(tests.integ.DATA_DIR, 'local_mode_lock')
DEFAULT_LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_lock')


@contextmanager
def lock():
 # Since Local Mode uses the same port for serving, we need a lock in order
 # to allow concurrent test execution.
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be good to have an explanation of why these locks are needed somewhere
@jesterhazyjesterhazyJun 19, 2019

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably at (multiple) points of use. or would a generic "lock for tests that need locks" comment work?
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a generic comment with specific callouts for the most common reasons (e.g. local mode)?
 local_mode_lock_fd = open(LOCK_PATH, 'w')
 local_mode_lock = local_mode_lock_fd.fileno()
def lock(path=DEFAULT_LOCK_PATH):
 """Create a file lock to control concurrent test execution. Certain tests or
 test operations need to limit concurrency to work reliably. Examples include
 local mode endpoint tests and vpc creation tests.
 """
 f = open(path, 'w')
 fd = f.fileno()

 fcntl.lockf(local_mode_lock, fcntl.LOCK_EX)
 fcntl.lockf(fd, fcntl.LOCK_EX)

 try:
 yield
 finally:
 time.sleep(5)
 fcntl.lockf(local_mode_lock, fcntl.LOCK_UN)
 fcntl.lockf(fd, fcntl.LOCK_UN)

24 changes: 14 additions & 10 deletions
tests/integ/test_local_mode.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,14 +18,18 @@
import boto3
import numpy
import pytest
import tests.integ.local_mode_utils as local_mode_utils
import tempfile

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ.timeout import timeout

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
from sagemaker.tensorflow import TensorFlow

# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_local_mode_lock')
DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data')
DEFAULT_REGION = 'us-west-2'

Expand DownExpand Up@@ -101,7 +105,7 @@ def test_tf_local_mode(tf_full_version, sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -140,7 +144,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):

 endpoint_name = estimator.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -178,7 +182,7 @@ def test_tf_local_data(sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -217,7 +221,7 @@ def test_tf_local_data_local_script():
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand All@@ -241,7 +245,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
 s3_model.sagemaker_session = sagemaker_local_session

 predictor = None
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = s3_model.deploy(initial_instance_count=1, instance_type='local')
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -255,7 +259,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mxnet_model):
 predictor = None

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 path = 'file://%s' % (str(tmpdir))
 model = mxnet_model(path)
Expand DownExpand Up@@ -285,7 +289,7 @@ def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -310,7 +314,7 @@ def test_mxnet_local_data_local_script(mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand DownExpand Up@@ -365,7 +369,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
 transformer = mx.transformer(1, 'local', assemble_with='Line', max_payload=1,
 strategy='SingleRecord', output_path=output_path)

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 transformer.transform(transform_input, content_type='text/csv', split_type='Line')
 transformer.wait()

Expand Down

5 changes: 3 additions & 2 deletions
tests/integ/test_source_dirs.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@

import pytest

import tests.integ.local_mode_utils as local_mode_utils
import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION

from sagemaker.pytorch.estimator import PyTorch
Expand All@@ -37,7 +37,8 @@ def test_source_dirs(tmpdir, sagemaker_local_session):
 sagemaker_session=sagemaker_local_session)
 estimator.fit()

 with local_mode_utils.lock():
 # endpoint tests all use the same port, so we use this lock to prevent concurrent execution
 with lock.lock():
 try:
 predictor = estimator.deploy(initial_instance_count=1, instance_type='local')
 predict_response = predictor.predict([7])
Expand Down

39 changes: 27 additions & 12 deletions
tests/integ/vpc_test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,13 @@
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import os
import tempfile

import tests.integ.lock as lock

VPC_NAME = 'sagemaker-python-sdk-test-vpc'
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_vpc_lock')


def _get_subnet_ids_by_name(ec2_client, name):
Expand DownExpand Up@@ -61,20 +67,24 @@ def _create_vpc_with_name(ec2_client, region, name):
 AvailabilityZone=(region + 'b'))['Subnet']['SubnetId']
 print('created subnet: {}'.format(subnet_id_b))

 s3_service = [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if s.endswith('s3')][0]
 s3_service = \
 [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if
 s.endswith('s3')][0]
 ec2_client.create_vpc_endpoint(VpcId=vpc_id, ServiceName=s3_service,
 RouteTableIds=[_get_route_table_id(ec2_client, vpc_id)])
 print('created s3 vpc endpoint')

 security_group_id = ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 security_group_id = \
 ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 print('created security group: {}'.format(security_group_id))

 # multi-host vpc jobs require communication among hosts
 ec2_client.authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': 'tcp',
 'FromPort': 0,
 'ToPort': 65535,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [{
 'GroupId': security_group_id}]}])

 ec2_client.create_tags(Resources=[vpc_id, subnet_id_a, subnet_id_b, security_group_id],
 Tags=[{'Key': 'Name', 'Value': name}])
Expand All@@ -83,23 +93,28 @@ def _create_vpc_with_name(ec2_client, region, name):


def get_or_create_vpc_resources(ec2_client, region, name=VPC_NAME):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client, name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)
 # use lock to prevent race condition when tests are running concurrently
 with lock.lock(LOCK_PATH):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client,
 name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)


def setup_security_group_for_encryption(ec2_client, security_group_id):
 sg_desc = ec2_client.describe_security_groups(GroupIds=[security_group_id])
 ingress_perms = sg_desc['SecurityGroups'][0]['IpPermissions']
 if len(ingress_perms) == 1:
 ec2_client.\
 ec2_client.\
 authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': '50',
 'UserIdGroupPairs': [{'GroupId': security_group_id}]},
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]},
 {'IpProtocol': 'udp',
 'FromPort': 500,
 'ToPort': 500,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]}])

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: prevent race condition in vpc tests#863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jesterhazy
merged 2 commits into

from

Jun 19, 2019
 Merged


fix: prevent race condition in vpc tests
#863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
429b8d2

fix: prevent race condition in vpc tests


jesterhazy Jun 19, 2019
0bdad3a

remove extra files


jesterhazy Jun 19, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.gitignore

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,4 +25,4 @@ venv/
*~
.pytest_cache/
*.swp
tests/data/local_mode_lock
.docker/

21 changes: 11 additions & 10 deletions
tests/integ/local_mode_utils.py → tests/integ/lock.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,24 +15,25 @@
import fcntl
import os
import time
import tempfile
from contextlib import contextmanager

import tests.integ

LOCK_PATH = os.path.join(tests.integ.DATA_DIR, 'local_mode_lock')
DEFAULT_LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_lock')


@contextmanager
def lock():
 # Since Local Mode uses the same port for serving, we need a lock in order
 # to allow concurrent test execution.
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be good to have an explanation of why these locks are needed somewhere
@jesterhazyjesterhazyJun 19, 2019

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably at (multiple) points of use. or would a generic "lock for tests that need locks" comment work?
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a generic comment with specific callouts for the most common reasons (e.g. local mode)?
 local_mode_lock_fd = open(LOCK_PATH, 'w')
 local_mode_lock = local_mode_lock_fd.fileno()
def lock(path=DEFAULT_LOCK_PATH):
 """Create a file lock to control concurrent test execution. Certain tests or
 test operations need to limit concurrency to work reliably. Examples include
 local mode endpoint tests and vpc creation tests.
 """
 f = open(path, 'w')
 fd = f.fileno()

 fcntl.lockf(local_mode_lock, fcntl.LOCK_EX)
 fcntl.lockf(fd, fcntl.LOCK_EX)

 try:
 yield
 finally:
 time.sleep(5)
 fcntl.lockf(local_mode_lock, fcntl.LOCK_UN)
 fcntl.lockf(fd, fcntl.LOCK_UN)

24 changes: 14 additions & 10 deletions
tests/integ/test_local_mode.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,14 +18,18 @@
import boto3
import numpy
import pytest
import tests.integ.local_mode_utils as local_mode_utils
import tempfile

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ.timeout import timeout

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
from sagemaker.tensorflow import TensorFlow

# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_local_mode_lock')
DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data')
DEFAULT_REGION = 'us-west-2'

Expand DownExpand Up@@ -101,7 +105,7 @@ def test_tf_local_mode(tf_full_version, sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -140,7 +144,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):

 endpoint_name = estimator.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -178,7 +182,7 @@ def test_tf_local_data(sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -217,7 +221,7 @@ def test_tf_local_data_local_script():
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand All@@ -241,7 +245,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
 s3_model.sagemaker_session = sagemaker_local_session

 predictor = None
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = s3_model.deploy(initial_instance_count=1, instance_type='local')
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -255,7 +259,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mxnet_model):
 predictor = None

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 path = 'file://%s' % (str(tmpdir))
 model = mxnet_model(path)
Expand DownExpand Up@@ -285,7 +289,7 @@ def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -310,7 +314,7 @@ def test_mxnet_local_data_local_script(mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand DownExpand Up@@ -365,7 +369,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
 transformer = mx.transformer(1, 'local', assemble_with='Line', max_payload=1,
 strategy='SingleRecord', output_path=output_path)

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 transformer.transform(transform_input, content_type='text/csv', split_type='Line')
 transformer.wait()

Expand Down

5 changes: 3 additions & 2 deletions
tests/integ/test_source_dirs.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@

import pytest

import tests.integ.local_mode_utils as local_mode_utils
import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION

from sagemaker.pytorch.estimator import PyTorch
Expand All@@ -37,7 +37,8 @@ def test_source_dirs(tmpdir, sagemaker_local_session):
 sagemaker_session=sagemaker_local_session)
 estimator.fit()

 with local_mode_utils.lock():
 # endpoint tests all use the same port, so we use this lock to prevent concurrent execution
 with lock.lock():
 try:
 predictor = estimator.deploy(initial_instance_count=1, instance_type='local')
 predict_response = predictor.predict([7])
Expand Down

39 changes: 27 additions & 12 deletions
tests/integ/vpc_test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,13 @@
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import os
import tempfile

import tests.integ.lock as lock

VPC_NAME = 'sagemaker-python-sdk-test-vpc'
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_vpc_lock')


def _get_subnet_ids_by_name(ec2_client, name):
Expand DownExpand Up@@ -61,20 +67,24 @@ def _create_vpc_with_name(ec2_client, region, name):
 AvailabilityZone=(region + 'b'))['Subnet']['SubnetId']
 print('created subnet: {}'.format(subnet_id_b))

 s3_service = [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if s.endswith('s3')][0]
 s3_service = \
 [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if
 s.endswith('s3')][0]
 ec2_client.create_vpc_endpoint(VpcId=vpc_id, ServiceName=s3_service,
 RouteTableIds=[_get_route_table_id(ec2_client, vpc_id)])
 print('created s3 vpc endpoint')

 security_group_id = ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 security_group_id = \
 ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 print('created security group: {}'.format(security_group_id))

 # multi-host vpc jobs require communication among hosts
 ec2_client.authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': 'tcp',
 'FromPort': 0,
 'ToPort': 65535,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [{
 'GroupId': security_group_id}]}])

 ec2_client.create_tags(Resources=[vpc_id, subnet_id_a, subnet_id_b, security_group_id],
 Tags=[{'Key': 'Name', 'Value': name}])
Expand All@@ -83,23 +93,28 @@ def _create_vpc_with_name(ec2_client, region, name):


def get_or_create_vpc_resources(ec2_client, region, name=VPC_NAME):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client, name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)
 # use lock to prevent race condition when tests are running concurrently
 with lock.lock(LOCK_PATH):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client,
 name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)


def setup_security_group_for_encryption(ec2_client, security_group_id):
 sg_desc = ec2_client.describe_security_groups(GroupIds=[security_group_id])
 ingress_perms = sg_desc['SecurityGroups'][0]['IpPermissions']
 if len(ingress_perms) == 1:
 ec2_client.\
 ec2_client.\
 authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': '50',
 'UserIdGroupPairs': [{'GroupId': security_group_id}]},
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]},
 {'IpProtocol': 'udp',
 'FromPort': 500,
 'ToPort': 500,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]}])

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
fix: prevent race condition in vpc tests#863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jesterhazy
merged 2 commits into

from

Jun 19, 2019
 Merged


fix: prevent race condition in vpc tests
#863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
429b8d2

fix: prevent race condition in vpc tests


jesterhazy Jun 19, 2019
0bdad3a

remove extra files


jesterhazy Jun 19, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.gitignore

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,4 +25,4 @@ venv/
*~
.pytest_cache/
*.swp
tests/data/local_mode_lock
.docker/

21 changes: 11 additions & 10 deletions
tests/integ/local_mode_utils.py → tests/integ/lock.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,24 +15,25 @@
import fcntl
import os
import time
import tempfile
from contextlib import contextmanager

import tests.integ

LOCK_PATH = os.path.join(tests.integ.DATA_DIR, 'local_mode_lock')
DEFAULT_LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_lock')


@contextmanager
def lock():
 # Since Local Mode uses the same port for serving, we need a lock in order
 # to allow concurrent test execution.
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be good to have an explanation of why these locks are needed somewhere
@jesterhazyjesterhazyJun 19, 2019

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably at (multiple) points of use. or would a generic "lock for tests that need locks" comment work?
@laurenyulaurenyuJun 19, 2019

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a generic comment with specific callouts for the most common reasons (e.g. local mode)?
 local_mode_lock_fd = open(LOCK_PATH, 'w')
 local_mode_lock = local_mode_lock_fd.fileno()
def lock(path=DEFAULT_LOCK_PATH):
 """Create a file lock to control concurrent test execution. Certain tests or
 test operations need to limit concurrency to work reliably. Examples include
 local mode endpoint tests and vpc creation tests.
 """
 f = open(path, 'w')
 fd = f.fileno()

 fcntl.lockf(local_mode_lock, fcntl.LOCK_EX)
 fcntl.lockf(fd, fcntl.LOCK_EX)

 try:
 yield
 finally:
 time.sleep(5)
 fcntl.lockf(local_mode_lock, fcntl.LOCK_UN)
 fcntl.lockf(fd, fcntl.LOCK_UN)

24 changes: 14 additions & 10 deletions
tests/integ/test_local_mode.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,14 +18,18 @@
import boto3
import numpy
import pytest
import tests.integ.local_mode_utils as local_mode_utils
import tempfile

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ.timeout import timeout

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
from sagemaker.tensorflow import TensorFlow

# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_local_mode_lock')
DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data')
DEFAULT_REGION = 'us-west-2'

Expand DownExpand Up@@ -101,7 +105,7 @@ def test_tf_local_mode(tf_full_version, sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -140,7 +144,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):

 endpoint_name = estimator.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -178,7 +182,7 @@ def test_tf_local_data(sagemaker_local_session):
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand DownExpand Up@@ -217,7 +221,7 @@ def test_tf_local_data_local_script():
 print('job succeeded: {}'.format(estimator.latest_training_job.name))

 endpoint_name = estimator.latest_training_job.name
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 json_predictor = estimator.deploy(initial_instance_count=1,
 instance_type='local',
Expand All@@ -241,7 +245,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
 s3_model.sagemaker_session = sagemaker_local_session

 predictor = None
 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = s3_model.deploy(initial_instance_count=1, instance_type='local')
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -255,7 +259,7 @@ def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model,
def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mxnet_model):
 predictor = None

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 path = 'file://%s' % (str(tmpdir))
 model = mxnet_model(path)
Expand DownExpand Up@@ -285,7 +289,7 @@ def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand All@@ -310,7 +314,7 @@ def test_mxnet_local_data_local_script(mxnet_full_version):
 mx.fit({'train': train_input, 'test': test_input})
 endpoint_name = mx.latest_training_job.name

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 try:
 predictor = mx.deploy(1, 'local', endpoint_name=endpoint_name)
 data = numpy.zeros(shape=(1, 1, 28, 28))
Expand DownExpand Up@@ -365,7 +369,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
 transformer = mx.transformer(1, 'local', assemble_with='Line', max_payload=1,
 strategy='SingleRecord', output_path=output_path)

 with local_mode_utils.lock():
 with lock.lock(LOCK_PATH):
 transformer.transform(transform_input, content_type='text/csv', split_type='Line')
 transformer.wait()

Expand Down

5 changes: 3 additions & 2 deletions
tests/integ/test_source_dirs.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@

import pytest

import tests.integ.local_mode_utils as local_mode_utils
import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION

from sagemaker.pytorch.estimator import PyTorch
Expand All@@ -37,7 +37,8 @@ def test_source_dirs(tmpdir, sagemaker_local_session):
 sagemaker_session=sagemaker_local_session)
 estimator.fit()

 with local_mode_utils.lock():
 # endpoint tests all use the same port, so we use this lock to prevent concurrent execution
 with lock.lock():
 try:
 predictor = estimator.deploy(initial_instance_count=1, instance_type='local')
 predict_response = predictor.predict([7])
Expand Down

39 changes: 27 additions & 12 deletions
tests/integ/vpc_test_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,13 @@
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import os
import tempfile

import tests.integ.lock as lock

VPC_NAME = 'sagemaker-python-sdk-test-vpc'
LOCK_PATH = os.path.join(tempfile.gettempdir(), 'sagemaker_test_vpc_lock')


def _get_subnet_ids_by_name(ec2_client, name):
Expand DownExpand Up@@ -61,20 +67,24 @@ def _create_vpc_with_name(ec2_client, region, name):
 AvailabilityZone=(region + 'b'))['Subnet']['SubnetId']
 print('created subnet: {}'.format(subnet_id_b))

 s3_service = [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if s.endswith('s3')][0]
 s3_service = \
 [s for s in ec2_client.describe_vpc_endpoint_services()['ServiceNames'] if
 s.endswith('s3')][0]
 ec2_client.create_vpc_endpoint(VpcId=vpc_id, ServiceName=s3_service,
 RouteTableIds=[_get_route_table_id(ec2_client, vpc_id)])
 print('created s3 vpc endpoint')

 security_group_id = ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 security_group_id = \
 ec2_client.create_security_group(VpcId=vpc_id, GroupName=name, Description=name)['GroupId']
 print('created security group: {}'.format(security_group_id))

 # multi-host vpc jobs require communication among hosts
 ec2_client.authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': 'tcp',
 'FromPort': 0,
 'ToPort': 65535,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [{
 'GroupId': security_group_id}]}])

 ec2_client.create_tags(Resources=[vpc_id, subnet_id_a, subnet_id_b, security_group_id],
 Tags=[{'Key': 'Name', 'Value': name}])
Expand All@@ -83,23 +93,28 @@ def _create_vpc_with_name(ec2_client, region, name):


def get_or_create_vpc_resources(ec2_client, region, name=VPC_NAME):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client, name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)
 # use lock to prevent race condition when tests are running concurrently
 with lock.lock(LOCK_PATH):
 if _vpc_exists(ec2_client, name):
 print('using existing vpc: {}'.format(name))
 return _get_subnet_ids_by_name(ec2_client, name), _get_security_id_by_name(ec2_client,
 name)
 else:
 print('creating new vpc: {}'.format(name))
 return _create_vpc_with_name(ec2_client, region, name)


def setup_security_group_for_encryption(ec2_client, security_group_id):
 sg_desc = ec2_client.describe_security_groups(GroupIds=[security_group_id])
 ingress_perms = sg_desc['SecurityGroups'][0]['IpPermissions']
 if len(ingress_perms) == 1:
 ec2_client.\
 ec2_client.\
 authorize_security_group_ingress(GroupId=security_group_id,
 IpPermissions=[{'IpProtocol': '50',
 'UserIdGroupPairs': [{'GroupId': security_group_id}]},
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]},
 {'IpProtocol': 'udp',
 'FromPort': 500,
 'ToPort': 500,
 'UserIdGroupPairs': [{'GroupId': security_group_id}]}])
 'UserIdGroupPairs': [
 {'GroupId': security_group_id}]}])