Skip to content

chore(ci): add the Idempotency feature to nox tests#4585

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
leandrodamascena merged 1 commit into from
Jun 20, 2024
Merged

chore(ci): add the Idempotency feature to nox tests #4585

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions noxfile.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,7 @@ def test_with_boto3_sdk_as_required_package(session: nox.Session):
f"{PREFIX_TESTS_FUNCTIONAL}/feature_flags/_boto3/",
f"{PREFIX_TESTS_UNIT}/data_classes/_boto3/",
f"{PREFIX_TESTS_FUNCTIONAL}/streaming/_boto3/",
f"{PREFIX_TESTS_FUNCTIONAL}/idempotency/_boto3/",
],
extras="aws-sdk",
)
Expand DownExpand Up@@ -158,3 +159,37 @@ def test_with_pydantic_required_package(session: nox.Session, pydantic: str):
f"{PREFIX_TESTS_UNIT}/parser/_pydantic/",
],
)


@nox.session()
@nox.parametrize("pydantic", ["1.10", "2.0"])
def test_with_boto3_and_pydantic_required_package(session: nox.Session, pydantic: str):

@leandrodamascenaleandrodamascenaJun 20, 2024

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I need to install pydantic + boto3 because we export the DynamoDBPersistenceLayer at top level and it requires boto3.

@heitorlessaheitorlessaJun 20, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you meant you only needed boto3 when using DynamoDBPersistenceLayer, whereas for this test you also need pydantic because it'll use serializers and models.

"""Tests that only depends for Boto3 + Pydantic library v1 and v2"""
# Idempotency with custom serializer

session.install(f"pydantic>={pydantic}")

build_and_run_test(
session,
folders=[
f"{PREFIX_TESTS_FUNCTIONAL}/idempotency/_pydantic/",
],
extras="aws-sdk",
)


@nox.session()
def test_with_redis_and_boto3_sdk_as_required_package(session: nox.Session):
Comment thread
heitorlessa marked this conversation as resolved.
"""Tests that depends on Redis library"""
# Idempotency - Redis backend

# Our Redis tests requires multiprocess library to simulate Race Condition
session.run("pip", "install", "multiprocess")

build_and_run_test(
session,
folders=[
f"{PREFIX_TESTS_FUNCTIONAL}/idempotency/_redis/",
],
extras="redis,aws-sdk",
)
Empty file addedtests/functional/idempotency/_boto3/__init__.py
View file
Open in desktop
Empty file.
0 tests/functional/idempotency/conftest.py → ...functional/idempotency/_boto3/conftest.py
View file
Open in desktop
File renamed without changes.
179 changes: 1 addition & 178 deletions ...unctional/idempotency/test_idempotency.py → ...al/idempotency/_boto3/test_idempotency.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,7 +8,6 @@
import pytest
from botocore import stub
from botocore.config import Config
from pydantic import BaseModel
from pytest import FixtureRequest
from pytest_mock import MockerFixture

Expand All@@ -32,7 +31,6 @@
IdempotencyInconsistentStateError,
IdempotencyInvalidStatusError,
IdempotencyKeyError,
IdempotencyModelTypeError,
IdempotencyNoSerializationModelError,
IdempotencyPersistenceLayerError,
IdempotencyValidationError,
Expand All@@ -47,9 +45,6 @@
from aws_lambda_powertools.utilities.idempotency.serialization.dataclass import (
DataclassSerializer,
)
from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import (
PydanticSerializer,
)
from aws_lambda_powertools.utilities.validation import envelopes, validator
from aws_lambda_powertools.warnings import PowertoolsUserWarning
from tests.functional.idempotency.utils import (
Expand All@@ -61,7 +56,7 @@
from tests.functional.utils import json_serialize, load_event

TABLE_NAME = "TEST_TABLE"
TESTS_MODULE_PREFIX = "test-func.tests.functional.idempotency.test_idempotency"
TESTS_MODULE_PREFIX = "test-func.tests.functional.idempotency._boto3.test_idempotency"


def get_dataclasses_lib():
Expand DownExpand Up@@ -1315,106 +1310,6 @@ def record_handler(record):
assert from_dict_called is False, "in case response is None, from_dict should not be called"


@pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"])
def test_idempotent_function_serialization_pydantic(output_serializer_type: str):
# GIVEN
config = IdempotencyConfig(use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic.<locals>.collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

class PaymentInput(BaseModel):
customer_id: str
transaction_id: str

class PaymentOutput(BaseModel):
customer_id: str
transaction_id: str

if output_serializer_type == "explicit":
output_serializer = PydanticSerializer(
model=PaymentOutput,
)
else:
output_serializer = PydanticSerializer

@idempotent_function(
data_keyword_argument="payment",
persistence_store=persistence_layer,
config=config,
output_serializer=output_serializer,
)
def collect_payment(payment: PaymentInput) -> PaymentOutput:
return PaymentOutput(**payment.dict())

# WHEN
payment = PaymentInput(**mock_event)
first_call: PaymentOutput = collect_payment(payment=payment)
assert first_call.customer_id == payment.customer_id
assert first_call.transaction_id == payment.transaction_id
assert isinstance(first_call, PaymentOutput)
second_call: PaymentOutput = collect_payment(payment=payment)
assert isinstance(second_call, PaymentOutput)
assert second_call.customer_id == payment.customer_id
assert second_call.transaction_id == payment.transaction_id


def test_idempotent_function_serialization_pydantic_failure_no_return_type():
# GIVEN
config = IdempotencyConfig(use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type.<locals>.collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

class PaymentInput(BaseModel):
customer_id: str
transaction_id: str

class PaymentOutput(BaseModel):
customer_id: str
transaction_id: str

idempotent_function_decorator = idempotent_function(
data_keyword_argument="payment",
persistence_store=persistence_layer,
config=config,
output_serializer=PydanticSerializer,
)
with pytest.raises(IdempotencyNoSerializationModelError, match="No serialization model was supplied"):

@idempotent_function_decorator
def collect_payment(payment: PaymentInput):
return PaymentOutput(**payment.dict())


def test_idempotent_function_serialization_pydantic_failure_bad_type():
# GIVEN
config = IdempotencyConfig(use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type.<locals>.collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

class PaymentInput(BaseModel):
customer_id: str
transaction_id: str

class PaymentOutput(BaseModel):
customer_id: str
transaction_id: str

idempotent_function_decorator = idempotent_function(
data_keyword_argument="payment",
persistence_store=persistence_layer,
config=config,
output_serializer=PydanticSerializer,
)
with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"):

@idempotent_function_decorator
def collect_payment(payment: PaymentInput) -> dict:
return PaymentOutput(**payment.dict())


@pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"])
def test_idempotent_function_serialization_dataclass(output_serializer_type: str):
# GIVEN
Expand DownExpand Up@@ -1493,37 +1388,6 @@ def collect_payment(payment: PaymentInput):
return PaymentOutput(**payment.dict())


def test_idempotent_function_serialization_dataclass_failure_bad_type():
# GIVEN
dataclasses = get_dataclasses_lib()
config = IdempotencyConfig(use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type.<locals>.collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

@dataclasses.dataclass
class PaymentInput:
customer_id: str
transaction_id: str

@dataclasses.dataclass
class PaymentOutput:
customer_id: str
transaction_id: str

idempotent_function_decorator = idempotent_function(
data_keyword_argument="payment",
persistence_store=persistence_layer,
config=config,
output_serializer=PydanticSerializer,
)
with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"):

@idempotent_function_decorator
def collect_payment(payment: PaymentInput) -> dict:
return PaymentOutput(**payment.dict())


def test_idempotent_function_arbitrary_args_kwargs():
# Scenario to validate we can use idempotent_function with a function
# with an arbitrary number of args and kwargs
Expand DownExpand Up@@ -1804,24 +1668,6 @@ class Foo:
assert as_dict == expected_result


def test_idempotent_function_pydantic():
# Scenario _prepare_data should convert a pydantic to a dict
class Foo(BaseModel):
name: str

expected_result = {"name": "Bar"}
data = Foo(name="Bar")
as_dict = _prepare_data(data)
assert as_dict == data.dict()
assert as_dict == expected_result


@pytest.mark.parametrize("data", [None, "foo", ["foo"], 1, True, {}])
def test_idempotent_function_other(data):
# All other data types should be left as is
assert _prepare_data(data) == data


def test_idempotent_function_dataclass_with_jmespath():
# GIVEN
dataclasses = get_dataclasses_lib()
Expand All@@ -1847,29 +1693,6 @@ def collect_payment(payment: Payment):
assert result == payment.transaction_id


def test_idempotent_function_pydantic_with_jmespath():
# GIVEN
config = IdempotencyConfig(event_key_jmespath="transaction_id", use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_pydantic_with_jmespath.<locals>.collect_payment#{hash_idempotency_key(mock_event['transaction_id'])}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

class Payment(BaseModel):
customer_id: str
transaction_id: str

@idempotent_function(data_keyword_argument="payment", persistence_store=persistence_layer, config=config)
def collect_payment(payment: Payment):
return payment.transaction_id

# WHEN
payment = Payment(**mock_event)
result = collect_payment(payment=payment)

# THEN idempotency key assertion happens at MockPersistenceLayer
assert result == payment.transaction_id


@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True)
def test_idempotent_lambda_compound_already_completed(
idempotency_config: IdempotencyConfig,
Expand Down
Empty file addedtests/functional/idempotency/_pydantic/__init__.py
View file
Open in desktop
Empty file.
Loading