diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 5ff678c586..289fa802a5 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -3,7 +3,7 @@ name: Build on: push: branches: - - master + - master pull_request: env: @@ -19,43 +19,43 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10"] spark-version-suffix: ["", "-spark2"] exclude: - - python-version: 3.8 - spark-version-suffix: "-spark2" - - python-version: 3.9 - spark-version-suffix: "-spark2" - - python-version: 3.10 - spark-version-suffix: "-spark2" - # Ignore this test because we failed to install docker-py - # docker-py will install pywin32==227, whereas pywin only added support for python 3.10 in version 301. - # For more detail, see https://github.com/flyteorg/flytekit/pull/856#issuecomment-1067152855 - - python-version: 3.10 - os: windows-latest + - python-version: 3.8 + spark-version-suffix: "-spark2" + - python-version: 3.9 + spark-version-suffix: "-spark2" + - python-version: 3.10 + spark-version-suffix: "-spark2" + # Ignore this test because we failed to install docker-py + # docker-py will install pywin32==227, whereas pywin only added support for python 3.10 in version 301. + # For more detail, see https://github.com/flyteorg/flytekit/pull/856#issuecomment-1067152855 + - python-version: 3.10 + os: windows-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Cache pip - uses: actions/cache@v2 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements files - key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }} - - name: Install dependencies - run: | - make setup${{ matrix.spark-version-suffix }} - pip freeze - - name: Test with coverage - env: - FLYTE_SDK_USE_STRUCTURED_DATASET: "TRUE" - run: | - coverage run -m pytest tests/flytekit/unit - - name: Codecov - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: true # optional (default = false) + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v2 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements files + key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }} + - name: Install dependencies + run: | + make setup${{ matrix.spark-version-suffix }} + pip freeze + - name: Test with coverage + env: + FLYTE_SDK_USE_STRUCTURED_DATASET: "TRUE" + run: | + coverage run -m pytest tests/flytekit/unit + - name: Codecov + uses: codecov/codecov-action@v1 + with: + fail_ci_if_error: true # optional (default = false) build-plugins: runs-on: ubuntu-latest @@ -84,35 +84,35 @@ jobs: - flytekit-spark - flytekit-sqlalchemy exclude: - # flytekit-modin depends on ray which does not have a 3.10 wheel yet. - # Issue tracked in https://github.com/ray-project/ray/issues/19116. - - python-version: 3.10 - plugin-names: "flytekit-modin" + # flytekit-modin depends on ray which does not have a 3.10 wheel yet. + # Issue tracked in https://github.com/ray-project/ray/issues/19116. + - python-version: 3.10 + plugin-names: "flytekit-modin" steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Cache pip - uses: actions/cache@v2 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements files - key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }} - - name: Install dependencies - run: | - make setup - cd plugins/${{ matrix.plugin-names }} - pip install -e . - if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi - pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit - pip freeze - - name: Test with coverage - run: | - cd plugins/${{ matrix.plugin-names }} - coverage run -m pytest tests + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v2 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements files + key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }} + - name: Install dependencies + run: | + make setup + cd plugins/${{ matrix.plugin-names }} + pip install -e . + if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi + pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit + pip freeze + - name: Test with coverage + run: | + cd plugins/${{ matrix.plugin-names }} + coverage run -m pytest tests lint: runs-on: ubuntu-latest @@ -139,8 +139,7 @@ jobs: - name: ShellCheck uses: ludeeus/action-shellcheck@master with: - ignore_paths: - boilerplate + ignore_paths: boilerplate docs: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index bd14c63aa5..d5d41a4af2 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ help: .PHONY: install-piptools install-piptools: - pip install -U pip-tools setuptools wheel pip==22.0.3 + pip install -U pip-tools setuptools wheel "pip>=22.0.3" .PHONY: update_boilerplate update_boilerplate: diff --git a/dev-requirements.txt b/dev-requirements.txt index 18ad44ccd2..550deb3d4b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -32,6 +32,7 @@ certifi==2021.10.8 # requests cffi==1.15.0 # via + # -c requirements.txt # bcrypt # cryptography # pynacl @@ -70,8 +71,11 @@ croniter==1.3.4 # via # -c requirements.txt # flytekit -cryptography==36.0.1 - # via paramiko +cryptography==36.0.2 + # via + # -c requirements.txt + # paramiko + # secretstorage dataclasses-json==0.5.6 # via # -c requirements.txt @@ -112,7 +116,7 @@ docstring-parser==0.13 # flytekit filelock==3.6.0 # via virtualenv -flyteidl==0.23.1 +flyteidl==0.24.0 # via # -c requirements.txt # flytekit @@ -121,7 +125,7 @@ google-api-core[grpc]==2.7.1 # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-core -google-auth==2.6.0 +google-auth==2.6.2 # via # google-api-core # google-cloud-core @@ -135,7 +139,7 @@ google-crc32c==1.3.0 # via google-resumable-media google-resumable-media==2.3.2 # via google-cloud-bigquery -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via # -c requirements.txt # flyteidl @@ -150,7 +154,7 @@ grpcio==1.44.0 # grpcio-status grpcio-status==1.44.0 # via google-api-core -identify==2.4.11 +identify==2.4.12 # via pre-commit idna==3.3 # via @@ -162,6 +166,11 @@ importlib-metadata==4.11.3 # keyring iniconfig==1.1.1 # via pytest +jeepney==0.7.1 + # via + # -c requirements.txt + # keyring + # secretstorage jinja2==3.0.3 # via # -c requirements.txt @@ -277,7 +286,9 @@ pyasn1==0.4.8 pyasn1-modules==0.2.8 # via google-auth pycparser==2.21 - # via cffi + # via + # -c requirements.txt + # cffi pynacl==1.5.0 # via paramiko pyparsing==3.0.7 @@ -329,7 +340,7 @@ pyyaml==5.4.1 # -c requirements.txt # docker-compose # pre-commit -regex==2022.3.2 +regex==2022.3.15 # via # -c requirements.txt # docker-image-py @@ -353,6 +364,10 @@ retry==0.9.2 # flytekit rsa==4.8 # via google-auth +secretstorage==3.3.1 + # via + # -c requirements.txt + # keyring six==1.16.0 # via # -c requirements.txt @@ -398,7 +413,7 @@ typing-inspect==0.7.1 # via # -c requirements.txt # dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # -c requirements.txt # flytekit diff --git a/doc-requirements.txt b/doc-requirements.txt index 6d50ca4b3b..15ee45abe1 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -41,8 +41,10 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 - # via -r doc-requirements.in +cryptography==36.0.2 + # via + # -r doc-requirements.in + # secretstorage css-html-js-minify==2.5.5 # via sphinx-material dataclasses-json==0.5.6 @@ -61,11 +63,11 @@ docutils==0.17.1 # via # sphinx # sphinx-panels -flyteidl==0.23.1 +flyteidl==0.24.0 # via flytekit furo @ git+https://github.com/flyteorg/furo@main # via -r doc-requirements.in -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via @@ -79,6 +81,10 @@ importlib-metadata==4.11.3 # via # keyring # sphinx +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -161,7 +167,7 @@ pytz==2021.3 # pandas pyyaml==6.0 # via sphinx-autoapi -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via @@ -173,6 +179,8 @@ responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -243,7 +251,7 @@ unidecode==1.3.4 # via # python-slugify # sphinx-autoapi -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 1a84c744a8..6e1dd9f043 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -41,13 +41,19 @@ Scalar, Schema, StructuredDatasetMetadata, + Union, + Void, ) -from flytekit.models.types import LiteralType, SimpleType, StructuredDatasetType +from flytekit.models.types import LiteralType, SimpleType, StructuredDatasetType, TypeStructure, UnionType T = typing.TypeVar("T") DEFINITIONS = "definitions" +class TypeTransformerFailedError(TypeError, AssertionError, ValueError): + ... + + class TypeTransformer(typing.Generic[T]): """ Base transformer type that should be implemented for every python native type that can be handled by flytekit @@ -85,7 +91,7 @@ def hash_overridable(self) -> bool: def assert_type(self, t: Type[T], v: T): if not hasattr(t, "__origin__") and not isinstance(v, t): - raise TypeError(f"Type of Val '{v}' is not an instance of {t}") + raise TypeTransformerFailedError(f"Type of Val '{v}' is not an instance of {t}") @abstractmethod def get_literal_type(self, t: Type[T]) -> LiteralType: @@ -147,18 +153,36 @@ def __init__( from_literal_transformer: typing.Callable[[Literal], T], ): super().__init__(name, t) + self._type = t self._lt = lt self._to_literal_transformer = to_literal_transformer self._from_literal_transformer = from_literal_transformer def get_literal_type(self, t: Type[T] = None) -> LiteralType: - return self._lt + return LiteralType.from_flyte_idl(self._lt.to_flyte_idl()) def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: + if type(python_val) != self._type: + raise TypeTransformerFailedError(f"Expected value of type {self._type} but got type {type(python_val)}") return self._to_literal_transformer(python_val) def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> T: - return self._from_literal_transformer(lv) + if get_origin(expected_python_type) is Annotated: + expected_python_type = get_args(expected_python_type)[0] + + if expected_python_type != self._type: + raise TypeTransformerFailedError( + f"Cannot convert to type {expected_python_type}, only {self._type} is supported" + ) + + try: # todo(maximsmol): this is quite ugly and each transformer should really check their Literal + res = self._from_literal_transformer(lv) + if type(res) != self._type: + raise TypeTransformerFailedError(f"Cannot convert literal {lv} to {self._type}") + return res + except AttributeError: + # Assume that this is because a property on `lv` was None + raise TypeTransformerFailedError(f"Cannot convert literal {lv}") def guess_python_type(self, literal_type: LiteralType) -> Type[T]: if literal_type.simple is not None and literal_type.simple == self._lt.simple: @@ -272,12 +296,12 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: if not dataclasses.is_dataclass(python_val): - raise AssertionError( + raise TypeTransformerFailedError( f"{type(python_val)} is not of type @dataclass, only Dataclasses are supported for " f"user defined datatypes in Flytekit" ) if not issubclass(type(python_val), DataClassJsonMixin): - raise AssertionError( + raise TypeTransformerFailedError( f"Dataclass {python_type} should be decorated with @dataclass_json to be " f"serialized correctly" ) self._serialize_flyte_type(python_val, python_type) @@ -435,12 +459,12 @@ def _fix_dataclass_int(self, dc_type: Type[DataClassJsonMixin], dc: DataClassJso def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> T: if not dataclasses.is_dataclass(expected_python_type): - raise AssertionError( + raise TypeTransformerFailedError( f"{expected_python_type} is not of type @dataclass, only Dataclasses are supported for " f"user defined datatypes in Flytekit" ) if not issubclass(expected_python_type, DataClassJsonMixin): - raise AssertionError( + raise TypeTransformerFailedError( f"Dataclass {expected_python_type} should be decorated with @dataclass_json to be " f"serialized correctly" ) @@ -472,12 +496,15 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: struct = Struct() - struct.update(_MessageToDict(python_val)) + try: + struct.update(_MessageToDict(python_val)) + except Exception: + raise TypeTransformerFailedError("Failed to convert to generic protobuf struct") return Literal(scalar=Scalar(generic=struct)) def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> T: if not (lv and lv.scalar and lv.scalar.generic is not None): - raise AssertionError("Can only convert a generic literal to a Protobuf") + raise TypeTransformerFailedError("Can only convert a generic literal to a Protobuf") pb_obj = expected_python_type() dictionary = _MessageToDict(lv.scalar.generic) @@ -622,16 +649,8 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: data = x.data if data is not None: idl_type_annotation = TypeAnnotationModel(annotations=data) - return LiteralType( - simple=res.simple, - schema=res.schema, - collection_type=res.collection_type, - map_value_type=res.map_value_type, - blob=res.blob, - enum_type=res.enum_type, - metadata=res.metadata, - annotation=idl_type_annotation, - ) + res = LiteralType.from_flyte_idl(res.to_flyte_idl()) + res._annotation = idl_type_annotation return res @classmethod @@ -639,8 +658,8 @@ def to_literal(cls, ctx: FlyteContext, python_val: typing.Any, python_type: Type """ Converts a python value of a given type and expected ``LiteralType`` into a resolved ``Literal`` value. """ - if python_val is None: - raise AssertionError(f"Python value cannot be None, expected {python_type}/{expected}") + if python_val is None and expected.union_type is None: + raise TypeTransformerFailedError(f"Python value cannot be None, expected {python_type}/{expected}") transformer = cls.get_transformer(python_type) if transformer.type_assertions_enabled: transformer.assert_type(python_type, python_val) @@ -801,13 +820,21 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: raise ValueError(f"Type of Generic List type is not supported, {e}") def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: + if type(python_val) != list: + raise TypeTransformerFailedError("Expected a list") + t = self.get_sub_type(python_type) lit_list = [TypeEngine.to_literal(ctx, x, t, expected.collection_type) for x in python_val] # type: ignore return Literal(collection=LiteralCollection(literals=lit_list)) def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[T]: + try: + lits = lv.collection.literals + except AttributeError: + raise TypeTransformerFailedError() + st = self.get_sub_type(expected_python_type) - return [TypeEngine.to_python_value(ctx, x, st) for x in lv.collection.literals] + return [TypeEngine.to_python_value(ctx, x, st) for x in lits] def guess_python_type(self, literal_type: LiteralType) -> Type[list]: if literal_type.collection_type: @@ -816,6 +843,198 @@ def guess_python_type(self, literal_type: LiteralType) -> Type[list]: raise ValueError(f"List transformer cannot reverse {literal_type}") +def _add_tag_to_type(x: LiteralType, tag: str) -> LiteralType: + x._structure = TypeStructure(tag=tag) + return x + + +def _type_essence(x: LiteralType) -> LiteralType: + if x.metadata is not None or x.structure is not None or x.annotation is not None: + x = LiteralType.from_flyte_idl(x.to_flyte_idl()) + x._metadata = None + x._structure = None + x._annotation = None + + return x + + +def _are_types_castable(upstream: LiteralType, downstream: LiteralType) -> bool: + if upstream.collection_type is not None: + if downstream.collection_type is None: + return False + + return _are_types_castable(upstream.collection_type, downstream.collection_type) + + if upstream.map_value_type is not None: + if downstream.map_value_type is None: + return False + + return _are_types_castable(upstream.map_value_type, downstream.map_value_type) + + # TODO: Structured dataset type matching requires that downstream structured datasets + # are a strict sub-set of the upstream structured dataset. + if upstream.structured_dataset_type is not None: + if downstream.structured_dataset_type is None: + return False + + usdt = upstream.structured_dataset_type + dsdt = downstream.structured_dataset_type + + if usdt.format != dsdt.format: + return False + + if usdt.external_schema_type != dsdt.external_schema_type: + return False + + if usdt.external_schema_bytes != dsdt.external_schema_bytes: + return False + + ucols = usdt.columns + dcols = dsdt.columns + + if len(ucols) != len(dcols): + return False + + for (u, d) in zip(ucols, dcols): + if u.name != d.name: + return False + + if not _are_types_castable(u.literal_type, d.literal_type): + return False + + return True + + if upstream.union_type is not None: + # for each upstream variant, there must be a compatible type downstream + for v in upstream.union_type.variants: + if not _are_types_castable(v, downstream): + return False + return True + + if downstream.union_type is not None: + # there must be a compatible downstream type + for v in downstream.union_type.variants: + if _are_types_castable(upstream, v): + return True + + if upstream.enum_type is not None: + # enums are castable to string + if downstream.simple == SimpleType.STRING: + return True + + if _type_essence(upstream) == _type_essence(downstream): + return True + + return False + + +class UnionTransformer(TypeTransformer[T]): + """ + Transformer that handles a typing.Union[T1, T2, ...] + """ + + def __init__(self): + super().__init__("Typed Union", typing.Union) + + def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: + if get_origin(t) is Annotated: + t = get_args(t)[0] + + try: + trans = [(TypeEngine.get_transformer(x), x) for x in get_args(t)] + # must go through TypeEngine.to_literal_type instead of trans.get_literal_type + # to handle Annotated + variants = [_add_tag_to_type(TypeEngine.to_literal_type(x), t.name) for (t, x) in trans] + return _type_models.LiteralType(union_type=UnionType(variants)) + except Exception as e: + raise ValueError(f"Type of Generic Union type is not supported, {e}") + + def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: + if get_origin(python_type) is Annotated: + python_type = get_args(python_type)[0] + + found_res = False + res = None + res_type = None + for t in get_args(python_type): + try: + trans = TypeEngine.get_transformer(t) + + res = trans.to_literal(ctx, python_val, t, expected) + res_type = _add_tag_to_type(trans.get_literal_type(t), trans.name) + if found_res: + # Should really never happen, sanity check + raise TypeError("Ambiguous choice of variant for union type") + found_res = True + except TypeTransformerFailedError as e: + logger.debug(f"Failed to convert from {python_val} to {t}", e) + continue + + if found_res: + return Literal(scalar=Scalar(union=Union(value=res, stored_type=res_type))) + + raise TypeTransformerFailedError(f"Cannot convert from {python_val} to {python_type}") + + def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> Optional[typing.Any]: + if get_origin(expected_python_type) is Annotated: + expected_python_type = get_args(expected_python_type)[0] + + union_tag = None + union_type = None + if lv.scalar is not None and lv.scalar.union is not None: + union_type = lv.scalar.union.stored_type + if union_type.structure is not None: + union_tag = union_type.structure.tag + + found_res = False + res = None + res_tag = None + for v in get_args(expected_python_type): + try: + trans = TypeEngine.get_transformer(v) + if union_tag is not None: + if trans.name != union_tag: + continue + + expected_literal_type = TypeEngine.to_literal_type(v) + if not _are_types_castable(union_type, expected_literal_type): + continue + + assert lv.scalar is not None # type checker + assert lv.scalar.union is not None # type checker + + res = trans.to_python_value(ctx, lv.scalar.union.value, v) + res_tag = trans.name + if found_res: + raise TypeError( + "Ambiguous choice of variant for union type. " + + f"Both {res_tag} and {trans.name} transformers match" + ) + found_res = True + else: + res = trans.to_python_value(ctx, lv, v) + if found_res: + raise TypeError( + "Ambiguous choice of variant for union type. " + + f"Both {res_tag} and {trans.name} transformers match" + ) + res_tag = trans.name + found_res = True + except TypeTransformerFailedError as e: + logger.debug(f"Failed to convert from {lv} to {v}", e) + + if found_res: + return res + + raise TypeError(f"Cannot convert from {lv} to {expected_python_type} (using tag {union_tag})") + + def guess_python_type(self, literal_type: LiteralType) -> type: + if literal_type.union_type is not None: + return typing.Union[tuple(TypeEngine.guess_python_type(v.type) for v in literal_type.union_type.variants)] + + raise ValueError(f"Union transformer cannot reverse {literal_type}") + + class DictTransformer(TypeTransformer[dict]): """ Transformer that transforms a univariate dictionary Dict[str, T] to a Literal Map or @@ -867,6 +1086,9 @@ def get_literal_type(self, t: Type[dict]) -> LiteralType: def to_literal( self, ctx: FlyteContext, python_val: typing.Any, python_type: Type[dict], expected: LiteralType ) -> Literal: + if type(python_val) != dict: + raise TypeTransformerFailedError("Expected a dict") + if expected and expected.simple and expected.simple == SimpleType.STRUCT: return self.dict_to_generic_literal(python_val) @@ -898,8 +1120,11 @@ def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: # for empty generic we have to explicitly test for lv.scalar.generic is not None as empty dict # evaluates to false if lv and lv.scalar and lv.scalar.generic is not None: - return _json.loads(_json_format.MessageToJson(lv.scalar.generic)) - raise TypeError(f"Cannot convert from {lv} to {expected_python_type}") + try: + return _json.loads(_json_format.MessageToJson(lv.scalar.generic)) + except TypeError: + raise TypeTransformerFailedError(f"Cannot convert from {lv} to {expected_python_type}") + raise TypeTransformerFailedError(f"Cannot convert from {lv} to {expected_python_type}") def guess_python_type(self, literal_type: LiteralType) -> Type[T]: if literal_type.map_value_type: @@ -998,10 +1223,15 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: values = [v.value for v in t] # type: ignore if not isinstance(values[0], str): - raise AssertionError("Only EnumTypes with value of string are supported") + raise TypeTransformerFailedError("Only EnumTypes with value of string are supported") return LiteralType(enum_type=_core_types.EnumType(values=values)) def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: + if type(python_val).__class__ != enum.EnumMeta: + raise TypeTransformerFailedError("Expected an enum") + if type(python_val.value) != str: + raise TypeTransformerFailedError("Only string-valued enums are supportedd") + return Literal(scalar=Scalar(primitive=Primitive(string_value=python_val.value))) # type: ignore def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> T: @@ -1075,7 +1305,13 @@ def _check_and_covert_float(lv: Literal) -> float: return lv.scalar.primitive.float_value elif lv.scalar.primitive.integer is not None: return float(lv.scalar.primitive.integer) - raise RuntimeError(f"Cannot convert literal {lv} to float") + raise TypeTransformerFailedError(f"Cannot convert literal {lv} to float") + + +def _check_and_convert_void(lv: Literal) -> None: + if lv.scalar.none_type is None: + raise TypeTransformerFailedError(f"Cannot conver literal {lv} to None") + return None def _register_default_type_transformers(): @@ -1142,13 +1378,15 @@ def _register_default_type_transformers(): TypeEngine.register( SimpleTransformer( "none", - None, + type(None), _type_models.LiteralType(simple=_type_models.SimpleType.NONE), - lambda x: None, - lambda x: None, - ) + lambda x: Literal(scalar=Scalar(none_type=Void())), + lambda x: _check_and_convert_void(x), + ), + [None], ) TypeEngine.register(ListTransformer()) + TypeEngine.register(UnionTransformer()) TypeEngine.register(DictTransformer()) TypeEngine.register(TextIOTransformer()) TypeEngine.register(BinaryIOTransformer()) diff --git a/flytekit/models/annotation.py b/flytekit/models/annotation.py index ced935c57e..bea6b1dc60 100644 --- a/flytekit/models/annotation.py +++ b/flytekit/models/annotation.py @@ -41,3 +41,8 @@ def from_flyte_idl(cls, proto): """ return cls(annotations=_json_format.MessageToDict(proto.annotations)) + + def __eq__(self, x: object) -> bool: + if not isinstance(x, self.__class__): + return False + return self.annotations == x.annotations diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py index 5c3d58bd40..1bc69ae41b 100644 --- a/flytekit/models/literals.py +++ b/flytekit/models/literals.py @@ -8,6 +8,7 @@ from flytekit.exceptions import user as _user_exceptions from flytekit.models import common as _common from flytekit.models.core import types as _core_types +from flytekit.models.types import LiteralType as _LiteralType from flytekit.models.types import OutputReference as _OutputReference from flytekit.models.types import SchemaType as _SchemaType from flytekit.models.types import StructuredDatasetType @@ -548,6 +549,48 @@ def from_flyte_idl(cls, pb2_object): return cls(uri=pb2_object.uri, type=_SchemaType.from_flyte_idl(pb2_object.type)) +class Union(_common.FlyteIdlEntity): + def __init__(self, value, stored_type): + """ + The runtime representation of a tagged union value. See `UnionType` for more details. + + :param flytekit.models.literals.Literal value: + :param flytekit.models.types.LiteralType stored_type: + """ + self._value = value + self._type = stored_type + + @property + def value(self): + """ + :rtype: flytekit.models.literals.Literal + """ + return self._value + + @property + def stored_type(self): + """ + :rtype: flytekit.models.types.LiteralType + """ + return self._type + + def to_flyte_idl(self): + """ + :rtype: flyteidl.core.literals_pb2.Union + """ + return _literals_pb2.Union(value=self.value.to_flyte_idl(), type=self._type.to_flyte_idl()) + + @classmethod + def from_flyte_idl(cls, pb2_object): + """ + :param flyteidl.core.literals_pb2.Schema pb2_object: + :rtype: Schema + """ + return cls( + value=Literal.from_flyte_idl(pb2_object.value), stored_type=_LiteralType.from_flyte_idl(pb2_object.type) + ) + + class StructuredDatasetMetadata(_common.FlyteIdlEntity): def __init__(self, structured_dataset_type: Optional[StructuredDatasetType] = None): self._structured_dataset_type = structured_dataset_type @@ -662,6 +705,7 @@ def __init__( blob: Blob = None, binary: Binary = None, schema: Schema = None, + union: Union = None, none_type: Void = None, error=None, generic: Struct = None, @@ -684,6 +728,7 @@ def __init__( self._blob = blob self._binary = binary self._schema = schema + self._union = union self._none_type = none_type self._error = error self._generic = generic @@ -717,6 +762,13 @@ def schema(self): """ return self._schema + @property + def union(self): + """ + :rtype: Union + """ + return self._union + @property def none_type(self): """ @@ -753,6 +805,7 @@ def value(self): or self.blob or self.binary or self.schema + or self.union or self.none_type or self.error or self.generic @@ -768,6 +821,7 @@ def to_flyte_idl(self): blob=self.blob.to_flyte_idl() if self.blob is not None else None, binary=self.binary.to_flyte_idl() if self.binary is not None else None, schema=self.schema.to_flyte_idl() if self.schema is not None else None, + union=self.union.to_flyte_idl() if self.union is not None else None, none_type=self.none_type.to_flyte_idl() if self.none_type is not None else None, error=self.error if self.error is not None else None, generic=self.generic, @@ -786,6 +840,7 @@ def from_flyte_idl(cls, pb2_object): blob=Blob.from_flyte_idl(pb2_object.blob) if pb2_object.HasField("blob") else None, binary=Binary.from_flyte_idl(pb2_object.binary) if pb2_object.HasField("binary") else None, schema=Schema.from_flyte_idl(pb2_object.schema) if pb2_object.HasField("schema") else None, + union=Union.from_flyte_idl(pb2_object.union) if pb2_object.HasField("union") else None, none_type=Void.from_flyte_idl(pb2_object.none_type) if pb2_object.HasField("none_type") else None, error=pb2_object.error if pb2_object.HasField("error") else None, generic=pb2_object.generic if pb2_object.HasField("generic") else None, diff --git a/flytekit/models/types.py b/flytekit/models/types.py index 0a0a2f8ced..4358d7229e 100644 --- a/flytekit/models/types.py +++ b/flytekit/models/types.py @@ -100,6 +100,50 @@ def from_flyte_idl(cls, proto): return cls(columns=[SchemaType.SchemaColumn.from_flyte_idl(c) for c in proto.columns]) +class UnionType(_common.FlyteIdlEntity): + """ + Models _types_pb2.UnionType + """ + + def __init__(self, variants: typing.List["LiteralType"]): + self._variants = variants + + @property + def variants(self) -> typing.List["LiteralType"]: + return self._variants + + def to_flyte_idl(self) -> _types_pb2.UnionType: + return _types_pb2.UnionType( + variants=[val.to_flyte_idl() if val else None for val in self._variants], + ) + + @classmethod + def from_flyte_idl(cls, proto: _types_pb2.UnionType): + return cls(variants=[LiteralType.from_flyte_idl(v) for v in proto.variants]) + + +class TypeStructure(_common.FlyteIdlEntity): + """ + Models _types_pb2.TypeStructure + """ + + def __init__(self, tag: str): + self._tag = tag + + @property + def tag(self) -> str: + return self._tag + + def to_flyte_idl(self) -> _types_pb2.TypeStructure: + return _types_pb2.TypeStructure( + tag=self._tag, + ) + + @classmethod + def from_flyte_idl(cls, proto: _types_pb2.TypeStructure): + return cls(tag=proto.tag) + + class StructuredDatasetType(_common.FlyteIdlEntity): class DatasetColumn(_common.FlyteIdlEntity): def __init__(self, name: str, literal_type: "LiteralType"): @@ -194,8 +238,10 @@ def __init__( map_value_type=None, blob=None, enum_type=None, + union_type=None, structured_dataset_type=None, metadata=None, + structure=None, annotation=None, ): """ @@ -208,9 +254,11 @@ def __init__( string. :param flytekit.models.core.types.BlobType blob: For blob objects, this describes the type. :param flytekit.models.core.types.EnumType enum_type: For enum objects, describes an enum + :param flytekit.models.core.types.UnionType union_type: For union objects, describes an python union type. + :param flytekit.models.core.types.TypeStructure structure: Type matching hints :param flytekit.models.core.types.StructuredDatasetType structured_dataset_type: structured dataset :param dict[Text, T] metadata: Additional data describing the type - :param flytekit.models.annotation.FlyteAnnotation annotation: Additional data + :param flytekit.models.annotation.TypeAnnotation annotation: Additional data describing the type _intended to be saturated by the client_ """ self._simple = simple @@ -219,6 +267,10 @@ def __init__( self._map_value_type = map_value_type self._blob = blob self._enum_type = enum_type + self._union_type = union_type + self._structured_dataset_type = structured_dataset_type + self._metadata = metadata + self._structure = structure self._structured_dataset_type = structured_dataset_type self._metadata = metadata self._annotation = annotation @@ -253,6 +305,14 @@ def blob(self) -> _core_types.BlobType: def enum_type(self) -> _core_types.EnumType: return self._enum_type + @property + def union_type(self) -> UnionType: + return self._union_type + + @property + def structure(self) -> TypeStructure: + return self._structure + @property def structured_dataset_type(self) -> StructuredDatasetType: return self._structured_dataset_type @@ -296,11 +356,13 @@ def to_flyte_idl(self): map_value_type=self.map_value_type.to_flyte_idl() if self.map_value_type is not None else None, blob=self.blob.to_flyte_idl() if self.blob is not None else None, enum_type=self.enum_type.to_flyte_idl() if self.enum_type else None, + union_type=self.union_type.to_flyte_idl() if self.union_type else None, structured_dataset_type=self.structured_dataset_type.to_flyte_idl() if self.structured_dataset_type else None, metadata=metadata, annotation=self.annotation.to_flyte_idl() if self.annotation else None, + structure=self.structure.to_flyte_idl() if self.structure else None, ) return t @@ -323,10 +385,12 @@ def from_flyte_idl(cls, proto): map_value_type=map_value_type, blob=_core_types.BlobType.from_flyte_idl(proto.blob) if proto.HasField("blob") else None, enum_type=_core_types.EnumType.from_flyte_idl(proto.enum_type) if proto.HasField("enum_type") else None, + union_type=UnionType.from_flyte_idl(proto.union_type) if proto.HasField("union_type") else None, structured_dataset_type=StructuredDatasetType.from_flyte_idl(proto.structured_dataset_type) if proto.HasField("structured_dataset_type") else None, metadata=_json_format.MessageToDict(proto.metadata) or None, + structure=TypeStructure.from_flyte_idl(proto.structure) if proto.HasField("structure") else None, annotation=TypeAnnotationModel.from_flyte_idl(proto.annotation) if proto.HasField("annotation") else None, ) diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 1fa673e955..744f56de6b 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -9,7 +9,7 @@ from marshmallow import fields from flytekit.core.context_manager import FlyteContext -from flytekit.core.type_engine import TypeEngine, TypeTransformer +from flytekit.core.type_engine import TypeEngine, TypeTransformer, TypeTransformerFailedError from flytekit.loggers import logger from flytekit.models.core.types import BlobType from flytekit.models.literals import Blob, BlobMetadata, Literal, Scalar @@ -263,7 +263,7 @@ def to_literal( should_upload = True if python_val is None: - raise AssertionError("None value cannot be converted to a file.") + raise TypeTransformerFailedError("None value cannot be converted to a file.") if not (python_type is os.PathLike or issubclass(python_type, FlyteFile)): raise ValueError(f"Incorrect type {python_type}, must be either a FlyteFile or os.PathLike") @@ -308,13 +308,13 @@ def to_literal( if isinstance(python_val, str): p = pathlib.Path(python_val) if not p.is_file(): - raise ValueError(f"Error converting {python_val} because it's not a file.") + raise TypeTransformerFailedError(f"Error converting {python_val} because it's not a file.") # python_type must be os.PathLike - see check at beginning of function else: should_upload = False else: - raise AssertionError(f"Expected FlyteFile or os.PathLike object, received {type(python_val)}") + raise TypeTransformerFailedError(f"Expected FlyteFile or os.PathLike object, received {type(python_val)}") # If we're uploading something, that means that the uri should always point to the upload destination. if should_upload: @@ -329,8 +329,10 @@ def to_literal( def to_python_value( self, ctx: FlyteContext, lv: Literal, expected_python_type: typing.Union[typing.Type[FlyteFile], os.PathLike] ) -> FlyteFile: - - uri = lv.scalar.blob.uri + try: + uri = lv.scalar.blob.uri + except AttributeError: + raise TypeTransformerFailedError(f"Cannot convert from {lv} to {expected_python_type}") # In this condition, we still return a FlyteFile instance, but it's a simple one that has no downloading tricks # Using is instead of issubclass because FlyteFile does actually subclass it if expected_python_type is os.PathLike: diff --git a/flytekit/types/schema/types.py b/flytekit/types/schema/types.py index b947ad7497..6f01cea085 100644 --- a/flytekit/types/schema/types.py +++ b/flytekit/types/schema/types.py @@ -15,7 +15,7 @@ from marshmallow import fields from flytekit.core.context_manager import FlyteContext, FlyteContextManager -from flytekit.core.type_engine import TypeEngine, TypeTransformer +from flytekit.core.type_engine import TypeEngine, TypeTransformer, TypeTransformerFailedError from flytekit.models.literals import Literal, Scalar, Schema from flytekit.models.types import LiteralType, SchemaType @@ -368,9 +368,14 @@ def to_literal( local_path=ctx.file_access.get_random_local_directory(), remote_path=ctx.file_access.get_random_remote_directory(), ) + try: + h = SchemaEngine.get_handler(type(python_val)) + except ValueError as e: + raise TypeTransformerFailedError( + f"DataFrames of type {type(python_val)} are not supported currently" + ) from e writer = schema.open(type(python_val)) writer.write(python_val) - h = SchemaEngine.get_handler(type(python_val)) if not h.handles_remote_io: ctx.file_access.put_data(schema.local_path, schema.remote_path, is_multipart=True) return Literal(scalar=Scalar(schema=Schema(schema.remote_path, self._get_schema_type(python_type)))) diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index bb210d8c13..1af8335a2c 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,17 +44,17 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-athena -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -68,9 +68,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -83,10 +83,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -105,6 +107,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -121,14 +125,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -151,14 +155,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt index 80a3fcf51f..db5606322f 100644 --- a/plugins/flytekit-aws-batch/requirements.txt +++ b/plugins/flytekit-aws-batch/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,17 +44,17 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-awsbatch -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -68,9 +68,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -83,10 +83,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -105,6 +107,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -121,14 +125,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -151,14 +155,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index f4b02da229..54b04b5a13 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -12,9 +12,9 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -boto3==1.21.10 +boto3==1.21.21 # via sagemaker-training -botocore==1.24.10 +botocore==1.24.21 # via # boto3 # s3transfer @@ -41,7 +41,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via # paramiko # secretstorage @@ -57,13 +57,13 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-awssagemaker gevent==21.12.0 # via sagemaker-training -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl greenlet==1.1.2 # via gevent @@ -71,9 +71,9 @@ grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring -inotify_simple==1.2.1 +inotify-simple==1.2.1 # via sagemaker-training jeepney==0.7.1 # via @@ -85,15 +85,15 @@ jinja2==3.0.3 # jinja2-time jinja2-time==0.2.0 # via cookiecutter -jmespath==0.10.0 +jmespath==1.0.0 # via # boto3 # botocore keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -106,15 +106,17 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow # sagemaker-training # scipy +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit -paramiko==2.9.2 +paramiko==2.10.2 # via sagemaker-training poyo==0.5.0 # via cookiecutter @@ -137,6 +139,8 @@ pycparser==2.21 # via cffi pynacl==1.5.0 # via paramiko +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -154,14 +158,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -180,6 +184,7 @@ six==1.16.0 # bcrypt # cookiecutter # grpcio + # paramiko # python-dateutil # retrying # sagemaker-training @@ -195,7 +200,7 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # botocore # flytekit @@ -205,15 +210,15 @@ werkzeug==2.0.3 # via sagemaker-training wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit zipp==3.7.0 # via importlib-metadata -zope.event==4.5.0 +zope-event==4.5.0 # via gevent -zope.interface==5.4.0 +zope-interface==5.4.0 # via gevent # The following packages are considered to be unsafe in a requirements file: diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt index aef0a38759..38576616bd 100644 --- a/plugins/flytekit-bigquery/requirements.txt +++ b/plugins/flytekit-bigquery/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -32,7 +32,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -46,27 +46,27 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-bigquery -google-api-core[grpc]==2.5.0 +google-api-core[grpc]==2.7.1 # via # google-cloud-bigquery # google-cloud-core -google-auth==2.6.0 +google-auth==2.6.2 # via # google-api-core # google-cloud-core -google-cloud-bigquery==2.34.0 +google-cloud-bigquery==2.34.2 # via flytekitplugins-bigquery -google-cloud-core==2.2.2 +google-cloud-core==2.2.3 # via google-cloud-bigquery google-crc32c==1.3.0 # via google-resumable-media -google-resumable-media==2.3.0 +google-resumable-media==2.3.2 # via google-cloud-bigquery -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via # flyteidl # google-api-core @@ -81,7 +81,7 @@ grpcio-status==1.44.0 # via google-api-core idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -95,9 +95,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -110,12 +110,14 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow packaging==21.3 - # via google-cloud-bigquery + # via + # google-cloud-bigquery + # marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -165,7 +167,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via @@ -174,7 +176,7 @@ requests==2.27.1 # google-api-core # google-cloud-bigquery # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -200,14 +202,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index b3f506d572..f66ca64fba 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -10,7 +10,7 @@ arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter -botocore==1.24.10 +botocore==1.24.21 # via flytekitplugins-data-fsspec certifi==2021.10.8 # via requests @@ -32,7 +32,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -46,19 +46,19 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-data-fsspec fsspec==2022.2.0 # via flytekitplugins-data-fsspec -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -70,13 +70,13 @@ jinja2==3.0.3 # jinja2-time jinja2-time==0.2.0 # via cookiecutter -jmespath==0.10.0 +jmespath==1.0.0 # via botocore keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -89,10 +89,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -111,6 +113,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -128,14 +132,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -158,7 +162,7 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # botocore # flytekit @@ -166,7 +170,7 @@ urllib3==1.26.8 # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index fc9dc9f8b7..bb61c3be8a 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via @@ -50,17 +50,17 @@ dolt-integrations==0.1.5 # via flytekitplugins-dolt doltcli==0.1.17 # via dolt-integrations -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-dolt -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -74,9 +74,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -89,10 +89,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via # dolt-integrations @@ -113,6 +115,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -129,14 +133,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -159,14 +163,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index 31b3be4b48..5c059de55f 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -20,6 +20,12 @@ attrs==21.4.0 # via jsonschema backcall==0.2.0 # via ipython +backports-zoneinfo==0.2.1 + # via + # pytz-deprecation-shim + # tzlocal +beautifulsoup4==4.10.0 + # via nbconvert binaryornot==0.4.4 # via cookiecutter bleach==4.1.0 @@ -47,7 +53,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -74,11 +80,11 @@ entrypoints==0.4 # nbconvert executing==0.8.3 # via stack-data -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-great-expectations -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl great-expectations==0.14.5 # via flytekitplugins-great-expectations @@ -88,24 +94,25 @@ grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via # great-expectations # keyring -ipykernel==6.9.1 +importlib-resources==5.4.0 + # via jsonschema +ipykernel==6.9.2 # via # ipywidgets # notebook -ipython==8.1.0 +ipython==8.1.1 # via # ipykernel # ipywidgets ipython-genutils==0.2.0 # via # ipywidgets - # nbformat # notebook -ipywidgets==7.6.5 +ipywidgets==7.7.0 # via great-expectations jedi==0.18.1 # via ipython @@ -145,13 +152,13 @@ jupyter-core==4.9.2 # notebook jupyterlab-pygments==0.1.2 # via nbconvert -jupyterlab-widgets==1.0.2 +jupyterlab-widgets==1.1.0 # via ipywidgets keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -172,11 +179,11 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -nbclient==0.5.11 +nbclient==0.5.13 # via nbconvert -nbconvert==6.4.2 +nbconvert==6.4.4 # via notebook -nbformat==5.1.3 +nbformat==5.2.0 # via # ipywidgets # nbclient @@ -188,9 +195,9 @@ nest-asyncio==1.5.4 # jupyter-client # nbclient # notebook -notebook==6.4.8 +notebook==6.4.10 # via widgetsnbextension -numpy==1.22.2 +numpy==1.22.3 # via # altair # great-expectations @@ -198,7 +205,9 @@ numpy==1.22.2 # pyarrow # scipy packaging==21.3 - # via bleach + # via + # bleach + # marshmallow pandas==1.4.1 # via # altair @@ -226,6 +235,8 @@ protobuf==3.19.4 # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl +psutil==5.9.0 + # via ipykernel ptyprocess==0.7.0 # via # pexpect @@ -274,7 +285,7 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via @@ -282,12 +293,14 @@ requests==2.27.1 # flytekit # great-expectations # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit -ruamel.yaml==0.17.17 +ruamel-yaml==0.17.17 # via great-expectations +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml scipy==1.8.0 # via great-expectations secretstorage==3.3.1 @@ -303,7 +316,9 @@ six==1.16.0 # python-dateutil sortedcontainers==2.4.0 # via flytekit -sqlalchemy==1.4.31 +soupsieve==2.3.1 + # via beautifulsoup4 +sqlalchemy==1.4.32 # via # -r requirements.in # flytekitplugins-great-expectations @@ -313,7 +328,7 @@ statsd==3.3.0 # via flytekit termcolor==1.1.0 # via great-expectations -terminado==0.13.1 +terminado==0.13.3 # via notebook testpath==0.6.0 # via nbconvert @@ -352,7 +367,7 @@ tzdata==2021.5 # via pytz-deprecation-shim tzlocal==4.1 # via great-expectations -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests @@ -363,14 +378,16 @@ webencodings==0.5.1 # via bleach wheel==0.37.1 # via flytekit -widgetsnbextension==3.5.2 +widgetsnbextension==3.6.0 # via ipywidgets -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit zipp==3.7.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index 69ed1e5f3e..e52eb728d1 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,17 +44,17 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-hive -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -68,9 +68,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -83,10 +83,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -105,6 +107,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -121,14 +125,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -151,14 +155,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index df398913af..eb236d223f 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -34,7 +34,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -48,19 +48,19 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-pod -google-auth==2.6.0 +google-auth==2.6.2 # via kubernetes -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -76,9 +76,9 @@ keyring==23.5.0 # via flytekit kubernetes==23.3.0 # via flytekitplugins-pod -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -91,12 +91,14 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow oauthlib==3.2.0 # via requests-oauthlib +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -121,6 +123,8 @@ pyasn1-modules==0.2.8 # via google-auth pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -140,7 +144,7 @@ pytz==2021.3 # pandas pyyaml==6.0 # via kubernetes -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via @@ -151,7 +155,7 @@ requests==2.27.1 # responses requests-oauthlib==1.3.1 # via kubernetes -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -178,7 +182,7 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # kubernetes @@ -188,7 +192,7 @@ websocket-client==1.3.1 # via kubernetes wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt index 82a2317f2f..6af7ac0ab6 100644 --- a/plugins/flytekit-kf-mpi/requirements.txt +++ b/plugins/flytekit-kf-mpi/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,19 +44,19 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via # flytekit # flytekitplugins-kfmpi -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-kfmpi -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -70,9 +70,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -85,10 +85,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -107,6 +109,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -123,14 +127,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -153,14 +157,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index 0a53a62333..5354e80db2 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,17 +44,17 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-kfpytorch -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -68,9 +68,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -83,10 +83,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -105,6 +107,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -121,14 +125,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -151,14 +155,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index b782c39630..749255fecf 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,17 +44,17 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-kftensorflow -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -68,9 +68,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -83,10 +83,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -105,6 +107,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -121,14 +125,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -151,14 +155,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-modin/requirements.txt b/plugins/flytekit-modin/requirements.txt index 4f3b01e5e4..fefbb4d944 100644 --- a/plugins/flytekit-modin/requirements.txt +++ b/plugins/flytekit-modin/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -16,13 +16,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -31,9 +33,9 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -49,24 +51,32 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -filelock==3.4.2 +filelock==3.6.0 # via ray -flyteidl==0.22.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.0 +flytekit==0.31.0 # via flytekitplugins-modin -fsspec==2022.1.0 +fsspec==2022.2.0 # via # flytekitplugins-modin # modin +googleapis-common-protos==1.56.0 + # via flyteidl grpcio==1.43.0 # via # flytekit # ray idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.3 # via keyring +importlib-resources==5.4.0 + # via jsonschema +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -77,9 +87,9 @@ jsonschema==4.4.0 # via ray keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -88,7 +98,7 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -modin==0.13.1 +modin==0.13.2 # via flytekitplugins-modin msgpack==1.0.3 # via ray @@ -96,7 +106,7 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # modin # pandas @@ -104,6 +114,7 @@ numpy==1.22.2 # ray packaging==21.3 # via + # marshmallow # modin # redis pandas==1.4.0 @@ -116,7 +127,11 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger # ray +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -135,7 +150,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -145,21 +160,23 @@ pytz==2021.3 # pandas pyyaml==6.0 # via ray -ray==1.10.0 +ray==1.11.0 # via flytekitplugins-modin -redis==4.1.2 +redis==4.1.4 # via ray -regex==2022.1.18 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -171,22 +188,24 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit zipp==3.7.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index 7b72a0b4c0..a4c86dba63 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,17 +44,17 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-pandera -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -68,9 +68,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -83,13 +83,15 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pandera # pyarrow packaging==21.3 - # via pandera + # via + # marshmallow + # pandera pandas==1.4.1 # via # flytekit @@ -134,14 +136,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -167,14 +169,14 @@ typing-inspect==0.7.1 # via # dataclasses-json # pandera -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in index f6a875fa41..c056eb983b 100644 --- a/plugins/flytekit-papermill/dev-requirements.in +++ b/plugins/flytekit-papermill/dev-requirements.in @@ -1,3 +1,3 @@ -flyteidl>=0.23.0 +flyteidl>=0.24.0 git+https://github.com/flyteorg/flytekit@master#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark # vcs+protocol://repo_url/#egg=pkg&subdirectory=flyte diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt index 54c6aff4cd..5e7a446b3b 100644 --- a/plugins/flytekit-papermill/dev-requirements.txt +++ b/plugins/flytekit-papermill/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile dev-requirements.in @@ -38,13 +38,13 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via # -r dev-requirements.in # flytekit flytekit==0.30.0 # via flytekitplugins-spark -git+https://github.com/flyteorg/flytekit@master#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark +flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@master#subdirectory=plugins/flytekit-spark # via -r dev-requirements.in googleapis-common-protos==1.55.0 # via flyteidl diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 8c3e4570e9..9463e5cbd7 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -16,10 +16,10 @@ attrs==21.4.0 # via jsonschema backcall==0.2.0 # via ipython +beautifulsoup4==4.10.0 + # via nbconvert binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via ipython bleach==4.1.0 # via nbconvert certifi==2021.10.8 @@ -28,13 +28,12 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via - # black # cookiecutter # flytekit # papermill @@ -42,9 +41,9 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -64,29 +63,31 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 +entrypoints==0.4 # via # jupyter-client # nbconvert # papermill -executing==0.8.2 +executing==0.8.3 # via stack-data -flyteidl==0.22.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.26.1 +flytekit==0.31.0 # via flytekitplugins-papermill -grpcio==1.43.0 +googleapis-common-protos==1.56.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.3 # via keyring -ipykernel==6.7.0 +importlib-resources==5.4.0 + # via jsonschema +ipykernel==6.9.2 # via flytekitplugins-papermill -ipython==8.0.1 +ipython==8.1.1 # via ipykernel -ipython-genutils==0.2.0 - # via nbformat jedi==0.18.1 # via ipython jeepney==0.7.1 @@ -106,7 +107,7 @@ jupyter-client==7.1.2 # via # ipykernel # nbclient -jupyter-core==4.9.1 +jupyter-core==4.9.2 # via # jupyter-client # nbconvert @@ -115,9 +116,9 @@ jupyterlab-pygments==0.1.2 # via nbconvert keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -133,18 +134,16 @@ matplotlib-inline==0.1.3 mistune==0.8.4 # via nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbclient==0.5.10 +nbclient==0.5.13 # via # nbconvert # papermill -nbconvert==6.4.1 +nbconvert==6.4.4 # via flytekitplugins-papermill -nbformat==5.1.3 +nbformat==5.2.0 # via # nbclient # nbconvert @@ -154,13 +153,15 @@ nest-asyncio==1.5.4 # ipykernel # jupyter-client # nbclient -numpy==1.22.1 +numpy==1.22.3 # via # pandas # pyarrow packaging==21.3 - # via bleach -pandas==1.4.0 + # via + # bleach + # marshmallow +pandas==1.4.1 # via flytekit pandocfilters==1.5.0 # via nbconvert @@ -168,22 +169,24 @@ papermill==2.3.4 # via flytekitplugins-papermill parso==0.8.3 # via jedi -pathspec==0.9.0 - # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.1 - # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.26 +prompt-toolkit==3.0.28 # via ipython protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl +psutil==5.9.0 + # via ipykernel ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 @@ -203,7 +206,7 @@ pyparsing==3.0.7 # via packaging pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow # croniter @@ -212,7 +215,7 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -224,7 +227,7 @@ pyyaml==6.0 # via papermill pyzmq==22.3.0 # via jupyter-client -regex==2022.1.18 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via @@ -232,7 +235,7 @@ requests==2.27.1 # flytekit # papermill # responses -responses==0.17.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -243,31 +246,29 @@ six==1.16.0 # asttokens # bleach # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit -stack-data==0.1.4 +soupsieve==2.3.1 + # via beautifulsoup4 +stack-data==0.2.0 # via ipython statsd==3.3.0 # via flytekit tenacity==8.0.1 # via papermill -testpath==0.5.0 +testpath==0.6.0 # via nbconvert text-unidecode==1.3 # via python-slugify textwrap3==0.9.2 # via ansiwrap -tomli==1.2.3 - # via black tornado==6.1 # via # ipykernel # jupyter-client -tqdm==4.62.3 +tqdm==4.63.0 # via papermill traitlets==5.1.1 # via @@ -279,13 +280,13 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via - # black + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests @@ -296,12 +297,14 @@ webencodings==0.5.1 # via bleach wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit zipp==3.7.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index 2d34ec966f..43c8f445e7 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,17 +44,17 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-snowflake -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -68,9 +68,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -83,10 +83,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -105,6 +107,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -121,14 +125,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -151,14 +155,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index 6e7c6e59f0..f53e809b77 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,17 +44,17 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-spark -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -68,9 +68,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -83,10 +83,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -107,6 +109,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging pyspark==3.2.1 # via flytekitplugins-spark python-dateutil==2.8.2 @@ -125,14 +129,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -155,14 +159,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index e617920549..77485ea9cb 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -30,7 +30,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage dataclasses-json==0.5.6 # via flytekit @@ -44,11 +44,11 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via flytekitplugins-sqlalchemy -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl greenlet==1.1.2 # via sqlalchemy @@ -56,7 +56,7 @@ grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -70,9 +70,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -85,10 +85,12 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # pandas # pyarrow +packaging==21.3 + # via marshmallow pandas==1.4.1 # via flytekit poyo==0.5.0 @@ -107,6 +109,8 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi +pyparsing==3.0.7 + # via packaging python-dateutil==2.8.2 # via # arrow @@ -123,14 +127,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -143,7 +147,7 @@ six==1.16.0 # python-dateutil sortedcontainers==2.4.0 # via flytekit -sqlalchemy==1.4.31 +sqlalchemy==1.4.32 # via flytekitplugins-sqlalchemy statsd==3.3.0 # via flytekit @@ -155,14 +159,14 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests # responses wheel==0.37.1 # via flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 56ab01553b..ee6690d1c5 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -18,6 +18,8 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.12 @@ -34,6 +36,8 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit +cryptography==36.0.2 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -46,9 +50,9 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.1 +flyteidl==0.24.0 # via flytekit -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit @@ -56,6 +60,10 @@ idna==3.3 # via requests importlib-metadata==4.11.3 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -106,6 +114,8 @@ py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi pyparsing==3.0.7 # via packaging pyrsistent==0.18.1 @@ -128,7 +138,7 @@ pytz==2021.3 # pandas pyyaml==5.4.1 # via -r requirements.in -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via @@ -139,6 +149,8 @@ responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -157,7 +169,7 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests diff --git a/requirements.txt b/requirements.txt index 7e81bf8124..739fd6efb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,8 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.12 @@ -32,6 +34,8 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit +cryptography==36.0.2 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -44,9 +48,9 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.1 +flyteidl==0.24.0 # via flytekit -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit @@ -54,6 +58,10 @@ idna==3.3 # via requests importlib-metadata==4.11.3 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -104,6 +112,8 @@ py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi pyparsing==3.0.7 # via packaging pyrsistent==0.18.1 @@ -126,7 +136,7 @@ pytz==2021.3 # pandas pyyaml==5.4.1 # via -r requirements.in -regex==2022.3.2 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via @@ -137,6 +147,8 @@ responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -155,7 +167,7 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests diff --git a/setup.py b/setup.py index e8767e8c48..69e2c8e340 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ ] }, install_requires=[ - "flyteidl>=0.23.1", + "flyteidl>=0.24.0", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=4.0.0,<7.0.0", diff --git a/tests/flytekit/common/parameterizers.py b/tests/flytekit/common/parameterizers.py index 27bd3ea1a7..4b48fbcfb9 100644 --- a/tests/flytekit/common/parameterizers.py +++ b/tests/flytekit/common/parameterizers.py @@ -51,6 +51,14 @@ dimensionality=_core_types.BlobType.BlobDimensionality.MULTIPART, ) ), + types.LiteralType( + union_type=types.UnionType( + variants=[ + types.LiteralType(simple=types.SimpleType.STRING, structure=types.TypeStructure(tag="str")), + types.LiteralType(simple=types.SimpleType.INTEGER, structure=types.TypeStructure(tag="int")), + ] + ) + ), ] @@ -170,6 +178,46 @@ timedelta(seconds=5), ), (literals.Scalar(none_type=literals.Void()), None), + ( + literals.Scalar( + union=literals.Union( + value=literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=10))), + stored_type=types.LiteralType( + simple=types.SimpleType.INTEGER, structure=types.TypeStructure(tag="int") + ), + ) + ), + 10, + ), + ( + literals.Scalar( + union=literals.Union( + value=literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=10))), + stored_type=types.LiteralType( + simple=types.SimpleType.INTEGER, structure=types.TypeStructure(tag="int") + ), + ) + ), + 10, + ), + ( + literals.Scalar( + union=literals.Union( + value=literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(string_value="test"))), + stored_type=types.LiteralType(simple=types.SimpleType.STRING, structure=types.TypeStructure(tag="str")), + ) + ), + "test", + ), + ( + literals.Scalar( + union=literals.Union( + value=literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(string_value="test"))), + stored_type=types.LiteralType(simple=types.SimpleType.STRING, structure=types.TypeStructure(tag="str")), + ) + ), + "test", + ), ] LIST_OF_SCALAR_LITERALS_AND_PYTHON_VALUE = [ diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 15a1f89700..cf110244fa 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -28,7 +28,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit -cryptography==36.0.1 +cryptography==36.0.2 # via secretstorage cycler==0.11.0 # via matplotlib @@ -44,19 +44,19 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.23.0 +flyteidl==0.24.0 # via flytekit -flytekit==0.30.3 +flytekit==0.31.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -fonttools==4.29.1 +fonttools==4.30.0 # via matplotlib -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.11.3 # via keyring jeepney==0.7.1 # via @@ -72,11 +72,11 @@ joblib==1.1.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in keyring==23.5.0 # via flytekit -kiwisolver==1.3.2 +kiwisolver==1.4.0 # via matplotlib -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -91,16 +91,18 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.22.3 # via # matplotlib # opencv-python # pandas # pyarrow -opencv-python==4.5.5.62 +opencv-python==4.5.5.64 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in packaging==21.3 - # via matplotlib + # via + # marshmallow + # matplotlib pandas==1.4.1 # via flytekit pillow==9.0.1 @@ -142,14 +144,14 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.19.0 # via flytekit retry==0.9.2 # via flytekit @@ -172,7 +174,7 @@ typing-extensions==4.1.1 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 +urllib3==1.26.9 # via # flytekit # requests @@ -181,7 +183,7 @@ wheel==0.37.1 # via # -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in # flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # deprecated # flytekit diff --git a/tests/flytekit/unit/core/test_are_types_castable.py b/tests/flytekit/unit/core/test_are_types_castable.py new file mode 100644 index 0000000000..2f7edcdf4c --- /dev/null +++ b/tests/flytekit/unit/core/test_are_types_castable.py @@ -0,0 +1,138 @@ +from flytekit.core.type_engine import _are_types_castable +from flytekit.models.annotation import TypeAnnotation +from flytekit.models.core.types import EnumType +from flytekit.models.types import LiteralType, SimpleType, StructuredDatasetType, TypeStructure, UnionType + +str_type = LiteralType(simple=SimpleType.STRING) +int_type = LiteralType(simple=SimpleType.INTEGER) +none_type = LiteralType(simple=SimpleType.NONE) +bool_type = LiteralType(simple=SimpleType.BOOLEAN) + +str_or_int = LiteralType(union_type=UnionType([str_type, int_type])) +int_or_str = LiteralType(union_type=UnionType([int_type, str_type])) +str_or_int_or_bool = LiteralType(union_type=UnionType([str_type, int_type, bool_type])) +optional_str = LiteralType(union_type=UnionType([str_type, none_type])) + + +def test_simple(): + assert _are_types_castable(str_type, str_type) + assert not _are_types_castable(str_type, int_type) + assert not _are_types_castable(int_type, str_type) + + +def test_metadata(): + a = LiteralType(simple=SimpleType.STRING, metadata={"test": 456}) + assert _are_types_castable( + a, + LiteralType(simple=SimpleType.STRING, metadata={"test": 123}), + ) + # must not clobber metadata + assert a.metadata == {"test": 456} + + +def test_annotation(): + a = LiteralType(simple=SimpleType.STRING, annotation=TypeAnnotation(annotations={"test": 456})) + assert _are_types_castable( + a, + LiteralType(simple=SimpleType.STRING, annotation=TypeAnnotation(annotations={"test": 123})), + ) + # must not clobber annotation + assert a.annotation.annotations == {"test": 456} + + +def test_structure(): + a = LiteralType(simple=SimpleType.STRING, structure=TypeStructure(tag="a")) + assert _are_types_castable( + a, + LiteralType(simple=SimpleType.STRING, structure=TypeStructure(tag="b")), + ) + # must not clobber annotation + assert a.structure.tag == "a" + + +def test_non_nullable(): + assert not _are_types_castable(none_type, str_type) + assert not _are_types_castable(str_type, none_type) + assert _are_types_castable(none_type, none_type) + + +def test_collection(): + assert _are_types_castable(LiteralType(collection_type=str_type), LiteralType(collection_type=str_type)) + assert not _are_types_castable(LiteralType(collection_type=str_type), LiteralType(collection_type=int_type)) + + +def test_map(): + assert _are_types_castable(LiteralType(map_value_type=str_type), LiteralType(map_value_type=str_type)) + assert not _are_types_castable(LiteralType(map_value_type=str_type), LiteralType(map_value_type=int_type)) + + +def test_structured_dataset(): + x = LiteralType( + structured_dataset_type=StructuredDatasetType( + columns=[ + StructuredDatasetType.DatasetColumn("a", str_type), + StructuredDatasetType.DatasetColumn("b", int_type), + ], + format="abc", + external_schema_type="zzz", + external_schema_bytes=b"zzz", + ) + ) + assert _are_types_castable(x, x) + + +def test_enum(): + e = LiteralType(enum_type=EnumType(["a", "b"])) + # enum is a str + assert _are_types_castable(e, str_type) + # a str is not necessarily an enum + assert not _are_types_castable(str_type, e) + + +def test_union(): + # str can be expanded to str | int + assert _are_types_castable(str_type, str_or_int) + # str can be expanded to int | str + assert _are_types_castable(str_type, int_or_str) + # str | int cannot be narrowed to str + assert not _are_types_castable(str_or_int, str_type) + # str | int == str | int + assert _are_types_castable(str_or_int, str_or_int) + # int | str == str | int + assert _are_types_castable(int_or_str, str_or_int) + # str is Optional[str] + assert _are_types_castable(str_type, optional_str) + # None is Optional[str] + assert _are_types_castable(none_type, optional_str) + # bool is not Optional[str] + assert not _are_types_castable(bool_type, optional_str) + + +def test_collection_union(): + # a list of str is a list of (str | int) + assert _are_types_castable(LiteralType(collection_type=str_type), LiteralType(collection_type=str_or_int)) + # a list of int is a list of (str | int) + assert _are_types_castable(LiteralType(collection_type=int_type), LiteralType(collection_type=str_or_int)) + # a list of str or a list of int is a list of (str | int) + assert _are_types_castable( + LiteralType( + union_type=UnionType([LiteralType(collection_type=int_type), LiteralType(collection_type=str_type)]) + ), + LiteralType(collection_type=str_or_int), + ) + assert _are_types_castable( + LiteralType( + union_type=UnionType([LiteralType(collection_type=int_type), LiteralType(collection_type=str_type)]) + ), + LiteralType(collection_type=str_or_int_or_bool), + ) + # a list of str or a list of bool is not a list of (str | int) + assert not _are_types_castable( + LiteralType( + union_type=UnionType([LiteralType(collection_type=int_type), LiteralType(collection_type=bool_type)]) + ), + LiteralType(collection_type=str_or_int), + ) + # not the other way around + assert not _are_types_castable(LiteralType(collection_type=str_or_int), LiteralType(collection_type=str_type)) + assert not _are_types_castable(LiteralType(collection_type=str_or_int), LiteralType(collection_type=int_type)) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 4414ad1de5..cbfa7f6d03 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -33,6 +33,8 @@ LiteralsResolver, SimpleTransformer, TypeEngine, + TypeTransformer, + TypeTransformerFailedError, convert_json_schema_to_python_class, dataclass_from_dict, ) @@ -40,8 +42,8 @@ from flytekit.models import types as model_types from flytekit.models.annotation import TypeAnnotation from flytekit.models.core.types import BlobType -from flytekit.models.literals import Blob, BlobMetadata, Literal, LiteralCollection, LiteralMap, Primitive, Scalar -from flytekit.models.types import LiteralType, SimpleType +from flytekit.models.literals import Blob, BlobMetadata, Literal, LiteralCollection, LiteralMap, Primitive, Scalar, Void +from flytekit.models.types import LiteralType, SimpleType, TypeStructure from flytekit.types.directory import TensorboardLogs from flytekit.types.directory.types import FlyteDirectory from flytekit.types.file import JPEGImageFile @@ -52,6 +54,8 @@ from flytekit.types.schema.types_pandas import PandasDataFrameTransformer from flytekit.types.structured.structured_dataset import StructuredDataset +T = typing.TypeVar("T") + def test_type_engine(): t = int @@ -368,7 +372,7 @@ def test_guessing_basic(): lt = model_types.LiteralType(simple=model_types.SimpleType.NONE) pt = TypeEngine.guess_python_type(lt) - assert pt is None + assert pt is type(None) # noqa: E721 lt = model_types.LiteralType( blob=BlobType( @@ -714,6 +718,283 @@ def test_enum_type(): TypeEngine.to_literal_type(UnsupportedEnumValues) +def union_type_tags_unique(t: LiteralType): + seen = set() + for x in t.union_type.variants: + if x.structure.tag in seen: + return False + seen.add(x.structure.tag) + + return True + + +def test_union_type(): + pt = typing.Union[str, int] + lt = TypeEngine.to_literal_type(pt) + assert lt.union_type.variants == [ + LiteralType(simple=SimpleType.STRING, structure=TypeStructure(tag="str")), + LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="int")), + ] + assert union_type_tags_unique(lt) + + ctx = FlyteContextManager.current_context() + lv = TypeEngine.to_literal(ctx, 3, pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "int" + assert lv.scalar.union.value.scalar.primitive.integer == 3 + assert v == 3 + + lv = TypeEngine.to_literal(ctx, "hello", pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "str" + assert lv.scalar.union.value.scalar.primitive.string_value == "hello" + assert v == "hello" + + +def test_union_type_with_annotated(): + pt = typing.Union[ + Annotated[str, FlyteAnnotation({"hello": "world"})], Annotated[int, FlyteAnnotation({"test": 123})] + ] + lt = TypeEngine.to_literal_type(pt) + assert lt.union_type.variants == [ + LiteralType( + simple=SimpleType.STRING, structure=TypeStructure(tag="str"), annotation=TypeAnnotation({"hello": "world"}) + ), + LiteralType( + simple=SimpleType.INTEGER, structure=TypeStructure(tag="int"), annotation=TypeAnnotation({"test": 123}) + ), + ] + assert union_type_tags_unique(lt) + + ctx = FlyteContextManager.current_context() + lv = TypeEngine.to_literal(ctx, 3, pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "int" + assert lv.scalar.union.value.scalar.primitive.integer == 3 + assert v == 3 + + lv = TypeEngine.to_literal(ctx, "hello", pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "str" + assert lv.scalar.union.value.scalar.primitive.string_value == "hello" + assert v == "hello" + + +def test_annotated_union_type(): + pt = Annotated[typing.Union[str, int], FlyteAnnotation({"hello": "world"})] + lt = TypeEngine.to_literal_type(pt) + assert lt.union_type.variants == [ + LiteralType(simple=SimpleType.STRING, structure=TypeStructure(tag="str")), + LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="int")), + ] + assert lt.annotation == TypeAnnotation({"hello": "world"}) + assert union_type_tags_unique(lt) + + ctx = FlyteContextManager.current_context() + lv = TypeEngine.to_literal(ctx, 3, pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "int" + assert lv.scalar.union.value.scalar.primitive.integer == 3 + assert v == 3 + + lv = TypeEngine.to_literal(ctx, "hello", pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "str" + assert lv.scalar.union.value.scalar.primitive.string_value == "hello" + assert v == "hello" + + +def test_optional_type(): + pt = typing.Optional[int] + lt = TypeEngine.to_literal_type(pt) + assert lt.union_type.variants == [ + LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="int")), + LiteralType(simple=SimpleType.NONE, structure=TypeStructure(tag="none")), + ] + assert union_type_tags_unique(lt) + + ctx = FlyteContextManager.current_context() + lv = TypeEngine.to_literal(ctx, 3, pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "int" + assert lv.scalar.union.value.scalar.primitive.integer == 3 + assert v == 3 + + lv = TypeEngine.to_literal(ctx, None, pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "none" + assert lv.scalar.union.value.scalar.none_type == Void() + assert v is None + + +def test_union_from_unambiguous_literal(): + pt = typing.Union[str, int] + lt = TypeEngine.to_literal_type(pt) + assert lt.union_type.variants == [ + LiteralType(simple=SimpleType.STRING, structure=TypeStructure(tag="str")), + LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="int")), + ] + assert union_type_tags_unique(lt) + + ctx = FlyteContextManager.current_context() + lv = TypeEngine.to_literal(ctx, 3, int, LiteralType(simple=SimpleType.INTEGER)) + assert lv.scalar.primitive.integer == 3 + + v = TypeEngine.to_python_value(ctx, lv, pt) + assert v == 3 + + +def test_union_custom_transformer(): + class MyInt: + def __init__(self, x: int): + self.val = x + + def __eq__(self, other): + if not isinstance(other, MyInt): + return False + return other.val == self.val + + TypeEngine.register( + SimpleTransformer( + "MyInt", + MyInt, + LiteralType(simple=SimpleType.INTEGER), + lambda x: Literal(scalar=Scalar(primitive=Primitive(integer=x.val))), + lambda x: MyInt(x.scalar.primitive.integer), + ) + ) + + pt = typing.Union[int, MyInt] + lt = TypeEngine.to_literal_type(pt) + assert lt.union_type.variants == [ + LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="int")), + LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="MyInt")), + ] + assert union_type_tags_unique(lt) + + ctx = FlyteContextManager.current_context() + lv = TypeEngine.to_literal(ctx, 3, pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "int" + assert lv.scalar.union.value.scalar.primitive.integer == 3 + assert v == 3 + + lv = TypeEngine.to_literal(ctx, MyInt(10), pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "MyInt" + assert lv.scalar.union.value.scalar.primitive.integer == 10 + assert v == MyInt(10) + + lv = TypeEngine.to_literal(ctx, 4, int, LiteralType(simple=SimpleType.INTEGER)) + assert lv.scalar.primitive.integer == 4 + try: + TypeEngine.to_python_value(ctx, lv, pt) + except TypeError as e: + assert "Ambiguous choice of variant" in str(e) + + del TypeEngine._REGISTRY[MyInt] + + +def test_union_custom_transformer_sanity_check(): + class UnsignedInt: + def __init__(self, x: int): + self.val = x + + def __eq__(self, other): + if not isinstance(other, UnsignedInt): + return False + return other.val == self.val + + # This transformer will not work in the implicit wrapping case + class UnsignedIntTransformer(TypeTransformer[UnsignedInt]): + def __init__(self): + super().__init__("UnsignedInt", UnsignedInt) + + def get_literal_type(self, t: typing.Type[T]) -> LiteralType: + return LiteralType(simple=SimpleType.INTEGER) + + def to_literal( + self, ctx: FlyteContext, python_val: T, python_type: typing.Type[T], expected: LiteralType + ) -> Literal: + if type(python_val) != int: + raise TypeTransformerFailedError("Expected an integer") + + if python_val < 0: + raise TypeTransformerFailedError("Expected a non-negative integer") + + return Literal(scalar=Scalar(primitive=Primitive(integer=python_val))) + + def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: typing.Type[T]) -> T: + val = lv.scalar.primitive.integer + return UnsignedInt(0 if val < 0 else val) + + TypeEngine.register(UnsignedIntTransformer()) + + pt = typing.Union[int, UnsignedInt] + lt = TypeEngine.to_literal_type(pt) + assert lt.union_type.variants == [ + LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="int")), + LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="UnsignedInt")), + ] + assert union_type_tags_unique(lt) + + ctx = FlyteContextManager.current_context() + with pytest.raises(TypeError, match="Ambiguous choice of variant for union type"): + TypeEngine.to_literal(ctx, 3, pt, lt) + + del TypeEngine._REGISTRY[UnsignedInt] + + +def test_union_of_lists(): + pt = typing.Union[typing.List[int], typing.List[str]] + lt = TypeEngine.to_literal_type(pt) + assert lt.union_type.variants == [ + LiteralType( + collection_type=LiteralType(simple=SimpleType.INTEGER), + structure=TypeStructure(tag="Typed List"), + ), + LiteralType( + collection_type=LiteralType(simple=SimpleType.STRING), + structure=TypeStructure(tag="Typed List"), + ), + ] + # Tags are deliberately NOT unique beacuse they are not required to encode the deep type structure, + # only the top-level type transformer choice + # + # The stored typed will be used to differentiate union variants and must produce a unique choice. + assert not union_type_tags_unique(lt) + + ctx = FlyteContextManager.current_context() + lv = TypeEngine.to_literal(ctx, ["hello", "world"], pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "Typed List" + assert [x.scalar.primitive.string_value for x in lv.scalar.union.value.collection.literals] == ["hello", "world"] + assert v == ["hello", "world"] + + lv = TypeEngine.to_literal(ctx, [1, 3], pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert lv.scalar.union.stored_type.structure.tag == "Typed List" + assert [x.scalar.primitive.integer for x in lv.scalar.union.value.collection.literals] == [1, 3] + assert v == [1, 3] + + +def test_list_of_unions(): + pt = typing.List[typing.Union[str, int]] + lt = TypeEngine.to_literal_type(pt) + # todo(maximsmol): seems like the order here is non-deterministic + assert lt.collection_type.union_type.variants == [ + LiteralType(simple=SimpleType.STRING, structure=TypeStructure(tag="str")), + LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="int")), + ] + assert union_type_tags_unique(lt.collection_type) # tags are deliberately NOT unique + + ctx = FlyteContextManager.current_context() + lv = TypeEngine.to_literal(ctx, ["hello", 123, "world"], pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert [x.scalar.union.stored_type.structure.tag for x in lv.collection.literals] == ["str", "int", "str"] + assert v == ["hello", 123, "world"] + + def test_pickle_type(): class Foo(object): def __init__(self, number: int): @@ -844,6 +1125,20 @@ def test_dict_to_literal_map_with_wrong_input_type(): TypeEngine.dict_to_literal_map(ctx, input, guessed_python_types) +def test_nested_annotated(): + """ + Test to show that nested Annotated types are flattened. + """ + pt = Annotated[Annotated[int, "inner-annotation"], "outer-annotation"] + lt = TypeEngine.to_literal_type(pt) + assert lt.simple == model_types.SimpleType.INTEGER + + ctx = FlyteContextManager.current_context() + lv = TypeEngine.to_literal(ctx, 42, pt, lt) + v = TypeEngine.to_python_value(ctx, lv, pt) + assert v == 42 + + def test_pass_annotated_to_downstream_tasks(): """ Test to confirm that the loaded dataframe is not affected and can be used in @dynamic. diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index 4ff531562e..ee4bc72756 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -3,10 +3,12 @@ import functools import os import random +import tempfile import typing from collections import OrderedDict from dataclasses import dataclass from enum import Enum +from textwrap import dedent import pandas import pandas as pd @@ -30,7 +32,7 @@ from flytekit.core.resources import Resources from flytekit.core.task import TaskMetadata, task from flytekit.core.testing import patch, task_mock -from flytekit.core.type_engine import RestrictedTypeError, TypeEngine +from flytekit.core.type_engine import RestrictedTypeError, SimpleTransformer, TypeEngine from flytekit.core.workflow import workflow from flytekit.models import literals as _literal_models from flytekit.models.core import types as _core_types @@ -1605,6 +1607,185 @@ def foo3(a: typing.Dict) -> typing.Dict: foo3(a=[{"hello": 2}]) +def test_union_type(): + ut = typing.Union[int, str, float, FlyteFile, FlyteSchema, typing.List[int], typing.Dict[str, int]] + + @task + def t1(a: ut) -> ut: + return a + + @workflow + def wf(a: ut) -> ut: + return t1(a=a) + + assert wf(a=2) == 2 + assert wf(a="2") == "2" + assert wf(a=2.0) == 2.0 + file = tempfile.NamedTemporaryFile(delete=False) + assert isinstance(wf(a=FlyteFile(file.name)), FlyteFile) + assert isinstance(wf(a=FlyteSchema()), FlyteSchema) + assert wf(a=[1, 2, 3]) == [1, 2, 3] + assert wf(a={"a": 1}) == {"a": 1} + + @task + def t2(a: typing.Union[float, dict]) -> typing.Union[float, dict]: + return a + + @workflow + def wf2(a: typing.Union[int, str]) -> typing.Union[int, str]: + return t2(a=a) + + with pytest.raises( + TypeError, + match=dedent( + r""" + Cannot convert from scalar { + union { + value { + scalar { + primitive { + string_value: "2" + } + } + } + type { + simple: STRING + structure { + tag: "str" + } + } + } + } + to typing.Union\[float, dict\] \(using tag str\) + """ + )[1:-1], + ): + assert wf2(a="2") == "2" + + +def test_optional_type(): + @task + def t1(a: typing.Optional[int]) -> typing.Optional[int]: + return a + + @workflow + def wf(a: typing.Optional[int]) -> typing.Optional[int]: + return t1(a=a) + + assert wf(a=2) == 2 + assert wf(a=None) is None + + +def test_optional_type_implicit_wrapping(): + @task + def t1(a: int) -> typing.Optional[int]: + return a if a > 0 else None + + @workflow + def wf(a: int) -> typing.Optional[int]: + return t1(a=a) + + assert wf(a=2) == 2 + assert wf(a=-10) is None + + +def test_union_type_implicit_wrapping(): + @task + def t1(a: int) -> typing.Union[int, str]: + return a if a > 0 else str(a) + + @workflow + def wf(a: int) -> typing.Union[int, str]: + return t1(a=a) + + assert wf(a=2) == 2 + assert wf(a=-10) == "-10" + + +def test_union_type_ambiguity_checking(): + class MyInt: + def __init__(self, x: int): + self.val = x + + def __eq__(self, other): + if not isinstance(other, MyInt): + return False + return other.val == self.val + + TypeEngine.register( + SimpleTransformer( + "MyInt", + MyInt, + LiteralType(simple=SimpleType.INTEGER), + lambda x: _literal_models.Literal( + scalar=_literal_models.Scalar(primitive=_literal_models.Primitive(integer=x.val)) + ), + lambda x: MyInt(x.scalar.primitive.integer), + ) + ) + + @task + def t1(a: typing.Union[int, MyInt]) -> int: + if isinstance(a, MyInt): + return a.val + return a + + @workflow + def wf(a: int) -> int: + return t1(a=a) + + with pytest.raises( + TypeError, match="Ambiguous choice of variant for union type. Both int and MyInt transformers match" + ): + assert wf(a=10) == 10 + + del TypeEngine._REGISTRY[MyInt] + + +def test_union_type_ambiguity_resolution(): + class MyInt: + def __init__(self, x: int): + self.val = x + + def __eq__(self, other): + if not isinstance(other, MyInt): + return False + return other.val == self.val + + TypeEngine.register( + SimpleTransformer( + "MyInt", + MyInt, + LiteralType(simple=SimpleType.INTEGER), + lambda x: _literal_models.Literal( + scalar=_literal_models.Scalar(primitive=_literal_models.Primitive(integer=x.val)) + ), + lambda x: MyInt(x.scalar.primitive.integer), + ) + ) + + @task + def t1(a: typing.Union[int, MyInt]) -> str: + if isinstance(a, MyInt): + return f"MyInt {str(a.val)}" + return str(a) + + @task + def t2(a: int) -> typing.Union[int, MyInt]: + if a < 0: + return MyInt(a) + return a + + @workflow + def wf(a: int) -> str: + return t1(a=t2(a=a)) + + assert wf(a=10) == "10" + assert wf(a=-10) == "MyInt -10" + + del TypeEngine._REGISTRY[MyInt] + + def test_task_annotate_primitive_type_has_no_effect(): @task def plus_two(