From 8fff9db523d0d7a82fe2a4ae182ad4ef16b5ea28 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 17 Jan 2022 14:17:12 +0800 Subject: [PATCH 1/5] Fixed flytekit-papermill ImportError Signed-off-by: Kevin Su --- plugins/flytekit-spark/flytekitplugins/spark/task.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/flytekit-spark/flytekitplugins/spark/task.py b/plugins/flytekit-spark/flytekitplugins/spark/task.py index 37ae03e913..7129ff2578 100644 --- a/plugins/flytekit-spark/flytekitplugins/spark/task.py +++ b/plugins/flytekit-spark/flytekitplugins/spark/task.py @@ -7,7 +7,11 @@ from pyspark.sql import SparkSession from flytekit import FlyteContextManager, PythonFunctionTask -from flytekit.core.context_manager import ExecutionParameters + +try: + from flytekit.core.context_manager import ExecutionParameters +except ImportError: + from flytekit.common.tasks.sdk_runnable import ExecutionParameters from flytekit.extend import ExecutionState, SerializationSettings, TaskPlugins from .models import SparkJob, SparkType From fc5b31ecfdc9f81c711c4407b6e2b8f5730bbf9e Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 17 Jan 2022 14:51:35 +0800 Subject: [PATCH 2/5] Fixed flytekit-papermill ImportError Signed-off-by: Kevin Su --- plugins/flytekit-papermill/requirements.in | 2 +- plugins/flytekit-papermill/requirements.txt | 20 +++++-------------- .../flytekitplugins/spark/task.py | 6 +----- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/plugins/flytekit-papermill/requirements.in b/plugins/flytekit-papermill/requirements.in index 668a9fe9fa..96daef6f78 100644 --- a/plugins/flytekit-papermill/requirements.in +++ b/plugins/flytekit-papermill/requirements.in @@ -1,3 +1,3 @@ . -e file:.#egg=flytekitplugins-papermill -flytekitplugins-spark +git+https://github.com/flyteorg/flytekit@master#subdirectory=plugins/flytekit-papermill diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index c31e0824bf..cef0f3c213 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -8,6 +8,10 @@ # via -r requirements.in ansiwrap==0.8.4 # via papermill +appnope==0.1.2 + # via + # ipykernel + # ipython arrow==1.2.1 # via jinja2-time attrs==21.2.0 @@ -22,8 +26,6 @@ bleach==4.1.0 # via nbconvert certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.7 @@ -42,8 +44,6 @@ cookiecutter==1.7.3 # via flytekit croniter==1.0.15 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit debugpy==1.5.1 @@ -74,9 +74,7 @@ flytekit==0.24.0 # flytekitplugins-papermill # flytekitplugins-spark flytekitplugins-spark==0.24.0 - # via - # -r requirements.in - # flytekitplugins-papermill + # via flytekitplugins-papermill grpcio==1.41.1 # via flytekit idna==3.3 @@ -91,10 +89,6 @@ ipython-genutils==0.2.0 # via nbformat jedi==0.18.0 # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -193,8 +187,6 @@ py4j==0.10.9.2 # via pyspark pyarrow==6.0.0 # via flytekit -pycparser==2.21 - # via cffi pygments==2.10.0 # via # ipython @@ -241,8 +233,6 @@ responses==0.15.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # bleach diff --git a/plugins/flytekit-spark/flytekitplugins/spark/task.py b/plugins/flytekit-spark/flytekitplugins/spark/task.py index 7129ff2578..37ae03e913 100644 --- a/plugins/flytekit-spark/flytekitplugins/spark/task.py +++ b/plugins/flytekit-spark/flytekitplugins/spark/task.py @@ -7,11 +7,7 @@ from pyspark.sql import SparkSession from flytekit import FlyteContextManager, PythonFunctionTask - -try: - from flytekit.core.context_manager import ExecutionParameters -except ImportError: - from flytekit.common.tasks.sdk_runnable import ExecutionParameters +from flytekit.core.context_manager import ExecutionParameters from flytekit.extend import ExecutionState, SerializationSettings, TaskPlugins from .models import SparkJob, SparkType From 917a01eb5f57b13e44454e6b1141c85c322fb7d6 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 17 Jan 2022 14:57:54 +0800 Subject: [PATCH 3/5] Updated requirement Signed-off-by: Kevin Su --- plugins/flytekit-papermill/requirements.in | 2 +- plugins/flytekit-papermill/requirements.txt | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/plugins/flytekit-papermill/requirements.in b/plugins/flytekit-papermill/requirements.in index 96daef6f78..68367ec6e6 100644 --- a/plugins/flytekit-papermill/requirements.in +++ b/plugins/flytekit-papermill/requirements.in @@ -1,3 +1,3 @@ . -e file:.#egg=flytekitplugins-papermill -git+https://github.com/flyteorg/flytekit@master#subdirectory=plugins/flytekit-papermill +git+https://github.com/flyteorg/flytekit@master#subdirectory=plugins/flytekit-spark diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index cef0f3c213..4bca5fd010 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -70,11 +70,11 @@ entrypoints==0.3 flyteidl==0.21.8 # via flytekit flytekit==0.24.0 + # via flytekitplugins-papermill +flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@master#subdirectory=plugins/flytekit-spark # via + # -r requirements.in # flytekitplugins-papermill - # flytekitplugins-spark -flytekitplugins-spark==0.24.0 - # via flytekitplugins-papermill grpcio==1.41.1 # via flytekit idna==3.3 @@ -183,8 +183,6 @@ ptyprocess==0.7.0 # via pexpect py==1.11.0 # via retry -py4j==0.10.9.2 - # via pyspark pyarrow==6.0.0 # via flytekit pygments==2.10.0 @@ -196,8 +194,6 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.18.0 # via jsonschema -pyspark==3.2.0 - # via flytekitplugins-spark python-dateutil==2.8.1 # via # arrow From 9de18d3dc2f8839bea1c49c74c329bd6b73cb985 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 18 Jan 2022 01:00:08 +0800 Subject: [PATCH 4/5] Updated requirement Signed-off-by: Kevin Su --- plugins/flytekit-papermill/requirements.in | 1 - plugins/flytekit-papermill/requirements.txt | 15 +++++++++------ plugins/flytekit-papermill/setup.py | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/plugins/flytekit-papermill/requirements.in b/plugins/flytekit-papermill/requirements.in index 68367ec6e6..a9dab93776 100644 --- a/plugins/flytekit-papermill/requirements.in +++ b/plugins/flytekit-papermill/requirements.in @@ -1,3 +1,2 @@ . -e file:.#egg=flytekitplugins-papermill -git+https://github.com/flyteorg/flytekit@master#subdirectory=plugins/flytekit-spark diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 4bca5fd010..4f78d91bbc 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -67,14 +67,14 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.8 +flyteidl==0.21.23 # via flytekit -flytekit==0.24.0 - # via flytekitplugins-papermill -flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@master#subdirectory=plugins/flytekit-spark +flytekit==0.30.0b4 # via - # -r requirements.in # flytekitplugins-papermill + # flytekitplugins-spark +flytekitplugins-spark==0.30.0b4 + # via flytekitplugins-papermill grpcio==1.41.1 # via flytekit idna==3.3 @@ -183,6 +183,8 @@ ptyprocess==0.7.0 # via pexpect py==1.11.0 # via retry +py4j==0.10.9.2 + # via pyspark pyarrow==6.0.0 # via flytekit pygments==2.10.0 @@ -194,6 +196,8 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.18.0 # via jsonschema +pyspark==3.2.0 + # via flytekitplugins-spark python-dateutil==2.8.1 # via # arrow @@ -233,7 +237,6 @@ six==1.16.0 # via # bleach # cookiecutter - # flytekit # grpcio # python-dateutil # responses diff --git a/plugins/flytekit-papermill/setup.py b/plugins/flytekit-papermill/setup.py index 4fa4943302..0c3b87a66e 100644 --- a/plugins/flytekit-papermill/setup.py +++ b/plugins/flytekit-papermill/setup.py @@ -5,8 +5,8 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" plugin_requires = [ - "flytekit>=0.16.0b0,<1.0.0", - "flytekitplugins-spark>=0.16.0b0,<1.0.0,!=0.24.0b0", + "flytekit>=0.30.0b4,<1.0.0", + "flytekitplugins-spark>=0.30.0b4,<1.0.0", "papermill>=1.2.0", "nbconvert>=6.0.7", "ipykernel>=5.0.0", From 9437d5ca89ee04a0bf40b1a5f4325ba9ee101fc5 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 18 Jan 2022 02:01:56 +0800 Subject: [PATCH 5/5] Added dev-requirements Signed-off-by: Kevin Su --- .github/workflows/pythonbuild.yml | 1 + .../flytekit-papermill/dev-requirements.in | 1 + .../flytekit-papermill/dev-requirements.txt | 152 ++++++++++++++++++ plugins/flytekit-papermill/requirements.txt | 11 +- plugins/flytekit-papermill/setup.py | 3 +- 5 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 plugins/flytekit-papermill/dev-requirements.in create mode 100644 plugins/flytekit-papermill/dev-requirements.txt diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index ce4b272426..ee688d4fe5 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -100,6 +100,7 @@ jobs: make setup cd plugins/${{ matrix.plugin-names }} pip install -e . + if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit pip freeze - name: Test with coverage diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in new file mode 100644 index 0000000000..8bf53e2636 --- /dev/null +++ b/plugins/flytekit-papermill/dev-requirements.in @@ -0,0 +1 @@ +flytekitplugins-spark>=0.30.0b4 diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt new file mode 100644 index 0000000000..849cef4c9f --- /dev/null +++ b/plugins/flytekit-papermill/dev-requirements.txt @@ -0,0 +1,152 @@ +# +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: +# +# pip-compile dev-requirements.in +# +arrow==1.2.1 + # via jinja2-time +binaryornot==0.4.4 + # via cookiecutter +certifi==2021.10.8 + # via requests +chardet==4.0.0 + # via binaryornot +charset-normalizer==2.0.10 + # via requests +checksumdir==1.2.0 + # via flytekit +click==7.1.2 + # via + # cookiecutter + # flytekit +cloudpickle==2.0.0 + # via flytekit +cookiecutter==1.7.3 + # via flytekit +croniter==1.2.0 + # via flytekit +dataclasses-json==0.5.6 + # via flytekit +decorator==5.1.1 + # via retry +deprecated==1.2.13 + # via flytekit +diskcache==5.4.0 + # via flytekit +docker-image-py==0.1.12 + # via flytekit +docstring-parser==0.13 + # via flytekit +flyteidl==0.21.23 + # via flytekit +flytekit==0.26.0 + # via flytekitplugins-spark +flytekitplugins-spark==0.30.0b4 + # via -r dev-requirements.in +grpcio==1.43.0 + # via flytekit +idna==3.3 + # via requests +importlib-metadata==4.10.1 + # via keyring +jinja2==3.0.3 + # via + # cookiecutter + # jinja2-time +jinja2-time==0.2.0 + # via cookiecutter +keyring==23.5.0 + # via flytekit +markupsafe==2.0.1 + # via jinja2 +marshmallow==3.14.1 + # via + # dataclasses-json + # marshmallow-enum + # marshmallow-jsonschema +marshmallow-enum==1.5.1 + # via dataclasses-json +marshmallow-jsonschema==0.13.0 + # via flytekit +mypy-extensions==0.4.3 + # via typing-inspect +natsort==8.0.2 + # via flytekit +numpy==1.22.1 + # via + # pandas + # pyarrow +pandas==1.3.5 + # via flytekit +poyo==0.5.0 + # via cookiecutter +protobuf==3.19.3 + # via + # flyteidl + # flytekit +py==1.11.0 + # via retry +py4j==0.10.9.2 + # via pyspark +pyarrow==6.0.1 + # via flytekit +pyspark==3.2.0 + # via flytekitplugins-spark +python-dateutil==2.8.1 + # via + # arrow + # croniter + # flytekit + # pandas +python-json-logger==2.0.2 + # via flytekit +python-slugify==5.0.2 + # via cookiecutter +pytimeparse==1.1.8 + # via flytekit +pytz==2021.3 + # via + # flytekit + # pandas +regex==2021.11.10 + # via docker-image-py +requests==2.27.1 + # via + # cookiecutter + # flytekit + # responses +responses==0.17.0 + # via flytekit +retry==0.9.2 + # via flytekit +six==1.16.0 + # via + # cookiecutter + # flytekit + # grpcio + # python-dateutil + # responses +sortedcontainers==2.4.0 + # via flytekit +statsd==3.3.0 + # via flytekit +text-unidecode==1.3 + # via python-slugify +typing-extensions==4.0.1 + # via typing-inspect +typing-inspect==0.7.1 + # via dataclasses-json +urllib3==1.26.8 + # via + # flytekit + # requests + # responses +wheel==0.37.1 + # via flytekit +wrapt==1.13.3 + # via + # deprecated + # flytekit +zipp==3.7.0 + # via importlib-metadata diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 4f78d91bbc..9c5a82d050 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -69,11 +69,7 @@ entrypoints==0.3 # papermill flyteidl==0.21.23 # via flytekit -flytekit==0.30.0b4 - # via - # flytekitplugins-papermill - # flytekitplugins-spark -flytekitplugins-spark==0.30.0b4 +flytekit==0.24.0 # via flytekitplugins-papermill grpcio==1.41.1 # via flytekit @@ -183,8 +179,6 @@ ptyprocess==0.7.0 # via pexpect py==1.11.0 # via retry -py4j==0.10.9.2 - # via pyspark pyarrow==6.0.0 # via flytekit pygments==2.10.0 @@ -196,8 +190,6 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.18.0 # via jsonschema -pyspark==3.2.0 - # via flytekitplugins-spark python-dateutil==2.8.1 # via # arrow @@ -237,6 +229,7 @@ six==1.16.0 # via # bleach # cookiecutter + # flytekit # grpcio # python-dateutil # responses diff --git a/plugins/flytekit-papermill/setup.py b/plugins/flytekit-papermill/setup.py index 0c3b87a66e..f8e92c92d0 100644 --- a/plugins/flytekit-papermill/setup.py +++ b/plugins/flytekit-papermill/setup.py @@ -5,8 +5,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" plugin_requires = [ - "flytekit>=0.30.0b4,<1.0.0", - "flytekitplugins-spark>=0.30.0b4,<1.0.0", + "flytekit>=0.16.0b0,<1.0.0", "papermill>=1.2.0", "nbconvert>=6.0.7", "ipykernel>=5.0.0",