Union Types

.github/workflows/pythonbuild.yml

@@ -3,7 +3,7 @@ name: Build
 on:
   push:
     branches:
-    - master
+      - master
   pull_request:
 
 env:
@@ -19,43 +19,43 @@ jobs:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
         spark-version-suffix: ["", "-spark2"]
         exclude:
-        - python-version: 3.8
-          spark-version-suffix: "-spark2"
-        - python-version: 3.9
-          spark-version-suffix: "-spark2"
-        - python-version: 3.10
-          spark-version-suffix: "-spark2"
-        # Ignore this test because we failed to install docker-py
-        # docker-py will install pywin32==227, whereas pywin only added support for python 3.10 in version 301.
-        # For more detail, see https://github.com/flyteorg/flytekit/pull/856#issuecomment-1067152855
-        - python-version: 3.10
-          os: windows-latest
+          - python-version: 3.8
+            spark-version-suffix: "-spark2"
+          - python-version: 3.9
+            spark-version-suffix: "-spark2"
+          - python-version: 3.10
+            spark-version-suffix: "-spark2"
+          # Ignore this test because we failed to install docker-py
+          # docker-py will install pywin32==227, whereas pywin only added support for python 3.10 in version 301.
+          # For more detail, see https://github.com/flyteorg/flytekit/pull/856#issuecomment-1067152855
+          - python-version: 3.10
+            os: windows-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Cache pip
-      uses: actions/cache@v2
-      with:
-        # This path is specific to Ubuntu
-        path: ~/.cache/pip
-        # Look to see if there is a cache hit for the corresponding requirements files
-        key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
-    - name: Install dependencies
-      run: |
-        make setup${{ matrix.spark-version-suffix }}
-        pip freeze
-    - name: Test with coverage
-      env:
-        FLYTE_SDK_USE_STRUCTURED_DATASET: "TRUE"
-      run: |
-        coverage run -m pytest tests/flytekit/unit
-    - name: Codecov
-      uses: codecov/codecov-action@v1
-      with:
-        fail_ci_if_error: true # optional (default = false)
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements files
+          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
+      - name: Install dependencies
+        run: |
+          make setup${{ matrix.spark-version-suffix }}
+          pip freeze
+      - name: Test with coverage
+        env:
+          FLYTE_SDK_USE_STRUCTURED_DATASET: "TRUE"
+        run: |
+          coverage run -m pytest tests/flytekit/unit
+      - name: Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          fail_ci_if_error: true # optional (default = false)
 
   build-plugins:
     runs-on: ubuntu-latest
@@ -84,35 +84,35 @@ jobs:
           - flytekit-spark
           - flytekit-sqlalchemy
         exclude:
-        # flytekit-modin depends on ray which does not have a 3.10 wheel yet.
-        # Issue tracked in https://github.com/ray-project/ray/issues/19116.
-        - python-version: 3.10
-          plugin-names: "flytekit-modin"
+          # flytekit-modin depends on ray which does not have a 3.10 wheel yet.
+          # Issue tracked in https://github.com/ray-project/ray/issues/19116.
+          - python-version: 3.10
+            plugin-names: "flytekit-modin"
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Cache pip
-      uses: actions/cache@v2
-      with:
-        # This path is specific to Ubuntu
-        path: ~/.cache/pip
-        # Look to see if there is a cache hit for the corresponding requirements files
-        key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }}
-    - name: Install dependencies
-      run: |
-        make setup
-        cd plugins/${{ matrix.plugin-names }}
-        pip install -e .
-        if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
-        pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
-        pip freeze
-    - name: Test with coverage
-      run: |
-        cd plugins/${{ matrix.plugin-names }}
-        coverage run -m pytest tests
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements files
+          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }}
+      - name: Install dependencies
+        run: |
+          make setup
+          cd plugins/${{ matrix.plugin-names }}
+          pip install -e .
+          if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
+          pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
+          pip freeze
+      - name: Test with coverage
+        run: |
+          cd plugins/${{ matrix.plugin-names }}
+          coverage run -m pytest tests
 
   lint:
     runs-on: ubuntu-latest
@@ -139,8 +139,7 @@ jobs:
       - name: ShellCheck
         uses: ludeeus/action-shellcheck@master
         with:
-          ignore_paths:
-            boilerplate
+          ignore_paths: boilerplate
 
   docs:
     runs-on: ubuntu-latest

Makefile

@@ -12,7 +12,7 @@ help:
 
 .PHONY: install-piptools
 install-piptools:
-	pip install -U pip-tools setuptools wheel pip==22.0.3
+	pip install -U pip-tools setuptools wheel "pip>=22.0.3"
 
 .PHONY: update_boilerplate
 update_boilerplate:

dev-requirements.txt

@@ -32,6 +32,7 @@ certifi==2021.10.8
     #   requests
 cffi==1.15.0
     # via
+    #   -c requirements.txt
     #   bcrypt
     #   cryptography
     #   pynacl
@@ -70,8 +71,11 @@ croniter==1.3.4
     # via
     #   -c requirements.txt
     #   flytekit
-cryptography==36.0.1
-    # via paramiko
+cryptography==36.0.2
+    # via
+    #   -c requirements.txt
+    #   paramiko
+    #   secretstorage
 dataclasses-json==0.5.6
     # via
     #   -c requirements.txt
@@ -112,7 +116,7 @@ docstring-parser==0.13
     #   flytekit
 filelock==3.6.0
     # via virtualenv
-flyteidl==0.23.1
+flyteidl==0.24.0
     # via
     #   -c requirements.txt
     #   flytekit
@@ -121,7 +125,7 @@ google-api-core[grpc]==2.7.1
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
     #   google-cloud-core
-google-auth==2.6.0
+google-auth==2.6.2
     # via
     #   google-api-core
     #   google-cloud-core
@@ -135,7 +139,7 @@ google-crc32c==1.3.0
     # via google-resumable-media
 google-resumable-media==2.3.2
     # via google-cloud-bigquery
-googleapis-common-protos==1.55.0
+googleapis-common-protos==1.56.0
     # via
     #   -c requirements.txt
     #   flyteidl
@@ -150,7 +154,7 @@ grpcio==1.44.0
     #   grpcio-status
 grpcio-status==1.44.0
     # via google-api-core
-identify==2.4.11
+identify==2.4.12
     # via pre-commit
 idna==3.3
     # via
@@ -162,6 +166,11 @@ importlib-metadata==4.11.3
     #   keyring
 iniconfig==1.1.1
     # via pytest
+jeepney==0.7.1
+    # via
+    #   -c requirements.txt
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   -c requirements.txt
@@ -277,7 +286,9 @@ pyasn1==0.4.8
 pyasn1-modules==0.2.8
     # via google-auth
 pycparser==2.21
-    # via cffi
+    # via
+    #   -c requirements.txt
+    #   cffi
 pynacl==1.5.0
     # via paramiko
 pyparsing==3.0.7
@@ -329,7 +340,7 @@ pyyaml==5.4.1
     #   -c requirements.txt
     #   docker-compose
     #   pre-commit
-regex==2022.3.2
+regex==2022.3.15
     # via
     #   -c requirements.txt
     #   docker-image-py
@@ -353,6 +364,10 @@ retry==0.9.2
     #   flytekit
 rsa==4.8
     # via google-auth
+secretstorage==3.3.1
+    # via
+    #   -c requirements.txt
+    #   keyring
 six==1.16.0
     # via
     #   -c requirements.txt
@@ -398,7 +413,7 @@ typing-inspect==0.7.1
     # via
     #   -c requirements.txt
     #   dataclasses-json
-urllib3==1.26.8
+urllib3==1.26.9
     # via
     #   -c requirements.txt
     #   flytekit

doc-requirements.txt

@@ -41,8 +41,10 @@ cookiecutter==1.7.3
     # via flytekit
 croniter==1.3.4
     # via flytekit
-cryptography==36.0.1
-    # via -r doc-requirements.in
+cryptography==36.0.2
+    # via
+    #   -r doc-requirements.in
+    #   secretstorage
 css-html-js-minify==2.5.5
     # via sphinx-material
 dataclasses-json==0.5.6
@@ -61,11 +63,11 @@ docutils==0.17.1
     # via
     #   sphinx
     #   sphinx-panels
-flyteidl==0.23.1
+flyteidl==0.24.0
     # via flytekit
 furo @ git+https://github.com/flyteorg/furo@main
     # via -r doc-requirements.in
-googleapis-common-protos==1.55.0
+googleapis-common-protos==1.56.0
     # via flyteidl
 grpcio==1.44.0
     # via
@@ -79,6 +81,10 @@ importlib-metadata==4.11.3
     # via
     #   keyring
     #   sphinx
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -161,7 +167,7 @@ pytz==2021.3
     #   pandas
 pyyaml==6.0
     # via sphinx-autoapi
-regex==2022.3.2
+regex==2022.3.15
     # via docker-image-py
 requests==2.27.1
     # via
@@ -173,6 +179,8 @@ responses==0.19.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -243,7 +251,7 @@ unidecode==1.3.4
     # via
     #   python-slugify
     #   sphinx-autoapi
-urllib3==1.26.8
+urllib3==1.26.9
     # via
     #   flytekit
     #   requests