From fa609f88b0abf7465e9a98e244f2b4c5466177b1 Mon Sep 17 00:00:00 2001 From: Victoria Bukta Date: Sun, 12 Feb 2023 19:24:11 -0500 Subject: [PATCH 1/3] pyiceberg: Add Google Cloud Storage support --- python/Makefile | 7 +- python/dev/docker-compose-gcs-server.yml | 30 ++++ python/dev/run-gcs-server.sh | 33 +++++ python/mkdocs/docs/configuration.md | 16 ++ python/mkdocs/docs/index.md | 3 +- python/poetry.lock | 41 ++++++ python/pyiceberg/io/fsspec.py | 21 +++ python/pyiceberg/io/pyarrow.py | 10 ++ python/pyproject.toml | 11 +- python/tests/conftest.py | 15 ++ python/tests/io/test_fsspec.py | 179 +++++++++++++++++++++++ 11 files changed, 363 insertions(+), 3 deletions(-) create mode 100644 python/dev/docker-compose-gcs-server.yml create mode 100644 python/dev/run-gcs-server.sh diff --git a/python/Makefile b/python/Makefile index 932c90dd775b..859df4c257eb 100644 --- a/python/Makefile +++ b/python/Makefile @@ -17,7 +17,8 @@ install: pip install poetry - poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb -E ray -E sql-postgres + poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb -E ray -E sql-postgres -E gcsfs + check-license: ./dev/check-license @@ -56,3 +57,7 @@ test-coverage: poetry run coverage report -m --fail-under=90 poetry run coverage html poetry run coverage xml + +test-gcs: + sh ./dev/run-gcs-server.sh + poetry run pytest tests/ -m gcs ${PYTEST_ARGS} diff --git a/python/dev/docker-compose-gcs-server.yml b/python/dev/docker-compose-gcs-server.yml new file mode 100644 index 000000000000..2a5164c81cf1 --- /dev/null +++ b/python/dev/docker-compose-gcs-server.yml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +version: "3" + +services: + gcs-server: + image: fsouza/fake-gcs-server + container_name: gcs-server + ports: + - 4443:4443 + entrypoint: > + /bin/sh -c " + mkdir -p /data/warehouse; + /bin/fake-gcs-server -data /data -scheme http; + exit 0; + " diff --git a/python/dev/run-gcs-server.sh b/python/dev/run-gcs-server.sh new file mode 100644 index 000000000000..289d89009ae3 --- /dev/null +++ b/python/dev/run-gcs-server.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -ex + +if [ $(docker ps -q --filter "name=gcs-server" --filter "status=running" ) ]; then + echo "Fake GCS Server running" +else + docker-compose -f dev/docker-compose-gcs-server.yml kill + docker-compose -f dev/docker-compose-gcs-server.yml up -d + while [ -z $(docker ps -q --filter "name=gcs-server" --filter "status=running" ) ] + do + echo "Waiting for Fake GCS Server" + sleep 1 + done +fi diff --git a/python/mkdocs/docs/configuration.md b/python/mkdocs/docs/configuration.md index f6b24745c1f6..72905c5ce5d8 100644 --- a/python/mkdocs/docs/configuration.md +++ b/python/mkdocs/docs/configuration.md @@ -74,6 +74,7 @@ For the FileIO there are several configuration options available: | Key | Example | Description | | ----------------------- | ----------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | adlfs.connection-string | AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqF...;BlobEndpoint=http://localhost/ | A [connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). | + | adlfs.account-name | devstoreaccount1 | The account that you want to connect to | | adlfs.account-key | Eby8vdM02xNOcqF... | The key to authentication against the account. | | adlfs.sas-token | NuHOuuzdQN7VRM%2FOpOeqBlawRCA845IY05h9eu1Yte4%3D | The shared access signature | @@ -81,6 +82,21 @@ For the FileIO there are several configuration options available: | adlfs.client-id | ad667be4-b811-11ed-afa1-0242ac120002 | The client-id | | adlfs.client-secret | oCA3R6P\*ka#oa1Sms2J74z... | The client-secret | +### Google Cloud Storage + +| Key | Example | Description | +|--------------------------|---------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| gs.project | my-gcp-project | Configure Google Cloud Project for GCS FileIO. | +| gs.endpoint-url | http://0.0.0.0:4443 | Configure an alternative endpoint for the GCS FileIO to access (format protocol://host:port) If not given, defaults to the value of environment variable "STORAGE_EMULATOR_HOST"; if that is not set either, will use the standard Google endpoint. | +| gs.token | google_default | Configure method authentication to GCS for FileIO. Can be the following, 'google_default', 'cache', 'anon', 'browser', 'cloud'. If not specified your credentials will be resolved in the following order: gcloud CLI default, gcsfs cached token, google compute metadata service, anonymous. | +| gs.access | read_only | Configure client to have specific access. Must be one of 'read_only', 'read_write', or 'full_control' | +| gs.consistency | md5 | Configure the check method when writing files. Must be one of 'none', 'size', or 'md5' | +| gs.cache-timeout | 60 | Configure the cache expiration time in seconds for object metadata cache | +| gs.requester-pays | False | Configure whether to use requester-pays requests | +| gs.session-kwargs | {} | Configure a dict of parameters to pass on to aiohttp.ClientSession; can contain, for example, proxy settings. | +| gs.default-location | US | Configure the default location where buckets are created, like 'US' or 'EUROPE-WEST3'. | +| gs.version-aware | False | Configure whether to support object versioning on the GCS bucket. | + ## REST Catalog ```yaml diff --git a/python/mkdocs/docs/index.md b/python/mkdocs/docs/index.md index 36fb656fcad5..89b66129e32d 100644 --- a/python/mkdocs/docs/index.md +++ b/python/mkdocs/docs/index.md @@ -60,7 +60,8 @@ You can mix and match optional dependencies depending on your needs: | s3fs | S3FS as a FileIO implementation to interact with the object store | | adlfs | ADLFS as a FileIO implementation to interact with the object store | | snappy | Support for snappy Avro compression | +| gcs | GCS as the FileIO implementation to interact with the object store | -You either need to install `s3fs`, `adlfs` or `pyarrow` for fetching files. +You either need to install `s3fs`, `adlfs`, `gcs`, or `pyarrow` for fetching files. There is both a [CLI](cli.md) and [Python API](api.md) available. diff --git a/python/poetry.lock b/python/poetry.lock index 9dc4bc9fd924..06734fa18b4d 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -342,6 +342,18 @@ test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-moc typing = ["importlib-metadata (>=5.1)", "mypy (==0.991)", "tomli", "typing-extensions (>=3.7.4.3)"] virtualenv = ["virtualenv (>=20.0.35)"] +[[package]] +name = "cachetools" +version = "5.3.0" +description = "Extensible memoizing collections and decorators" +category = "main" +optional = true +python-versions = "~=3.7" +files = [ + {file = "cachetools-5.3.0-py3-none-any.whl", hash = "sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4"}, + {file = "cachetools-5.3.0.tar.gz", hash = "sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14"}, +] + [[package]] name = "certifi" version = "2023.7.22" @@ -1921,6 +1933,33 @@ files = [ [package.dependencies] numpy = ">=1.16.6" +[[package]] +name = "pyasn1" +version = "0.4.8" +description = "ASN.1 types and codecs" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "pyasn1-0.4.8-py2.py3-none-any.whl", hash = "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d"}, + {file = "pyasn1-0.4.8.tar.gz", hash = "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.2.8" +description = "A collection of ASN.1-based protocols modules." +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "pyasn1-modules-0.2.8.tar.gz", hash = "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e"}, + {file = "pyasn1_modules-0.2.8-py2.py3-none-any.whl", hash = "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.5.0" + [[package]] name = "pycparser" version = "2.21" @@ -2525,6 +2564,7 @@ files = [ {file = "rpds_py-0.9.2.tar.gz", hash = "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945"}, ] + [[package]] name = "s3fs" version = "2023.6.0" @@ -3066,6 +3106,7 @@ cffi = ["cffi (>=1.11)"] adlfs = ["adlfs"] duckdb = ["duckdb", "pyarrow"] dynamodb = ["boto3"] +gcsfs = ["gcsfs"] glue = ["boto3"] hive = ["thrift"] pandas = ["pandas", "pyarrow"] diff --git a/python/pyiceberg/io/fsspec.py b/python/pyiceberg/io/fsspec.py index 65472904d049..98d9e485754a 100644 --- a/python/pyiceberg/io/fsspec.py +++ b/python/pyiceberg/io/fsspec.py @@ -16,6 +16,7 @@ # under the License. """FileIO implementation for reading and writing table files that uses fsspec compatible filesystems.""" import errno +import json import logging import os from functools import lru_cache, partial @@ -124,6 +125,24 @@ def _s3(properties: Properties) -> AbstractFileSystem: return fs +def _gs(properties: Properties) -> AbstractFileSystem: + from gcsfs import GCSFileSystem + + fs_kwargs = { + "project": properties.get("gs.project"), + "access": properties.get("gs.access", "full_control"), + "token": properties.get("gs.token"), + "consistency": properties.get("gs.consistency"), + "cache_timeout": properties.get("gs.cache-timeout"), + "requester_pays": properties.get("gs.requester-pays"), + "session_kwargs": json.loads(properties.get("gs.session-kwargs", "{}")), + "endpoint_url": properties.get("gs.endpoint-url"), + "default_location": properties.get("gs.default-location"), + "version_aware": json.loads(properties.get("gs.version-aware", "false").lower()), + } + return GCSFileSystem(**fs_kwargs) + + def _adlfs(properties: Properties) -> AbstractFileSystem: from adlfs import AzureBlobFileSystem @@ -145,6 +164,8 @@ def _adlfs(properties: Properties) -> AbstractFileSystem: "s3n": _s3, "abfs": _adlfs, "abfss": _adlfs, + "gs": _gs, + "gcs": _gs, } diff --git a/python/pyiceberg/io/pyarrow.py b/python/pyiceberg/io/pyarrow.py index c501278492b1..bfe835068ff6 100644 --- a/python/pyiceberg/io/pyarrow.py +++ b/python/pyiceberg/io/pyarrow.py @@ -57,6 +57,7 @@ FileSystem, FileType, FSSpecHandler, + GcsFileSystem, LocalFileSystem, PyFileSystem, S3FileSystem, @@ -300,6 +301,15 @@ def _get_fs(self, scheme: str) -> FileSystem: client_kwargs["proxy_options"] = proxy_uri return S3FileSystem(**client_kwargs) + elif scheme in {"gs", "gcs"}: + if access_token := self.properties.get("gs.access-token"): + client_kwargs = { + "access_token": access_token, + "credential_token_expiration": self.properties.get("gs.credential-token-expiration"), + } + return GcsFileSystem(**client_kwargs) + else: + return GcsFileSystem() elif scheme == "file": return LocalFileSystem() else: diff --git a/python/pyproject.toml b/python/pyproject.toml index be0bfaf73a86..93667a6896a9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -64,6 +64,7 @@ s3fs = { version = ">=2021.08.0,<2024.1.0", optional = true } # Upper bound set adlfs = { version = ">=2021.07.0,<2024.1.0", optional = true } # Upper bound set arbitrarily, to be reassessed in early 2024. psycopg2-binary = { version = ">=2.9.6", optional = true } sqlalchemy = { version = "^2.0.18", optional = true } +gcsfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.4.0" @@ -99,14 +100,18 @@ glue = ["boto3"] adlfs = ["adlfs"] dynamodb = ["boto3"] zstandard = ["zstandard"] +<<<<<<< HEAD sql-postgres = ["sqlalchemy", "psycopg2-binary"] +======= +gcsfs = ["gcsfs"] +>>>>>>> c5ab4da8e (pyiceberg: Add Google Cloud Storage support) [tool.pytest.ini_options] markers = [ "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint-url args)", "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)", "integration: marks integration tests against Apache Spark" -] + "gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint-url)" [tool.black] line-length = 130 @@ -199,6 +204,10 @@ ignore_missing_imports = true module = "adlfs.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "gcsfs.*" +ignore_missing_imports = true + [[tool.mypy.overrides]] module = "packaging.*" ignore_missing_imports = true diff --git a/python/tests/conftest.py b/python/tests/conftest.py index e5d08604897e..27faa0340488 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -113,6 +113,11 @@ def pytest_addoption(parser: pytest.Parser) -> None: default="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", help="The ADLS secret account key for tests marked as adlfs", ) + parser.addoption( + "--gs.endpoint-url", action="store", default="http://0.0.0.0:4443", help="The GCS endpoint URL for tests marked gcs" + ) + parser.addoption("--gs.token", action="store", default="anon", help="The GCS authentication method for tests marked gcs") + parser.addoption("--gs.project", action="store", default="test", help="The GCP project for tests marked gcs") @pytest.fixture(scope="session") @@ -1297,6 +1302,16 @@ def fsspec_fileio(request: pytest.FixtureRequest) -> FsspecFileIO: return fsspec.FsspecFileIO(properties=properties) +@pytest.fixture +def fsspec_fileio_gcs(request: pytest.FixtureRequest) -> FsspecFileIO: + properties = { + "gs.endpoint-url": request.config.getoption("--gs.endpoint-url"), + "gs.token": request.config.getoption("--gs.token"), + "gs.project": request.config.getoption("--gs.project"), + } + return fsspec.FsspecFileIO(properties=properties) + + class MockAWSResponse(aiobotocore.awsrequest.AioAWSResponse): """A mocked aws response implementation (for test use only). diff --git a/python/tests/io/test_fsspec.py b/python/tests/io/test_fsspec.py index a92e383e7916..d53c2a8d3701 100644 --- a/python/tests/io/test_fsspec.py +++ b/python/tests/io/test_fsspec.py @@ -387,6 +387,185 @@ def test_writing_avro_file_adlfs(generated_manifest_entry_file: str, adlfs_fsspe adlfs_fsspec_fileio.delete(f"abfss://tests/{filename}") +@pytest.mark.gcs +def test_fsspec_new_input_file_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test creating a new input file from a fsspec file-io""" + filename = str(uuid.uuid4()) + + input_file = fsspec_fileio_gcs.new_input(f"gs://warehouse/{filename}") + + assert isinstance(input_file, fsspec.FsspecInputFile) + assert input_file.location == f"gs://warehouse/{filename}" + + +@pytest.mark.gcs +def test_fsspec_new_output_file_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test creating a new output file from an fsspec file-io""" + filename = str(uuid.uuid4()) + + output_file = fsspec_fileio_gcs.new_output(f"gs://warehouse/{filename}") + + assert isinstance(output_file, fsspec.FsspecOutputFile) + assert output_file.location == f"gs://warehouse/{filename}" + + +@pytest.mark.gcs +def test_fsspec_write_and_read_file_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test writing and reading a file using FsspecInputFile and FsspecOutputFile""" + filename = str(uuid.uuid4()) + output_file = fsspec_fileio_gcs.new_output(location=f"gs://warehouse/{filename}") + with output_file.create() as f: + f.write(b"foo") + + input_file = fsspec_fileio_gcs.new_input(f"gs://warehouse/{filename}") + assert input_file.open().read() == b"foo" + + fsspec_fileio_gcs.delete(input_file) + + +@pytest.mark.gcs +def test_fsspec_getting_length_of_file_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test getting the length of an FsspecInputFile and FsspecOutputFile""" + filename = str(uuid.uuid4()) + + output_file = fsspec_fileio_gcs.new_output(location=f"gs://warehouse/{filename}") + with output_file.create() as f: + f.write(b"foobar") + + assert len(output_file) == 6 + + input_file = fsspec_fileio_gcs.new_input(location=f"gs://warehouse/{filename}") + assert len(input_file) == 6 + + fsspec_fileio_gcs.delete(output_file) + + +@pytest.mark.gcs +def test_fsspec_file_tell_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test finding cursor position for an fsspec file-io file""" + + filename = str(uuid.uuid4()) + + output_file = fsspec_fileio_gcs.new_output(location=f"gs://warehouse/{filename}") + with output_file.create() as write_file: + write_file.write(b"foobar") + + input_file = fsspec_fileio_gcs.new_input(location=f"gs://warehouse/{filename}") + f = input_file.open() + + f.seek(0) + assert f.tell() == 0 + f.seek(1) + assert f.tell() == 1 + f.seek(3) + assert f.tell() == 3 + f.seek(0) + assert f.tell() == 0 + + +@pytest.mark.gcs +def test_fsspec_read_specified_bytes_for_file_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test reading a specified number of bytes from an fsspec file-io file""" + + filename = str(uuid.uuid4()) + output_file = fsspec_fileio_gcs.new_output(location=f"gs://warehouse/{filename}") + with output_file.create() as write_file: + write_file.write(b"foo") + + input_file = fsspec_fileio_gcs.new_input(location=f"gs://warehouse/{filename}") + f = input_file.open() + + f.seek(0) + assert b"f" == f.read(1) + f.seek(0) + assert b"fo" == f.read(2) + f.seek(1) + assert b"o" == f.read(1) + f.seek(1) + assert b"oo" == f.read(2) + f.seek(0) + assert b"foo" == f.read(999) # test reading amount larger than entire content length + + fsspec_fileio_gcs.delete(input_file) + + +@pytest.mark.gcs +def test_fsspec_raise_on_opening_file_not_found_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test that an fsspec input file raises appropriately when the gcs file is not found""" + + filename = str(uuid.uuid4()) + input_file = fsspec_fileio_gcs.new_input(location=f"gs://warehouse/{filename}") + with pytest.raises(FileNotFoundError) as exc_info: + input_file.open().read() + + assert filename in str(exc_info.value) + + +@pytest.mark.gcs +def test_checking_if_a_file_exists_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test checking if a file exists""" + + non_existent_file = fsspec_fileio_gcs.new_input(location="gs://warehouse/does-not-exist.txt") + assert not non_existent_file.exists() + + filename = str(uuid.uuid4()) + output_file = fsspec_fileio_gcs.new_output(location=f"gs://warehouse/{filename}") + assert not output_file.exists() + with output_file.create() as f: + f.write(b"foo") + + existing_input_file = fsspec_fileio_gcs.new_input(location=f"gs://warehouse/{filename}") + assert existing_input_file.exists() + + existing_output_file = fsspec_fileio_gcs.new_output(location=f"gs://warehouse/{filename}") + assert existing_output_file.exists() + + fsspec_fileio_gcs.delete(existing_output_file) + + +@pytest.mark.gcs +def test_closing_a_file_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test closing an output file and input file""" + filename = str(uuid.uuid4()) + output_file = fsspec_fileio_gcs.new_output(location=f"gs://warehouse/{filename}") + with output_file.create() as write_file: + write_file.write(b"foo") + assert not write_file.closed # type: ignore + assert write_file.closed # type: ignore + + input_file = fsspec_fileio_gcs.new_input(location=f"gs://warehouse/{filename}") + f = input_file.open() + assert not f.closed # type: ignore + f.close() + assert f.closed # type: ignore + + fsspec_fileio_gcs.delete(f"gs://warehouse/{filename}") + + +@pytest.mark.gcs +def test_fsspec_converting_an_outputfile_to_an_inputfile_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test converting an output file to an input file""" + filename = str(uuid.uuid4()) + output_file = fsspec_fileio_gcs.new_output(location=f"gs://warehouse/{filename}") + input_file = output_file.to_input_file() + assert input_file.location == output_file.location + + +@pytest.mark.gcs +def test_writing_avro_file_gcs(generated_manifest_entry_file: str, fsspec_fileio_gcs: FsspecFileIO) -> None: + """Test that bytes match when reading a local avro file, writing it using fsspec file-io, and then reading it again""" + filename = str(uuid.uuid4()) + with PyArrowFileIO().new_input(location=generated_manifest_entry_file).open() as f: + b1 = f.read() + with fsspec_fileio_gcs.new_output(location=f"gs://warehouse/{filename}").create() as out_f: + out_f.write(b1) + with fsspec_fileio_gcs.new_input(location=f"gs://warehouse/{filename}").open() as in_f: + b2 = in_f.read() + assert b1 == b2 # Check that bytes of read from local avro file match bytes written to s3 + + fsspec_fileio_gcs.delete(f"gs://warehouse/{filename}") + + TEST_URI = "https://iceberg-test-signer" From 8b2ba338360c7cc1f0f3b3c125c13d7ad5e5e291 Mon Sep 17 00:00:00 2001 From: Victoria Bukta Date: Mon, 26 Jun 2023 13:03:37 -0400 Subject: [PATCH 2/3] updated project dependecies --- python/mkdocs/docs/configuration.md | 25 +- python/poetry.lock | 397 ++++++++++++++++++++++++---- python/pyiceberg/io/pyarrow.py | 6 +- python/pyproject.toml | 6 +- 4 files changed, 366 insertions(+), 68 deletions(-) diff --git a/python/mkdocs/docs/configuration.md b/python/mkdocs/docs/configuration.md index 72905c5ce5d8..fecdb784a254 100644 --- a/python/mkdocs/docs/configuration.md +++ b/python/mkdocs/docs/configuration.md @@ -84,18 +84,19 @@ For the FileIO there are several configuration options available: ### Google Cloud Storage -| Key | Example | Description | -|--------------------------|---------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| gs.project | my-gcp-project | Configure Google Cloud Project for GCS FileIO. | -| gs.endpoint-url | http://0.0.0.0:4443 | Configure an alternative endpoint for the GCS FileIO to access (format protocol://host:port) If not given, defaults to the value of environment variable "STORAGE_EMULATOR_HOST"; if that is not set either, will use the standard Google endpoint. | -| gs.token | google_default | Configure method authentication to GCS for FileIO. Can be the following, 'google_default', 'cache', 'anon', 'browser', 'cloud'. If not specified your credentials will be resolved in the following order: gcloud CLI default, gcsfs cached token, google compute metadata service, anonymous. | -| gs.access | read_only | Configure client to have specific access. Must be one of 'read_only', 'read_write', or 'full_control' | -| gs.consistency | md5 | Configure the check method when writing files. Must be one of 'none', 'size', or 'md5' | -| gs.cache-timeout | 60 | Configure the cache expiration time in seconds for object metadata cache | -| gs.requester-pays | False | Configure whether to use requester-pays requests | -| gs.session-kwargs | {} | Configure a dict of parameters to pass on to aiohttp.ClientSession; can contain, for example, proxy settings. | -| gs.default-location | US | Configure the default location where buckets are created, like 'US' or 'EUROPE-WEST3'. | -| gs.version-aware | False | Configure whether to support object versioning on the GCS bucket. | +| Key | Example | Description | +|--------------------------------|---------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| gs.project | my-gcp-project | Configure Google Cloud Project for GCS FileIO. | +| gs.endpoint-url | http://0.0.0.0:4443 | Configure an alternative endpoint for the GCS FileIO to access (format protocol://host:port) If not given, defaults to the value of environment variable "STORAGE_EMULATOR_HOST"; if that is not set either, will use the standard Google endpoint. | +| gs.token | google_default | Configure method authentication to GCS for FileIO. Can be the following, 'google_default', 'cache', 'anon', 'browser', 'cloud'. If not specified your credentials will be resolved in the following order: gcloud CLI default, gcsfs cached token, google compute metadata service, anonymous. | +| gs.access | read_only | Configure client to have specific access. Must be one of 'read_only', 'read_write', or 'full_control' | +| gs.consistency | md5 | Configure the check method when writing files. Must be one of 'none', 'size', or 'md5' | +| gs.cache-timeout | 60 | Configure the cache expiration time in seconds for object metadata cache | +| gs.requester-pays | False | Configure whether to use requester-pays requests | +| gs.session-kwargs | {} | Configure a dict of parameters to pass on to aiohttp.ClientSession; can contain, for example, proxy settings. | +| gs.default-location | US | Configure the default location where buckets are created, like 'US' or 'EUROPE-WEST3'. | +| gs.version-aware | False | Configure whether to support object versioning on the GCS bucket. | +| gs.credential-token-expiration | None | Configure expiration for credential generated with an access token. Type datetime. Must be specified if `token` is specified. | ## REST Catalog diff --git a/python/poetry.lock b/python/poetry.lock index 06734fa18b4d..f6169d9cd51e 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -24,24 +24,24 @@ docs = ["furo", "myst-parser", "numpydoc", "sphinx"] [[package]] name = "aiobotocore" -version = "2.5.2" +version = "2.4.2" description = "Async client for aws services using botocore and aiohttp" optional = true python-versions = ">=3.7" files = [ - {file = "aiobotocore-2.5.2-py3-none-any.whl", hash = "sha256:337429ffd3cc367532572d40be809a84c7b5335f3f8eca2f23e09dfaa9a9ef90"}, - {file = "aiobotocore-2.5.2.tar.gz", hash = "sha256:e7399f21570db1c287f1c0c814dd3475dfe1c8166722e2c77ce67f172cbcfa89"}, + {file = "aiobotocore-2.4.2-py3-none-any.whl", hash = "sha256:4acd1ebe2e44be4b100aa553910bda899f6dc090b3da2bc1cf3d5de2146ed208"}, + {file = "aiobotocore-2.4.2.tar.gz", hash = "sha256:0603b74a582dffa7511ce7548d07dc9b10ec87bc5fb657eb0b34f9bd490958bf"}, ] [package.dependencies] -aiohttp = ">=3.3.1,<4.0.0" -aioitertools = ">=0.5.1,<1.0.0" -botocore = ">=1.29.161,<1.29.162" -wrapt = ">=1.10.10,<2.0.0" +aiohttp = ">=3.3.1" +aioitertools = ">=0.5.1" +botocore = ">=1.27.59,<1.27.60" +wrapt = ">=1.10.10" [package.extras] -awscli = ["awscli (>=1.27.161,<1.27.162)"] -boto3 = ["boto3 (>=1.26.161,<1.26.162)"] +awscli = ["awscli (>=1.25.60,<1.25.61)"] +boto3 = ["boto3 (>=1.24.59,<1.24.60)"] [[package]] name = "aiohttp" @@ -283,17 +283,17 @@ aio = ["azure-core[aio] (>=1.28.0,<2.0.0)"] [[package]] name = "boto3" -version = "1.26.161" +version = "1.24.59" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.26.161-py3-none-any.whl", hash = "sha256:f66e5c9dbe7f34383bcf64fa6070771355c11a44dd75c7f1279f2f37e1c89183"}, - {file = "boto3-1.26.161.tar.gz", hash = "sha256:662731e464d14af1035f44fc6a46b0e3112ee011ac0a5ed416d205daa3e15f25"}, + {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, + {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, ] [package.dependencies] -botocore = ">=1.29.161,<1.30.0" +botocore = ">=1.27.59,<1.28.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -302,13 +302,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.29.161" +version = "1.27.59" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.29.161-py3-none-any.whl", hash = "sha256:b906999dd53dda2ef0ef6f7f55fcc81a4b06b9f1c8a9f65c546e0b981f959f5f"}, - {file = "botocore-1.29.161.tar.gz", hash = "sha256:a50edd715eb510343e27849f36483804aae4b871590db4d4996aa53368dcac40"}, + {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, + {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, ] [package.dependencies] @@ -317,7 +317,7 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.16.9)"] +crt = ["awscrt (==0.14.0)"] [[package]] name = "build" @@ -344,14 +344,13 @@ virtualenv = ["virtualenv (>=20.0.35)"] [[package]] name = "cachetools" -version = "5.3.0" +version = "5.3.1" description = "Extensible memoizing collections and decorators" -category = "main" optional = true -python-versions = "~=3.7" +python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.0-py3-none-any.whl", hash = "sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4"}, - {file = "cachetools-5.3.0.tar.gz", hash = "sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14"}, + {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, + {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, ] [[package]] @@ -681,6 +680,17 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +optional = true +python-versions = ">=3.5" +files = [ + {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, + {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, +] + [[package]] name = "distlib" version = "0.3.7" @@ -905,13 +915,13 @@ files = [ [[package]] name = "fsspec" -version = "2023.6.0" +version = "2023.1.0" description = "File-system specification" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"}, - {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"}, + {file = "fsspec-2023.1.0-py3-none-any.whl", hash = "sha256:b833e2e541e9e8cde0ab549414187871243177feb3d344f9d27b25a93f5d8139"}, + {file = "fsspec-2023.1.0.tar.gz", hash = "sha256:fbae7f20ff801eb5f7d0bedf81f25c787c0dfac5e982d98fa3884a9cde2b5411"}, ] [package.extras] @@ -919,9 +929,8 @@ abfs = ["adlfs"] adl = ["adlfs"] arrow = ["pyarrow (>=1)"] dask = ["dask", "distributed"] -devel = ["pytest", "pytest-cov"] dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +entrypoints = ["importlib-metadata"] fuse = ["fusepy"] gcs = ["gcsfs"] git = ["pygit2"] @@ -938,6 +947,249 @@ smb = ["smbprotocol"] ssh = ["paramiko"] tqdm = ["tqdm"] +[[package]] +name = "gcsfs" +version = "2023.1.0" +description = "Convenient Filesystem interface over GCS" +optional = true +python-versions = ">=3.7" +files = [ + {file = "gcsfs-2023.1.0-py2.py3-none-any.whl", hash = "sha256:62c491b9e2a8e9e58b8a899eec2ce111f827718a65539019ff3cadf447e48f41"}, + {file = "gcsfs-2023.1.0.tar.gz", hash = "sha256:0a7b7ca8c1affa126a14ba35d7b7dff81c49e2aaceedda9732c7f159a4837a26"}, +] + +[package.dependencies] +aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" +decorator = ">4.1.2" +fsspec = "2023.1.0" +google-auth = ">=1.2" +google-auth-oauthlib = "*" +google-cloud-storage = "*" +requests = "*" + +[package.extras] +crc = ["crcmod"] +gcsfuse = ["fusepy"] + +[[package]] +name = "google-api-core" +version = "2.11.1" +description = "Google API client core library" +optional = true +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a"}, + {file = "google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.22.0" +description = "Google Authentication Library" +optional = true +python-versions = ">=3.6" +files = [ + {file = "google-auth-2.22.0.tar.gz", hash = "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce"}, + {file = "google_auth-2.22.0-py2.py3-none-any.whl", hash = "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" +six = ">=1.9.0" +urllib3 = "<2.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-auth-oauthlib" +version = "1.0.0" +description = "Google Authentication Library" +optional = true +python-versions = ">=3.6" +files = [ + {file = "google-auth-oauthlib-1.0.0.tar.gz", hash = "sha256:e375064964820b47221a7e1b7ee1fd77051b6323c3f9e3e19785f78ab67ecfc5"}, + {file = "google_auth_oauthlib-1.0.0-py2.py3-none-any.whl", hash = "sha256:95880ca704928c300f48194d1770cf5b1462835b6e49db61445a520f793fd5fb"}, +] + +[package.dependencies] +google-auth = ">=2.15.0" +requests-oauthlib = ">=0.7.0" + +[package.extras] +tool = ["click (>=6.0.0)"] + +[[package]] +name = "google-cloud-core" +version = "2.3.3" +description = "Google Cloud API client core library" +optional = true +python-versions = ">=3.7" +files = [ + {file = "google-cloud-core-2.3.3.tar.gz", hash = "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb"}, + {file = "google_cloud_core-2.3.3-py2.py3-none-any.whl", hash = "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863"}, +] + +[package.dependencies] +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" + +[package.extras] +grpc = ["grpcio (>=1.38.0,<2.0dev)"] + +[[package]] +name = "google-cloud-storage" +version = "2.10.0" +description = "Google Cloud Storage API client library" +optional = true +python-versions = ">=3.7" +files = [ + {file = "google-cloud-storage-2.10.0.tar.gz", hash = "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7"}, + {file = "google_cloud_storage-2.10.0-py2.py3-none-any.whl", hash = "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" +google-cloud-core = ">=2.3.0,<3.0dev" +google-resumable-media = ">=2.3.2" +requests = ">=2.18.0,<3.0.0dev" + +[package.extras] +protobuf = ["protobuf (<5.0.0dev)"] + +[[package]] +name = "google-crc32c" +version = "1.5.0" +description = "A python wrapper of the C library 'Google CRC32C'" +optional = true +python-versions = ">=3.7" +files = [ + {file = "google-crc32c-1.5.0.tar.gz", hash = "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win32.whl", hash = "sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win32.whl", hash = "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win32.whl", hash = "sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win32.whl", hash = "sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win32.whl", hash = "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93"}, +] + +[package.extras] +testing = ["pytest"] + +[[package]] +name = "google-resumable-media" +version = "2.5.0" +description = "Utilities for Google Media Downloads and Resumable Uploads" +optional = true +python-versions = ">= 3.7" +files = [ + {file = "google-resumable-media-2.5.0.tar.gz", hash = "sha256:218931e8e2b2a73a58eb354a288e03a0fd5fb1c4583261ac6e4c078666468c93"}, + {file = "google_resumable_media-2.5.0-py2.py3-none-any.whl", hash = "sha256:da1bd943e2e114a56d85d6848497ebf9be6a14d3db23e9fc57581e7c3e8170ec"}, +] + +[package.dependencies] +google-crc32c = ">=1.0,<2.0dev" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] +requests = ["requests (>=2.18.0,<3.0.0dev)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.59.1" +description = "Common protobufs used in Google APIs" +optional = true +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.59.1.tar.gz", hash = "sha256:b35d530fe825fb4227857bc47ad84c33c809ac96f312e13182bdeaa2abe1178a"}, + {file = "googleapis_common_protos-1.59.1-py2.py3-none-any.whl", hash = "sha256:0cbedb6fb68f1c07e18eb4c48256320777707e7d0c55063ae56c15db3224a61e"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + [[package]] name = "greenlet" version = "2.0.2" @@ -1647,6 +1899,22 @@ files = [ {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, ] +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = true +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + [[package]] name = "packaging" version = "23.1" @@ -1935,30 +2203,28 @@ numpy = ">=1.16.6" [[package]] name = "pyasn1" -version = "0.4.8" -description = "ASN.1 types and codecs" -category = "main" +version = "0.5.0" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = true -python-versions = "*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "pyasn1-0.4.8-py2.py3-none-any.whl", hash = "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d"}, - {file = "pyasn1-0.4.8.tar.gz", hash = "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba"}, + {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, + {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, ] [[package]] name = "pyasn1-modules" -version = "0.2.8" -description = "A collection of ASN.1-based protocols modules." -category = "main" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" optional = true -python-versions = "*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "pyasn1-modules-0.2.8.tar.gz", hash = "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e"}, - {file = "pyasn1_modules-0.2.8-py2.py3-none-any.whl", hash = "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74"}, + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, ] [package.dependencies] -pyasn1 = ">=0.4.6,<0.5.0" +pyasn1 = ">=0.4.6,<0.6.0" [[package]] name = "pycparser" @@ -2419,6 +2685,24 @@ six = "*" fixture = ["fixtures"] test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + [[package]] name = "responses" version = "0.23.1" @@ -2564,26 +2848,39 @@ files = [ {file = "rpds_py-0.9.2.tar.gz", hash = "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945"}, ] +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = true +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" [[package]] name = "s3fs" -version = "2023.6.0" +version = "2023.1.0" description = "Convenient Filesystem interface over S3" optional = true -python-versions = ">= 3.8" +python-versions = ">= 3.7" files = [ - {file = "s3fs-2023.6.0-py3-none-any.whl", hash = "sha256:d1a0a423d0d2e17fb2a193d9531935dc3f45ba742693448a461b6b34f6a92a24"}, - {file = "s3fs-2023.6.0.tar.gz", hash = "sha256:63fd8ddf05eb722de784b7b503196107f2a518061298cf005a8a4715b4d49117"}, + {file = "s3fs-2023.1.0-py3-none-any.whl", hash = "sha256:a549ae518fff4388bd6fca5248575c29f521e7e39efcd2bfab476651701fd114"}, + {file = "s3fs-2023.1.0.tar.gz", hash = "sha256:8b2e28372423e93f26312208a9272e22a962ddd0a79d63f9a68693b6af5ff187"}, ] [package.dependencies] -aiobotocore = ">=2.5.0,<2.6.0" +aiobotocore = ">=2.4.2,<2.5.0" aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" -fsspec = "2023.6.0" +fsspec = "2023.1.0" [package.extras] -awscli = ["aiobotocore[awscli] (>=2.5.0,<2.6.0)"] -boto3 = ["aiobotocore[boto3] (>=2.5.0,<2.6.0)"] +awscli = ["aiobotocore[awscli] (>=2.4.2,<2.5.0)"] +boto3 = ["aiobotocore[boto3] (>=2.4.2,<2.5.0)"] [[package]] name = "s3transfer" @@ -3120,4 +3417,4 @@ zstandard = ["zstandard"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "5d2066d7196f6defe9f350a0e20f46e54753b3b2a2ad8de0e655e6f0901d8fea" +content-hash = "31fc3bf4ad0d8fb4f43ef7146e24e45bb333f68427c681f897147c2d7b4c02e5" diff --git a/python/pyiceberg/io/pyarrow.py b/python/pyiceberg/io/pyarrow.py index bfe835068ff6..c2f451906d97 100644 --- a/python/pyiceberg/io/pyarrow.py +++ b/python/pyiceberg/io/pyarrow.py @@ -302,10 +302,12 @@ def _get_fs(self, scheme: str) -> FileSystem: return S3FileSystem(**client_kwargs) elif scheme in {"gs", "gcs"}: - if access_token := self.properties.get("gs.access-token"): + access_token = self.properties.get("gs.token") + expiration = self.properties.get("gs.credential-token-expiration") + if (access_token is not None) and (expiration is not None): client_kwargs = { "access_token": access_token, - "credential_token_expiration": self.properties.get("gs.credential-token-expiration"), + "credential_token_expiration": expiration, } return GcsFileSystem(**client_kwargs) else: diff --git a/python/pyproject.toml b/python/pyproject.toml index 93667a6896a9..4d14f9c38063 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -100,18 +100,16 @@ glue = ["boto3"] adlfs = ["adlfs"] dynamodb = ["boto3"] zstandard = ["zstandard"] -<<<<<<< HEAD sql-postgres = ["sqlalchemy", "psycopg2-binary"] -======= gcsfs = ["gcsfs"] ->>>>>>> c5ab4da8e (pyiceberg: Add Google Cloud Storage support) [tool.pytest.ini_options] markers = [ "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint-url args)", "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)", - "integration: marks integration tests against Apache Spark" + "integration: marks integration tests against Apache Spark", "gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint-url)" +] [tool.black] line-length = 130 From 3740c698bd401ebb9bd1b5b5120aeb4e4cf917fa Mon Sep 17 00:00:00 2001 From: Victoria Bukta <3473392+Buktoria@users.noreply.github.com> Date: Sun, 23 Jul 2023 20:08:28 -0400 Subject: [PATCH 3/3] Update python/pyiceberg/io/fsspec.py Co-authored-by: Fokko Driesprong --- python/pyiceberg/io/fsspec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyiceberg/io/fsspec.py b/python/pyiceberg/io/fsspec.py index 98d9e485754a..a2d07dcad3f6 100644 --- a/python/pyiceberg/io/fsspec.py +++ b/python/pyiceberg/io/fsspec.py @@ -138,7 +138,7 @@ def _gs(properties: Properties) -> AbstractFileSystem: "session_kwargs": json.loads(properties.get("gs.session-kwargs", "{}")), "endpoint_url": properties.get("gs.endpoint-url"), "default_location": properties.get("gs.default-location"), - "version_aware": json.loads(properties.get("gs.version-aware", "false").lower()), + "version_aware": properties.get("gs.version-aware", "false").lower() == 'true', } return GCSFileSystem(**fs_kwargs)