Python: Add Google Cloud Storage support

python/Makefile

@@ -17,7 +17,8 @@
 
 install:
 	pip install poetry
-	poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb -E ray -E sql-postgres
+	poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb -E ray -E sql-postgres -E gcsfs
+
 
 check-license:
 	./dev/check-license
@@ -56,3 +57,7 @@ test-coverage:
 	poetry run coverage report -m --fail-under=90
 	poetry run coverage html
 	poetry run coverage xml
+
+test-gcs:
+	sh ./dev/run-gcs-server.sh
+	poetry run pytest tests/ -m gcs ${PYTEST_ARGS}

python/dev/docker-compose-gcs-server.yml

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+version: "3"
+
+services:
+ gcs-server:
+ image: fsouza/fake-gcs-server
+ container_name: gcs-server
+ ports:
+ - 4443:4443
+ entrypoint: >
+ /bin/sh -c "
+ mkdir -p /data/warehouse;
+ /bin/fake-gcs-server -data /data -scheme http;
+ exit 0;
+ "

python/dev/run-gcs-server.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -ex
+
+if [ $(docker ps -q --filter "name=gcs-server" --filter "status=running" ) ]; then
+ echo "Fake GCS Server running"
+else
+ docker-compose -f dev/docker-compose-gcs-server.yml kill
+ docker-compose -f dev/docker-compose-gcs-server.yml up -d
+ while [ -z $(docker ps -q --filter "name=gcs-server" --filter "status=running" ) ]
+ do
+ echo "Waiting for Fake GCS Server"
+ sleep 1
+ done
+fi

python/mkdocs/docs/configuration.md

@@ -74,13 +74,30 @@ For the FileIO there are several configuration options available:
 | Key | Example | Description |
 | ----------------------- | ----------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | adlfs.connection-string | AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqF...;BlobEndpoint=http://localhost/ | A [connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). |
+
 | adlfs.account-name | devstoreaccount1 | The account that you want to connect to |
 | adlfs.account-key | Eby8vdM02xNOcqF... | The key to authentication against the account. |
 | adlfs.sas-token | NuHOuuzdQN7VRM%2FOpOeqBlawRCA845IY05h9eu1Yte4%3D | The shared access signature |
 | adlfs.tenant-id | ad667be4-b811-11ed-afa1-0242ac120002 | The tenant-id |
 | adlfs.client-id | ad667be4-b811-11ed-afa1-0242ac120002 | The client-id |
 | adlfs.client-secret | oCA3R6P\*ka#oa1Sms2J74z... | The client-secret |
 
+### Google Cloud Storage
+
+| Key | Example | Description |
+|--------------------------------|---------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| gs.project | my-gcp-project | Configure Google Cloud Project for GCS FileIO. |
+| gs.endpoint-url | http://0.0.0.0:4443 | Configure an alternative endpoint for the GCS FileIO to access (format protocol://host:port) If not given, defaults to the value of environment variable "STORAGE_EMULATOR_HOST"; if that is not set either, will use the standard Google endpoint. |
+| gs.token | google_default | Configure method authentication to GCS for FileIO. Can be the following, 'google_default', 'cache', 'anon', 'browser', 'cloud'. If not specified your credentials will be resolved in the following order: gcloud CLI default, gcsfs cached token, google compute metadata service, anonymous. |
+| gs.access | read_only | Configure client to have specific access. Must be one of 'read_only', 'read_write', or 'full_control' |
+| gs.consistency | md5 | Configure the check method when writing files. Must be one of 'none', 'size', or 'md5' |
+| gs.cache-timeout | 60 | Configure the cache expiration time in seconds for object metadata cache |
+| gs.requester-pays | False | Configure whether to use requester-pays requests |
+| gs.session-kwargs | {} | Configure a dict of parameters to pass on to aiohttp.ClientSession; can contain, for example, proxy settings. |
+| gs.default-location | US | Configure the default location where buckets are created, like 'US' or 'EUROPE-WEST3'. |
+| gs.version-aware | False | Configure whether to support object versioning on the GCS bucket. |
+| gs.credential-token-expiration | None | Configure expiration for credential generated with an access token. Type datetime. Must be specified if `token` is specified. |
+
 ## REST Catalog
 
 ```yaml

python/mkdocs/docs/index.md

@@ -60,7 +60,8 @@ You can mix and match optional dependencies depending on your needs:
 | s3fs | S3FS as a FileIO implementation to interact with the object store |
 | adlfs | ADLFS as a FileIO implementation to interact with the object store |
 | snappy | Support for snappy Avro compression |
+| gcs | GCS as the FileIO implementation to interact with the object store |
 
-You either need to install `s3fs`, `adlfs` or `pyarrow` for fetching files.
+You either need to install `s3fs`, `adlfs`, `gcs`, or `pyarrow` for fetching files.
 
 There is both a [CLI](cli.md) and [Python API](api.md) available.