ARROW-16887: [R][Docs] Update Filesystem Vignette for GCS

cpp/src/arrow/filesystem/s3fs_test.cc

@@ -322,7 +322,7 @@ TEST_F(S3OptionsTest, FromAssumeRole) {
 class S3RegionResolutionTest : public AwsTestMixin {};
 
 TEST_F(S3RegionResolutionTest, PublicBucket) {
- ASSERT_OK_AND_EQ("us-east-2", ResolveS3BucketRegion("ursa-labs-taxi-data"));
+ ASSERT_OK_AND_EQ("us-east-2", ResolveS3BucketRegion("voltrondata-labs-datasets"));
 
  // Taken from a registry of open S3-hosted datasets
  // at https://github.com/awslabs/open-data-registry

docs/source/python/dataset.rst

@@ -355,7 +355,7 @@ specifying a S3 path:
 
 .. code-block:: python
 
- dataset = ds.dataset("s3://ursa-labs-taxi-data/", partitioning=["year", "month"])
+ dataset = ds.dataset("s3://voltrondata-labs-datasets/nyc-taxi/")
 
 Typically, you will want to customize the connection parameters, and then
 a file system object can be created and passed to the ``filesystem`` keyword:
@@ -365,8 +365,7 @@ a file system object can be created and passed to the ``filesystem`` keyword:
  from pyarrow import fs
 
  s3 = fs.S3FileSystem(region="us-east-2")
- dataset = ds.dataset("ursa-labs-taxi-data/", filesystem=s3,
- partitioning=["year", "month"])
+ dataset = ds.dataset("voltrondata-labs-datasets/nyc-taxi/", filesystem=s3)
 
 The currently available classes are :class:`~pyarrow.fs.S3FileSystem` and
 :class:`~pyarrow.fs.HadoopFileSystem`. See the :ref:`filesystem` docs for more
@@ -387,8 +386,7 @@ useful for testing or benchmarking.
 
  # By default, MinIO will listen for unencrypted HTTP traffic.
  minio = fs.S3FileSystem(scheme="http", endpoint_override="localhost:9000")
- dataset = ds.dataset("ursa-labs-taxi-data/", filesystem=minio,
- partitioning=["year", "month"])
+ dataset = ds.dataset("voltrondata-labs-datasets/nyc-taxi/", filesystem=minio)
 
 
 Working with Parquet Datasets

python/pyarrow/_s3fs.pyx

@@ -74,8 +74,8 @@ def resolve_s3_region(bucket):
 
  Examples
  --------
- >>> fs.resolve_s3_region('registry.opendata.aws')
- 'us-east-1'
+ >>> fs.resolve_s3_region('voltrondata-labs-datasets')
+ 'us-east-2'
  """
  cdef:
  c_string c_bucket

python/pyarrow/tests/test_fs.py

@@ -1616,15 +1616,17 @@ def test_s3_real_aws():
  assert fs.region == default_region
 
  fs = S3FileSystem(anonymous=True, region='us-east-2')
- entries = fs.get_file_info(FileSelector('ursa-labs-taxi-data'))
+ entries = fs.get_file_info(FileSelector(
+ 'voltrondata-labs-datasets/nyc-taxi'))
  assert len(entries) > 0
- with fs.open_input_stream('ursa-labs-taxi-data/2019/06/data.parquet') as f:
+ key = 'voltrondata-labs-datasets/nyc-taxi/year=2019/month=6/part-0.parquet'
+ with fs.open_input_stream(key) as f:
  md = f.metadata()
  assert 'Content-Type' in md
- assert md['Last-Modified'] == b'2020-01-17T16:26:28Z'
+ assert md['Last-Modified'] == b'2022-07-12T23:32:00Z'
  # For some reason, the header value is quoted
  # (both with AWS and Minio)
- assert md['ETag'] == b'"f1efd5d76cb82861e1542117bfa52b90-8"'
+ assert md['ETag'] == b'"4c6a76826a695c6ac61592bc30cda3df-16"'
 
 
 @pytest.mark.s3
@@ -1653,7 +1655,7 @@ def test_s3_real_aws_region_selection():
 @pytest.mark.s3
 def test_resolve_s3_region():
  from pyarrow.fs import resolve_s3_region
- assert resolve_s3_region('ursa-labs-taxi-data') == 'us-east-2'
+ assert resolve_s3_region('voltrondata-labs-datasets') == 'us-east-2'
  assert resolve_s3_region('mf-nwp-models') == 'eu-west-1'
 
  with pytest.raises(ValueError, match="Not a valid bucket name"):

r/.gitignore

@@ -18,6 +18,7 @@ vignettes/nyc-taxi/
 arrow_*.tar.gz
 arrow_*.tgz
 extra-tests/files
+.deps
 
 # C++ sources for an offline build. They're copied from the ../cpp directory, so ignore them here.
 /tools/cpp/

r/NAMESPACE

@@ -301,6 +301,7 @@ export(float)
 export(float16)
 export(float32)
 export(float64)
+export(gs_bucket)
 export(halffloat)
 export(hive_partition)
 export(infer_type)

r/R/filesystem.R

@@ -155,6 +155,26 @@ FileSelector$create <- function(base_dir, allow_not_found = FALSE, recursive = F
 #' - `allow_bucket_deletion`: logical, if TRUE, the filesystem will delete
 #' buckets if`$DeleteDir()` is called on the bucket level (default `FALSE`).
 #'
+#' `GcsFileSystem$create()` optionally takes arguments:
+#'
+#' - `anonymous`: logical, default `FALSE`. If true, will not attempt to look up
+#' credentials using standard GCS configuration methods.
+#' - `access_token`: optional string for authentication. Should be provided along
+#' with `expiration`
+#' - `expiration`: optional date representing point at which `access_token` will
+#' expire.
+#' - `json_credentials`: optional string for authentication. Point to a JSON
+#' credentials file downloaded from GCS.
+#' - `endpoint_override`: if non-empty, will connect to provided host name / port,
+#' such as "localhost:9001", instead of default GCS ones. This is primarily useful
+#' for testing purposes.
+#' - `scheme`: connection transport (default "https")
+#' - `default_bucket_location`: the default location (or "region") to create new
+#' buckets in.
+#' - `retry_limit_seconds`: the maximum amount of time to spend retrying if
+#' the filesystem encounters errors. Default is 15 seconds.
+#' - `default_metadata`: default metadata to write in new objects.
+#'
 #' @section Methods:
 #'
 #' - `$GetFileInfo(x)`: `x` may be a [FileSelector][FileSelector] or a character
@@ -426,7 +446,7 @@ default_s3_options <- list(
 #' relative path. Note that this function's success does not guarantee that you
 #' are authorized to access the bucket's contents.
 #' @examplesIf FALSE
-#' bucket <- s3_bucket("ursa-labs-taxi-data")
+#' bucket <- s3_bucket("voltrondata-labs-datasets")
 #' @export
 s3_bucket <- function(bucket, ...) {
  assert_that(is.string(bucket))
@@ -448,6 +468,28 @@ s3_bucket <- function(bucket, ...) {
  SubTreeFileSystem$create(fs_and_path$path, fs)
 }
 
+#' Connect to a Google Cloud Storage (GCS) bucket
+#'
+#' `gs_bucket()` is a convenience function to create an `GcsFileSystem` object
+#' that holds onto its relative path
+#'
+#' @param bucket string GCS bucket name or path
+#' @param ... Additional connection options, passed to `GcsFileSystem$create()`
+#' @return A `SubTreeFileSystem` containing an `GcsFileSystem` and the bucket's
+#' relative path. Note that this function's success does not guarantee that you
+#' are authorized to access the bucket's contents.
+#' @examplesIf FALSE
+#' bucket <- gs_bucket("voltrondata-labs-datasets")
+#' @export
+gs_bucket <- function(bucket, ...) {
+ assert_that(is.string(bucket))
+ args <- list2(...)
+
+ fs <- exec(GcsFileSystem$create, !!!args)
+
+ SubTreeFileSystem$create(bucket, fs)
+}
+
 #' @usage NULL
 #' @format NULL
 #' @rdname FileSystem

r/_pkgdown.yml

@@ -90,7 +90,7 @@ navbar:
  href: articles/install.html
  - text: Working with Arrow Datasets and dplyr
  href: articles/dataset.html
- - text: Working with Cloud Storage (S3)
+ - text: Working with Cloud Storage (S3, GCS)
  href: articles/fs.html
  - text: Apache Arrow in Python and R with reticulate
  href: articles/python.html
@@ -198,6 +198,7 @@ reference:
  - title: File systems
  contents:
  - s3_bucket
+ - gs_bucket
  - FileSystem
  - FileInfo
  - FileSelector

r/tests/testthat/test-filesystem.R

@@ -147,7 +147,7 @@ test_that("FileSystem$from_uri", {
  skip_on_cran()
  skip_if_not_available("s3")
  skip_if_offline()
- fs_and_path <- FileSystem$from_uri("s3://ursa-labs-taxi-data")
+ fs_and_path <- FileSystem$from_uri("s3://voltrondata-labs-datasets")
  expect_r6_class(fs_and_path$fs, "S3FileSystem")
  expect_identical(fs_and_path$fs$region, "us-east-2")
 })
@@ -156,11 +156,11 @@ test_that("SubTreeFileSystem$create() with URI", {
  skip_on_cran()
  skip_if_not_available("s3")
  skip_if_offline()
- fs <- SubTreeFileSystem$create("s3://ursa-labs-taxi-data")
+ fs <- SubTreeFileSystem$create("s3://voltrondata-labs-datasets")
  expect_r6_class(fs, "SubTreeFileSystem")
  expect_identical(
  capture.output(print(fs)),
- "SubTreeFileSystem: s3://ursa-labs-taxi-data/"
+ "SubTreeFileSystem: s3://voltrondata-labs-datasets/"
  )
 })
 
@@ -187,6 +187,19 @@ test_that("s3_bucket", {
  capture.output(print(bucket)),
  "SubTreeFileSystem: s3://ursa-labs-r-test/"
  )
- skip_on_os("windows") # FIXME
  expect_identical(bucket$base_path, "ursa-labs-r-test/")
 })
+
+test_that("gs_bucket", {
+ skip_on_cran()
+ skip_if_not_available("gcs")
+ skip_if_offline()
+ bucket <- gs_bucket("voltrondata-labs-datasets")
+ expect_r6_class(bucket, "SubTreeFileSystem")
+ expect_r6_class(bucket$base_fs, "GcsFileSystem")
+ expect_identical(
+ capture.output(print(bucket)),
+ "SubTreeFileSystem: gs://voltrondata-labs-datasets/"
+ )
+ expect_identical(bucket$base_path, "voltrondata-labs-datasets/")
+})