From 8de916c51027ccbfcbb50e74271f649e4837f02d Mon Sep 17 00:00:00 2001 From: Dmitry Vukolov Date: Sun, 30 Dec 2018 04:51:03 +0300 Subject: [PATCH] Strip s3:// scheme in S3FSWrapper isdir() and isfile() Only strip s3:// if path starts with that Change-Id: I302e4327fed02de4c4d4a9f6f212347390bb84fa --- python/pyarrow/filesystem.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py index 98efb1e3ec37..92a65ce69892 100644 --- a/python/pyarrow/filesystem.py +++ b/python/pyarrow/filesystem.py @@ -319,7 +319,7 @@ class S3FSWrapper(DaskFileSystem): @implements(FileSystem.isdir) def isdir(self, path): - path = _stringify_path(path) + path = _sanitize_s3(_stringify_path(path)) try: contents = self.fs.ls(path) if len(contents) == 1 and contents[0] == path: @@ -331,7 +331,7 @@ def isdir(self, path): @implements(FileSystem.isfile) def isfile(self, path): - path = _stringify_path(path) + path = _sanitize_s3(_stringify_path(path)) try: contents = self.fs.ls(path) return len(contents) == 1 and contents[0] == path @@ -345,7 +345,7 @@ def walk(self, path, refresh=False): Generator version of what is in s3fs, which yields a flattened list of files """ - path = _stringify_path(path).replace('s3://', '') + path = _sanitize_s3(_stringify_path(path)) directories = set() files = set() @@ -371,6 +371,13 @@ def walk(self, path, refresh=False): yield tup +def _sanitize_s3(path): + if path.startswith('s3://'): + return path.replace('s3://', '') + else: + return path + + def _ensure_filesystem(fs): fs_type = type(fs)