From 5200af16fda2234f7ad08df3730ddbaaa74db33f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 18 Sep 2019 18:12:35 +0200 Subject: [PATCH 01/39] s3 filesystem bindings --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 + python/CMakeLists.txt | 4 + python/pyarrow/_csv.pyx | 4 +- python/pyarrow/_cuda.pyx | 4 +- python/pyarrow/_fs.pyx | 13 +- python/pyarrow/_json.pyx | 4 +- python/pyarrow/_orc.pxd | 4 +- python/pyarrow/_orc.pyx | 2 +- python/pyarrow/_parquet.pxd | 10 +- python/pyarrow/_parquet.pyx | 8 +- python/pyarrow/_s3.pyx | 52 ++++++ python/pyarrow/feather.pxi | 4 +- python/pyarrow/fs.py | 5 + python/pyarrow/includes/libarrow.pxd | 166 ++++++++++++++------ python/pyarrow/includes/libarrow_fs.pxd | 92 ----------- python/pyarrow/includes/libarrow_s3.pxd | 36 +++++ python/pyarrow/io-hdfs.pxi | 4 +- python/pyarrow/io.pxi | 56 +++---- python/pyarrow/ipc.pxi | 28 ++-- python/pyarrow/lib.pxd | 24 +-- python/pyarrow/serialization.pxi | 6 +- python/pyarrow/tests/conftest.py | 6 + python/pyarrow/tests/test_fs.py | 31 ++++ python/setup.py | 6 + 24 files changed, 345 insertions(+), 230 deletions(-) create mode 100644 python/pyarrow/_s3.pyx delete mode 100644 python/pyarrow/includes/libarrow_fs.pxd create mode 100644 python/pyarrow/includes/libarrow_s3.pxd diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index e8e4fa6f7bc7..95c48172baca 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2535,6 +2535,12 @@ if(ARROW_S3) include_directories(SYSTEM ${AWSSDK_INCLUDE_DIR}) message(STATUS "Found AWS SDK headers: ${AWSSDK_INCLUDE_DIR}") message(STATUS "Found AWS SDK libraries: ${AWSSDK_LINK_LIBRARIES}") + + if(APPLE) + set_target_properties(AWS::aws-c-common PROPERTIES + INTERFACE_LINK_LIBRARIES "-pthread;pthread;-framework CoreFoundation" + ) + endif() endif() # Write out the package configurations. diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b2282a6b69bc..67ce446b5c8f 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -385,6 +385,10 @@ set(CYTHON_EXTENSIONS lib _fs _csv _json) set(LINK_LIBS arrow_shared arrow_python_shared) +if(PYARROW_BUILD_S3) + set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _s3) +endif() + if(PYARROW_BUILD_CUDA) # Arrow CUDA find_package(ArrowCuda) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index 25ff47d39b54..5dccad760f42 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -466,7 +466,7 @@ cdef class ConvertOptions: self.options.include_missing_columns = value -cdef _get_reader(input_file, shared_ptr[InputStream]* out): +cdef _get_reader(input_file, shared_ptr[CInputStream]* out): use_memory_map = False get_input_stream(input_file, use_memory_map, out) @@ -522,7 +522,7 @@ def read_csv(input_file, read_options=None, parse_options=None, Contents of the CSV file as a in-memory table. """ cdef: - shared_ptr[InputStream] stream + shared_ptr[CInputStream] stream CCSVReadOptions c_read_options CCSVParseOptions c_parse_options CCSVConvertOptions c_convert_options diff --git a/python/pyarrow/_cuda.pyx b/python/pyarrow/_cuda.pyx index a9f51b0a654f..e0cad68b2bfe 100644 --- a/python/pyarrow/_cuda.pyx +++ b/python/pyarrow/_cuda.pyx @@ -729,7 +729,7 @@ cdef class BufferReader(NativeFile): self.buffer = obj self.reader = new CCudaBufferReader(self.buffer.buffer) self.set_random_access_file( - shared_ptr[RandomAccessFile](self.reader)) + shared_ptr[CRandomAccessFile](self.reader)) self.is_readable = True def read_buffer(self, nbytes=None): @@ -776,7 +776,7 @@ cdef class BufferWriter(NativeFile): def __cinit__(self, CudaBuffer buffer): self.buffer = buffer self.writer = new CCudaBufferWriter(self.buffer.cuda_buffer) - self.set_output_stream(shared_ptr[OutputStream](self.writer)) + self.set_output_stream(shared_ptr[COutputStream](self.writer)) self.is_writable = True def writeat(self, int64_t position, object data): diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index 769ca8d7391f..db41051eaeb1 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -22,17 +22,8 @@ import six from pyarrow.compat import frombytes, tobytes from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint -from pyarrow.includes.libarrow_fs cimport * -from pyarrow.util import _stringify_path from pyarrow.lib import _detect_compression -from pyarrow.lib cimport ( - check_status, - NativeFile, - BufferedOutputStream, - BufferedInputStream, - CompressedInputStream, - CompressedOutputStream -) +from pyarrow.lib cimport * cdef inline c_string _path_as_bytes(path) except *: @@ -523,4 +514,4 @@ cdef class SubTreeFileSystem(FileSystem): cdef init(self, const shared_ptr[CFileSystem]& wrapped): FileSystem.init(self, wrapped) - self.subtreefs = wrapped.get() + self.subtreefs = wrapped.get() \ No newline at end of file diff --git a/python/pyarrow/_json.pyx b/python/pyarrow/_json.pyx index ffbf01c09e7c..da3588a5e625 100644 --- a/python/pyarrow/_json.pyx +++ b/python/pyarrow/_json.pyx @@ -135,7 +135,7 @@ cdef class ParseOptions: self.options.newlines_in_values = value -cdef _get_reader(input_file, shared_ptr[InputStream]* out): +cdef _get_reader(input_file, shared_ptr[CInputStream]* out): use_memory_map = False get_input_stream(input_file, use_memory_map, out) @@ -175,7 +175,7 @@ def read_json(input_file, read_options=None, parse_options=None, Contents of the JSON file as a in-memory table. """ cdef: - shared_ptr[InputStream] stream + shared_ptr[CInputStream] stream CJSONReadOptions c_read_options CJSONParseOptions c_parse_options shared_ptr[CJSONReader] reader diff --git a/python/pyarrow/_orc.pxd b/python/pyarrow/_orc.pxd index ebbf8beda828..6c18ca503a70 100644 --- a/python/pyarrow/_orc.pxd +++ b/python/pyarrow/_orc.pxd @@ -28,7 +28,7 @@ from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus, CKeyValueMetadata, CRecordBatch, CTable, - RandomAccessFile, OutputStream, + RandomAccessFile, COutputStream, TimeUnit) @@ -37,7 +37,7 @@ cdef extern from "arrow/adapters/orc/adapter.h" \ cdef cppclass ORCFileReader: @staticmethod - CStatus Open(const shared_ptr[RandomAccessFile]& file, + CStatus Open(const shared_ptr[CRandomAccessFile]& file, CMemoryPool* pool, unique_ptr[ORCFileReader]* reader) diff --git a/python/pyarrow/_orc.pyx b/python/pyarrow/_orc.pyx index c9f5b2e158d6..0ee3ca632e46 100644 --- a/python/pyarrow/_orc.pyx +++ b/python/pyarrow/_orc.pyx @@ -46,7 +46,7 @@ cdef class ORCReader: def open(self, object source, c_bool use_memory_map=True): cdef: - shared_ptr[RandomAccessFile] rd_handle + shared_ptr[CRandomAccessFile] rd_handle self.source = source diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 19fb214c8f04..5cf3ff5931b4 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -24,7 +24,7 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport (CChunkedArray, CSchema, CStatus, CTable, CMemoryPool, CBuffer, CKeyValueMetadata, - RandomAccessFile, OutputStream, + CRandomAccessFile, COutputStream, TimeUnit) @@ -316,7 +316,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil: unique_ptr[CRowGroupMetaData] RowGroup(int i) const SchemaDescriptor* schema() shared_ptr[const CKeyValueMetadata] key_value_metadata() const - void WriteTo(OutputStream* dst) const + void WriteTo(COutputStream* dst) const cdef shared_ptr[CFileMetaData] CFileMetaData_Make \ " parquet::FileMetaData::Make"(const void* serialized_metadata, @@ -406,7 +406,7 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil: cdef cppclass FileReaderBuilder: FileReaderBuilder() - CStatus Open(const shared_ptr[RandomAccessFile]& file, + CStatus Open(const shared_ptr[CRandomAccessFile]& file, const CReaderProperties& properties, const shared_ptr[CFileMetaData]& metadata) @@ -435,7 +435,7 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: @staticmethod CStatus Open(const CSchema& schema, CMemoryPool* pool, - const shared_ptr[OutputStream]& sink, + const shared_ptr[COutputStream]& sink, const shared_ptr[WriterProperties]& properties, const shared_ptr[ArrowWriterProperties]& arrow_properties, unique_ptr[FileWriter]* writer) @@ -448,4 +448,4 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: CStatus WriteMetaDataFile( const CFileMetaData& file_metadata, - const OutputStream* sink) + const COutputStream* sink) diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 3d62c0eb13e6..f93def8a67ee 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -568,7 +568,7 @@ cdef class FileMetaData: def __reduce__(self): cdef: NativeFile sink = BufferOutputStream() - OutputStream* c_sink = sink.get_output_stream().get() + COutputStream* c_sink = sink.get_output_stream().get() with nogil: self._metadata.WriteTo(c_sink) @@ -694,7 +694,7 @@ cdef class FileMetaData: Write the metadata object to a metadata-only file """ cdef: - shared_ptr[OutputStream] sink + shared_ptr[COutputStream] sink c_string c_where try: @@ -1010,7 +1010,7 @@ cdef class ParquetReader: read_dictionary=None, FileMetaData metadata=None, int buffer_size=0): cdef: - shared_ptr[RandomAccessFile] rd_handle + shared_ptr[CRandomAccessFile] rd_handle shared_ptr[CFileMetaData] c_metadata CReaderProperties properties = default_reader_properties() ArrowReaderProperties arrow_props = ( @@ -1202,7 +1202,7 @@ cdef class ParquetReader: cdef class ParquetWriter: cdef: unique_ptr[FileWriter] writer - shared_ptr[OutputStream] sink + shared_ptr[COutputStream] sink bint own_sink cdef readonly: diff --git a/python/pyarrow/_s3.pyx b/python/pyarrow/_s3.pyx new file mode 100644 index 000000000000..d363a61644ec --- /dev/null +++ b/python/pyarrow/_s3.pyx @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: language_level = 3 + +import six + +from pyarrow.compat import frombytes, tobytes +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint +from pyarrow.lib import _detect_compression +from pyarrow.lib cimport * + + +cdef class S3FileSystem(FileSystem): + + cdef: + CS3FileSystem* s3fs + + def __init__(self, str access_key, str secret_key, str region='us-east-1', + str scheme='https', str endpoint_override=None): + cdef: + CS3Options options + shared_ptr[CS3FileSystem] wrapped + + options.access_key = tobytes(access_key) + options.secret_key = tobytes(secret_key) + options.region = tobytes(region) + options.scheme = tobytes(scheme) + if endpoint_override is not None: + options.endpoint_override = endpoint_override + + check_status(CS3FileSystem.Make(options, &wrapped)) + self.init( wrapped) + + cdef init(self, const shared_ptr[CFileSystem]& wrapped): + FileSystem.init(self, wrapped) + self.s3fs = wrapped.get() diff --git a/python/pyarrow/feather.pxi b/python/pyarrow/feather.pxi index 6fd13bc04b46..8700f67ae621 100644 --- a/python/pyarrow/feather.pxi +++ b/python/pyarrow/feather.pxi @@ -34,7 +34,7 @@ cdef class FeatherWriter: self.num_rows = -1 def open(self, object dest): - cdef shared_ptr[OutputStream] sink + cdef shared_ptr[COutputStream] sink get_writer(dest, &sink) with nogil: @@ -76,7 +76,7 @@ cdef class FeatherReader: pass def open(self, source, c_bool use_memory_map=True): - cdef shared_ptr[RandomAccessFile] reader + cdef shared_ptr[CRandomAccessFile] reader get_reader(source, use_memory_map, &reader) with nogil: diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index cd5263acbcad..5b94a37c8873 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -18,3 +18,8 @@ from __future__ import absolute_import from pyarrow._fs import * # noqa + +try: + from pyarrow._s3 import * # noqa +except ImportError: + pass diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 69dafa4e46f2..3cb4f70bafea 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -765,13 +765,16 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: CStatus Write(const uint8_t* data, int64_t nbytes) CStatus Flush() - cdef cppclass OutputStream(FileInterface, Writable): + cdef cppclass COutputStream" arrow::io::OutputStream"(FileInterface, + Writable): pass - cdef cppclass InputStream(FileInterface, Readable): + cdef cppclass CInputStream" arrow::io::InputStream"(FileInterface, + Readable): pass - cdef cppclass RandomAccessFile(InputStream, Seekable): + cdef cppclass CRandomAccessFile" arrow::io::RandomAccessFile"(CInputStream, + Seekable): CStatus GetSize(int64_t* size) CStatus ReadAt(int64_t position, int64_t nbytes, @@ -780,24 +783,24 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[CBuffer]* out) c_bool supports_zero_copy() - cdef cppclass WritableFile(OutputStream, Seekable): + cdef cppclass WritableFile(COutputStream, Seekable): CStatus WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) - cdef cppclass ReadWriteFileInterface(RandomAccessFile, + cdef cppclass ReadWriteFileInterface(CRandomAccessFile, WritableFile): pass - cdef cppclass FileSystem: + cdef cppclass CIOFileSystem" arrow::io::FileSystem": CStatus Stat(const c_string& path, FileStatistics* stat) - cdef cppclass FileOutputStream(OutputStream): + cdef cppclass FileOutputStream(COutputStream): @staticmethod - CStatus Open(const c_string& path, shared_ptr[OutputStream]* file) + CStatus Open(const c_string& path, shared_ptr[COutputStream]* file) int file_descriptor() - cdef cppclass ReadableFile(RandomAccessFile): + cdef cppclass ReadableFile(CRandomAccessFile): @staticmethod CStatus Open(const c_string& path, shared_ptr[ReadableFile]* file) @@ -823,46 +826,46 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int file_descriptor() cdef cppclass CCompressedInputStream \ - " arrow::io::CompressedInputStream"(InputStream): + " arrow::io::CompressedInputStream"(CInputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, - shared_ptr[InputStream] raw, + shared_ptr[CInputStream] raw, shared_ptr[CCompressedInputStream]* out) @staticmethod - CStatus Make(CCodec* codec, shared_ptr[InputStream] raw, + CStatus Make(CCodec* codec, shared_ptr[CInputStream] raw, shared_ptr[CCompressedInputStream]* out) cdef cppclass CCompressedOutputStream \ - " arrow::io::CompressedOutputStream"(OutputStream): + " arrow::io::CompressedOutputStream"(COutputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, - shared_ptr[OutputStream] raw, + shared_ptr[COutputStream] raw, shared_ptr[CCompressedOutputStream]* out) @staticmethod - CStatus Make(CCodec* codec, shared_ptr[OutputStream] raw, + CStatus Make(CCodec* codec, shared_ptr[COutputStream] raw, shared_ptr[CCompressedOutputStream]* out) cdef cppclass CBufferedInputStream \ - " arrow::io::BufferedInputStream"(InputStream): + " arrow::io::BufferedInputStream"(CInputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, - shared_ptr[InputStream] raw, + shared_ptr[CInputStream] raw, shared_ptr[CBufferedInputStream]* out) - shared_ptr[InputStream] Detach() + shared_ptr[CInputStream] Detach() cdef cppclass CBufferedOutputStream \ - " arrow::io::BufferedOutputStream"(OutputStream): + " arrow::io::BufferedOutputStream"(COutputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, - shared_ptr[OutputStream] raw, + shared_ptr[COutputStream] raw, shared_ptr[CBufferedOutputStream]* out) - CStatus Detach(shared_ptr[OutputStream]* raw) + CStatus Detach(shared_ptr[COutputStream]* raw) # ---------------------------------------------------------------------- # HDFS @@ -894,13 +897,13 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int64_t block_size int16_t permissions - cdef cppclass HdfsReadableFile(RandomAccessFile): + cdef cppclass HdfsReadableFile(CRandomAccessFile): pass - cdef cppclass HdfsOutputStream(OutputStream): + cdef cppclass HdfsOutputStream(COutputStream): pass - cdef cppclass CHadoopFileSystem" arrow::io::HadoopFileSystem"(FileSystem): + cdef cppclass CHadoopFileSystem" arrow::io::HadoopFileSystem"(CIOFileSystem): @staticmethod CStatus Connect(const HdfsConnectionConfig* config, shared_ptr[CHadoopFileSystem]* client) @@ -936,16 +939,16 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[HdfsOutputStream]* handle) cdef cppclass CBufferReader \ - " arrow::io::BufferReader"(RandomAccessFile): + " arrow::io::BufferReader"(CRandomAccessFile): CBufferReader(const shared_ptr[CBuffer]& buffer) CBufferReader(const uint8_t* data, int64_t nbytes) cdef cppclass CBufferOutputStream \ - " arrow::io::BufferOutputStream"(OutputStream): + " arrow::io::BufferOutputStream"(COutputStream): CBufferOutputStream(const shared_ptr[CResizableBuffer]& buffer) cdef cppclass CMockOutputStream \ - " arrow::io::MockOutputStream"(OutputStream): + " arrow::io::MockOutputStream"(COutputStream): CMockOutputStream() int64_t GetExtentBytesWritten() @@ -958,6 +961,71 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: void set_memcopy_threshold(int64_t threshold) +cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: + + enum CFileType "arrow::fs::FileType": + CFileType_NonExistent "arrow::fs::FileType::NonExistent" + CFileType_Unknown "arrow::fs::FileType::Unknown" + CFileType_File "arrow::fs::FileType::File" + CFileType_Directory "arrow::fs::FileType::Directory" + + cdef cppclass CTimePoint "arrow::fs::TimePoint": + pass + + cdef cppclass CFileStats "arrow::fs::FileStats": + CFileStats() + CFileStats(CFileStats&&) + CFileStats& operator=(CFileStats&&) + CFileStats(const CFileStats&) + CFileStats& operator=(const CFileStats&) + + CFileType type() + void set_type(CFileType type) + c_string path() + void set_path(const c_string& path) + c_string base_name() + int64_t size() + void set_size(int64_t size) + c_string extension() + CTimePoint mtime() + void set_mtime(CTimePoint mtime) + + cdef cppclass CSelector "arrow::fs::Selector": + CSelector() + c_string base_dir + c_bool allow_non_existent + c_bool recursive + + cdef cppclass CFileSystem "arrow::fs::FileSystem": + CStatus GetTargetStats(const c_string& path, CFileStats* out) + CStatus GetTargetStats(const vector[c_string]& paths, + vector[CFileStats]* out) + CStatus GetTargetStats(const CSelector& select, + vector[CFileStats]* out) + CStatus CreateDir(const c_string& path, c_bool recursive) + CStatus DeleteDir(const c_string& path) + CStatus DeleteFile(const c_string& path) + CStatus DeleteFiles(const vector[c_string]& paths) + CStatus Move(const c_string& src, const c_string& dest) + CStatus CopyFile(const c_string& src, const c_string& dest) + CStatus OpenInputStream(const c_string& path, + shared_ptr[CInputStream]* out) + CStatus OpenInputFile(const c_string& path, + shared_ptr[CRandomAccessFile]* out) + CStatus OpenOutputStream(const c_string& path, + shared_ptr[COutputStream]* out) + CStatus OpenAppendStream(const c_string& path, + shared_ptr[COutputStream]* out) + + cdef cppclass CLocalFileSystem "arrow::fs::LocalFileSystem"(CFileSystem): + LocalFileSystem() + + cdef cppclass CSubTreeFileSystem \ + "arrow::fs::SubTreeFileSystem"(CFileSystem): + CSubTreeFileSystem(const c_string& base_path, + shared_ptr[CFileSystem] base_fs) + + cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: enum MessageType" arrow::ipc::Message::Type": MessageType_SCHEMA" arrow::ipc::Message::SCHEMA" @@ -1001,14 +1069,14 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: MetadataVersion metadata_version() MessageType type() - CStatus SerializeTo(OutputStream* stream, const CIpcOptions& options, + CStatus SerializeTo(COutputStream* stream, const CIpcOptions& options, int64_t* output_length) c_string FormatMessageType(MessageType type) cdef cppclass CMessageReader" arrow::ipc::MessageReader": @staticmethod - unique_ptr[CMessageReader] Open(const shared_ptr[InputStream]& stream) + unique_ptr[CMessageReader] Open(const shared_ptr[CInputStream]& stream) CStatus ReadNextMessage(unique_ptr[CMessage]* out) @@ -1020,7 +1088,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: cdef cppclass CRecordBatchStreamReader \ " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader): @staticmethod - CStatus Open(const InputStream* stream, + CStatus Open(const CInputStream* stream, shared_ptr[CRecordBatchReader]* out) @staticmethod @@ -1031,24 +1099,24 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: " arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter): @staticmethod CResult[shared_ptr[CRecordBatchWriter]] Open( - OutputStream* sink, const shared_ptr[CSchema]& schema, + COutputStream* sink, const shared_ptr[CSchema]& schema, CIpcOptions& options) cdef cppclass CRecordBatchFileWriter \ " arrow::ipc::RecordBatchFileWriter"(CRecordBatchWriter): @staticmethod CResult[shared_ptr[CRecordBatchWriter]] Open( - OutputStream* sink, const shared_ptr[CSchema]& schema, + COutputStream* sink, const shared_ptr[CSchema]& schema, CIpcOptions& options) cdef cppclass CRecordBatchFileReader \ " arrow::ipc::RecordBatchFileReader": @staticmethod - CStatus Open(RandomAccessFile* file, + CStatus Open(CRandomAccessFile* file, shared_ptr[CRecordBatchFileReader]* out) @staticmethod - CStatus Open2" Open"(RandomAccessFile* file, + CStatus Open2" Open"(CRandomAccessFile* file, int64_t footer_offset, shared_ptr[CRecordBatchFileReader]* out) @@ -1058,16 +1126,16 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: CStatus ReadRecordBatch(int i, shared_ptr[CRecordBatch]* batch) - CStatus ReadMessage(InputStream* stream, unique_ptr[CMessage]* message) + CStatus ReadMessage(CInputStream* stream, unique_ptr[CMessage]* message) CStatus GetRecordBatchSize(const CRecordBatch& batch, int64_t* size) CStatus GetTensorSize(const CTensor& tensor, int64_t* size) - CStatus WriteTensor(const CTensor& tensor, OutputStream* dst, + CStatus WriteTensor(const CTensor& tensor, COutputStream* dst, int32_t* metadata_length, int64_t* body_length) - CStatus ReadTensor(InputStream* stream, shared_ptr[CTensor]* out) + CStatus ReadTensor(CInputStream* stream, shared_ptr[CTensor]* out) CStatus ReadRecordBatch(const CMessage& message, const shared_ptr[CSchema]& schema, @@ -1082,16 +1150,16 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: CMemoryPool* pool, shared_ptr[CBuffer]* out) - CStatus ReadSchema(InputStream* stream, CDictionaryMemo* dictionary_memo, + CStatus ReadSchema(CInputStream* stream, CDictionaryMemo* dictionary_memo, shared_ptr[CSchema]* out) CStatus ReadRecordBatch(const shared_ptr[CSchema]& schema, CDictionaryMemo* dictionary_memo, - InputStream* stream, + CInputStream* stream, shared_ptr[CRecordBatch]* out) - CStatus AlignStream(InputStream* stream, int64_t alignment) - CStatus AlignStream(OutputStream* stream, int64_t alignment) + CStatus AlignStream(CInputStream* stream, int64_t alignment) + CStatus AlignStream(COutputStream* stream, int64_t alignment) cdef CStatus GetRecordBatchPayload\ " arrow::ipc::internal::GetRecordBatchPayload"( @@ -1102,7 +1170,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: cdef cppclass CFeatherWriter" arrow::ipc::feather::TableWriter": @staticmethod - CStatus Open(const shared_ptr[OutputStream]& stream, + CStatus Open(const shared_ptr[COutputStream]& stream, unique_ptr[CFeatherWriter]* out) void SetDescription(const c_string& desc) @@ -1113,7 +1181,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: cdef cppclass CFeatherReader" arrow::ipc::feather::TableReader": @staticmethod - CStatus Open(const shared_ptr[RandomAccessFile]& file, + CStatus Open(const shared_ptr[CRandomAccessFile]& file, unique_ptr[CFeatherReader]* out) c_string GetDescription() @@ -1172,7 +1240,7 @@ cdef extern from "arrow/csv/api.h" namespace "arrow::csv" nogil: cdef cppclass CCSVReader" arrow::csv::TableReader": @staticmethod - CStatus Make(CMemoryPool*, shared_ptr[InputStream], + CStatus Make(CMemoryPool*, shared_ptr[CInputStream], CCSVReadOptions, CCSVParseOptions, CCSVConvertOptions, shared_ptr[CCSVReader]* out) @@ -1200,7 +1268,7 @@ cdef extern from "arrow/json/reader.h" namespace "arrow::json" nogil: cdef cppclass CJSONReader" arrow::json::TableReader": @staticmethod - CStatus Make(CMemoryPool*, shared_ptr[InputStream], + CStatus Make(CMemoryPool*, shared_ptr[CInputStream], CJSONReadOptions, CJSONParseOptions, shared_ptr[CJSONReader]* out) @@ -1379,10 +1447,10 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: CStatus Make(const uint8_t* data, int64_t size, object base, shared_ptr[CBuffer]* out) - cdef cppclass PyReadableFile(RandomAccessFile): + cdef cppclass PyReadableFile(CRandomAccessFile): PyReadableFile(object fo) - cdef cppclass PyOutputStream(OutputStream): + cdef cppclass PyOutputStream(COutputStream): PyOutputStream(object fo) cdef cppclass PandasOptions: @@ -1398,7 +1466,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: shared_ptr[CRecordBatch] batch vector[shared_ptr[CTensor]] tensors - CStatus WriteTo(OutputStream* dst) + CStatus WriteTo(COutputStream* dst) CStatus GetComponents(CMemoryPool* pool, PyObject** dst) CStatus SerializeObject(object context, object sequence, @@ -1408,7 +1476,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: const CSerializedPyObject& obj, PyObject* base, PyObject** out) - CStatus ReadSerializedObject(RandomAccessFile* src, + CStatus ReadSerializedObject(CRandomAccessFile* src, CSerializedPyObject* out) CStatus GetSerializedFromComponents(int num_tensors, int num_ndarrays, diff --git a/python/pyarrow/includes/libarrow_fs.pxd b/python/pyarrow/includes/libarrow_fs.pxd deleted file mode 100644 index f54a2e50357e..000000000000 --- a/python/pyarrow/includes/libarrow_fs.pxd +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# distutils: language = c++ - -from libcpp.functional cimport function - -from pyarrow.includes.common cimport * -from pyarrow.includes.libarrow cimport ( - InputStream as CInputStream, - OutputStream as COutputStream, - RandomAccessFile as CRandomAccessFile -) - - -cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: - - enum CFileType "arrow::fs::FileType": - CFileType_NonExistent "arrow::fs::FileType::NonExistent" - CFileType_Unknown "arrow::fs::FileType::Unknown" - CFileType_File "arrow::fs::FileType::File" - CFileType_Directory "arrow::fs::FileType::Directory" - - cdef cppclass CTimePoint "arrow::fs::TimePoint": - pass - - cdef cppclass CFileStats "arrow::fs::FileStats": - CFileStats() - CFileStats(CFileStats&&) - CFileStats& operator=(CFileStats&&) - CFileStats(const CFileStats&) - CFileStats& operator=(const CFileStats&) - - CFileType type() - void set_type(CFileType type) - c_string path() - void set_path(const c_string& path) - c_string base_name() - int64_t size() - void set_size(int64_t size) - c_string extension() - CTimePoint mtime() - void set_mtime(CTimePoint mtime) - - cdef cppclass CSelector "arrow::fs::Selector": - CSelector() - c_string base_dir - c_bool allow_non_existent - c_bool recursive - - cdef cppclass CFileSystem "arrow::fs::FileSystem": - CStatus GetTargetStats(const c_string& path, CFileStats* out) - CStatus GetTargetStats(const vector[c_string]& paths, - vector[CFileStats]* out) - CStatus GetTargetStats(const CSelector& select, - vector[CFileStats]* out) - CStatus CreateDir(const c_string& path, c_bool recursive) - CStatus DeleteDir(const c_string& path) - CStatus DeleteFile(const c_string& path) - CStatus DeleteFiles(const vector[c_string]& paths) - CStatus Move(const c_string& src, const c_string& dest) - CStatus CopyFile(const c_string& src, const c_string& dest) - CStatus OpenInputStream(const c_string& path, - shared_ptr[CInputStream]* out) - CStatus OpenInputFile(const c_string& path, - shared_ptr[CRandomAccessFile]* out) - CStatus OpenOutputStream(const c_string& path, - shared_ptr[COutputStream]* out) - CStatus OpenAppendStream(const c_string& path, - shared_ptr[COutputStream]* out) - - cdef cppclass CLocalFileSystem "arrow::fs::LocalFileSystem"(CFileSystem): - LocalFileSystem() - - cdef cppclass CSubTreeFileSystem \ - "arrow::fs::SubTreeFileSystem"(CFileSystem): - CSubTreeFileSystem(const c_string& base_path, - shared_ptr[CFileSystem] base_fs) diff --git a/python/pyarrow/includes/libarrow_s3.pxd b/python/pyarrow/includes/libarrow_s3.pxd new file mode 100644 index 000000000000..8586132ab131 --- /dev/null +++ b/python/pyarrow/includes/libarrow_s3.pxd @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from libcpp.functional cimport function + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport CFileSystem + +cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: + + cdef struct CS3Options "arrow::fs::S3Options": + c_string region + c_string endpoint_override + c_string scheme + c_string access_key + c_string secret_key + + cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem): + @staticmethod + CStatus Make(const CS3Options& options, shared_ptr[CS3FileSystem]* out) diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi index e9655e4a7f74..b224abbead82 100644 --- a/python/pyarrow/io-hdfs.pxi +++ b/python/pyarrow/io-hdfs.pxi @@ -424,7 +424,7 @@ cdef class HadoopFileSystem: c_replication, c_default_block_size, &wr_handle)) - out.set_output_stream( wr_handle) + out.set_output_stream( wr_handle) out.is_writable = True else: with nogil: @@ -432,7 +432,7 @@ cdef class HadoopFileSystem: .OpenReadable(c_path, &rd_handle)) out.set_random_access_file( - rd_handle) + rd_handle) out.is_readable = True assert not out.closed diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index 6ff1403006ef..0cdf21efc010 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -133,29 +133,29 @@ cdef class NativeFile: else: check_status(self.output_stream.get().Close()) - cdef set_random_access_file(self, shared_ptr[RandomAccessFile] handle): - self.input_stream = handle + cdef set_random_access_file(self, shared_ptr[CRandomAccessFile] handle): + self.input_stream = handle self.random_access = handle self.is_seekable = True - cdef set_input_stream(self, shared_ptr[InputStream] handle): + cdef set_input_stream(self, shared_ptr[CInputStream] handle): self.input_stream = handle self.random_access.reset() self.is_seekable = False - cdef set_output_stream(self, shared_ptr[OutputStream] handle): + cdef set_output_stream(self, shared_ptr[COutputStream] handle): self.output_stream = handle - cdef shared_ptr[RandomAccessFile] get_random_access_file(self) except *: + cdef shared_ptr[CRandomAccessFile] get_random_access_file(self) except *: self._assert_readable() self._assert_seekable() return self.random_access - cdef shared_ptr[InputStream] get_input_stream(self) except *: + cdef shared_ptr[CInputStream] get_input_stream(self) except *: self._assert_readable() return self.input_stream - cdef shared_ptr[OutputStream] get_output_stream(self) except *: + cdef shared_ptr[COutputStream] get_output_stream(self) except *: self._assert_writable() return self.output_stream @@ -679,11 +679,11 @@ cdef class PythonFile(NativeFile): if kind == 'r': self.set_random_access_file( - shared_ptr[RandomAccessFile](new PyReadableFile(handle))) + shared_ptr[CRandomAccessFile](new PyReadableFile(handle))) self.is_readable = True else: self.set_output_stream( - shared_ptr[OutputStream](new PyOutputStream(handle))) + shared_ptr[COutputStream](new PyOutputStream(handle))) self.is_writable = True def truncate(self, pos=None): @@ -720,8 +720,8 @@ cdef class MemoryMappedFile(NativeFile): result.path = path result.is_readable = True result.is_writable = True - result.set_output_stream( handle) - result.set_random_access_file( handle) + result.set_output_stream( handle) + result.set_random_access_file( handle) result.handle = handle return result @@ -750,8 +750,8 @@ cdef class MemoryMappedFile(NativeFile): with nogil: check_status(CMemoryMappedFile.Open(c_path, c_mode, &handle)) - self.set_output_stream( handle) - self.set_random_access_file( handle) + self.set_output_stream( handle) + self.set_random_access_file( handle) self.handle = handle def resize(self, new_size): @@ -836,7 +836,7 @@ cdef class OSFile(NativeFile): check_status(ReadableFile.Open(path, pool, &handle)) self.is_readable = True - self.set_random_access_file( handle) + self.set_random_access_file( handle) cdef _open_writable(self, c_string path): with nogil: @@ -1174,7 +1174,7 @@ cdef class BufferReader(NativeFile): def __cinit__(self, object obj): self.buffer = as_buffer(obj) - self.set_random_access_file(shared_ptr[RandomAccessFile]( + self.set_random_access_file(shared_ptr[CRandomAccessFile]( new CBufferReader(self.buffer.buffer))) self.is_readable = True @@ -1204,7 +1204,7 @@ cdef class CompressedInputStream(NativeFile): check_status(CCompressedInputStream.Make( codec.get(), stream.get_input_stream(), &compressed_stream)) - self.set_input_stream( compressed_stream) + self.set_input_stream( compressed_stream) self.is_readable = True @@ -1234,13 +1234,13 @@ cdef class CompressedOutputStream(NativeFile): check_status(CCompressedOutputStream.Make( codec.get(), stream.get_output_stream(), &compressed_stream)) - self.set_output_stream( compressed_stream) + self.set_output_stream( compressed_stream) self.is_writable = True ctypedef CBufferedInputStream* _CBufferedInputStreamPtr ctypedef CBufferedOutputStream* _CBufferedOutputStreamPtr -ctypedef RandomAccessFile* _RandomAccessFilePtr +ctypedef CRandomAccessFile* _RandomAccessFilePtr cdef class BufferedInputStream(NativeFile): @@ -1255,7 +1255,7 @@ cdef class BufferedInputStream(NativeFile): buffer_size, maybe_unbox_memory_pool(memory_pool), stream.get_input_stream(), &buffered_stream)) - self.set_input_stream( buffered_stream) + self.set_input_stream( buffered_stream) self.is_readable = True def detach(self): @@ -1269,7 +1269,7 @@ cdef class BufferedInputStream(NativeFile): The underlying raw input stream """ cdef: - shared_ptr[InputStream] c_raw + shared_ptr[CInputStream] c_raw _CBufferedInputStreamPtr buffered NativeFile raw @@ -1287,7 +1287,7 @@ cdef class BufferedInputStream(NativeFile): # selectively. if dynamic_cast[_RandomAccessFilePtr](c_raw.get()) != nullptr: raw.set_random_access_file( - static_pointer_cast[RandomAccessFile, InputStream](c_raw)) + static_pointer_cast[CRandomAccessFile, CInputStream](c_raw)) else: raw.set_input_stream(c_raw) return raw @@ -1305,7 +1305,7 @@ cdef class BufferedOutputStream(NativeFile): buffer_size, maybe_unbox_memory_pool(memory_pool), stream.get_output_stream(), &buffered_stream)) - self.set_output_stream( buffered_stream) + self.set_output_stream( buffered_stream) self.is_writable = True def detach(self): @@ -1319,7 +1319,7 @@ cdef class BufferedOutputStream(NativeFile): The underlying raw output stream """ cdef: - shared_ptr[OutputStream] c_raw + shared_ptr[COutputStream] c_raw _CBufferedOutputStreamPtr buffered NativeFile raw @@ -1400,7 +1400,7 @@ cdef NativeFile _get_native_file(object source, c_bool use_memory_map): cdef get_reader(object source, c_bool use_memory_map, - shared_ptr[RandomAccessFile]* reader): + shared_ptr[CRandomAccessFile]* reader): cdef NativeFile nf nf = _get_native_file(source, use_memory_map) @@ -1408,7 +1408,7 @@ cdef get_reader(object source, c_bool use_memory_map, cdef get_input_stream(object source, c_bool use_memory_map, - shared_ptr[InputStream]* out): + shared_ptr[CInputStream]* out): """ Like get_reader(), but can automatically decompress, and returns an InputStream. @@ -1416,7 +1416,7 @@ cdef get_input_stream(object source, c_bool use_memory_map, cdef: NativeFile nf unique_ptr[CCodec] codec - shared_ptr[InputStream] input_stream + shared_ptr[CInputStream] input_stream shared_ptr[CCompressedInputStream] compressed_stream CompressionType compression_type @@ -1435,12 +1435,12 @@ cdef get_input_stream(object source, c_bool use_memory_map, check_status(CCodec.Create(compression_type, &codec)) check_status(CCompressedInputStream.Make(codec.get(), input_stream, &compressed_stream)) - input_stream = compressed_stream + input_stream = compressed_stream out[0] = input_stream -cdef get_writer(object source, shared_ptr[OutputStream]* writer): +cdef get_writer(object source, shared_ptr[COutputStream]* writer): cdef NativeFile nf try: diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi index 3d8b3f4af4db..c9684f13e5bb 100644 --- a/python/pyarrow/ipc.pxi +++ b/python/pyarrow/ipc.pxi @@ -17,6 +17,7 @@ import warnings + cdef class Message: """ Container for an Arrow IPC message with metadata and optional body @@ -76,7 +77,7 @@ cdef class Message: """ cdef: int64_t output_length = 0 - OutputStream* out + COutputStream* out CIpcOptions options options.alignment = alignment @@ -136,9 +137,11 @@ cdef class MessageReader: @staticmethod def open_stream(source): - cdef MessageReader result = MessageReader.__new__(MessageReader) - cdef shared_ptr[InputStream] in_stream - cdef unique_ptr[CMessageReader] reader + cdef: + MessageReader result = MessageReader.__new__(MessageReader) + shared_ptr[CInputStream] in_stream + unique_ptr[CMessageReader] reader + _get_input_stream(source, &in_stream) with nogil: reader = CMessageReader.Open(in_stream) @@ -250,7 +253,7 @@ cdef class _CRecordBatchWriter: cdef class _RecordBatchStreamWriter(_CRecordBatchWriter): cdef: - shared_ptr[OutputStream] sink + shared_ptr[COutputStream] sink CIpcOptions options bint closed @@ -276,7 +279,7 @@ cdef class _RecordBatchStreamWriter(_CRecordBatchWriter): self.writer = GetResultValue(result) -cdef _get_input_stream(object source, shared_ptr[InputStream]* out): +cdef _get_input_stream(object source, shared_ptr[CInputStream]* out): try: source = as_buffer(source) except TypeError: @@ -332,7 +335,7 @@ cdef class _CRecordBatchReader: cdef class _RecordBatchStreamReader(_CRecordBatchReader): cdef: - shared_ptr[InputStream] in_stream + shared_ptr[CInputStream] in_stream cdef readonly: Schema schema @@ -367,7 +370,7 @@ cdef class _RecordBatchFileWriter(_RecordBatchStreamWriter): cdef class _RecordBatchFileReader: cdef: shared_ptr[CRecordBatchFileReader] reader - shared_ptr[RandomAccessFile] file + shared_ptr[CRandomAccessFile] file cdef readonly: Schema schema @@ -516,9 +519,8 @@ def read_tensor(source): """ cdef: shared_ptr[CTensor] sp_tensor - InputStream* c_stream - - cdef NativeFile nf = as_native_file(source) + CInputStream* c_stream + NativeFile nf = as_native_file(source) c_stream = nf.get_input_stream().get() with nogil: @@ -540,7 +542,7 @@ def read_message(source): """ cdef: Message result = Message.__new__(Message) - InputStream* c_stream + CInputStream* c_stream cdef NativeFile nf = as_native_file(source) c_stream = nf.get_input_stream().get() @@ -571,7 +573,7 @@ def read_schema(obj, DictionaryMemo dictionary_memo=None): """ cdef: shared_ptr[CSchema] result - shared_ptr[RandomAccessFile] cpp_file + shared_ptr[CRandomAccessFile] cpp_file CDictionaryMemo temp_memo CDictionaryMemo* arg_dict_memo diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 553227a4e559..571dfaa9449d 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -436,9 +436,9 @@ cdef class ResizableBuffer(Buffer): cdef class NativeFile: cdef: - shared_ptr[InputStream] input_stream - shared_ptr[RandomAccessFile] random_access - shared_ptr[OutputStream] output_stream + shared_ptr[CInputStream] input_stream + shared_ptr[CRandomAccessFile] random_access + shared_ptr[COutputStream] output_stream bint is_readable bint is_writable bint is_seekable @@ -449,13 +449,13 @@ cdef class NativeFile: # extension classes are technically virtual in the C++ sense) we can expose # the arrow::io abstract file interfaces to other components throughout the # suite of Arrow C++ libraries - cdef set_random_access_file(self, shared_ptr[RandomAccessFile] handle) - cdef set_input_stream(self, shared_ptr[InputStream] handle) - cdef set_output_stream(self, shared_ptr[OutputStream] handle) + cdef set_random_access_file(self, shared_ptr[CRandomAccessFile] handle) + cdef set_input_stream(self, shared_ptr[CInputStream] handle) + cdef set_output_stream(self, shared_ptr[COutputStream] handle) - cdef shared_ptr[RandomAccessFile] get_random_access_file(self) except * - cdef shared_ptr[InputStream] get_input_stream(self) except * - cdef shared_ptr[OutputStream] get_output_stream(self) except * + cdef shared_ptr[CRandomAccessFile] get_random_access_file(self) except * + cdef shared_ptr[CInputStream] get_input_stream(self) except * + cdef shared_ptr[COutputStream] get_output_stream(self) except * cdef class BufferedInputStream(NativeFile): @@ -485,10 +485,10 @@ cdef class _CRecordBatchReader: cdef get_input_stream(object source, c_bool use_memory_map, - shared_ptr[InputStream]* reader) + shared_ptr[CInputStream]* reader) cdef get_reader(object source, c_bool use_memory_map, - shared_ptr[RandomAccessFile]* reader) -cdef get_writer(object source, shared_ptr[OutputStream]* writer) + shared_ptr[CRandomAccessFile]* reader) +cdef get_writer(object source, shared_ptr[COutputStream]* writer) # Default is allow_none=False cdef DataType ensure_type(object type, c_bool allow_none=*) diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi index fba834434d02..cf343b0ba320 100644 --- a/python/pyarrow/serialization.pxi +++ b/python/pyarrow/serialization.pxi @@ -254,11 +254,11 @@ cdef class SerializedPyObject: """ Write serialized object to a sink """ - cdef shared_ptr[OutputStream] stream + cdef shared_ptr[COutputStream] stream get_writer(sink, &stream) self._write_to(stream.get()) - cdef _write_to(self, OutputStream* stream): + cdef _write_to(self, COutputStream* stream): with nogil: check_status(self.data.WriteTo(stream)) @@ -399,7 +399,7 @@ def read_serialized(source, base=None): ------- serialized : the serialized data """ - cdef shared_ptr[RandomAccessFile] stream + cdef shared_ptr[CRandomAccessFile] stream get_reader(source, True, &stream) cdef SerializedPyObject serialized = SerializedPyObject() diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index a095d81bcf70..e4920252c184 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -126,6 +126,12 @@ except ImportError: pass +try: + from pyarrow.fs import S3FileSystem + defaults['s3'] = True +except ImportError: + pass + def pytest_configure(config): for mark in groups: diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index f897e0d36f97..e93e5e54283d 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import os from datetime import datetime try: import pathlib @@ -307,3 +308,33 @@ def test_open_append_stream(fs, tempdir, testpath, compression, buffer_size, f.write(b'\nnewly added') assert decompressor(file_.read_bytes()) == b'already existing\nnewly added' + + +import subprocess + + +@pytest.fixture +def minio_server(tempdir): + host, port = '127.0.0.1', 9000 + access_key, secret_key = 'arrow', 'apachearrow' + + datadir = tempdir / 'minio' + address = '{}:{}'.format(host, port) + + args = ['minio', '--compat', 'server', '--address', address, str(datadir)] + env = os.environ.copy() + env.update({ + 'MINIO_ACCESS_KEY': access_key, + 'MINIO_SECRET_KEY': secret_key + }) + + try: + with subprocess.Popen(args, env=env) as proc: + yield host, port + proc.terminate() + except FileNotFoundError as e: + pytest.skip('Minio executable cannot be located') + + +def test_minio(minio_server): + host, port = minio_server diff --git a/python/setup.py b/python/setup.py index 5e88352a8f9b..6d5fd2013a65 100755 --- a/python/setup.py +++ b/python/setup.py @@ -141,6 +141,8 @@ def initialize_options(self): if not hasattr(sys, 'gettotalrefcount'): self.build_type = 'release' + self.with_s3 = strtobool( + os.environ.get('PYARROW_WITH_S3', '0')) self.with_cuda = strtobool( os.environ.get('PYARROW_WITH_CUDA', '0')) self.with_flight = strtobool( @@ -215,6 +217,8 @@ def _run_cmake(self): if self.cmake_generator: cmake_options += ['-G', self.cmake_generator] + if self.with_s3: + cmake_options.append('-DPYARROW_BUILD_S3=on') if self.with_cuda: cmake_options.append('-DPYARROW_BUILD_CUDA=on') if self.with_flight: @@ -414,6 +418,8 @@ def _failure_permitted(self, name): return True if name == '_flight' and not self.with_flight: return True + if name == '_s3' and not self.with_s3: + return True if name == '_cuda' and not self.with_cuda: return True if name == 'gandiva' and not self.with_gandiva: From dd41d21c2fad93a82da6ea8e2c5d4a82c7a918a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 18 Sep 2019 18:32:52 +0200 Subject: [PATCH 02/39] imports [skip ci] --- python/pyarrow/_fs.pxd | 68 ++++++++++++++++++++++++++++++++++++++++++ python/pyarrow/_fs.pyx | 20 ------------- python/pyarrow/_s3.pyx | 7 +++-- 3 files changed, 72 insertions(+), 23 deletions(-) create mode 100644 python/pyarrow/_fs.pxd diff --git a/python/pyarrow/_fs.pxd b/python/pyarrow/_fs.pxd new file mode 100644 index 000000000000..31c758604282 --- /dev/null +++ b/python/pyarrow/_fs.pxd @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: language_level = 3 + +import six + +from pyarrow.compat import frombytes, tobytes +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint +from pyarrow.lib import _detect_compression +from pyarrow.lib cimport * + + +cpdef enum FileType: + NonExistent = CFileType_NonExistent + Unknown = CFileType_Unknown + File = CFileType_File + Directory = CFileType_Directory + + +cdef class FileStats: + cdef: + CFileStats stats + + @staticmethod + cdef FileStats wrap(CFileStats stats) + + +cdef class Selector: + cdef: + CSelector selector + + +cdef class FileSystem: + cdef: + shared_ptr[CFileSystem] wrapped + CFileSystem* fs + + cdef init(self, const shared_ptr[CFileSystem]& wrapped) + + +cdef class LocalFileSystem(FileSystem): + cdef: + CLocalFileSystem* localfs + + cdef init(self, const shared_ptr[CFileSystem]& wrapped) + + +cdef class SubTreeFileSystem(FileSystem): + cdef: + CSubTreeFileSystem* subtreefs + + cdef init(self, const shared_ptr[CFileSystem]& wrapped) \ No newline at end of file diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index db41051eaeb1..90ab3ae9bc1d 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -37,18 +37,9 @@ cdef inline c_string _path_as_bytes(path) except *: return tobytes(path) -cpdef enum FileType: - NonExistent = CFileType_NonExistent - Unknown = CFileType_Unknown - File = CFileType_File - Directory = CFileType_Directory - - cdef class FileStats: """FileSystem entry stats""" - cdef CFileStats stats - def __init__(self): raise TypeError('dont initialize me') @@ -147,7 +138,6 @@ cdef class Selector: recursive : bool, default False Whether to recurse into subdirectories. """ - cdef CSelector selector def __init__(self, base_dir, bint allow_non_existent=False, bint recursive=False): @@ -183,10 +173,6 @@ cdef class Selector: cdef class FileSystem: """Abstract file system API""" - cdef: - shared_ptr[CFileSystem] wrapped - CFileSystem* fs - def __init__(self): raise TypeError("FileSystem is an abstract class, instantiate one of " "the subclasses instead: LocalFileSystem or " @@ -475,9 +461,6 @@ cdef class LocalFileSystem(FileSystem): except when deleting an entry). """ - cdef: - CLocalFileSystem* localfs - def __init__(self): cdef shared_ptr[CLocalFileSystem] wrapped wrapped = make_shared[CLocalFileSystem]() @@ -499,9 +482,6 @@ cdef class SubTreeFileSystem(FileSystem): filesystem. """ - cdef: - CSubTreeFileSystem* subtreefs - def __init__(self, base_path, FileSystem base_fs): cdef: c_string pathstr diff --git a/python/pyarrow/_s3.pyx b/python/pyarrow/_s3.pyx index d363a61644ec..e4fb9ba5bb8e 100644 --- a/python/pyarrow/_s3.pyx +++ b/python/pyarrow/_s3.pyx @@ -21,9 +21,10 @@ import six from pyarrow.compat import frombytes, tobytes from pyarrow.includes.common cimport * -from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint -from pyarrow.lib import _detect_compression -from pyarrow.lib cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_s3 cimport * +from pyarrow._fs cimport FileSystem +from pyarrow.lib cimport check_status cdef class S3FileSystem(FileSystem): From 1551b525c43d7823828e76f3f935b1fba913a0aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 19 Sep 2019 19:54:09 +0200 Subject: [PATCH 03/39] wip [skip ci] --- cpp/src/arrow/filesystem/s3fs.h | 2 +- python/pyarrow/_fs.pyx | 3 +- python/pyarrow/_s3.pyx | 36 ++- python/pyarrow/fs.py | 13 +- python/pyarrow/includes/libarrow_s3.pxd | 29 ++- python/pyarrow/tests/test_fs.py | 312 ++++++++++++++++-------- python/setup.py | 1 + 7 files changed, 275 insertions(+), 121 deletions(-) diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h index c02c4c154028..25720784622e 100644 --- a/cpp/src/arrow/filesystem/s3fs.h +++ b/cpp/src/arrow/filesystem/s3fs.h @@ -129,7 +129,7 @@ class ARROW_EXPORT S3FileSystem : public FileSystem { std::unique_ptr impl_; }; -enum class S3LogLevel { Off, Fatal, Error, Warn, Info, Debug, Trace }; +enum class S3LogLevel : int8_t { Off, Fatal, Error, Warn, Info, Debug, Trace }; struct ARROW_EXPORT S3GlobalOptions { S3LogLevel log_level; diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index 90ab3ae9bc1d..5bd2b531de5e 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -372,6 +372,7 @@ cdef class FileSystem: stream.set_input_stream(in_handle) stream.is_readable = True + stream.is_seekable = True return self._wrap_input_stream( stream, path=path, compression=compression, buffer_size=buffer_size @@ -494,4 +495,4 @@ cdef class SubTreeFileSystem(FileSystem): cdef init(self, const shared_ptr[CFileSystem]& wrapped): FileSystem.init(self, wrapped) - self.subtreefs = wrapped.get() \ No newline at end of file + self.subtreefs = wrapped.get() diff --git a/python/pyarrow/_s3.pyx b/python/pyarrow/_s3.pyx index e4fb9ba5bb8e..b145516ba79d 100644 --- a/python/pyarrow/_s3.pyx +++ b/python/pyarrow/_s3.pyx @@ -27,23 +27,47 @@ from pyarrow._fs cimport FileSystem from pyarrow.lib cimport check_status +cpdef enum S3LogLevel: + Off = CS3LogLevel_Off + Fatal = CS3LogLevel_Fatal + Error = CS3LogLevel_Error + Warn = CS3LogLevel_Warn + Info = CS3LogLevel_Info + Debug = CS3LogLevel_Debug + Trace = CS3LogLevel_Trace + + +def initialize_s3(S3LogLevel log_level=S3LogLevel.Error): + cdef CS3GlobalOptions options + options.log_level = log_level + check_status(CInitializeS3(options)) + + +def finalize_s3(): + check_status(CFinalizeS3()) + + cdef class S3FileSystem(FileSystem): cdef: CS3FileSystem* s3fs - def __init__(self, str access_key, str secret_key, str region='us-east-1', - str scheme='https', str endpoint_override=None): + def __init__(self, str access_key=None, str secret_key=None, + str region='us-east-1', str scheme='https', + str endpoint_override=None, bint background_writes=True): cdef: - CS3Options options + CS3Options options = CS3Options.Defaults() shared_ptr[CS3FileSystem] wrapped - options.access_key = tobytes(access_key) - options.secret_key = tobytes(secret_key) + if access_key is not None or secret_key is not None: + options.ConfigureAccessKey(tobytes(access_key), + tobytes(secret_key)) + options.region = tobytes(region) options.scheme = tobytes(scheme) + options.background_writes = background_writes if endpoint_override is not None: - options.endpoint_override = endpoint_override + options.endpoint_override = tobytes(endpoint_override) check_status(CS3FileSystem.Make(options, &wrapped)) self.init( wrapped) diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index 5b94a37c8873..c38d9df20fea 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -17,9 +17,16 @@ from __future__ import absolute_import -from pyarrow._fs import * # noqa - +from pyarrow._fs import ( + Selector, + FileType, + FileStats, + FileSystem, + LocalFileSystem, + SubTreeFileSystem +) +from pyarrow._s3 import S3FileSystem try: - from pyarrow._s3 import * # noqa + from pyarrow._s3 import S3FileSystem, initialize_s3, finalize_s3 except ImportError: pass diff --git a/python/pyarrow/includes/libarrow_s3.pxd b/python/pyarrow/includes/libarrow_s3.pxd index 8586132ab131..ec4a145eb7fd 100644 --- a/python/pyarrow/includes/libarrow_s3.pxd +++ b/python/pyarrow/includes/libarrow_s3.pxd @@ -24,13 +24,36 @@ from pyarrow.includes.libarrow cimport CFileSystem cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: - cdef struct CS3Options "arrow::fs::S3Options": + enum CS3LogLevel "arrow::fs::S3LogLevel": + CS3LogLevel_Off "arrow::fs::S3LogLevel::Off" + CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal" + CS3LogLevel_Error "arrow::fs::S3LogLevel::Error" + CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn" + CS3LogLevel_Info "arrow::fs::S3LogLevel::Info" + CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug" + CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace" + + cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions": + CS3LogLevel log_level + + cdef cppclass CS3Options "arrow::fs::S3Options": c_string region c_string endpoint_override c_string scheme - c_string access_key - c_string secret_key + c_bool background_writes + void ConfigureDefaultCredentials() + void ConfigureAccessKey(const c_string& access_key, + const c_string& secret_key) + @staticmethod + CS3Options Defaults() + @staticmethod + CS3Options FromAccessKey(const c_string& access_key, + const c_string& secret_key) cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem): @staticmethod CStatus Make(const CS3Options& options, shared_ptr[CS3FileSystem]* out) + + cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"( + const CS3GlobalOptions& options) + cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"() \ No newline at end of file diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index e93e5e54283d..1db9adc0a9a0 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -15,9 +15,13 @@ # specific language governing permissions and limitations # under the License. +import io import os +import subprocess +import tempfile from datetime import datetime try: + import pathlib except ImportError: import pathlib2 as pathlib # py2 compat @@ -25,38 +29,168 @@ import pytest from pyarrow import ArrowIOError +from pyarrow.tests.test_io import gzip_compress, gzip_decompress from pyarrow.fs import (FileType, Selector, FileSystem, LocalFileSystem, SubTreeFileSystem) -from pyarrow.tests.test_io import gzip_compress, gzip_decompress + + +class S3Path: + + def __init__(self, path, minio_client): + self.client = minio_client + self.path = path + + @property + def bucket_name(self): + return self.path.split('/', 1)[0] + + @property + def object_name(self): + return self.path.split('/', 1)[1] + + @property + def parent(self): + parent, _ = self.path.rsplit('/', 1) + return S3Path(parent, minio_client=self.client) + + def as_posix(self): + return self.path # always posix + + def touch(self): + object_name = self.object_name.rstrip('/') + self.client.put_object( + bucket_name=self.bucket_name, + object_name=object_name, + data=io.BytesIO(b''), + length=0 + ) + + def mkdir(self, parents=True): + object_name = self.object_name + if not object_name.endswith('/'): + object_name += '/' + self.client.put_object( + bucket_name=self.bucket_name, + object_name=object_name, + data=io.BytesIO(b''), + length=0 + ) + + def exists(self): + from minio.error import ResponseError, NoSuchKey, NoSuchBucket + + try: + self.client.get_object( + bucket_name=self.bucket_name, + object_name=self.object_name + ) + except (NoSuchBucket, NoSuchKey): + return False + else: + return True + + def write_bytes(self, content): + assert not self.object_name.endswith('/') + self.client.put_object( + bucket_name=self.bucket_name, + object_name=self.object_name, + data=io.BytesIO(content), + length=len(content) + ) + + def read_bytes(self): + assert not self.object_name.endswith('/') + data = self.client.get_object( + bucket_name=self.bucket_name, + object_name=self.object_name + ) + return data.read() + + +@pytest.fixture(scope='module') +@pytest.mark.s3 +def minio_server(): + host, port = 'localhost', 9000 + access_key, secret_key = 'arrow', 'apachearrow' + + address = '{}:{}'.format(host, port) + env = os.environ.copy() + env.update({ + 'MINIO_ACCESS_KEY': access_key, + 'MINIO_SECRET_KEY': secret_key + }) + + try: + with tempfile.TemporaryDirectory() as tempdir: + args = ['minio', '--compat', 'server', '--address', address, + tempdir] + with subprocess.Popen(args, env=env) as proc: + yield address, access_key, secret_key + proc.terminate() + except FileNotFoundError: + pytest.skip('Minio executable cannot be located') + + +@pytest.fixture(scope='module') +def minio_client(minio_server): + from minio import Minio + address, access_key, secret_key = minio_server + client = Minio(address, access_key=access_key, secret_key=secret_key, + secure=False) + client.make_bucket('bucket') + return client + + +@pytest.fixture +def localfs(tempdir): + def local_paths(p): + path = tempdir / p + return (path.as_posix(), path) + return (LocalFileSystem(), local_paths) + + +@pytest.fixture +def s3fs(minio_server, minio_client): + from pyarrow.fs import S3FileSystem, initialize_s3 + + def s3_paths(p): + path = S3Path('bucket/{}'.format(p), minio_client=minio_client) + return (path.as_posix(), path) + + initialize_s3() + address, access_key, secret_key = minio_server + fs = S3FileSystem(access_key=access_key, secret_key=secret_key, + endpoint_override=address, scheme='http') + + return (fs, s3_paths) + + +@pytest.fixture(params=[ + pytest.lazy_fixture('localfs'), + pytest.lazy_fixture('s3fs') +]) +def subtreefs(request): + fs = SubTreeFileSystem('', request.param[0]) + return (fs, lambda p: p) @pytest.fixture(params=[ - pytest.param( - lambda tmp: LocalFileSystem(), - id='LocalFileSystem' - ), - pytest.param( - lambda tmp: SubTreeFileSystem(tmp, LocalFileSystem()), - id='SubTreeFileSystem(LocalFileSystem)' - ) + pytest.lazy_fixture('localfs'), + pytest.lazy_fixture('s3fs'), + # pytest.lazy_fixture('subtreefs'), ]) -def fs(request, tempdir): - return request.param(tempdir.as_posix()) +def filesystem(request): + return request.param @pytest.fixture -def testpath(request, fs, tempdir): - # we always use the tempdir for reading and writing test artifacts, but - # if the filesystem is wrapped in a SubTreeFileSystem then we don't need - # to prepend the path with the tempdir, we also test the API with both - # pathlib.Path objects and plain python strings - def convert(path): - if isinstance(fs, SubTreeFileSystem): - path = pathlib.Path(path) - else: - path = tempdir / path - return path.as_posix() - return convert +def fs(request, filesystem): + return filesystem[0] + + +@pytest.fixture +def paths(request, filesystem): + return filesystem[1] def test_cannot_instantiate_base_filesystem(): @@ -75,6 +209,7 @@ class Path: fs.create_dir(path) +@pytest.mark.skip() def test_get_target_stats(fs, tempdir, testpath): aaa, aaa_ = testpath('a/aa/aaa'), tempdir / 'a' / 'aa' / 'aaa' bb, bb_ = testpath('a/bb'), tempdir / 'a' / 'bb' @@ -117,6 +252,7 @@ def mtime_almost_equal(fs_dt, pathlib_ts): assert mtime_almost_equal(c_stat.mtime, c_.stat().st_mtime) +@pytest.mark.skip() def test_get_target_stats_with_selector(fs, tempdir, testpath): base_dir = testpath('.') base_dir_ = tempdir @@ -139,16 +275,15 @@ def test_get_target_stats_with_selector(fs, tempdir, testpath): assert st.type == FileType.File -def test_create_dir(fs, tempdir, testpath): - directory = testpath('directory') - directory_ = tempdir / 'directory' +def test_create_dir(fs, paths): + directory, directory_ = paths('test-directory/') assert not directory_.exists() fs.create_dir(directory) + from pyarrow.fs import S3FileSystem assert directory_.exists() # recursive - directory = testpath('deeply/nested/directory') - directory_ = tempdir / 'deeply' / 'nested' / 'directory' + directory, directory_ = paths('deeply/nested/directory/') assert not directory_.exists() with pytest.raises(ArrowIOError): fs.create_dir(directory, recursive=False) @@ -156,11 +291,9 @@ def test_create_dir(fs, tempdir, testpath): assert directory_.exists() -def test_delete_dir(fs, tempdir, testpath): - folder = testpath('directory') - nested = testpath('nested/directory') - folder_ = tempdir / 'directory' - nested_ = tempdir / 'nested' / 'directory' +def test_delete_dir(fs, paths): + folder, folder_ = paths('directory/') + nested, nested_ = paths('nested/directory/') folder_.mkdir() nested_.mkdir(parents=True) @@ -174,53 +307,50 @@ def test_delete_dir(fs, tempdir, testpath): assert not nested_.exists() -def test_copy_file(fs, tempdir, testpath): +def test_copy_file(fs, paths): # copy file - source = testpath('source-file') - source_ = tempdir / 'source-file' + source, source_ = paths('test-copy-source-file') source_.touch() - target = testpath('target-file') - target_ = tempdir / 'target-file' + target, target_ = paths('test-copy-target-file') + assert not target_.exists() fs.copy_file(source, target) assert source_.exists() assert target_.exists() -def test_move(fs, tempdir, testpath): - # move directory - source = testpath('source-dir') - source_ = tempdir / 'source-dir' - source_.mkdir() - target = testpath('target-dir') - target_ = tempdir / 'target-dir' - assert not target_.exists() - fs.move(source, target) - assert not source_.exists() - assert target_.exists() +def test_move(fs, paths): + # # move directory (doesn't work with S3) + # source, source_ = paths('source-dir/') + # source_.mkdir() + # target, target_ = paths('target-dir/') + # assert source_.exists() + # assert not target_.exists() + + # fs.move(source, target) + # assert not source_.exists() + # assert target_.exists() # move file - source = testpath('source-file') - source_ = tempdir / 'source-file' + source, source_ = paths('test-move-source-file') source_.touch() - target = testpath('target-file') - target_ = tempdir / 'target-file' + target, target_ = paths('test-move-target-file') + assert source_.exists() assert not target_.exists() + fs.move(source, target) assert not source_.exists() assert target_.exists() -def test_delete_file(fs, tempdir, testpath): - target = testpath('target-file') - target_ = tempdir / 'target-file' +def test_delete_file(fs, paths): + target, target_ = paths('test-delete-target-file') target_.touch() assert target_.exists() fs.delete_file(target) assert not target_.exists() - nested = testpath('nested/target-file') - nested_ = tempdir / 'nested/target-file' + nested, nested_ = paths('test-delete-nested/target-file') nested_.parent.mkdir() nested_.touch() assert nested_.exists() @@ -241,22 +371,20 @@ def identity(v): ('gzip', 256, gzip_compress), ] ) -def test_open_input_stream(fs, tempdir, testpath, compression, buffer_size, - compressor): - file = testpath('abc') - file_ = tempdir / 'abc' - data = b'some data' * 1024 +def test_open_input_stream(fs, paths, compression, buffer_size, compressor): + file, file_ = paths('open-input-stream') + + data = b'some data for reading' * 1024 file_.write_bytes(compressor(data)) with fs.open_input_stream(file, compression, buffer_size) as f: - result = f.read() + result = f.read(len(data)) assert result == data -def test_open_input_file(fs, tempdir, testpath): - file = testpath('abc') - file_ = tempdir / 'abc' +def test_open_input_file(fs, paths): + file, file_ = paths('open-input-file') data = b'some data' * 1024 file_.write_bytes(data) @@ -277,15 +405,16 @@ def test_open_input_file(fs, tempdir, testpath): ('gzip', 256, gzip_decompress), ] ) -def test_open_output_stream(fs, tempdir, testpath, compression, buffer_size, - decompressor): - file = testpath('abc') - file_ = tempdir / 'abc' +def test_open_output_stream(fs, paths, compression, buffer_size, decompressor): + file, file_ = paths('open-output-stream-1') - data = b'some data' * 1024 + data = b'some data for writing' * 1024 with fs.open_output_stream(file, compression, buffer_size) as f: f.write(data) + with fs.open_input_stream(file, compression, buffer_size) as f: + assert f.read(len(data)) == data + assert decompressor(file_.read_bytes()) == data @@ -298,43 +427,12 @@ def test_open_output_stream(fs, tempdir, testpath, compression, buffer_size, ('gzip', 256, gzip_compress, gzip_decompress), ] ) -def test_open_append_stream(fs, tempdir, testpath, compression, buffer_size, - compressor, decompressor): - file = testpath('abc') - file_ = tempdir / 'abc' +def test_open_append_stream(fs, paths, compression, buffer_size, compressor, + decompressor): + file, file_ = paths('open-append-stream') file_.write_bytes(compressor(b'already existing')) with fs.open_append_stream(file, compression, buffer_size) as f: f.write(b'\nnewly added') - assert decompressor(file_.read_bytes()) == b'already existing\nnewly added' - - -import subprocess - - -@pytest.fixture -def minio_server(tempdir): - host, port = '127.0.0.1', 9000 - access_key, secret_key = 'arrow', 'apachearrow' - - datadir = tempdir / 'minio' - address = '{}:{}'.format(host, port) - - args = ['minio', '--compat', 'server', '--address', address, str(datadir)] - env = os.environ.copy() - env.update({ - 'MINIO_ACCESS_KEY': access_key, - 'MINIO_SECRET_KEY': secret_key - }) - - try: - with subprocess.Popen(args, env=env) as proc: - yield host, port - proc.terminate() - except FileNotFoundError as e: - pytest.skip('Minio executable cannot be located') - - -def test_minio(minio_server): - host, port = minio_server + assert decompressor(file_.read_bytes()) == b'already existing\nnewly added' \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index 6d5fd2013a65..285331190ca4 100755 --- a/python/setup.py +++ b/python/setup.py @@ -170,6 +170,7 @@ def initialize_options(self): CYTHON_MODULE_NAMES = [ 'lib', + '_s3', '_fs', '_csv', '_json', From a343950e2f919970d1faffdfd81c4deb515fde3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 20 Sep 2019 20:01:45 +0200 Subject: [PATCH 04/39] testing suite --- python/pyarrow/tests/test_fs.py | 383 ++++++++++++++++++-------------- 1 file changed, 212 insertions(+), 171 deletions(-) diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 1db9adc0a9a0..cd99c6e65a28 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -34,79 +34,133 @@ SubTreeFileSystem) -class S3Path: +class Local: - def __init__(self, path, minio_client): - self.client = minio_client - self.path = path + def __init__(self, tempdir): + self.impl = LocalFileSystem() + self.tempdir = tempdir + + def pathpair(self, p): + path_for_wrapper = str(self.tempdir / p) + path_for_impl = (self.tempdir / p).as_posix() + return (path_for_wrapper, path_for_impl) + + def mkdir(self, p): + return pathlib.Path(p).mkdir(parents=True) + + def touch(self, p): + return pathlib.Path(p).touch() + + def exists(self, p): + return pathlib.Path(p).exists() + + def write_bytes(self, p, data): + return pathlib.Path(p).write_bytes(data) - @property - def bucket_name(self): - return self.path.split('/', 1)[0] + def read_bytes(self, p): + return pathlib.Path(p).read_bytes() + + +class SubTreeLocal(Local): + + def __init__(self, tempdir, prefix='local/prefix'): + prefix_absolute = tempdir / prefix + prefix_absolute.mkdir(parents=True) + + self.impl = SubTreeFileSystem( + prefix_absolute.as_posix(), + LocalFileSystem() + ) + self.prefix = prefix + self.tempdir = tempdir - @property - def object_name(self): - return self.path.split('/', 1)[1] + def pathpair(self, p): + path_for_wrapper = str(self.tempdir / self.prefix / p) + path_for_impl = p + return (path_for_wrapper, path_for_impl) - @property - def parent(self): - parent, _ = self.path.rsplit('/', 1) - return S3Path(parent, minio_client=self.client) - def as_posix(self): - return self.path # always posix +class S3: - def touch(self): - object_name = self.object_name.rstrip('/') + def __init__(self, minio_client, bucket='test-bucket', **kwargs): + from pyarrow.fs import S3FileSystem, initialize_s3 + initialize_s3() + self.impl = S3FileSystem(**kwargs) + self.client = minio_client + self.bucket = bucket + + def pathpair(self, p): + path_for_wrapper = p + path_for_impl = '/'.join([self.bucket, p]) + return (path_for_wrapper, path_for_impl) + + def touch(self, p): self.client.put_object( - bucket_name=self.bucket_name, - object_name=object_name, + bucket_name=self.bucket, + object_name=p.rstrip('/'), data=io.BytesIO(b''), length=0 ) - def mkdir(self, parents=True): - object_name = self.object_name - if not object_name.endswith('/'): - object_name += '/' + def mkdir(self, p): + if not p.endswith('/'): + p += '/' self.client.put_object( - bucket_name=self.bucket_name, - object_name=object_name, + bucket_name=self.bucket, + object_name=p, data=io.BytesIO(b''), length=0 ) - def exists(self): - from minio.error import ResponseError, NoSuchKey, NoSuchBucket - + def exists(self, p): + from minio.error import NoSuchKey, NoSuchBucket try: self.client.get_object( - bucket_name=self.bucket_name, - object_name=self.object_name + bucket_name=self.bucket, + object_name=p ) except (NoSuchBucket, NoSuchKey): return False else: return True - def write_bytes(self, content): - assert not self.object_name.endswith('/') + def write_bytes(self, p, data): + assert not p.endswith('/') self.client.put_object( - bucket_name=self.bucket_name, - object_name=self.object_name, - data=io.BytesIO(content), - length=len(content) + bucket_name=self.bucket, + object_name=p, + data=io.BytesIO(data), + length=len(data) ) - def read_bytes(self): - assert not self.object_name.endswith('/') + def read_bytes(self, p): + assert not p.endswith('/') data = self.client.get_object( - bucket_name=self.bucket_name, - object_name=self.object_name + bucket_name=self.bucket, + object_name=p ) return data.read() +class SubTreeS3(S3): + + def __init__(self, minio_client, bucket='test-bucket', prefix='s3/prefix', + **kwargs): + from pyarrow.fs import S3FileSystem + self.impl = SubTreeFileSystem( + '/'.join([bucket, prefix]), + S3FileSystem(**kwargs) + ) + self.client = minio_client + self.bucket = bucket + self.prefix = prefix + + def pathpair(self, p): + path_for_wrapper = '/'.join([self.prefix, p]) + path_for_impl = p + return (path_for_wrapper, path_for_impl) + + @pytest.fixture(scope='module') @pytest.mark.s3 def minio_server(): @@ -122,8 +176,8 @@ def minio_server(): try: with tempfile.TemporaryDirectory() as tempdir: - args = ['minio', '--compat', 'server', '--address', address, - tempdir] + args = ['minio', '--compat', 'server', '--quiet', '--address', + address, tempdir] with subprocess.Popen(args, env=env) as proc: yield address, access_key, secret_key proc.terminate() @@ -135,64 +189,45 @@ def minio_server(): def minio_client(minio_server): from minio import Minio address, access_key, secret_key = minio_server - client = Minio(address, access_key=access_key, secret_key=secret_key, - secure=False) - client.make_bucket('bucket') - return client + return Minio( + address, + access_key=access_key, + secret_key=secret_key, + secure=False + ) -@pytest.fixture -def localfs(tempdir): - def local_paths(p): - path = tempdir / p - return (path.as_posix(), path) - return (LocalFileSystem(), local_paths) - - -@pytest.fixture -def s3fs(minio_server, minio_client): - from pyarrow.fs import S3FileSystem, initialize_s3 - - def s3_paths(p): - path = S3Path('bucket/{}'.format(p), minio_client=minio_client) - return (path.as_posix(), path) - - initialize_s3() - address, access_key, secret_key = minio_server - fs = S3FileSystem(access_key=access_key, secret_key=secret_key, - endpoint_override=address, scheme='http') - - return (fs, s3_paths) +@pytest.fixture(params=[ + Local, + SubTreeLocal +]) +def localfs(request, tempdir): + return request.param(tempdir) @pytest.fixture(params=[ - pytest.lazy_fixture('localfs'), - pytest.lazy_fixture('s3fs') + S3, + SubTreeS3 ]) -def subtreefs(request): - fs = SubTreeFileSystem('', request.param[0]) - return (fs, lambda p: p) +def s3fs(request, minio_server, minio_client): + address, access_key, secret_key = minio_server + return request.param( + minio_client=minio_client, + endpoint_override=address, + access_key=access_key, + secret_key=secret_key, + scheme='http' + ) @pytest.fixture(params=[ pytest.lazy_fixture('localfs'), pytest.lazy_fixture('s3fs'), - # pytest.lazy_fixture('subtreefs'), ]) -def filesystem(request): +def fs(request): return request.param -@pytest.fixture -def fs(request, filesystem): - return filesystem[0] - - -@pytest.fixture -def paths(request, filesystem): - return filesystem[1] - - def test_cannot_instantiate_base_filesystem(): with pytest.raises(TypeError): FileSystem() @@ -206,14 +241,14 @@ class Path: pathlib.Path()] for path in invalid_paths: with pytest.raises(TypeError): - fs.create_dir(path) + fs.impl.create_dir(path) @pytest.mark.skip() -def test_get_target_stats(fs, tempdir, testpath): - aaa, aaa_ = testpath('a/aa/aaa'), tempdir / 'a' / 'aa' / 'aaa' - bb, bb_ = testpath('a/bb'), tempdir / 'a' / 'bb' - c, c_ = testpath('c.txt'), tempdir / 'c.txt' +def test_get_target_stats(fs, paths): + aaa, aaa_ = paths('a/aa/aaa') + bb, bb_ = paths('a/bb') + c, c_ = paths('c.txt') aaa_.mkdir(parents=True) bb_.touch() @@ -275,87 +310,88 @@ def test_get_target_stats_with_selector(fs, tempdir, testpath): assert st.type == FileType.File -def test_create_dir(fs, paths): - directory, directory_ = paths('test-directory/') - assert not directory_.exists() - fs.create_dir(directory) - from pyarrow.fs import S3FileSystem - assert directory_.exists() +def test_create_dir(fs): + _d, d = fs.pathpair('test-directory/') + assert not fs.exists(_d) + fs.impl.create_dir(d) + assert fs.exists(_d) # recursive - directory, directory_ = paths('deeply/nested/directory/') - assert not directory_.exists() + _r, r = fs.pathpair('deeply/nested/directory/') + assert not fs.exists(_r) with pytest.raises(ArrowIOError): - fs.create_dir(directory, recursive=False) - fs.create_dir(directory) - assert directory_.exists() + fs.impl.create_dir(r, recursive=False) + fs.impl.create_dir(r) + assert fs.exists(_r) + +def test_delete_dir(fs): + _d, d = fs.pathpair('directory/') + _nd, nd = fs.pathpair('directory/nested/') + fs.mkdir(_nd) -def test_delete_dir(fs, paths): - folder, folder_ = paths('directory/') - nested, nested_ = paths('nested/directory/') + assert fs.exists(_nd) + fs.impl.delete_dir(nd) + assert not fs.exists(_nd) - folder_.mkdir() - nested_.mkdir(parents=True) + assert fs.exists(_d) + fs.impl.delete_dir(d) + assert not fs.exists(_d) - assert folder_.exists() - fs.delete_dir(folder) - assert not folder_.exists() - assert nested_.exists() - fs.delete_dir(nested) - assert not nested_.exists() +def test_copy_file(fs): + _s, s = fs.pathpair('test-copy-source-file') + _t, t = fs.pathpair('test-copy-target-file') + fs.touch(_s) + assert not fs.exists(_t) + fs.impl.copy_file(s, t) + assert fs.exists(_s) + assert fs.exists(_t) -def test_copy_file(fs, paths): - # copy file - source, source_ = paths('test-copy-source-file') - source_.touch() - target, target_ = paths('test-copy-target-file') - assert not target_.exists() - fs.copy_file(source, target) - assert source_.exists() - assert target_.exists() +def test_move_directory(localfs): + # move directory (doesn't work with S3) + _s, s = localfs.pathpair('source-dir/') + _t, t = localfs.pathpair('target-dir/') + localfs.mkdir(_s) + assert localfs.exists(_s) + assert not localfs.exists(_t) + localfs.impl.move(s, t) + assert not localfs.exists(_s) + assert localfs.exists(_t) -def test_move(fs, paths): - # # move directory (doesn't work with S3) - # source, source_ = paths('source-dir/') - # source_.mkdir() - # target, target_ = paths('target-dir/') - # assert source_.exists() - # assert not target_.exists() - # fs.move(source, target) - # assert not source_.exists() - # assert target_.exists() +def test_move_file(fs): + _s, s = fs.pathpair('test-move-source-file') + _t, t = fs.pathpair('test-move-target-file') + fs.touch(_s) - # move file - source, source_ = paths('test-move-source-file') - source_.touch() - target, target_ = paths('test-move-target-file') - assert source_.exists() - assert not target_.exists() + assert fs.exists(_s) + assert not fs.exists(_t) + fs.impl.move(s, t) + assert not fs.exists(_s) + assert fs.exists(_t) - fs.move(source, target) - assert not source_.exists() - assert target_.exists() +def test_delete_file(fs): + _p, p = fs.pathpair('test-delete-target-file') + fs.touch(_p) -def test_delete_file(fs, paths): - target, target_ = paths('test-delete-target-file') - target_.touch() - assert target_.exists() - fs.delete_file(target) - assert not target_.exists() + assert fs.exists(_p) + fs.impl.delete_file(p) + assert not fs.exists(_p) - nested, nested_ = paths('test-delete-nested/target-file') - nested_.parent.mkdir() - nested_.touch() - assert nested_.exists() - fs.delete_file(nested) - assert not nested_.exists() + _p, p = fs.pathpair('test-delete-nested') + fs.mkdir(_p) + + _p, p = fs.pathpair('test-delete-nested/target-file') + fs.touch(_p) + + assert fs.exists(_p) + fs.impl.delete_file(p) + assert not fs.exists(_p) def identity(v): @@ -371,25 +407,26 @@ def identity(v): ('gzip', 256, gzip_compress), ] ) -def test_open_input_stream(fs, paths, compression, buffer_size, compressor): - file, file_ = paths('open-input-stream') +def test_open_input_stream(fs, compression, buffer_size, compressor): + _p, p = fs.pathpair('open-input-stream') data = b'some data for reading' * 1024 - file_.write_bytes(compressor(data)) + fs.write_bytes(_p, compressor(data)) - with fs.open_input_stream(file, compression, buffer_size) as f: + with fs.impl.open_input_stream(p, compression, buffer_size) as f: result = f.read(len(data)) assert result == data -def test_open_input_file(fs, paths): - file, file_ = paths('open-input-file') +def test_open_input_file(fs): + _p, p = fs.pathpair('open-input-file') + data = b'some data' * 1024 - file_.write_bytes(data) + fs.write_bytes(_p, data) read_from = len(b'some data') * 512 - with fs.open_input_file(file) as f: + with fs.impl.open_input_file(p) as f: f.seek(read_from) result = f.read() @@ -405,17 +442,18 @@ def test_open_input_file(fs, paths): ('gzip', 256, gzip_decompress), ] ) -def test_open_output_stream(fs, paths, compression, buffer_size, decompressor): - file, file_ = paths('open-output-stream-1') +def test_open_output_stream(fs, compression, buffer_size, decompressor): + _p, p = fs.pathpair('open-output-stream') data = b'some data for writing' * 1024 - with fs.open_output_stream(file, compression, buffer_size) as f: + with fs.impl.open_output_stream(p, compression, buffer_size) as f: f.write(data) - with fs.open_input_stream(file, compression, buffer_size) as f: + with fs.impl.open_input_stream(p, compression, buffer_size) as f: assert f.read(len(data)) == data - assert decompressor(file_.read_bytes()) == data + result = decompressor(fs.read_bytes(_p)) + assert result == data @pytest.mark.parametrize( @@ -427,12 +465,15 @@ def test_open_output_stream(fs, paths, compression, buffer_size, decompressor): ('gzip', 256, gzip_compress, gzip_decompress), ] ) -def test_open_append_stream(fs, paths, compression, buffer_size, compressor, +def test_open_append_stream(localfs, compression, buffer_size, compressor, decompressor): - file, file_ = paths('open-append-stream') - file_.write_bytes(compressor(b'already existing')) + _p, p = localfs.pathpair('open-append-stream') + + data = compressor(b'already existing') + localfs.write_bytes(_p, data) - with fs.open_append_stream(file, compression, buffer_size) as f: + with localfs.impl.open_append_stream(p, compression, buffer_size) as f: f.write(b'\nnewly added') - assert decompressor(file_.read_bytes()) == b'already existing\nnewly added' \ No newline at end of file + result = decompressor(localfs.read_bytes(_p)) + assert result == b'already existing\nnewly added' \ No newline at end of file From 44aedfd10df9a76c8c8f685b4d2374ba49471a48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 20 Sep 2019 21:23:55 +0200 Subject: [PATCH 05/39] stat test --- python/pyarrow/_fs.pyx | 4 +- python/pyarrow/tests/test_fs.py | 73 ++++++++++++++++++++++----------- python/requirements-test.txt | 2 + 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index 5bd2b531de5e..cb44a05b1e9f 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -97,9 +97,7 @@ cdef class FileStats: Only regular files are guaranteed to have a size. """ if self.stats.type() != CFileType_File: - raise ValueError( - 'Only regular files are guaranteed to have a size' - ) + return None return self.stats.size() @property diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index cd99c6e65a28..71441fc606d2 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -17,6 +17,7 @@ import io import os +import calendar import subprocess import tempfile from datetime import datetime @@ -42,7 +43,7 @@ def __init__(self, tempdir): def pathpair(self, p): path_for_wrapper = str(self.tempdir / p) - path_for_impl = (self.tempdir / p).as_posix() + path_for_impl = '/'.join([self.tempdir.as_posix(), p]) return (path_for_wrapper, path_for_impl) def mkdir(self, p): @@ -54,6 +55,11 @@ def touch(self, p): def exists(self, p): return pathlib.Path(p).exists() + def mtime(self, p): + path = pathlib.Path(p) + mtime = path.stat().st_mtime + return datetime.utcfromtimestamp(mtime) + def write_bytes(self, p, data): return pathlib.Path(p).write_bytes(data) @@ -124,6 +130,14 @@ def exists(self, p): else: return True + def mtime(self, p): + stat = self.client.stat_object( + bucket_name=self.bucket, + object_name=p + ) + ts = calendar.timegm(stat.last_modified) + return datetime.utcfromtimestamp(ts) + def write_bytes(self, p, data): assert not p.endswith('/') self.client.put_object( @@ -189,12 +203,15 @@ def minio_server(): def minio_client(minio_server): from minio import Minio address, access_key, secret_key = minio_server - return Minio( + bucket = 'test-bucket' + client = Minio( address, access_key=access_key, secret_key=secret_key, secure=False ) + client.make_bucket(bucket) + return client, bucket @pytest.fixture(params=[ @@ -211,8 +228,10 @@ def localfs(request, tempdir): ]) def s3fs(request, minio_server, minio_client): address, access_key, secret_key = minio_server + client, bucket = minio_client return request.param( - minio_client=minio_client, + minio_client=client, + bucket=bucket, endpoint_override=address, access_key=access_key, secret_key=secret_key, @@ -244,47 +263,50 @@ class Path: fs.impl.create_dir(path) -@pytest.mark.skip() -def test_get_target_stats(fs, paths): - aaa, aaa_ = paths('a/aa/aaa') - bb, bb_ = paths('a/bb') - c, c_ = paths('c.txt') +def test_file_stat_repr(): + # TODO(kszucs) + pass + + +def test_get_target_stats(fs): + _aaa, aaa = fs.pathpair('a/aa/aaa/') + _bb, bb = fs.pathpair('a/bb') + _c, c = fs.pathpair('c.txt') - aaa_.mkdir(parents=True) - bb_.touch() - c_.write_bytes(b'test') + fs.mkdir(_aaa) + fs.touch(_bb) + fs.write_bytes(_c, b'test') - def mtime_almost_equal(fs_dt, pathlib_ts): + def mtime_almost_equal(a, b): # arrow's filesystem implementation truncates mtime to microsends # resolution whereas pathlib rounds - pathlib_dt = datetime.utcfromtimestamp(pathlib_ts) - difference = (fs_dt - pathlib_dt).total_seconds() - return abs(difference) <= 10**-6 + diff = (a - b).total_seconds() + return abs(diff) <= 10**-6 - aaa_stat, bb_stat, c_stat = fs.get_target_stats([aaa, bb, c]) + aaa_stat, bb_stat, c_stat = fs.impl.get_target_stats([aaa, bb, c]) assert aaa_stat.path == aaa assert 'aaa' in repr(aaa_stat) - assert aaa_stat.base_name == 'aaa' + # type is inconsistent base_name has a trailing slas for 'aaa' and 'aaa/' + # assert aaa_stat.base_name == 'aaa' assert aaa_stat.extension == '' - assert aaa_stat.type == FileType.Directory - assert mtime_almost_equal(aaa_stat.mtime, aaa_.stat().st_mtime) - with pytest.raises(ValueError): - aaa_stat.size + # assert aaa_stat.type == FileType.Directory + assert mtime_almost_equal(aaa_stat.mtime, fs.mtime(_aaa)) + # assert aaa_stat is None assert bb_stat.path == str(bb) assert bb_stat.base_name == 'bb' assert bb_stat.extension == '' assert bb_stat.type == FileType.File assert bb_stat.size == 0 - assert mtime_almost_equal(bb_stat.mtime, bb_.stat().st_mtime) + assert mtime_almost_equal(bb_stat.mtime, fs.mtime(_bb)) assert c_stat.path == str(c) assert c_stat.base_name == 'c.txt' assert c_stat.extension == 'txt' assert c_stat.type == FileType.File assert c_stat.size == 4 - assert mtime_almost_equal(c_stat.mtime, c_.stat().st_mtime) + assert mtime_almost_equal(c_stat.mtime, fs.mtime(_c)) @pytest.mark.skip() @@ -476,4 +498,7 @@ def test_open_append_stream(localfs, compression, buffer_size, compressor, f.write(b'\nnewly added') result = decompressor(localfs.read_bytes(_p)) - assert result == b'already existing\nnewly added' \ No newline at end of file + assert result == b'already existing\nnewly added' + + +# TODO(kszucs): test that open_append_stream raises for s3 \ No newline at end of file diff --git a/python/requirements-test.txt b/python/requirements-test.txt index 73eabfebd288..89921fa29f50 100644 --- a/python/requirements-test.txt +++ b/python/requirements-test.txt @@ -1,6 +1,8 @@ cython hypothesis +minio pandas pathlib2; python_version < "3.4" pytest +pytest-lazy-fixture pytz From c0b91621e52a7134a6496d24df83e65d3180a4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 12:36:33 +0200 Subject: [PATCH 06/39] test requirements; flake8 --- ci/conda_env_python.yml | 2 + ci/travis_script_python.sh | 3 + python/pyarrow/_fs.pxd | 2 +- python/pyarrow/fs.py | 4 +- python/pyarrow/includes/libarrow.pxd | 21 ++-- python/pyarrow/includes/libarrow_s3.pxd | 3 +- python/pyarrow/tests/conftest.py | 2 +- python/pyarrow/tests/test_fs.py | 123 ++++++++++++++++-------- 8 files changed, 107 insertions(+), 53 deletions(-) diff --git a/ci/conda_env_python.yml b/ci/conda_env_python.yml index a0cd737b326c..65ab7d8afb8f 100644 --- a/ci/conda_env_python.yml +++ b/ci/conda_env_python.yml @@ -19,9 +19,11 @@ cython=0.29.7 cloudpickle hypothesis numpy>=1.14 +minio pandas pytest pytest-faulthandler +pytest-lazy-fixture pytz setuptools setuptools_scm=3.2.0 diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 202c24f0a58c..965195a1df32 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -135,6 +135,7 @@ cmake -GNinja \ -DARROW_TENSORFLOW=on \ -DARROW_PYTHON=on \ -DARROW_ORC=on \ + -DARROW_S3=on \ -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ $ARROW_CPP_DIR @@ -164,6 +165,7 @@ export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_ORC=1 +export PYARROW_WITH_S3=1 if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then export PYARROW_WITH_FLIGHT=1 fi @@ -177,6 +179,7 @@ python setup.py develop python -c "import pyarrow.parquet" python -c "import pyarrow.plasma" python -c "import pyarrow.orc" +python -c "import pyarrow.fs" # Ensure we do eagerly import pandas (or other expensive imports) python < scripts/test_imports.py diff --git a/python/pyarrow/_fs.pxd b/python/pyarrow/_fs.pxd index 31c758604282..11b5769f854e 100644 --- a/python/pyarrow/_fs.pxd +++ b/python/pyarrow/_fs.pxd @@ -65,4 +65,4 @@ cdef class SubTreeFileSystem(FileSystem): cdef: CSubTreeFileSystem* subtreefs - cdef init(self, const shared_ptr[CFileSystem]& wrapped) \ No newline at end of file + cdef init(self, const shared_ptr[CFileSystem]& wrapped) diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index c38d9df20fea..76b382e3f127 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -17,7 +17,7 @@ from __future__ import absolute_import -from pyarrow._fs import ( +from pyarrow._fs import ( # noqa Selector, FileType, FileStats, @@ -27,6 +27,6 @@ ) from pyarrow._s3 import S3FileSystem try: - from pyarrow._s3 import S3FileSystem, initialize_s3, finalize_s3 + from pyarrow._s3 import S3FileSystem, initialize_s3, finalize_s3 # noqa except ImportError: pass diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 3cb4f70bafea..2fd15d1f9781 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -811,7 +811,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int file_descriptor() cdef cppclass CMemoryMappedFile \ - " arrow::io::MemoryMappedFile"(ReadWriteFileInterface): + "arrow::io::MemoryMappedFile"(ReadWriteFileInterface): @staticmethod CStatus Create(const c_string& path, int64_t size, @@ -826,7 +826,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int file_descriptor() cdef cppclass CCompressedInputStream \ - " arrow::io::CompressedInputStream"(CInputStream): + "arrow::io::CompressedInputStream"(CInputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, shared_ptr[CInputStream] raw, @@ -837,7 +837,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[CCompressedInputStream]* out) cdef cppclass CCompressedOutputStream \ - " arrow::io::CompressedOutputStream"(COutputStream): + "arrow::io::CompressedOutputStream"(COutputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, shared_ptr[COutputStream] raw, @@ -848,7 +848,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[CCompressedOutputStream]* out) cdef cppclass CBufferedInputStream \ - " arrow::io::BufferedInputStream"(CInputStream): + "arrow::io::BufferedInputStream"(CInputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, @@ -858,7 +858,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[CInputStream] Detach() cdef cppclass CBufferedOutputStream \ - " arrow::io::BufferedOutputStream"(COutputStream): + "arrow::io::BufferedOutputStream"(COutputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, @@ -903,7 +903,8 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: cdef cppclass HdfsOutputStream(COutputStream): pass - cdef cppclass CHadoopFileSystem" arrow::io::HadoopFileSystem"(CIOFileSystem): + cdef cppclass CHadoopFileSystem \ + "arrow::io::HadoopFileSystem"(CIOFileSystem): @staticmethod CStatus Connect(const HdfsConnectionConfig* config, shared_ptr[CHadoopFileSystem]* client) @@ -939,21 +940,21 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[HdfsOutputStream]* handle) cdef cppclass CBufferReader \ - " arrow::io::BufferReader"(CRandomAccessFile): + "arrow::io::BufferReader"(CRandomAccessFile): CBufferReader(const shared_ptr[CBuffer]& buffer) CBufferReader(const uint8_t* data, int64_t nbytes) cdef cppclass CBufferOutputStream \ - " arrow::io::BufferOutputStream"(COutputStream): + "arrow::io::BufferOutputStream"(COutputStream): CBufferOutputStream(const shared_ptr[CResizableBuffer]& buffer) cdef cppclass CMockOutputStream \ - " arrow::io::MockOutputStream"(COutputStream): + "arrow::io::MockOutputStream"(COutputStream): CMockOutputStream() int64_t GetExtentBytesWritten() cdef cppclass CFixedSizeBufferWriter \ - " arrow::io::FixedSizeBufferWriter"(WritableFile): + "arrow::io::FixedSizeBufferWriter"(WritableFile): CFixedSizeBufferWriter(const shared_ptr[CBuffer]& buffer) void set_memcopy_threads(int num_threads) diff --git a/python/pyarrow/includes/libarrow_s3.pxd b/python/pyarrow/includes/libarrow_s3.pxd index ec4a145eb7fd..d2cde972f2bb 100644 --- a/python/pyarrow/includes/libarrow_s3.pxd +++ b/python/pyarrow/includes/libarrow_s3.pxd @@ -44,6 +44,7 @@ cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: void ConfigureDefaultCredentials() void ConfigureAccessKey(const c_string& access_key, const c_string& secret_key) + @staticmethod CS3Options Defaults() @staticmethod @@ -56,4 +57,4 @@ cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"( const CS3GlobalOptions& options) - cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"() \ No newline at end of file + cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"() diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index e4920252c184..213e2bab4ab2 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -127,7 +127,7 @@ pass try: - from pyarrow.fs import S3FileSystem + from pyarrow.fs import S3FileSystem # noqa defaults['s3'] = True except ImportError: pass diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 71441fc606d2..dccbfa633f00 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -22,20 +22,65 @@ import tempfile from datetime import datetime try: - import pathlib except ImportError: import pathlib2 as pathlib # py2 compat import pytest +import pyarrow as pa from pyarrow import ArrowIOError from pyarrow.tests.test_io import gzip_compress, gzip_decompress from pyarrow.fs import (FileType, Selector, FileSystem, LocalFileSystem, SubTreeFileSystem) -class Local: +class FileSystemWrapper: + + # Whether the filesystem may "implicitly" create intermediate directories + have_implicit_directories = False + # Whether the filesystem may allow writing a file "over" a directory + allow_write_file_over_dir = False + # Whether the filesystem allows moving a directory + allow_move_dir = True + # Whether the filesystem allows appending to a file + allow_append_to_file = False + # Whether the filesystem supports directory modification times + have_directory_mtimes = True + + @property + def impl(self): + return self._impl + + @impl.setter + def impl(self, impl): + self._impl = impl + + def pathpair(self, p): + raise NotImplementedError() + + def mkdir(self, p): + raise NotImplementedError() + + def touch(self, p): + raise NotImplementedError() + + def exists(self, p): + raise NotImplementedError() + + def mtime(self, p): + raise NotImplementedError() + + def write_bytes(self, p, data): + raise NotImplementedError() + + def read_bytes(self, p): + raise NotImplementedError() + + +class LocalWrapper(FileSystemWrapper): + + allow_append_to_file = True def __init__(self, tempdir): self.impl = LocalFileSystem() @@ -67,7 +112,7 @@ def read_bytes(self, p): return pathlib.Path(p).read_bytes() -class SubTreeLocal(Local): +class SubTreeLocalWrapper(LocalWrapper): def __init__(self, tempdir, prefix='local/prefix'): prefix_absolute = tempdir / prefix @@ -86,11 +131,12 @@ def pathpair(self, p): return (path_for_wrapper, path_for_impl) -class S3: +class S3Wrapper(FileSystemWrapper): + + allow_move_dir = False def __init__(self, minio_client, bucket='test-bucket', **kwargs): - from pyarrow.fs import S3FileSystem, initialize_s3 - initialize_s3() + from pyarrow.fs import S3FileSystem self.impl = S3FileSystem(**kwargs) self.client = minio_client self.bucket = bucket @@ -156,7 +202,7 @@ def read_bytes(self, p): return data.read() -class SubTreeS3(S3): +class SubTreeS3Wrapper(S3Wrapper): def __init__(self, minio_client, bucket='test-bucket', prefix='s3/prefix', **kwargs): @@ -215,18 +261,20 @@ def minio_client(minio_server): @pytest.fixture(params=[ - Local, - SubTreeLocal + LocalWrapper, + SubTreeLocalWrapper ]) def localfs(request, tempdir): return request.param(tempdir) @pytest.fixture(params=[ - S3, - SubTreeS3 + S3Wrapper, + SubTreeS3Wrapper ]) def s3fs(request, minio_server, minio_client): + from pyarrow.fs import initialize_s3 + initialize_s3() address, access_key, secret_key = minio_server client, bucket = minio_client return request.param( @@ -263,11 +311,6 @@ class Path: fs.impl.create_dir(path) -def test_file_stat_repr(): - # TODO(kszucs) - pass - - def test_get_target_stats(fs): _aaa, aaa = fs.pathpair('a/aa/aaa/') _bb, bb = fs.pathpair('a/bb') @@ -287,11 +330,11 @@ def mtime_almost_equal(a, b): assert aaa_stat.path == aaa assert 'aaa' in repr(aaa_stat) + assert aaa_stat.extension == '' + assert mtime_almost_equal(aaa_stat.mtime, fs.mtime(_aaa)) # type is inconsistent base_name has a trailing slas for 'aaa' and 'aaa/' # assert aaa_stat.base_name == 'aaa' - assert aaa_stat.extension == '' # assert aaa_stat.type == FileType.Directory - assert mtime_almost_equal(aaa_stat.mtime, fs.mtime(_aaa)) # assert aaa_stat is None assert bb_stat.path == str(bb) @@ -372,17 +415,21 @@ def test_copy_file(fs): assert fs.exists(_t) -def test_move_directory(localfs): +def test_move_directory(fs): # move directory (doesn't work with S3) - _s, s = localfs.pathpair('source-dir/') - _t, t = localfs.pathpair('target-dir/') - localfs.mkdir(_s) + _s, s = fs.pathpair('source-dir/') + _t, t = fs.pathpair('target-dir/') + fs.mkdir(_s) - assert localfs.exists(_s) - assert not localfs.exists(_t) - localfs.impl.move(s, t) - assert not localfs.exists(_s) - assert localfs.exists(_t) + if fs.allow_move_dir: + assert fs.exists(_s) + assert not fs.exists(_t) + fs.impl.move(s, t) + assert not fs.exists(_s) + assert fs.exists(_t) + else: + with pytest.raises(pa.ArrowIOError): + fs.impl.move(s, t) def test_move_file(fs): @@ -487,18 +534,18 @@ def test_open_output_stream(fs, compression, buffer_size, decompressor): ('gzip', 256, gzip_compress, gzip_decompress), ] ) -def test_open_append_stream(localfs, compression, buffer_size, compressor, +def test_open_append_stream(fs, compression, buffer_size, compressor, decompressor): - _p, p = localfs.pathpair('open-append-stream') + _p, p = fs.pathpair('open-append-stream') data = compressor(b'already existing') - localfs.write_bytes(_p, data) - - with localfs.impl.open_append_stream(p, compression, buffer_size) as f: - f.write(b'\nnewly added') - - result = decompressor(localfs.read_bytes(_p)) - assert result == b'already existing\nnewly added' - + fs.write_bytes(_p, data) -# TODO(kszucs): test that open_append_stream raises for s3 \ No newline at end of file + if fs.allow_append_to_file: + with fs.impl.open_append_stream(p, compression, buffer_size) as f: + f.write(b'\nnewly added') + result = decompressor(fs.read_bytes(_p)) + assert result == b'already existing\nnewly added' + else: + with pytest.raises(pa.ArrowNotImplementedError): + fs.impl.open_append_stream(p, compression, buffer_size) From 45a2a17ba38a9bd3388970b064d6cae7429c580f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 12:51:45 +0200 Subject: [PATCH 07/39] docstrings --- python/pyarrow/_fs.pyx | 31 +++++++++++++++++++------------ python/pyarrow/_s3.pyx | 23 +++++++++++++++++++++++ python/pyarrow/fs.py | 2 +- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index cb44a05b1e9f..c977e46f2bf2 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -126,7 +126,7 @@ cdef class Selector: Parameters ---------- - base_dir : str or pathlib.Path + base_dir : str The directory in which to select files. Relative paths also work, use '.' for the current directory and '..' for the parent. allow_non_existent : bool, default False @@ -224,7 +224,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The path of the new directory. recursive: bool, default True Create nested directories as well. @@ -238,7 +238,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The path of the directory to be deleted. """ cdef c_string directory = _path_as_bytes(path) @@ -255,9 +255,9 @@ cdef class FileSystem: Parameters ---------- - src : str or pathlib.Path + src : str The path of the file or the directory to be moved. - dest : str or pathlib.Path + dest : str The destination path where the file or directory is moved to. """ cdef: @@ -274,9 +274,9 @@ cdef class FileSystem: Parameters ---------- - src : str or pathlib.Path + src : str The path of the file to be copied from. - dest : str or pathlib.Path + dest : str The destination path where the file is copied to. """ cdef: @@ -290,7 +290,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The path of the file to be deleted. """ cdef c_string file = _path_as_bytes(path) @@ -320,7 +320,7 @@ cdef class FileSystem: Parameters ---------- - path : Union[str, pathlib.Path] + path : str The source to open for reading. Returns @@ -344,7 +344,7 @@ cdef class FileSystem: Parameters ---------- - source: str or pathlib.Path + source: str The source to open for reading. compression: str optional, default 'detect' The compression algorithm to use for on-the-fly decompression. @@ -383,7 +383,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The source to open for writing. compression: str optional, default 'detect' The compression algorithm to use for on-the-fly compression. @@ -421,7 +421,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The source to open for writing. compression: str optional, default 'detect' The compression algorithm to use for on-the-fly compression. @@ -479,6 +479,13 @@ cdef class SubTreeFileSystem(FileSystem): Note, that this makes no security guarantee. For example, symlinks may allow to "escape" the subtree and access other parts of the underlying filesystem. + + Parameters + ---------- + base_path: str + The root of the subtree. + base_fs: FileSystem + FileSystem object the operations delegated to. """ def __init__(self, base_path, FileSystem base_fs): diff --git a/python/pyarrow/_s3.pyx b/python/pyarrow/_s3.pyx index b145516ba79d..d7bfc79cc97f 100644 --- a/python/pyarrow/_s3.pyx +++ b/python/pyarrow/_s3.pyx @@ -48,6 +48,29 @@ def finalize_s3(): cdef class S3FileSystem(FileSystem): + """S3-backed FileSystem implementation + + Note: S3 buckets are special and the operations available on them may be + limited or more expensive than desired. + + Parameters + ---------- + access_key: str, default None + AWS Access Key ID. Pass None to use the standard AWS environment + variables and/or configuration file. + secret_key: str, default None + AWS Secret Access key. Pass None to use the standard AWS environment + variables and/or configuration file. + region: str, default 'us-east-1' + AWS region to connect to. + scheme: str, default 'https' + S3 connection transport scheme. + endpoint_override: str, default None + Override region with a connect string such as "localhost:9000" + background_writes: boolean, default True + Whether OutputStream writes will be issued in the background, without + blocking. + """ cdef: CS3FileSystem* s3fs diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index 76b382e3f127..23291203818d 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -25,7 +25,7 @@ LocalFileSystem, SubTreeFileSystem ) -from pyarrow._s3 import S3FileSystem + try: from pyarrow._s3 import S3FileSystem, initialize_s3, finalize_s3 # noqa except ImportError: From 9ce7180d1eb700f32b0fe0ef3122a7771f08267a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 13:15:59 +0200 Subject: [PATCH 08/39] cmake format; fix orc cimport --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 +++--- python/pyarrow/_orc.pxd | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 95c48172baca..b9126bfff5bc 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2537,9 +2537,9 @@ if(ARROW_S3) message(STATUS "Found AWS SDK libraries: ${AWSSDK_LINK_LIBRARIES}") if(APPLE) - set_target_properties(AWS::aws-c-common PROPERTIES - INTERFACE_LINK_LIBRARIES "-pthread;pthread;-framework CoreFoundation" - ) + set_target_properties(AWS::aws-c-common + PROPERTIES INTERFACE_LINK_LIBRARIES + "-pthread;pthread;-framework CoreFoundation") endif() endif() diff --git a/python/pyarrow/_orc.pxd b/python/pyarrow/_orc.pxd index 6c18ca503a70..649fe8248f41 100644 --- a/python/pyarrow/_orc.pxd +++ b/python/pyarrow/_orc.pxd @@ -28,7 +28,7 @@ from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus, CKeyValueMetadata, CRecordBatch, CTable, - RandomAccessFile, COutputStream, + CRandomAccessFile, COutputStream, TimeUnit) From 9042c7e4c123901838d08482f250198521b25978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 13:27:56 +0200 Subject: [PATCH 09/39] use S3FS_DIR --- python/pyarrow/tests/test_fs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index dccbfa633f00..cbe0752a578b 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -234,9 +234,11 @@ def minio_server(): 'MINIO_SECRET_KEY': secret_key }) + minio_dir = os.environ.get('S3FS_DIR', '') + minio_bin = os.path.join(minio_dir, 'minio') if minio_dir else 'minio' try: with tempfile.TemporaryDirectory() as tempdir: - args = ['minio', '--compat', 'server', '--quiet', '--address', + args = [minio_bin, '--compat', 'server', '--quiet', '--address', address, tempdir] with subprocess.Popen(args, env=env) as proc: yield address, access_key, secret_key From f25ae5aede0aa17170845b1b7358c2e5017ea30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 18:28:13 +0200 Subject: [PATCH 10/39] travis --- .travis.yml | 2 ++ ci/travis_script_python.sh | 3 ++- python/pyarrow/tests/test_fs.py | 16 +++++++--------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4fc143a493db..b840b26163f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,6 +110,7 @@ matrix: - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - ARROW_TRAVIS_USE_TOOLCHAIN=1 - ARROW_TRAVIS_VALGRIND=1 + - ARROW_TRAVIS_S3=1 # TODO(wesm): Run the benchmarks outside of Travis # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1 before_script: @@ -136,6 +137,7 @@ matrix: - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_FLIGHT=1 - ARROW_TRAVIS_ORC=1 + - ARROW_TRAVIS_S3=1 - ARROW_TRAVIS_PARQUET=1 # TODO(ARROW-4763): llvm and llvmdev packages are in conflict: # https://github.com/conda-forge/llvmdev-feedstock/issues/60 diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 965195a1df32..73973d7ace9e 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -135,7 +135,6 @@ cmake -GNinja \ -DARROW_TENSORFLOW=on \ -DARROW_PYTHON=on \ -DARROW_ORC=on \ - -DARROW_S3=on \ -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ $ARROW_CPP_DIR @@ -166,6 +165,8 @@ export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_ORC=1 export PYARROW_WITH_S3=1 +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + export PYARROW_WITH_S3=1 if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then export PYARROW_WITH_FLIGHT=1 fi diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index cbe0752a578b..f078185b2772 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -236,15 +236,13 @@ def minio_server(): minio_dir = os.environ.get('S3FS_DIR', '') minio_bin = os.path.join(minio_dir, 'minio') if minio_dir else 'minio' - try: - with tempfile.TemporaryDirectory() as tempdir: - args = [minio_bin, '--compat', 'server', '--quiet', '--address', - address, tempdir] - with subprocess.Popen(args, env=env) as proc: - yield address, access_key, secret_key - proc.terminate() - except FileNotFoundError: - pytest.skip('Minio executable cannot be located') + + with tempfile.TemporaryDirectory() as tempdir: + args = [minio_bin, '--compat', 'server', '--quiet', '--address', + address, tempdir] + with subprocess.Popen(args, env=env) as proc: + yield address, access_key, secret_key + proc.terminate() @pytest.fixture(scope='module') From efa05d28e7e0e0d7a5a52484379e795d1b314123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 19:43:36 +0200 Subject: [PATCH 11/39] use minio for dask.s3fs test too --- python/pyarrow/tests/conftest.py | 46 +++++++++++++++++++++++++ python/pyarrow/tests/test_fs.py | 50 +++------------------------- python/pyarrow/tests/test_parquet.py | 38 +++++++++++++-------- 3 files changed, 75 insertions(+), 59 deletions(-) diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index 213e2bab4ab2..bcf0ef947bfa 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -16,6 +16,9 @@ # under the License. import os +import subprocess +import tempfile + import pytest import hypothesis as h @@ -225,3 +228,46 @@ def tempdir(tmpdir): @pytest.fixture(scope='session') def datadir(): return pathlib.Path(__file__).parent / 'data' + + +@pytest.fixture(scope='module') +@pytest.mark.s3 +def minio_server(): + host, port = 'localhost', 9000 + access_key, secret_key = 'arrow', 'apachearrow' + + address = '{}:{}'.format(host, port) + env = os.environ.copy() + env.update({ + 'MINIO_ACCESS_KEY': access_key, + 'MINIO_SECRET_KEY': secret_key + }) + + minio_dir = os.environ.get('S3FS_DIR', '') + minio_bin = os.path.join(minio_dir, 'minio') if minio_dir else 'minio' + + with tempfile.TemporaryDirectory() as tempdir: + args = [minio_bin, '--compat', 'server', '--quiet', '--address', + address, tempdir] + with subprocess.Popen(args, env=env) as proc: + yield address, access_key, secret_key + proc.terminate() + + +@pytest.fixture(scope='module') +def minio_client(minio_server): + from minio import Minio + address, access_key, secret_key = minio_server + return Minio( + address, + access_key=access_key, + secret_key=secret_key, + secure=False + ) + + +@pytest.fixture(scope='module') +def minio_bucket(minio_client): + bucket_name = 'pyarrow-bucket' + minio_client.make_bucket(bucket_name) + return bucket_name diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index f078185b2772..4dfae43fd0a1 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -16,10 +16,7 @@ # under the License. import io -import os import calendar -import subprocess -import tempfile from datetime import datetime try: import pathlib @@ -221,45 +218,6 @@ def pathpair(self, p): return (path_for_wrapper, path_for_impl) -@pytest.fixture(scope='module') -@pytest.mark.s3 -def minio_server(): - host, port = 'localhost', 9000 - access_key, secret_key = 'arrow', 'apachearrow' - - address = '{}:{}'.format(host, port) - env = os.environ.copy() - env.update({ - 'MINIO_ACCESS_KEY': access_key, - 'MINIO_SECRET_KEY': secret_key - }) - - minio_dir = os.environ.get('S3FS_DIR', '') - minio_bin = os.path.join(minio_dir, 'minio') if minio_dir else 'minio' - - with tempfile.TemporaryDirectory() as tempdir: - args = [minio_bin, '--compat', 'server', '--quiet', '--address', - address, tempdir] - with subprocess.Popen(args, env=env) as proc: - yield address, access_key, secret_key - proc.terminate() - - -@pytest.fixture(scope='module') -def minio_client(minio_server): - from minio import Minio - address, access_key, secret_key = minio_server - bucket = 'test-bucket' - client = Minio( - address, - access_key=access_key, - secret_key=secret_key, - secure=False - ) - client.make_bucket(bucket) - return client, bucket - - @pytest.fixture(params=[ LocalWrapper, SubTreeLocalWrapper @@ -272,14 +230,14 @@ def localfs(request, tempdir): S3Wrapper, SubTreeS3Wrapper ]) -def s3fs(request, minio_server, minio_client): +def s3fs(request, minio_server, minio_client, minio_bucket): from pyarrow.fs import initialize_s3 initialize_s3() + address, access_key, secret_key = minio_server - client, bucket = minio_client return request.param( - minio_client=client, - bucket=bucket, + minio_client=minio_client, + bucket=minio_bucket, endpoint_override=address, access_key=access_key, secret_key=secret_key, diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index f8a3563e25f9..0a3365db2c70 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1843,18 +1843,24 @@ def test_filters_read_table(tempdir): assert table.num_rows == 3 -@pytest.yield_fixture -def s3_example(): - access_key = os.environ['PYARROW_TEST_S3_ACCESS_KEY'] - secret_key = os.environ['PYARROW_TEST_S3_SECRET_KEY'] - bucket_name = os.environ['PYARROW_TEST_S3_BUCKET'] - +@pytest.fixture +def s3_example(minio_server, minio_bucket): import s3fs - fs = s3fs.S3FileSystem(key=access_key, secret=secret_key) - test_dir = guid() + address, access_key, secret_key = minio_server + bucket_name = minio_bucket + + fs = s3fs.S3FileSystem( + key=access_key, + secret=secret_key, + client_kwargs={ + 'endpoint_url': 'http://{}'.format(address) + } + ) + test_dir = guid() bucket_uri = 's3://{0}/{1}'.format(bucket_name, test_dir) + fs.mkdir(bucket_uri) yield fs, bucket_uri fs.rm(bucket_uri, recursive=True) @@ -1920,23 +1926,29 @@ def _visit_level(base_dir, level, part_keys): for value in values: this_part_keys = part_keys + [(name, value)] - level_dir = base_dir / '{0}={1}'.format(name, value) + level_dir = fs._path_join( + str(base_dir), + '{0}={1}'.format(name, value) + ) fs.mkdir(level_dir) if level == DEPTH - 1: # Generate example data - file_path = level_dir / guid() - + file_path = fs._path_join(level_dir, guid()) filtered_df = _filter_partition(df, this_part_keys) part_table = pa.Table.from_pandas(filtered_df) with fs.open(file_path, 'wb') as f: _write_table(part_table, f) assert fs.exists(file_path) - (level_dir / '_SUCCESS').touch() + file_success = fs._path_join(level_dir, '_SUCCESS') + with fs.open(file_success, 'wb') as f: + pass else: _visit_level(level_dir, level + 1, this_part_keys) - (level_dir / '_SUCCESS').touch() + file_success = fs._path_join(level_dir, '_SUCCESS') + with fs.open(file_success, 'wb') as f: + pass _visit_level(base_dir, 0, []) From 2cb19d1ff8fdc230a5db557113c0ac745e0e218d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 19:58:46 +0200 Subject: [PATCH 12/39] conditional import of test dependencies --- python/pyarrow/tests/conftest.py | 19 +++++++++++-------- python/pyarrow/tests/test_parquet.py | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index bcf0ef947bfa..99044273890a 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -246,19 +246,22 @@ def minio_server(): minio_dir = os.environ.get('S3FS_DIR', '') minio_bin = os.path.join(minio_dir, 'minio') if minio_dir else 'minio' - with tempfile.TemporaryDirectory() as tempdir: - args = [minio_bin, '--compat', 'server', '--quiet', '--address', - address, tempdir] - with subprocess.Popen(args, env=env) as proc: - yield address, access_key, secret_key - proc.terminate() + try: + with tempfile.TemporaryDirectory() as tempdir: + args = [minio_bin, '--compat', 'server', '--quiet', '--address', + address, tempdir] + with subprocess.Popen(args, env=env) as proc: + yield address, access_key, secret_key + proc.terminate() + except IOError: + pytest.skip('`minio` command cannot be located, try to set S3FS_DIR') @pytest.fixture(scope='module') def minio_client(minio_server): - from minio import Minio + minio = pytest.importorskip('minio') address, access_key, secret_key = minio_server - return Minio( + return minio.Minio( address, access_key=access_key, secret_key=secret_key, diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 0a3365db2c70..cebb87a9f9bb 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1845,7 +1845,7 @@ def test_filters_read_table(tempdir): @pytest.fixture def s3_example(minio_server, minio_bucket): - import s3fs + s3fs = pytest.importorskip('s3fs') address, access_key, secret_key = minio_server bucket_name = minio_bucket From 68eb591614c863debdd4ad584702fee897b7a6af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 20:07:51 +0200 Subject: [PATCH 13/39] enable PYARROW_WITH_S3 on appveyor --- ci/cpp-msvc-build-main.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat index b088e2eec763..5427742b9e0a 100644 --- a/ci/cpp-msvc-build-main.bat +++ b/ci/cpp-msvc-build-main.bat @@ -98,6 +98,9 @@ pip install -r requirements.txt pickle5 set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS% set PYARROW_CMAKE_GENERATOR=%GENERATOR% +if "%ARROW_S3%" == "ON" ( + set PYARROW_WITH_S3="ON" +) if "%ARROW_BUILD_FLIGHT%" == "ON" ( @rem ARROW-5441: bundling Arrow Flight libraries not implemented set PYARROW_BUNDLE_ARROW_CPP=OFF From 7daf5668dd7fefe0409ed0f9b3b46d8f4950d5d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 20:09:35 +0200 Subject: [PATCH 14/39] fix syntax error in travis script --- ci/travis_script_python.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 73973d7ace9e..2a8a7c96e4d4 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -167,6 +167,7 @@ export PYARROW_WITH_ORC=1 export PYARROW_WITH_S3=1 if [ "$ARROW_TRAVIS_S3" == "1" ]; then export PYARROW_WITH_S3=1 +fi if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then export PYARROW_WITH_FLIGHT=1 fi From 7800c75d8bbc99ab201116542d7ab9bb845b657e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 22 Sep 2019 20:27:46 +0200 Subject: [PATCH 15/39] appveyor flag --- ci/cpp-msvc-build-main.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat index 5427742b9e0a..b6d1b20a5109 100644 --- a/ci/cpp-msvc-build-main.bat +++ b/ci/cpp-msvc-build-main.bat @@ -99,7 +99,7 @@ pip install -r requirements.txt pickle5 set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS% set PYARROW_CMAKE_GENERATOR=%GENERATOR% if "%ARROW_S3%" == "ON" ( - set PYARROW_WITH_S3="ON" + set PYARROW_WITH_S3=ON ) if "%ARROW_BUILD_FLIGHT%" == "ON" ( @rem ARROW-5441: bundling Arrow Flight libraries not implemented From 72e56a68f89114199b2ca53efb92118587e05039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 23 Sep 2019 11:01:07 +0200 Subject: [PATCH 16/39] enable S3 in travis python builds --- ci/travis_script_python.sh | 4 ++++ python/pyarrow/_fs.pyx | 3 ++- python/pyarrow/tests/conftest.py | 7 ++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 2a8a7c96e4d4..b1767dfabc05 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -100,6 +100,10 @@ CMAKE_COMMON_FLAGS="-DARROW_EXTRA_ERROR_CONTEXT=ON" PYTHON_CPP_BUILD_TARGETS="arrow_python-all plasma parquet" +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_S3=ON" +fi + if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_FLIGHT=ON" fi diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index c977e46f2bf2..dc12659322e3 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -41,7 +41,8 @@ cdef class FileStats: """FileSystem entry stats""" def __init__(self): - raise TypeError('dont initialize me') + raise TypeError("FileStats cannot be instantiated directly, use " + "FileSystem.get_target_stats method instead.") @staticmethod cdef FileStats wrap(CFileStats stats): diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index 99044273890a..ace175458a2b 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -243,18 +243,15 @@ def minio_server(): 'MINIO_SECRET_KEY': secret_key }) - minio_dir = os.environ.get('S3FS_DIR', '') - minio_bin = os.path.join(minio_dir, 'minio') if minio_dir else 'minio' - try: with tempfile.TemporaryDirectory() as tempdir: - args = [minio_bin, '--compat', 'server', '--quiet', '--address', + args = ['minio', '--compat', 'server', '--quiet', '--address', address, tempdir] with subprocess.Popen(args, env=env) as proc: yield address, access_key, secret_key proc.terminate() except IOError: - pytest.skip('`minio` command cannot be located, try to set S3FS_DIR') + pytest.skip('`minio` command cannot be located') @pytest.fixture(scope='module') From 8cbe0eeef21bd74b18f33254f92e1c6eb5be322d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 23 Sep 2019 11:53:43 +0200 Subject: [PATCH 17/39] travis osx --- .travis.yml | 3 +++ ci/travis_install_osx.sh | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/.travis.yml b/.travis.yml index b840b26163f0..1ed388580372 100644 --- a/.travis.yml +++ b/.travis.yml @@ -151,6 +151,7 @@ matrix: - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi # If either C++ or Python changed, we must install the C++ libraries - git submodule update --init + - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 @@ -163,6 +164,7 @@ matrix: cache: addons: env: + - ARROW_TRAVIS_S3=1 - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_USE_TOOLCHAIN=1 - ARROW_BUILD_WARNING_LEVEL=CHECKIN @@ -172,6 +174,7 @@ matrix: before_script: script: - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - name: "Java OpenJDK8 and OpenJDK11" language: cpp diff --git a/ci/travis_install_osx.sh b/ci/travis_install_osx.sh index 38e971710dea..5ffabb247c5d 100755 --- a/ci/travis_install_osx.sh +++ b/ci/travis_install_osx.sh @@ -41,3 +41,11 @@ if [ "$ARROW_CI_RUBY_AFFECTED" = "1" ]; then run_brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile --verbose rm ${brew_log_path} fi + +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + # Download the Minio S3 server into PATH + S3FS_DIR=~/.local/bin/ + mkdir -p $S3FS_DIR + wget --directory-prefix $S3FS_DIR https://dl.min.io/server/minio/release/darwin-amd64/minio + chmod +x $S3FS_DIR/minio +fi \ No newline at end of file From fb0f2813ab2a03aa6966c2b6c736824575e67a48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 23 Sep 2019 14:53:35 +0200 Subject: [PATCH 18/39] travis minio install script --- .travis.yml | 5 +++-- ci/travis_install_linux.sh | 8 -------- ci/travis_install_minio.sh | 35 +++++++++++++++++++++++++++++++++++ ci/travis_install_osx.sh | 8 -------- 4 files changed, 38 insertions(+), 18 deletions(-) create mode 100644 ci/travis_install_minio.sh diff --git a/.travis.yml b/.travis.yml index 1ed388580372..7144e13b239a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -117,6 +117,7 @@ matrix: - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1 @@ -151,7 +152,7 @@ matrix: - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi # If either C++ or Python changed, we must install the C++ libraries - git submodule update --init - - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 @@ -174,7 +175,7 @@ matrix: before_script: script: - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - name: "Java OpenJDK8 and OpenJDK11" language: cpp diff --git a/ci/travis_install_linux.sh b/ci/travis_install_linux.sh index a5283139a833..441d50d9b16c 100755 --- a/ci/travis_install_linux.sh +++ b/ci/travis_install_linux.sh @@ -42,14 +42,6 @@ if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then sudo apt-get install -y -qq llvm-$ARROW_LLVM_MAJOR_VERSION-dev fi -if [ "$ARROW_TRAVIS_S3" == "1" ]; then - # Download the Minio S3 server into PATH - S3FS_DIR=~/.local/bin/ - mkdir -p $S3FS_DIR - wget --directory-prefix $S3FS_DIR https://dl.min.io/server/minio/release/linux-amd64/minio - chmod +x $S3FS_DIR/minio -fi - if [ "$ARROW_TRAVIS_USE_SYSTEM" == "1" ]; then if [ "$DISTRO_CODENAME" == "xenial" ]; then # TODO(ARROW-4761): Install libzstd-dev once we support zstd<1 diff --git a/ci/travis_install_minio.sh b/ci/travis_install_minio.sh new file mode 100644 index 000000000000..5459bb8bfc4a --- /dev/null +++ b/ci/travis_install_minio.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + # Download the Minio S3 server into PATH + if [ $TRAVIS_OS_NAME = "osx" ]; then + MINIO_URL=https://dl.min.io/server/minio/release/darwin-amd64/minio + else + MINIO_URL=https://dl.min.io/server/minio/release/linux-amd64/minio + fi + + S3FS_DIR=~/.local/bin/ + mkdir -p $S3FS_DIR + wget --directory-prefix $S3FS_DIR $MINIO_URL + chmod +x $S3FS_DIR/minio +fi \ No newline at end of file diff --git a/ci/travis_install_osx.sh b/ci/travis_install_osx.sh index 5ffabb247c5d..2d79eb017ed5 100755 --- a/ci/travis_install_osx.sh +++ b/ci/travis_install_osx.sh @@ -40,12 +40,4 @@ if [ "$ARROW_CI_RUBY_AFFECTED" = "1" ]; then run_brew bundle --file=$TRAVIS_BUILD_DIR/cpp/Brewfile --verbose run_brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile --verbose rm ${brew_log_path} -fi - -if [ "$ARROW_TRAVIS_S3" == "1" ]; then - # Download the Minio S3 server into PATH - S3FS_DIR=~/.local/bin/ - mkdir -p $S3FS_DIR - wget --directory-prefix $S3FS_DIR https://dl.min.io/server/minio/release/darwin-amd64/minio - chmod +x $S3FS_DIR/minio fi \ No newline at end of file From 041cad42a5887ffbb4981e60e80b6e34dc714eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 23 Sep 2019 15:10:22 +0200 Subject: [PATCH 19/39] executable flag --- ci/travis_install_minio.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/travis_install_minio.sh diff --git a/ci/travis_install_minio.sh b/ci/travis_install_minio.sh old mode 100644 new mode 100755 From d3722871184a9fac4c1f57d6709fb023b7f2cc5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 23 Sep 2019 18:46:05 +0200 Subject: [PATCH 20/39] install minio in the conda-toolchain build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7144e13b239a..990fd2256674 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,6 +87,7 @@ matrix: - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh # If either C++ or Python changed, we must install the C++ libraries - git submodule update --init - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh From 8585a6085a978c3ea2ab325060784b1489b7d1e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 23 Sep 2019 19:07:51 +0200 Subject: [PATCH 21/39] py2 compat --- python/pyarrow/fs.py | 2 ++ python/pyarrow/tests/conftest.py | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index 23291203818d..a8d9e02c6100 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -30,3 +30,5 @@ from pyarrow._s3 import S3FileSystem, initialize_s3, finalize_s3 # noqa except ImportError: pass +else: + pass # initialize_s3? diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index ace175458a2b..1fb57ffffb27 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -230,6 +230,22 @@ def datadir(): return pathlib.Path(__file__).parent / 'data' +try: + from tempfile import TemporaryDirectory +except ImportError: + import shutil + + class TemporaryDirectory(object): + """Temporary directory implementation for python 2""" + + def __enter__(self): + self.tmp = tempfile.mkdtemp() + return self.tmp + + def __exit__(self, exc_type, exc_value, traceback): + shutil.rmtree(self.tmp) + + @pytest.fixture(scope='module') @pytest.mark.s3 def minio_server(): @@ -244,7 +260,7 @@ def minio_server(): }) try: - with tempfile.TemporaryDirectory() as tempdir: + with TemporaryDirectory() as tempdir: args = ['minio', '--compat', 'server', '--quiet', '--address', address, tempdir] with subprocess.Popen(args, env=env) as proc: From 88e0c9f792e414735c913139c33e59d030779fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 23 Sep 2019 22:32:03 +0200 Subject: [PATCH 22/39] py2 compat --- python/pyarrow/tests/conftest.py | 16 ++++++++++------ python/pyarrow/tests/test_flight.py | 12 +----------- python/pyarrow/util.py | 10 ++++++++++ 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index 1fb57ffffb27..59a276327c87 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -27,6 +27,8 @@ except ImportError: import pathlib2 as pathlib # py2 compat +from pyarrow.util import find_free_port + # setup hypothesis profiles h.settings.register_profile('ci', max_examples=1000) @@ -246,10 +248,10 @@ def __exit__(self, exc_type, exc_value, traceback): shutil.rmtree(self.tmp) -@pytest.fixture(scope='module') +@pytest.fixture(scope='session') @pytest.mark.s3 def minio_server(): - host, port = 'localhost', 9000 + host, port = 'localhost', find_free_port() access_key, secret_key = 'arrow', 'apachearrow' address = '{}:{}'.format(host, port) @@ -263,14 +265,16 @@ def minio_server(): with TemporaryDirectory() as tempdir: args = ['minio', '--compat', 'server', '--quiet', '--address', address, tempdir] - with subprocess.Popen(args, env=env) as proc: + try: + proc = subprocess.Popen(args, env=env) yield address, access_key, secret_key - proc.terminate() + finally: + proc.kill() except IOError: pytest.skip('`minio` command cannot be located') -@pytest.fixture(scope='module') +@pytest.fixture(scope='session') def minio_client(minio_server): minio = pytest.importorskip('minio') address, access_key, secret_key = minio_server @@ -282,7 +286,7 @@ def minio_client(minio_server): ) -@pytest.fixture(scope='module') +@pytest.fixture(scope='session') def minio_bucket(minio_client): bucket_name = 'pyarrow-bucket' minio_client.make_bucket(bucket_name) diff --git a/python/pyarrow/tests/test_flight.py b/python/pyarrow/tests/test_flight.py index 83afba6de65e..40099b06aec6 100644 --- a/python/pyarrow/tests/test_flight.py +++ b/python/pyarrow/tests/test_flight.py @@ -17,9 +17,7 @@ # under the License. import base64 -import contextlib import os -import socket import struct import tempfile import threading @@ -30,7 +28,7 @@ import pyarrow as pa from pyarrow.compat import tobytes -from pyarrow.util import pathlib +from pyarrow.util import pathlib, find_free_port try: from pyarrow import flight @@ -48,14 +46,6 @@ pytestmark = pytest.mark.flight -def find_free_port(): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - with contextlib.closing(sock) as sock: - sock.bind(('', 0)) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - return sock.getsockname()[1] - - def test_import(): # So we see the ImportError somewhere import pyarrow.flight # noqa diff --git a/python/pyarrow/util.py b/python/pyarrow/util.py index 5e4fb3579372..7219a447f356 100644 --- a/python/pyarrow/util.py +++ b/python/pyarrow/util.py @@ -19,8 +19,10 @@ from __future__ import absolute_import +import contextlib import functools import six +import socket import warnings @@ -125,3 +127,11 @@ def get_contiguous_span(shape, strides, itemsize): if end - start != itemsize * product(shape): raise ValueError('array data is non-contiguous') return start, end + + +def find_free_port(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with contextlib.closing(sock) as sock: + sock.bind(('', 0)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return sock.getsockname()[1] From 2be25ce291e33f589135b1a8053ae4414eb24a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 23 Sep 2019 22:37:34 +0200 Subject: [PATCH 23/39] auto initialize s3 on import --- cpp/src/arrow/filesystem/s3fs.h | 10 +++++++++- python/pyarrow/fs.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h index 25720784622e..ecdcf9284197 100644 --- a/cpp/src/arrow/filesystem/s3fs.h +++ b/cpp/src/arrow/filesystem/s3fs.h @@ -129,7 +129,15 @@ class ARROW_EXPORT S3FileSystem : public FileSystem { std::unique_ptr impl_; }; -enum class S3LogLevel : int8_t { Off, Fatal, Error, Warn, Info, Debug, Trace }; +enum class ARROW_EXPORT S3LogLevel : int8_t { + Off, + Fatal, + Error, + Warn, + Info, + Debug, + Trace +}; struct ARROW_EXPORT S3GlobalOptions { S3LogLevel log_level; diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index a8d9e02c6100..d62176cf3026 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -31,4 +31,4 @@ except ImportError: pass else: - pass # initialize_s3? + initialize_s3() \ No newline at end of file From 00340ed4c664ec249f0d6ec9148d9ef8b07fb839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 23 Sep 2019 23:02:43 +0200 Subject: [PATCH 24/39] fixture error handling --- python/pyarrow/fs.py | 2 +- python/pyarrow/tests/conftest.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index d62176cf3026..0fba639a17e5 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -31,4 +31,4 @@ except ImportError: pass else: - initialize_s3() \ No newline at end of file + initialize_s3() diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index 59a276327c87..e9fbd86398bc 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -261,17 +261,19 @@ def minio_server(): 'MINIO_SECRET_KEY': secret_key }) - try: - with TemporaryDirectory() as tempdir: - args = ['minio', '--compat', 'server', '--quiet', '--address', - address, tempdir] - try: - proc = subprocess.Popen(args, env=env) - yield address, access_key, secret_key - finally: + with TemporaryDirectory() as tempdir: + args = ['minio', '--compat', 'server', '--quiet', '--address', + address, tempdir] + proc = None + try: + proc = subprocess.Popen(args, env=env) + except IOError: + pytest.skip('`minio` command cannot be located') + else: + yield address, access_key, secret_key + finally: + if proc is not None: proc.kill() - except IOError: - pytest.skip('`minio` command cannot be located') @pytest.fixture(scope='session') From 098048a8a8da74c8679744dc1c24cb05006283d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 24 Sep 2019 00:03:53 +0200 Subject: [PATCH 25/39] more compat --- cpp/src/arrow/filesystem/s3fs.h | 10 +- .../Untitled-checkpoint.ipynb | 6 + python/Untitled.ipynb | 387 ++++++++++++++++++ python/open-append-stream | 2 + python/pyarrow/_s3.pyx | 6 +- python/pyarrow/tests/test_fs.py | 98 ++++- python/run_test.sh | 70 ++++ python/test.parquet | Bin 0 -> 106391 bytes python/test.txt | 0 9 files changed, 545 insertions(+), 34 deletions(-) create mode 100644 python/.ipynb_checkpoints/Untitled-checkpoint.ipynb create mode 100644 python/Untitled.ipynb create mode 100644 python/open-append-stream create mode 100755 python/run_test.sh create mode 100644 python/test.parquet create mode 100644 python/test.txt diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h index ecdcf9284197..25720784622e 100644 --- a/cpp/src/arrow/filesystem/s3fs.h +++ b/cpp/src/arrow/filesystem/s3fs.h @@ -129,15 +129,7 @@ class ARROW_EXPORT S3FileSystem : public FileSystem { std::unique_ptr impl_; }; -enum class ARROW_EXPORT S3LogLevel : int8_t { - Off, - Fatal, - Error, - Warn, - Info, - Debug, - Trace -}; +enum class S3LogLevel : int8_t { Off, Fatal, Error, Warn, Info, Debug, Trace }; struct ARROW_EXPORT S3GlobalOptions { S3LogLevel log_level; diff --git a/python/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/python/.ipynb_checkpoints/Untitled-checkpoint.ipynb new file mode 100644 index 000000000000..2fd64429bf42 --- /dev/null +++ b/python/.ipynb_checkpoints/Untitled-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/Untitled.ipynb b/python/Untitled.ipynb new file mode 100644 index 000000000000..a6546e3b758d --- /dev/null +++ b/python/Untitled.ipynb @@ -0,0 +1,387 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import pyarrow as pa\n", + "import pyarrow.parquet as pq\n", + "from s3fs import S3File, S3FileSystem\n", + "\n", + "\n", + "df = pd.DataFrame({'col0': []})\n", + "s3_filepath = 's3://some-bogus-bucket/df.parquet'" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Exception ignored in: \n", + "Traceback (most recent call last):\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1127, in __del__\n", + " self.close()\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1104, in close\n", + " self.flush(force=True)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 976, in flush\n", + " self._initiate_upload()\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 984, in _initiate_upload\n", + " Bucket=self.bucket, Key=self.key, ACL=self.acl)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 971, in _call_s3\n", + " **kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 189, in _call_s3\n", + " return method(**additional_kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 357, in _api_call\n", + " return self._make_api_call(operation_name, kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 648, in _make_api_call\n", + " operation_model, request_dict, request_context)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 667, in _make_request\n", + " return self._endpoint.make_request(operation_model, request_dict)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 102, in make_request\n", + " return self._send_request(request_dict, operation_model)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 132, in _send_request\n", + " request = self.create_request(request_dict, operation_model)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 116, in create_request\n", + " operation_name=operation_model.name)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 356, in emit\n", + " return self._emitter.emit(aliased_event_name, **kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 228, in emit\n", + " return self._emit(event_name, kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 211, in _emit\n", + " response = handler(**kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 90, in handler\n", + " return self.sign(operation_name, request)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 157, in sign\n", + " auth.add_auth(request)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 425, in add_auth\n", + " super(S3SigV4Auth, self).add_auth(request)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 357, in add_auth\n", + " raise NoCredentialsError\n", + "botocore.exceptions.NoCredentialsError: Unable to locate credentials\n" + ] + } + ], + "source": [ + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Exception ignored in: \n", + "Traceback (most recent call last):\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1127, in __del__\n", + " self.close()\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1104, in close\n", + " self.flush(force=True)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 976, in flush\n", + " self._initiate_upload()\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 984, in _initiate_upload\n", + " Bucket=self.bucket, Key=self.key, ACL=self.acl)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 971, in _call_s3\n", + " **kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 189, in _call_s3\n", + " return method(**additional_kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 357, in _api_call\n", + " return self._make_api_call(operation_name, kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 648, in _make_api_call\n", + " operation_model, request_dict, request_context)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 667, in _make_request\n", + " return self._endpoint.make_request(operation_model, request_dict)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 102, in make_request\n", + " return self._send_request(request_dict, operation_model)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 132, in _send_request\n", + " request = self.create_request(request_dict, operation_model)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 116, in create_request\n", + " operation_name=operation_model.name)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 356, in emit\n", + " return self._emitter.emit(aliased_event_name, **kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 228, in emit\n", + " return self._emit(event_name, kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 211, in _emit\n", + " response = handler(**kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 90, in handler\n", + " return self.sign(operation_name, request)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 157, in sign\n", + " auth.add_auth(request)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 425, in add_auth\n", + " super(S3SigV4Auth, self).add_auth(request)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 357, in add_auth\n", + " raise NoCredentialsError\n", + "botocore.exceptions.NoCredentialsError: Unable to locate credentials\n" + ] + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "out.flush()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CMakeLists.txt \u001b[34mbuild\u001b[m\u001b[m requirements-wheel.txt\r\n", + "Dockerfile \u001b[35mcmake_modules\u001b[m\u001b[m requirements.txt\r\n", + "Dockerfile.alpine \u001b[34mdist\u001b[m\u001b[m \u001b[31mrun_test.sh\u001b[m\u001b[m\r\n", + "Dockerfile.nopandas \u001b[34mexamples\u001b[m\u001b[m \u001b[34mscripts\u001b[m\u001b[m\r\n", + "MANIFEST.in \u001b[34mmanylinux1\u001b[m\u001b[m setup.cfg\r\n", + "README.md \u001b[34mmanylinux2010\u001b[m\u001b[m \u001b[31msetup.py\u001b[m\u001b[m\r\n", + "Untitled.ipynb nm_arrow.log \u001b[34msource-dir\u001b[m\u001b[m\r\n", + "\u001b[31masv-build.sh\u001b[m\u001b[m \u001b[34mpyarrow\u001b[m\u001b[m test.parquet\r\n", + "\u001b[31masv-install.sh\u001b[m\u001b[m \u001b[34mpyarrow.egg-info\u001b[m\u001b[m test.txt\r\n", + "\u001b[31masv-uninstall.sh\u001b[m\u001b[m pyproject.toml visible_symbols.log\r\n", + "asv.conf.json requirements-build.txt\r\n", + "\u001b[34mbenchmarks\u001b[m\u001b[m requirements-test.txt\r\n" + ] + } + ], + "source": [ + "!ls" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "f = open('test.txt', 'wb')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'_io.BufferedWriter' object has no attribute '__mro__'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__mro__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m: '_io.BufferedWriter' object has no attribute '__mro__'" + ] + } + ], + "source": [ + "f." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "ename": "NoCredentialsError", + "evalue": "Unable to locate credentials", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNoCredentialsError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\u001b[0m in \u001b[0;36mclose\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1102\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1103\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforced\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1104\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mforce\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1105\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1106\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\u001b[0m in \u001b[0;36mflush\u001b[0;34m(self, force)\u001b[0m\n\u001b[1;32m 974\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 975\u001b[0m \u001b[0;31m# Initialize a multipart upload\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 976\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_initiate_upload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 977\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 978\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_upload_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfinal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mforce\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36m_initiate_upload\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 982\u001b[0m self.mpu = self._call_s3(\n\u001b[1;32m 983\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ms3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_multipart_upload\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 984\u001b[0;31m Bucket=self.bucket, Key=self.key, ACL=self.acl)\n\u001b[0m\u001b[1;32m 985\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mClientError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 986\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mtranslate_boto_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36m_call_s3\u001b[0;34m(self, method, *kwarglist, **kwargs)\u001b[0m\n\u001b[1;32m 969\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_call_s3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mkwarglist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 970\u001b[0m return self.fs._call_s3(method, self.s3_additional_kwargs, *kwarglist,\n\u001b[0;32m--> 971\u001b[0;31m **kwargs)\n\u001b[0m\u001b[1;32m 972\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 973\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_initiate_upload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36m_call_s3\u001b[0;34m(self, method, *akwarglist, **kwargs)\u001b[0m\n\u001b[1;32m 187\u001b[0m additional_kwargs = self._get_s3_method_kwargs(method, *akwarglist,\n\u001b[1;32m 188\u001b[0m **kwargs)\n\u001b[0;32m--> 189\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0madditional_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_s3_method_kwargs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0makwarglist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 355\u001b[0m \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[1;32m 356\u001b[0m \u001b[0;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 358\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[0m_api_call\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 646\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 647\u001b[0m http, parsed_response = self._make_request(\n\u001b[0;32m--> 648\u001b[0;31m operation_model, request_dict, request_context)\n\u001b[0m\u001b[1;32m 649\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 650\u001b[0m self.meta.events.emit(\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, operation_model, request_dict, request_context)\u001b[0m\n\u001b[1;32m 665\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 666\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 667\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_endpoint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 668\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 669\u001b[0m self.meta.events.emit(\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\u001b[0m in \u001b[0;36mmake_request\u001b[0;34m(self, operation_model, request_dict)\u001b[0m\n\u001b[1;32m 100\u001b[0m logger.debug(\"Making request for %s with params: %s\",\n\u001b[1;32m 101\u001b[0m operation_model, request_dict)\n\u001b[0;32m--> 102\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcreate_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\u001b[0m in \u001b[0;36m_send_request\u001b[0;34m(self, request_dict, operation_model)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_send_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0mattempts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m \u001b[0mrequest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 133\u001b[0m \u001b[0mcontext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'context'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 134\u001b[0m success_response, exception = self._get_response(\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\u001b[0m in \u001b[0;36mcreate_request\u001b[0;34m(self, params, operation_model)\u001b[0m\n\u001b[1;32m 114\u001b[0m op_name=operation_model.name)\n\u001b[1;32m 115\u001b[0m self._event_emitter.emit(event_name, request=request,\n\u001b[0;32m--> 116\u001b[0;31m operation_name=operation_model.name)\n\u001b[0m\u001b[1;32m 117\u001b[0m \u001b[0mprepared_request\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mprepared_request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\u001b[0m in \u001b[0;36memit\u001b[0;34m(self, event_name, **kwargs)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0memit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0maliased_event_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_alias_event_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevent_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_emitter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0memit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maliased_event_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 357\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 358\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0memit_until_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\u001b[0m in \u001b[0;36memit\u001b[0;34m(self, event_name, **kwargs)\u001b[0m\n\u001b[1;32m 226\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 227\u001b[0m \"\"\"\n\u001b[0;32m--> 228\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_emit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 229\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 230\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0memit_until_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\u001b[0m in \u001b[0;36m_emit\u001b[0;34m(self, event_name, kwargs, stop_on_response)\u001b[0m\n\u001b[1;32m 209\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers_to_call\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Event %s: calling handler %s'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhandler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 212\u001b[0m \u001b[0mresponses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstop_on_response\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\u001b[0m in \u001b[0;36mhandler\u001b[0;34m(self, operation_name, request, **kwargs)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0;31m# this method is invoked to sign the request.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;31m# Don't call this method directly.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m def sign(self, operation_name, request, region_name=None,\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\u001b[0m in \u001b[0;36msign\u001b[0;34m(self, operation_name, request, region_name, signing_type, expires_in, signing_name)\u001b[0m\n\u001b[1;32m 155\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m \u001b[0mauth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_auth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_choose_signer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msigning_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\u001b[0m in \u001b[0;36madd_auth\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 423\u001b[0m self._region_name = signing_context.get(\n\u001b[1;32m 424\u001b[0m 'region', self._default_region_name)\n\u001b[0;32m--> 425\u001b[0;31m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mS3SigV4Auth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_auth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 427\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_modify_request_before_signing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\u001b[0m in \u001b[0;36madd_auth\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0madd_auth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 356\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcredentials\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mNoCredentialsError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 358\u001b[0m \u001b[0mdatetime_now\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutcnow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'timestamp'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatetime_now\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrftime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSIGV4_TIMESTAMP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNoCredentialsError\u001b[0m: Unable to locate credentials" + ] + } + ], + "source": [ + "out.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Exception ignored in: \n", + "Traceback (most recent call last):\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1127, in __del__\n", + " self.close()\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1104, in close\n", + " self.flush(force=True)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 976, in flush\n", + " self._initiate_upload()\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 984, in _initiate_upload\n", + " Bucket=self.bucket, Key=self.key, ACL=self.acl)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 971, in _call_s3\n", + " **kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 189, in _call_s3\n", + " return method(**additional_kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 357, in _api_call\n", + " return self._make_api_call(operation_name, kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 648, in _make_api_call\n", + " operation_model, request_dict, request_context)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 667, in _make_request\n", + " return self._endpoint.make_request(operation_model, request_dict)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 102, in make_request\n", + " return self._send_request(request_dict, operation_model)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 132, in _send_request\n", + " request = self.create_request(request_dict, operation_model)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 116, in create_request\n", + " operation_name=operation_model.name)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 356, in emit\n", + " return self._emitter.emit(aliased_event_name, **kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 228, in emit\n", + " return self._emit(event_name, kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 211, in _emit\n", + " response = handler(**kwargs)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 90, in handler\n", + " return self.sign(operation_name, request)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 157, in sign\n", + " auth.add_auth(request)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 425, in add_auth\n", + " super(S3SigV4Auth, self).add_auth(request)\n", + " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 357, in add_auth\n", + " raise NoCredentialsError\n", + "botocore.exceptions.NoCredentialsError: Unable to locate credentials\n" + ] + } + ], + "source": [ + "out = S3File(S3FileSystem(), s3_filepath, mode='wb')\n", + "table = pa.Table.from_pandas(df.copy())\n", + "try:\n", + " pq.write_table(table, out)\n", + "except:\n", + " print('EEEEEEEEEEEEEe')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/open-append-stream b/python/open-append-stream new file mode 100644 index 000000000000..e7b3f4697c0e --- /dev/null +++ b/python/open-append-stream @@ -0,0 +1,2 @@ + +newly added \ No newline at end of file diff --git a/python/pyarrow/_s3.pyx b/python/pyarrow/_s3.pyx index d7bfc79cc97f..bd783f25bb67 100644 --- a/python/pyarrow/_s3.pyx +++ b/python/pyarrow/_s3.pyx @@ -75,9 +75,9 @@ cdef class S3FileSystem(FileSystem): cdef: CS3FileSystem* s3fs - def __init__(self, str access_key=None, str secret_key=None, - str region='us-east-1', str scheme='https', - str endpoint_override=None, bint background_writes=True): + def __init__(self, access_key=None, secret_key=None, region='us-east-1', + scheme='https', endpoint_override=None, + bint background_writes=True): cdef: CS3Options options = CS3Options.Defaults() shared_ptr[CS3FileSystem] wrapped diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 4dfae43fd0a1..811f4ffe1bf2 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -56,6 +56,15 @@ def impl(self, impl): def pathpair(self, p): raise NotImplementedError() + def rmdir(self, p): + raise NotImplementedError() + + def unlink(self, p): + raise NotImplementedError() + + def iterdir(self, p): + raise NotImplementedError() + def mkdir(self, p): raise NotImplementedError() @@ -88,9 +97,19 @@ def pathpair(self, p): path_for_impl = '/'.join([self.tempdir.as_posix(), p]) return (path_for_wrapper, path_for_impl) + def unlink(self, p): + return pathlib.Path(p).unlink() + + def rmdir(self, p): + return pathlib.Path(p).rmdir() + def mkdir(self, p): return pathlib.Path(p).mkdir(parents=True) + def iterdir(self, p): + for path in pathlib.Path(p).iterdir(): + yield (path, path.is_dir()) + def touch(self, p): return pathlib.Path(p).touch() @@ -151,6 +170,20 @@ def touch(self, p): length=0 ) + def unlink(self, p): + self.client.remove_object( + bucket_name=self.bucket, + object_name=p.rstrip('/') + ) + + def rmdir(self, p): + if not p.endswith('/'): + p += '/' + self.client.remove_object( + bucket_name=self.bucket, + object_name=p + ) + def mkdir(self, p): if not p.endswith('/'): p += '/' @@ -161,6 +194,17 @@ def mkdir(self, p): length=0 ) + def iterdir(self, p): + if not p.endswith('/'): + p += '/' + objs = self.client.list_objects( + bucket_name=self.bucket, + prefix=p, + recursive=False + ) + for obj in objs: + yield (obj.object_name, obj.is_dir) + def exists(self, p): from minio.error import NoSuchKey, NoSuchBucket try: @@ -228,7 +272,7 @@ def localfs(request, tempdir): @pytest.fixture(params=[ S3Wrapper, - SubTreeS3Wrapper + #SubTreeS3Wrapper ]) def s3fs(request, minio_server, minio_client, minio_bucket): from pyarrow.fs import initialize_s3 @@ -310,27 +354,37 @@ def mtime_almost_equal(a, b): assert mtime_almost_equal(c_stat.mtime, fs.mtime(_c)) -@pytest.mark.skip() -def test_get_target_stats_with_selector(fs, tempdir, testpath): - base_dir = testpath('.') - base_dir_ = tempdir - - selector = Selector(base_dir, allow_non_existent=False, recursive=True) - assert selector.base_dir == str(base_dir) - - (tempdir / 'test_file').touch() - (tempdir / 'test_directory').mkdir() - - stats = fs.get_target_stats(selector) - expected = list(base_dir_.iterdir()) - assert len(stats) == len(expected) - - for st in stats: - p = base_dir_ / st.path - if p.is_dir(): - assert st.type == FileType.Directory - if p.is_file(): - assert st.type == FileType.File +def test_get_target_stats_with_selector(fs): + _base_dir, base_dir = fs.pathpair('selector-dir/') + _file_a, file_a = fs.pathpair('selector-dir/test_file_a') + _file_b, file_b = fs.pathpair('selector-dir/test_file_b') + _dir_a, dir_a = fs.pathpair('selector-dir/test_dir_a') + try: + fs.mkdir(_base_dir) + fs.touch(_file_a) + fs.touch(_file_b) + fs.mkdir(_dir_a) + + selector = Selector(base_dir, allow_non_existent=False, recursive=True) + assert selector.base_dir == base_dir + + stats = fs.impl.get_target_stats(selector) + expected = list(fs.iterdir(_base_dir)) + assert len(stats) == len(expected) + + left = sorted(stats, key=lambda st: st.path) + right = sorted(expected, key=lambda tpl: tpl[0]) + + for l, r in zip(left, right): + if r[1] is True: + assert l.type == FileType.Directory + else: + assert l.type == FileType.File + finally: + fs.unlink(_file_a) + fs.unlink(_file_b) + fs.rmdir(_dir_a) + fs.rmdir(_base_dir) def test_create_dir(fs): diff --git a/python/run_test.sh b/python/run_test.sh new file mode 100755 index 000000000000..47e93e15a203 --- /dev/null +++ b/python/run_test.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +export CXXFLAGS="" +export ARROW_BUILD_TYPE=debug +export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX +export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX +export ARROW_HOME=$CONDA_PREFIX +export PARQUET_HOME=$CONDA_PREFIX +export PARQUET_TEST_DATA=`pwd`/../cpp/submodules/parquet-testing/data +export ARROW_TEST_DATA=`pwd`/../testing/data + + +mkdir -p ../cpp/build +pushd ../cpp/build + +cmake -GNinja \ + -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DARROW_PYTHON=ON \ + -DARROW_PLASMA=OFF \ + -DARROW_PARQUET=ON \ + -DARROW_GANDIVA=OFF \ + -DARROW_ORC=ON \ + -DARROW_FLIGHT=OFF \ + -DARROW_S3=ON \ + -DARROW_TENSORFLOW=OFF \ + -DARROW_DEPENDENCY_SOURCE=CONDA \ + -DARROW_EXTRA_ERROR_CONTEXT=ON \ + -DARROW_BUILD_TESTS=ON \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=YES \ + -DCMAKE_CXX_FLAGS=$CXXFLAGS \ + .. + +ninja +# ninja test +ninja install + +popd + +export PYARROW_CMAKE_GENERATOR=Ninja +export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_PLASMA=0 +export PYARROW_WITH_GANDIVA=0 +export PYARROW_WITH_DATASET=0 +export PYARROW_WITH_FLIGHT=0 +export PYARROW_WITH_S3=1 +export PYARROW_WITH_ORC=1 + +python setup.py develop + +py.test -sv "$@" diff --git a/python/test.parquet b/python/test.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8548b7e899f3509e0ca346c589ab409ded4ea932 GIT binary patch literal 106391 zcmeI*e`wVA9mnza_j}*(*UNg0wCneJ$1a;2dPs?haSOPzxf#m2m0-n|4&3UDCD0V@ zY)x#bo8K@*P}6O!8x%KWG#xY{b2dn!X7%hU{^1{krJH{URj^Q4m>uyU(0(w2|j~nUrH*E!-zhpSf?{yyyBtPxrU>pZGW~>z^GzfBpJ5uf8`nKK`?V zV-u5;y&p{+4r;BbwD5%h0tg_000IagfB*v3AaFb_yB9`+VpgaI0DeXU5I_I{1Q0*~ z0R#|Gfs>jHiX#LIhYk=z009ILKmY**5I_KdDiJWrV9}XGCMQ&h0Y4@J2q1s}0tg_0 z00Ib@WH7g|pk&wp9t03T009ILKmY**5U3IXlML>(0C|0t81Q2vfB*srAbCP;a|^{rzGMPfy3h7yoR3jsNdW^?vtjS>N$J-^-*N(Ocm@`HSb)&3mfH zPyWs6y!Jbh&y7t?PWFB@aX2X5y{rt&(9$IY5I_I{1Q56z0V}=c4@X2ErItuF{rj*I z|7V*%m#x4y9dzIqNY(!W*{mM}YxJ$Pk{!u=89$zGcEn3>vf7-z9S1s{j?VUdzMbs; z;B6;)ap0Jhypi{^8?y0e5WR3RQIbS8uEzaDt8e?x10qhsq*e{Iw+0URpN_klXwRH* z7K^k4XYMPx(z{o_v3F=&T!L%k6BAp9#wVwy{xbUGd_FjP==#{$7jK?=NF;ctQCb@G z@y}gSmy_U=A#wHLjS+Dh_4xS>*5;hPI(*Pd>*0|h{ zwuanZ2}a+u6MF5SX6p@e)}J=U{2{kff~zmb?$`5*hfm*(-QQKGhcA7_*=X9CH$5jo z!>e++N1qm(nI37Q8?U%Exv2{-UkD(800IagfB*srAW-H4qZi!oo3s&V+$c>UR_ZPS zhy*)R61FoDq$e6dBIid!h-RFb7@=Wha9C5tOwb9##^B9Z^TDq*nyhiU7(%pN-L9i1 z&S=YAgA_x_@i!&cC55IMYRcT3?je8x0tg_000IagfPe~^AYw*7g2K>8B#Yd05OH%F z8fsE4Bl46PN|{M2nnc_ot1ui=s{<}U-YtxW=%~t5f1lT|7%_A_WG0M+!zS9SE*u}3 zx!~OR$f?4ZNH{{m-zx$LAbQEX>sU>6@~dB*L6D1Q0*~0R#|0009Il zO28x$vnV8(zbd2{MixsVBhkenb0Z!)H>LF1Cxz%DSZ)WN5sYLAvIahD2s%g$q^YQix8?Y9WPG)F-a|(>Xmv009ILKmY** z5LjselS0hu^q^#Ux(OnSrH}=S)FV0M?$?Oun)v8=$hDVaO&V0@W@Qytsn=}%Bqw3G znOUH&Ip|9xg(dSWP$z#NfB*srAbR7?T@>tg*;bT}w8nlF!2?zQx zQy(jg9$E=2j0dJV)^M>p)}5ze={opm?3f!VnM)p#^&kt!K#C0+09om5%JmUI009IL zKmY**5Lk8rlR->*tYDEUy^#!Zrz*YoH~^wGfD1|F(toMb+t67EH^s=Y@m0y~wh#Y6 z=XVv$^j`j|+ptxKO{mjbIjMKqKf+uP0R#|0009ILK%lY&Oaw7|f&>ey^qxHwMiAZV zqs7$5U1XaOt(ChgB|CL55K{yzx>LwwA4)O7oNuxpDee{G>i~+`DMSkcn}L)_Ni5wd z#LU4m%R$Z_GRr{d+2_GeZ} zjxDjOeoL&f-Lj32A%Fk^2vm^3Ro!5*ufNrfRZySlDgp=~fB*srAbyx378aMLO_27^_i|BfB*srAb7Wc+x#*%2?j$=c)W?Ksfs zbab}w^X+8!2X8ycfBpW5l{`P_Wlz8FWskmJk~%M(Oq9e<-%kIiSyJACXP#}}*XBFY zY^SU?-}b%MuWKr8c2ePc__p}*wHjW{dxdTG{Es%t?qjc{GwJ&5NZy;*<+g=gzBq8q zO5VtO*$vrvG$b$V?(zd9e|sxo`YP#6DzRn9j>o_INWmiYb30*))SA#PpOJ)k;!0e< z)0-93=c3QzdhZnPMQnYxH#{OTFg-o34HJ**b8`)o_VsdM+u!JGuZOQoETzZHwZi`m z(8r^N^TOvLwjML*?a@EA@}QpY$*S;-+G5|V#BP1B0AuWBZmJP5%Ke7zJ$r literal 0 HcmV?d00001 diff --git a/python/test.txt b/python/test.txt new file mode 100644 index 000000000000..e69de29bb2d1 From c541b3e15e4a70b4c0be979457df573cdea9b420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 24 Sep 2019 00:04:44 +0200 Subject: [PATCH 26/39] comment left --- python/pyarrow/tests/test_fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 811f4ffe1bf2..076faf205e04 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -272,7 +272,7 @@ def localfs(request, tempdir): @pytest.fixture(params=[ S3Wrapper, - #SubTreeS3Wrapper + SubTreeS3Wrapper ]) def s3fs(request, minio_server, minio_client, minio_bucket): from pyarrow.fs import initialize_s3 From 38dcb88d09bf6a18a78ec80e849648de70f29725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 24 Sep 2019 09:51:37 +0200 Subject: [PATCH 27/39] rat --- .../Untitled-checkpoint.ipynb | 6 - python/Untitled.ipynb | 387 ------------------ python/open-append-stream | 2 - python/pyarrow/includes/libarrow.pxd | 21 +- python/test.txt | 0 5 files changed, 10 insertions(+), 406 deletions(-) delete mode 100644 python/.ipynb_checkpoints/Untitled-checkpoint.ipynb delete mode 100644 python/Untitled.ipynb delete mode 100644 python/open-append-stream delete mode 100644 python/test.txt diff --git a/python/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/python/.ipynb_checkpoints/Untitled-checkpoint.ipynb deleted file mode 100644 index 2fd64429bf42..000000000000 --- a/python/.ipynb_checkpoints/Untitled-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/python/Untitled.ipynb b/python/Untitled.ipynb deleted file mode 100644 index a6546e3b758d..000000000000 --- a/python/Untitled.ipynb +++ /dev/null @@ -1,387 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import pyarrow as pa\n", - "import pyarrow.parquet as pq\n", - "from s3fs import S3File, S3FileSystem\n", - "\n", - "\n", - "df = pd.DataFrame({'col0': []})\n", - "s3_filepath = 's3://some-bogus-bucket/df.parquet'" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "3" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Exception ignored in: \n", - "Traceback (most recent call last):\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1127, in __del__\n", - " self.close()\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1104, in close\n", - " self.flush(force=True)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 976, in flush\n", - " self._initiate_upload()\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 984, in _initiate_upload\n", - " Bucket=self.bucket, Key=self.key, ACL=self.acl)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 971, in _call_s3\n", - " **kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 189, in _call_s3\n", - " return method(**additional_kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 357, in _api_call\n", - " return self._make_api_call(operation_name, kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 648, in _make_api_call\n", - " operation_model, request_dict, request_context)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 667, in _make_request\n", - " return self._endpoint.make_request(operation_model, request_dict)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 102, in make_request\n", - " return self._send_request(request_dict, operation_model)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 132, in _send_request\n", - " request = self.create_request(request_dict, operation_model)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 116, in create_request\n", - " operation_name=operation_model.name)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 356, in emit\n", - " return self._emitter.emit(aliased_event_name, **kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 228, in emit\n", - " return self._emit(event_name, kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 211, in _emit\n", - " response = handler(**kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 90, in handler\n", - " return self.sign(operation_name, request)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 157, in sign\n", - " auth.add_auth(request)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 425, in add_auth\n", - " super(S3SigV4Auth, self).add_auth(request)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 357, in add_auth\n", - " raise NoCredentialsError\n", - "botocore.exceptions.NoCredentialsError: Unable to locate credentials\n" - ] - } - ], - "source": [ - "del out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "3" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Exception ignored in: \n", - "Traceback (most recent call last):\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1127, in __del__\n", - " self.close()\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1104, in close\n", - " self.flush(force=True)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 976, in flush\n", - " self._initiate_upload()\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 984, in _initiate_upload\n", - " Bucket=self.bucket, Key=self.key, ACL=self.acl)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 971, in _call_s3\n", - " **kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 189, in _call_s3\n", - " return method(**additional_kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 357, in _api_call\n", - " return self._make_api_call(operation_name, kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 648, in _make_api_call\n", - " operation_model, request_dict, request_context)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 667, in _make_request\n", - " return self._endpoint.make_request(operation_model, request_dict)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 102, in make_request\n", - " return self._send_request(request_dict, operation_model)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 132, in _send_request\n", - " request = self.create_request(request_dict, operation_model)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 116, in create_request\n", - " operation_name=operation_model.name)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 356, in emit\n", - " return self._emitter.emit(aliased_event_name, **kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 228, in emit\n", - " return self._emit(event_name, kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 211, in _emit\n", - " response = handler(**kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 90, in handler\n", - " return self.sign(operation_name, request)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 157, in sign\n", - " auth.add_auth(request)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 425, in add_auth\n", - " super(S3SigV4Auth, self).add_auth(request)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 357, in add_auth\n", - " raise NoCredentialsError\n", - "botocore.exceptions.NoCredentialsError: Unable to locate credentials\n" - ] - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "out.flush()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CMakeLists.txt \u001b[34mbuild\u001b[m\u001b[m requirements-wheel.txt\r\n", - "Dockerfile \u001b[35mcmake_modules\u001b[m\u001b[m requirements.txt\r\n", - "Dockerfile.alpine \u001b[34mdist\u001b[m\u001b[m \u001b[31mrun_test.sh\u001b[m\u001b[m\r\n", - "Dockerfile.nopandas \u001b[34mexamples\u001b[m\u001b[m \u001b[34mscripts\u001b[m\u001b[m\r\n", - "MANIFEST.in \u001b[34mmanylinux1\u001b[m\u001b[m setup.cfg\r\n", - "README.md \u001b[34mmanylinux2010\u001b[m\u001b[m \u001b[31msetup.py\u001b[m\u001b[m\r\n", - "Untitled.ipynb nm_arrow.log \u001b[34msource-dir\u001b[m\u001b[m\r\n", - "\u001b[31masv-build.sh\u001b[m\u001b[m \u001b[34mpyarrow\u001b[m\u001b[m test.parquet\r\n", - "\u001b[31masv-install.sh\u001b[m\u001b[m \u001b[34mpyarrow.egg-info\u001b[m\u001b[m test.txt\r\n", - "\u001b[31masv-uninstall.sh\u001b[m\u001b[m pyproject.toml visible_symbols.log\r\n", - "asv.conf.json requirements-build.txt\r\n", - "\u001b[34mbenchmarks\u001b[m\u001b[m requirements-test.txt\r\n" - ] - } - ], - "source": [ - "!ls" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "f = open('test.txt', 'wb')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "'_io.BufferedWriter' object has no attribute '__mro__'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__mro__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m: '_io.BufferedWriter' object has no attribute '__mro__'" - ] - } - ], - "source": [ - "f." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "ename": "NoCredentialsError", - "evalue": "Unable to locate credentials", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNoCredentialsError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\u001b[0m in \u001b[0;36mclose\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1102\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1103\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforced\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1104\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mforce\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1105\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1106\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\u001b[0m in \u001b[0;36mflush\u001b[0;34m(self, force)\u001b[0m\n\u001b[1;32m 974\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 975\u001b[0m \u001b[0;31m# Initialize a multipart upload\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 976\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_initiate_upload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 977\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 978\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_upload_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfinal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mforce\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36m_initiate_upload\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 982\u001b[0m self.mpu = self._call_s3(\n\u001b[1;32m 983\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ms3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_multipart_upload\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 984\u001b[0;31m Bucket=self.bucket, Key=self.key, ACL=self.acl)\n\u001b[0m\u001b[1;32m 985\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mClientError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 986\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mtranslate_boto_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36m_call_s3\u001b[0;34m(self, method, *kwarglist, **kwargs)\u001b[0m\n\u001b[1;32m 969\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_call_s3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mkwarglist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 970\u001b[0m return self.fs._call_s3(method, self.s3_additional_kwargs, *kwarglist,\n\u001b[0;32m--> 971\u001b[0;31m **kwargs)\n\u001b[0m\u001b[1;32m 972\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 973\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_initiate_upload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36m_call_s3\u001b[0;34m(self, method, *akwarglist, **kwargs)\u001b[0m\n\u001b[1;32m 187\u001b[0m additional_kwargs = self._get_s3_method_kwargs(method, *akwarglist,\n\u001b[1;32m 188\u001b[0m **kwargs)\n\u001b[0;32m--> 189\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0madditional_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_s3_method_kwargs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0makwarglist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 355\u001b[0m \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[1;32m 356\u001b[0m \u001b[0;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 358\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[0m_api_call\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 646\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 647\u001b[0m http, parsed_response = self._make_request(\n\u001b[0;32m--> 648\u001b[0;31m operation_model, request_dict, request_context)\n\u001b[0m\u001b[1;32m 649\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 650\u001b[0m self.meta.events.emit(\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, operation_model, request_dict, request_context)\u001b[0m\n\u001b[1;32m 665\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 666\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 667\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_endpoint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 668\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 669\u001b[0m self.meta.events.emit(\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\u001b[0m in \u001b[0;36mmake_request\u001b[0;34m(self, operation_model, request_dict)\u001b[0m\n\u001b[1;32m 100\u001b[0m logger.debug(\"Making request for %s with params: %s\",\n\u001b[1;32m 101\u001b[0m operation_model, request_dict)\n\u001b[0;32m--> 102\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcreate_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\u001b[0m in \u001b[0;36m_send_request\u001b[0;34m(self, request_dict, operation_model)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_send_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0mattempts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m \u001b[0mrequest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 133\u001b[0m \u001b[0mcontext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'context'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 134\u001b[0m success_response, exception = self._get_response(\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\u001b[0m in \u001b[0;36mcreate_request\u001b[0;34m(self, params, operation_model)\u001b[0m\n\u001b[1;32m 114\u001b[0m op_name=operation_model.name)\n\u001b[1;32m 115\u001b[0m self._event_emitter.emit(event_name, request=request,\n\u001b[0;32m--> 116\u001b[0;31m operation_name=operation_model.name)\n\u001b[0m\u001b[1;32m 117\u001b[0m \u001b[0mprepared_request\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mprepared_request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\u001b[0m in \u001b[0;36memit\u001b[0;34m(self, event_name, **kwargs)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0memit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0maliased_event_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_alias_event_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevent_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_emitter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0memit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maliased_event_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 357\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 358\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0memit_until_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\u001b[0m in \u001b[0;36memit\u001b[0;34m(self, event_name, **kwargs)\u001b[0m\n\u001b[1;32m 226\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 227\u001b[0m \"\"\"\n\u001b[0;32m--> 228\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_emit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 229\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 230\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0memit_until_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\u001b[0m in \u001b[0;36m_emit\u001b[0;34m(self, event_name, kwargs, stop_on_response)\u001b[0m\n\u001b[1;32m 209\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers_to_call\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Event %s: calling handler %s'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevent_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhandler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 212\u001b[0m \u001b[0mresponses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstop_on_response\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\u001b[0m in \u001b[0;36mhandler\u001b[0;34m(self, operation_name, request, **kwargs)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0;31m# this method is invoked to sign the request.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;31m# Don't call this method directly.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m def sign(self, operation_name, request, region_name=None,\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\u001b[0m in \u001b[0;36msign\u001b[0;34m(self, operation_name, request, region_name, signing_type, expires_in, signing_name)\u001b[0m\n\u001b[1;32m 155\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m \u001b[0mauth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_auth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_choose_signer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msigning_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\u001b[0m in \u001b[0;36madd_auth\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 423\u001b[0m self._region_name = signing_context.get(\n\u001b[1;32m 424\u001b[0m 'region', self._default_region_name)\n\u001b[0;32m--> 425\u001b[0;31m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mS3SigV4Auth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_auth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 427\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_modify_request_before_signing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\u001b[0m in \u001b[0;36madd_auth\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0madd_auth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 356\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcredentials\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mNoCredentialsError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 358\u001b[0m \u001b[0mdatetime_now\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutcnow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'timestamp'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatetime_now\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrftime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSIGV4_TIMESTAMP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mNoCredentialsError\u001b[0m: Unable to locate credentials" - ] - } - ], - "source": [ - "out.close()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Exception ignored in: \n", - "Traceback (most recent call last):\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1127, in __del__\n", - " self.close()\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 1104, in close\n", - " self.flush(force=True)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/fsspec/spec.py\", line 976, in flush\n", - " self._initiate_upload()\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 984, in _initiate_upload\n", - " Bucket=self.bucket, Key=self.key, ACL=self.acl)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 971, in _call_s3\n", - " **kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/s3fs/core.py\", line 189, in _call_s3\n", - " return method(**additional_kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 357, in _api_call\n", - " return self._make_api_call(operation_name, kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 648, in _make_api_call\n", - " operation_model, request_dict, request_context)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/client.py\", line 667, in _make_request\n", - " return self._endpoint.make_request(operation_model, request_dict)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 102, in make_request\n", - " return self._send_request(request_dict, operation_model)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 132, in _send_request\n", - " request = self.create_request(request_dict, operation_model)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/endpoint.py\", line 116, in create_request\n", - " operation_name=operation_model.name)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 356, in emit\n", - " return self._emitter.emit(aliased_event_name, **kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 228, in emit\n", - " return self._emit(event_name, kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/hooks.py\", line 211, in _emit\n", - " response = handler(**kwargs)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 90, in handler\n", - " return self.sign(operation_name, request)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/signers.py\", line 157, in sign\n", - " auth.add_auth(request)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 425, in add_auth\n", - " super(S3SigV4Auth, self).add_auth(request)\n", - " File \"/Users/krisz/.conda/envs/arrow37/lib/python3.7/site-packages/botocore/auth.py\", line 357, in add_auth\n", - " raise NoCredentialsError\n", - "botocore.exceptions.NoCredentialsError: Unable to locate credentials\n" - ] - } - ], - "source": [ - "out = S3File(S3FileSystem(), s3_filepath, mode='wb')\n", - "table = pa.Table.from_pandas(df.copy())\n", - "try:\n", - " pq.write_table(table, out)\n", - "except:\n", - " print('EEEEEEEEEEEEEe')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/python/open-append-stream b/python/open-append-stream deleted file mode 100644 index e7b3f4697c0e..000000000000 --- a/python/open-append-stream +++ /dev/null @@ -1,2 +0,0 @@ - -newly added \ No newline at end of file diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 2fd15d1f9781..3cb4f70bafea 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -811,7 +811,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int file_descriptor() cdef cppclass CMemoryMappedFile \ - "arrow::io::MemoryMappedFile"(ReadWriteFileInterface): + " arrow::io::MemoryMappedFile"(ReadWriteFileInterface): @staticmethod CStatus Create(const c_string& path, int64_t size, @@ -826,7 +826,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int file_descriptor() cdef cppclass CCompressedInputStream \ - "arrow::io::CompressedInputStream"(CInputStream): + " arrow::io::CompressedInputStream"(CInputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, shared_ptr[CInputStream] raw, @@ -837,7 +837,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[CCompressedInputStream]* out) cdef cppclass CCompressedOutputStream \ - "arrow::io::CompressedOutputStream"(COutputStream): + " arrow::io::CompressedOutputStream"(COutputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, shared_ptr[COutputStream] raw, @@ -848,7 +848,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[CCompressedOutputStream]* out) cdef cppclass CBufferedInputStream \ - "arrow::io::BufferedInputStream"(CInputStream): + " arrow::io::BufferedInputStream"(CInputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, @@ -858,7 +858,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[CInputStream] Detach() cdef cppclass CBufferedOutputStream \ - "arrow::io::BufferedOutputStream"(COutputStream): + " arrow::io::BufferedOutputStream"(COutputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, @@ -903,8 +903,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: cdef cppclass HdfsOutputStream(COutputStream): pass - cdef cppclass CHadoopFileSystem \ - "arrow::io::HadoopFileSystem"(CIOFileSystem): + cdef cppclass CHadoopFileSystem" arrow::io::HadoopFileSystem"(CIOFileSystem): @staticmethod CStatus Connect(const HdfsConnectionConfig* config, shared_ptr[CHadoopFileSystem]* client) @@ -940,21 +939,21 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[HdfsOutputStream]* handle) cdef cppclass CBufferReader \ - "arrow::io::BufferReader"(CRandomAccessFile): + " arrow::io::BufferReader"(CRandomAccessFile): CBufferReader(const shared_ptr[CBuffer]& buffer) CBufferReader(const uint8_t* data, int64_t nbytes) cdef cppclass CBufferOutputStream \ - "arrow::io::BufferOutputStream"(COutputStream): + " arrow::io::BufferOutputStream"(COutputStream): CBufferOutputStream(const shared_ptr[CResizableBuffer]& buffer) cdef cppclass CMockOutputStream \ - "arrow::io::MockOutputStream"(COutputStream): + " arrow::io::MockOutputStream"(COutputStream): CMockOutputStream() int64_t GetExtentBytesWritten() cdef cppclass CFixedSizeBufferWriter \ - "arrow::io::FixedSizeBufferWriter"(WritableFile): + " arrow::io::FixedSizeBufferWriter"(WritableFile): CFixedSizeBufferWriter(const shared_ptr[CBuffer]& buffer) void set_memcopy_threads(int num_threads) diff --git a/python/test.txt b/python/test.txt deleted file mode 100644 index e69de29bb2d1..000000000000 From 45436f7b3f5bf21544f5fbd1865f3cd8c9d54845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 24 Sep 2019 11:04:04 +0200 Subject: [PATCH 28/39] cython flake8 --- python/pyarrow/includes/libarrow.pxd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 3cb4f70bafea..12cd1c3808b5 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -903,7 +903,8 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: cdef cppclass HdfsOutputStream(COutputStream): pass - cdef cppclass CHadoopFileSystem" arrow::io::HadoopFileSystem"(CIOFileSystem): + cdef cppclass CHadoopFileSystem \ + "arrow::io::HadoopFileSystem"(CIOFileSystem): @staticmethod CStatus Connect(const HdfsConnectionConfig* config, shared_ptr[CHadoopFileSystem]* client) From 751cfd42910384ae15d5e8fe94766a838cef640d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 24 Sep 2019 13:00:00 +0200 Subject: [PATCH 29/39] resolve a couple of review comments; enum workaround --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 ++++ python/pyarrow/_fs.pyx | 1 - python/pyarrow/_s3.pyx | 2 +- python/pyarrow/includes/libarrow.pxd | 2 +- python/pyarrow/includes/libarrow_s3.pxd | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b9126bfff5bc..49fb104502fd 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2537,6 +2537,10 @@ if(ARROW_S3) message(STATUS "Found AWS SDK libraries: ${AWSSDK_LINK_LIBRARIES}") if(APPLE) + # CoreFoundation's path is hardcoded in the CMake files provided by + # aws-sdk-cpp to use the MacOSX SDK provided by XCode which makes + # XCode a hard dependency. Command Line Tools is often used instead + # of the full XCode suite, so let the linker to find it. set_target_properties(AWS::aws-c-common PROPERTIES INTERFACE_LINK_LIBRARIES "-pthread;pthread;-framework CoreFoundation") diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index dc12659322e3..39079aee1051 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -371,7 +371,6 @@ cdef class FileSystem: stream.set_input_stream(in_handle) stream.is_readable = True - stream.is_seekable = True return self._wrap_input_stream( stream, path=path, compression=compression, buffer_size=buffer_size diff --git a/python/pyarrow/_s3.pyx b/python/pyarrow/_s3.pyx index bd783f25bb67..2c537b6c536b 100644 --- a/python/pyarrow/_s3.pyx +++ b/python/pyarrow/_s3.pyx @@ -19,12 +19,12 @@ import six +from pyarrow.lib cimport check_status from pyarrow.compat import frombytes, tobytes from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * from pyarrow.includes.libarrow_s3 cimport * from pyarrow._fs cimport FileSystem -from pyarrow.lib cimport check_status cpdef enum S3LogLevel: diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 12cd1c3808b5..82085487eda3 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -964,7 +964,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: - enum CFileType "arrow::fs::FileType": + ctypedef enum CFileType "arrow::fs::FileType": CFileType_NonExistent "arrow::fs::FileType::NonExistent" CFileType_Unknown "arrow::fs::FileType::Unknown" CFileType_File "arrow::fs::FileType::File" diff --git a/python/pyarrow/includes/libarrow_s3.pxd b/python/pyarrow/includes/libarrow_s3.pxd index d2cde972f2bb..8dc109c5e6e3 100644 --- a/python/pyarrow/includes/libarrow_s3.pxd +++ b/python/pyarrow/includes/libarrow_s3.pxd @@ -24,7 +24,7 @@ from pyarrow.includes.libarrow cimport CFileSystem cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: - enum CS3LogLevel "arrow::fs::S3LogLevel": + ctypedef enum CS3LogLevel "arrow::fs::S3LogLevel": CS3LogLevel_Off "arrow::fs::S3LogLevel::Off" CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal" CS3LogLevel_Error "arrow::fs::S3LogLevel::Error" From fee57a9a4caa8c8ebe132c7056dfcabd0ce4fad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 24 Sep 2019 13:09:39 +0200 Subject: [PATCH 30/39] remove accidentally committed files --- python/run_test.sh | 70 -------------------------------------------- python/test.parquet | Bin 106391 -> 0 bytes 2 files changed, 70 deletions(-) delete mode 100755 python/run_test.sh delete mode 100644 python/test.parquet diff --git a/python/run_test.sh b/python/run_test.sh deleted file mode 100755 index 47e93e15a203..000000000000 --- a/python/run_test.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -export CXXFLAGS="" -export ARROW_BUILD_TYPE=debug -export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX -export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX -export ARROW_HOME=$CONDA_PREFIX -export PARQUET_HOME=$CONDA_PREFIX -export PARQUET_TEST_DATA=`pwd`/../cpp/submodules/parquet-testing/data -export ARROW_TEST_DATA=`pwd`/../testing/data - - -mkdir -p ../cpp/build -pushd ../cpp/build - -cmake -GNinja \ - -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DARROW_PYTHON=ON \ - -DARROW_PLASMA=OFF \ - -DARROW_PARQUET=ON \ - -DARROW_GANDIVA=OFF \ - -DARROW_ORC=ON \ - -DARROW_FLIGHT=OFF \ - -DARROW_S3=ON \ - -DARROW_TENSORFLOW=OFF \ - -DARROW_DEPENDENCY_SOURCE=CONDA \ - -DARROW_EXTRA_ERROR_CONTEXT=ON \ - -DARROW_BUILD_TESTS=ON \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=YES \ - -DCMAKE_CXX_FLAGS=$CXXFLAGS \ - .. - -ninja -# ninja test -ninja install - -popd - -export PYARROW_CMAKE_GENERATOR=Ninja -export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE -export PYARROW_WITH_PARQUET=1 -export PYARROW_WITH_PLASMA=0 -export PYARROW_WITH_GANDIVA=0 -export PYARROW_WITH_DATASET=0 -export PYARROW_WITH_FLIGHT=0 -export PYARROW_WITH_S3=1 -export PYARROW_WITH_ORC=1 - -python setup.py develop - -py.test -sv "$@" diff --git a/python/test.parquet b/python/test.parquet deleted file mode 100644 index 8548b7e899f3509e0ca346c589ab409ded4ea932..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 106391 zcmeI*e`wVA9mnza_j}*(*UNg0wCneJ$1a;2dPs?haSOPzxf#m2m0-n|4&3UDCD0V@ zY)x#bo8K@*P}6O!8x%KWG#xY{b2dn!X7%hU{^1{krJH{URj^Q4m>uyU(0(w2|j~nUrH*E!-zhpSf?{yyyBtPxrU>pZGW~>z^GzfBpJ5uf8`nKK`?V zV-u5;y&p{+4r;BbwD5%h0tg_000IagfB*v3AaFb_yB9`+VpgaI0DeXU5I_I{1Q0*~ z0R#|Gfs>jHiX#LIhYk=z009ILKmY**5I_KdDiJWrV9}XGCMQ&h0Y4@J2q1s}0tg_0 z00Ib@WH7g|pk&wp9t03T009ILKmY**5U3IXlML>(0C|0t81Q2vfB*srAbCP;a|^{rzGMPfy3h7yoR3jsNdW^?vtjS>N$J-^-*N(Ocm@`HSb)&3mfH zPyWs6y!Jbh&y7t?PWFB@aX2X5y{rt&(9$IY5I_I{1Q56z0V}=c4@X2ErItuF{rj*I z|7V*%m#x4y9dzIqNY(!W*{mM}YxJ$Pk{!u=89$zGcEn3>vf7-z9S1s{j?VUdzMbs; z;B6;)ap0Jhypi{^8?y0e5WR3RQIbS8uEzaDt8e?x10qhsq*e{Iw+0URpN_klXwRH* z7K^k4XYMPx(z{o_v3F=&T!L%k6BAp9#wVwy{xbUGd_FjP==#{$7jK?=NF;ctQCb@G z@y}gSmy_U=A#wHLjS+Dh_4xS>*5;hPI(*Pd>*0|h{ zwuanZ2}a+u6MF5SX6p@e)}J=U{2{kff~zmb?$`5*hfm*(-QQKGhcA7_*=X9CH$5jo z!>e++N1qm(nI37Q8?U%Exv2{-UkD(800IagfB*srAW-H4qZi!oo3s&V+$c>UR_ZPS zhy*)R61FoDq$e6dBIid!h-RFb7@=Wha9C5tOwb9##^B9Z^TDq*nyhiU7(%pN-L9i1 z&S=YAgA_x_@i!&cC55IMYRcT3?je8x0tg_000IagfPe~^AYw*7g2K>8B#Yd05OH%F z8fsE4Bl46PN|{M2nnc_ot1ui=s{<}U-YtxW=%~t5f1lT|7%_A_WG0M+!zS9SE*u}3 zx!~OR$f?4ZNH{{m-zx$LAbQEX>sU>6@~dB*L6D1Q0*~0R#|0009Il zO28x$vnV8(zbd2{MixsVBhkenb0Z!)H>LF1Cxz%DSZ)WN5sYLAvIahD2s%g$q^YQix8?Y9WPG)F-a|(>Xmv009ILKmY** z5LjselS0hu^q^#Ux(OnSrH}=S)FV0M?$?Oun)v8=$hDVaO&V0@W@Qytsn=}%Bqw3G znOUH&Ip|9xg(dSWP$z#NfB*srAbR7?T@>tg*;bT}w8nlF!2?zQx zQy(jg9$E=2j0dJV)^M>p)}5ze={opm?3f!VnM)p#^&kt!K#C0+09om5%JmUI009IL zKmY**5Lk8rlR->*tYDEUy^#!Zrz*YoH~^wGfD1|F(toMb+t67EH^s=Y@m0y~wh#Y6 z=XVv$^j`j|+ptxKO{mjbIjMKqKf+uP0R#|0009ILK%lY&Oaw7|f&>ey^qxHwMiAZV zqs7$5U1XaOt(ChgB|CL55K{yzx>LwwA4)O7oNuxpDee{G>i~+`DMSkcn}L)_Ni5wd z#LU4m%R$Z_GRr{d+2_GeZ} zjxDjOeoL&f-Lj32A%Fk^2vm^3Ro!5*ufNrfRZySlDgp=~fB*srAbyx378aMLO_27^_i|BfB*srAb7Wc+x#*%2?j$=c)W?Ksfs zbab}w^X+8!2X8ycfBpW5l{`P_Wlz8FWskmJk~%M(Oq9e<-%kIiSyJACXP#}}*XBFY zY^SU?-}b%MuWKr8c2ePc__p}*wHjW{dxdTG{Es%t?qjc{GwJ&5NZy;*<+g=gzBq8q zO5VtO*$vrvG$b$V?(zd9e|sxo`YP#6DzRn9j>o_INWmiYb30*))SA#PpOJ)k;!0e< z)0-93=c3QzdhZnPMQnYxH#{OTFg-o34HJ**b8`)o_VsdM+u!JGuZOQoETzZHwZi`m z(8r^N^TOvLwjML*?a@EA@}QpY$*S;-+G5|V#BP1B0AuWBZmJP5%Ke7zJ$r From d399643dc716437a0059e3ed2f7807dcf5f559dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 24 Sep 2019 18:53:49 +0200 Subject: [PATCH 31/39] simplify test suite --- python/pyarrow/tests/test_fs.py | 593 ++++++++++++-------------------- 1 file changed, 214 insertions(+), 379 deletions(-) diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 076faf205e04..bbdef4f22260 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -import io -import calendar from datetime import datetime try: import pathlib @@ -26,275 +24,107 @@ import pytest import pyarrow as pa -from pyarrow import ArrowIOError from pyarrow.tests.test_io import gzip_compress, gzip_decompress from pyarrow.fs import (FileType, Selector, FileSystem, LocalFileSystem, SubTreeFileSystem) -class FileSystemWrapper: - - # Whether the filesystem may "implicitly" create intermediate directories - have_implicit_directories = False - # Whether the filesystem may allow writing a file "over" a directory - allow_write_file_over_dir = False - # Whether the filesystem allows moving a directory - allow_move_dir = True - # Whether the filesystem allows appending to a file - allow_append_to_file = False - # Whether the filesystem supports directory modification times - have_directory_mtimes = True - - @property - def impl(self): - return self._impl - - @impl.setter - def impl(self, impl): - self._impl = impl - - def pathpair(self, p): - raise NotImplementedError() - - def rmdir(self, p): - raise NotImplementedError() - - def unlink(self, p): - raise NotImplementedError() - - def iterdir(self, p): - raise NotImplementedError() - - def mkdir(self, p): - raise NotImplementedError() - - def touch(self, p): - raise NotImplementedError() - - def exists(self, p): - raise NotImplementedError() - - def mtime(self, p): - raise NotImplementedError() - - def write_bytes(self, p, data): - raise NotImplementedError() - - def read_bytes(self, p): - raise NotImplementedError() +@pytest.fixture +def localfs(request, tempdir): + return dict( + fs=LocalFileSystem(), + pathfn=lambda p: (tempdir / p).as_posix(), + allow_move_dir=True, + allow_append_to_file=True, + ) -class LocalWrapper(FileSystemWrapper): +@pytest.fixture +def subtree_localfs(request, tempdir, localfs): + prefix = 'subtree/prefix/' + (tempdir / prefix).mkdir(parents=True) + return dict( + fs=SubTreeFileSystem(prefix, localfs['fs']), + pathfn=prefix.__add__, + allow_move_dir=True, + allow_append_to_file=True, + ) - allow_append_to_file = True - def __init__(self, tempdir): - self.impl = LocalFileSystem() - self.tempdir = tempdir +@pytest.fixture +def s3fs(request, minio_server): + from pyarrow.fs import S3FileSystem + # initialize_s3() - def pathpair(self, p): - path_for_wrapper = str(self.tempdir / p) - path_for_impl = '/'.join([self.tempdir.as_posix(), p]) - return (path_for_wrapper, path_for_impl) + address, access_key, secret_key = minio_server + bucket = 'pyarrow-filesystem/' + fs = S3FileSystem( + endpoint_override=address, + access_key=access_key, + secret_key=secret_key, + scheme='http' + ) + fs.create_dir(bucket) - def unlink(self, p): - return pathlib.Path(p).unlink() + return dict( + fs=fs, + pathfn=bucket.__add__, + allow_move_dir=False, + allow_append_to_file=False, + ) - def rmdir(self, p): - return pathlib.Path(p).rmdir() - def mkdir(self, p): - return pathlib.Path(p).mkdir(parents=True) +@pytest.fixture +def subtree_s3fs(request, s3fs): + prefix = 'pyarrow-filesystem/prefix/' + return dict( + fs=SubTreeFileSystem(prefix, s3fs['fs']), + pathfn=prefix.__add__, + allow_move_dir=False, + allow_append_to_file=False, + ) - def iterdir(self, p): - for path in pathlib.Path(p).iterdir(): - yield (path, path.is_dir()) - def touch(self, p): - return pathlib.Path(p).touch() +@pytest.fixture(params=[ + pytest.param( + pytest.lazy_fixture('localfs'), + id='LocalFileSystem()' + ), + pytest.param( + pytest.lazy_fixture('subtree_localfs'), + id='SubTreeFileSystem(LocalFileSystem())' + ), + pytest.param( + pytest.lazy_fixture('s3fs'), + id='S3FileSystem' + ), + pytest.param( + pytest.lazy_fixture('subtree_s3fs'), + id='SubTreeFileSystem(S3FileSystem())' + ) +]) +def filesystem_config(request): + return request.param - def exists(self, p): - return pathlib.Path(p).exists() - - def mtime(self, p): - path = pathlib.Path(p) - mtime = path.stat().st_mtime - return datetime.utcfromtimestamp(mtime) - - def write_bytes(self, p, data): - return pathlib.Path(p).write_bytes(data) - - def read_bytes(self, p): - return pathlib.Path(p).read_bytes() - - -class SubTreeLocalWrapper(LocalWrapper): - - def __init__(self, tempdir, prefix='local/prefix'): - prefix_absolute = tempdir / prefix - prefix_absolute.mkdir(parents=True) - - self.impl = SubTreeFileSystem( - prefix_absolute.as_posix(), - LocalFileSystem() - ) - self.prefix = prefix - self.tempdir = tempdir - - def pathpair(self, p): - path_for_wrapper = str(self.tempdir / self.prefix / p) - path_for_impl = p - return (path_for_wrapper, path_for_impl) - - -class S3Wrapper(FileSystemWrapper): - - allow_move_dir = False - - def __init__(self, minio_client, bucket='test-bucket', **kwargs): - from pyarrow.fs import S3FileSystem - self.impl = S3FileSystem(**kwargs) - self.client = minio_client - self.bucket = bucket - - def pathpair(self, p): - path_for_wrapper = p - path_for_impl = '/'.join([self.bucket, p]) - return (path_for_wrapper, path_for_impl) - - def touch(self, p): - self.client.put_object( - bucket_name=self.bucket, - object_name=p.rstrip('/'), - data=io.BytesIO(b''), - length=0 - ) - - def unlink(self, p): - self.client.remove_object( - bucket_name=self.bucket, - object_name=p.rstrip('/') - ) - - def rmdir(self, p): - if not p.endswith('/'): - p += '/' - self.client.remove_object( - bucket_name=self.bucket, - object_name=p - ) - - def mkdir(self, p): - if not p.endswith('/'): - p += '/' - self.client.put_object( - bucket_name=self.bucket, - object_name=p, - data=io.BytesIO(b''), - length=0 - ) - - def iterdir(self, p): - if not p.endswith('/'): - p += '/' - objs = self.client.list_objects( - bucket_name=self.bucket, - prefix=p, - recursive=False - ) - for obj in objs: - yield (obj.object_name, obj.is_dir) - - def exists(self, p): - from minio.error import NoSuchKey, NoSuchBucket - try: - self.client.get_object( - bucket_name=self.bucket, - object_name=p - ) - except (NoSuchBucket, NoSuchKey): - return False - else: - return True - - def mtime(self, p): - stat = self.client.stat_object( - bucket_name=self.bucket, - object_name=p - ) - ts = calendar.timegm(stat.last_modified) - return datetime.utcfromtimestamp(ts) - - def write_bytes(self, p, data): - assert not p.endswith('/') - self.client.put_object( - bucket_name=self.bucket, - object_name=p, - data=io.BytesIO(data), - length=len(data) - ) - - def read_bytes(self, p): - assert not p.endswith('/') - data = self.client.get_object( - bucket_name=self.bucket, - object_name=p - ) - return data.read() - - -class SubTreeS3Wrapper(S3Wrapper): - - def __init__(self, minio_client, bucket='test-bucket', prefix='s3/prefix', - **kwargs): - from pyarrow.fs import S3FileSystem - self.impl = SubTreeFileSystem( - '/'.join([bucket, prefix]), - S3FileSystem(**kwargs) - ) - self.client = minio_client - self.bucket = bucket - self.prefix = prefix - - def pathpair(self, p): - path_for_wrapper = '/'.join([self.prefix, p]) - path_for_impl = p - return (path_for_wrapper, path_for_impl) +@pytest.fixture +def fs(request, filesystem_config): + return filesystem_config['fs'] -@pytest.fixture(params=[ - LocalWrapper, - SubTreeLocalWrapper -]) -def localfs(request, tempdir): - return request.param(tempdir) +@pytest.fixture +def pathfn(request, filesystem_config): + return filesystem_config['pathfn'] -@pytest.fixture(params=[ - S3Wrapper, - SubTreeS3Wrapper -]) -def s3fs(request, minio_server, minio_client, minio_bucket): - from pyarrow.fs import initialize_s3 - initialize_s3() - address, access_key, secret_key = minio_server - return request.param( - minio_client=minio_client, - bucket=minio_bucket, - endpoint_override=address, - access_key=access_key, - secret_key=secret_key, - scheme='http' - ) +@pytest.fixture +def allow_move_dir(request, filesystem_config): + return filesystem_config['allow_move_dir'] -@pytest.fixture(params=[ - pytest.lazy_fixture('localfs'), - pytest.lazy_fixture('s3fs'), -]) -def fs(request): - return request.param +@pytest.fixture +def allow_append_to_file(request, filesystem_config): + return filesystem_config['allow_append_to_file'] def test_cannot_instantiate_base_filesystem(): @@ -310,31 +140,27 @@ class Path: pathlib.Path()] for path in invalid_paths: with pytest.raises(TypeError): - fs.impl.create_dir(path) + fs.create_dir(path) -def test_get_target_stats(fs): - _aaa, aaa = fs.pathpair('a/aa/aaa/') - _bb, bb = fs.pathpair('a/bb') - _c, c = fs.pathpair('c.txt') +def test_get_target_stats(fs, pathfn): + aaa = pathfn('a/aa/aaa/') + bb = pathfn('a/bb') + c = pathfn('c.txt') - fs.mkdir(_aaa) - fs.touch(_bb) - fs.write_bytes(_c, b'test') + fs.create_dir(aaa) + with fs.open_output_stream(bb): + pass # touch + with fs.open_output_stream(c) as fp: + fp.write(b'test') - def mtime_almost_equal(a, b): - # arrow's filesystem implementation truncates mtime to microsends - # resolution whereas pathlib rounds - diff = (a - b).total_seconds() - return abs(diff) <= 10**-6 - - aaa_stat, bb_stat, c_stat = fs.impl.get_target_stats([aaa, bb, c]) + aaa_stat, bb_stat, c_stat = fs.get_target_stats([aaa, bb, c]) assert aaa_stat.path == aaa assert 'aaa' in repr(aaa_stat) assert aaa_stat.extension == '' - assert mtime_almost_equal(aaa_stat.mtime, fs.mtime(_aaa)) - # type is inconsistent base_name has a trailing slas for 'aaa' and 'aaa/' + assert isinstance(aaa_stat.mtime, datetime) + # assert mtime_almost_equal(aaa_stat.mtime, fs.mtime(_aaa)) # assert aaa_stat.base_name == 'aaa' # assert aaa_stat.type == FileType.Directory # assert aaa_stat is None @@ -344,135 +170,138 @@ def mtime_almost_equal(a, b): assert bb_stat.extension == '' assert bb_stat.type == FileType.File assert bb_stat.size == 0 - assert mtime_almost_equal(bb_stat.mtime, fs.mtime(_bb)) + assert isinstance(bb_stat.mtime, datetime) assert c_stat.path == str(c) assert c_stat.base_name == 'c.txt' assert c_stat.extension == 'txt' assert c_stat.type == FileType.File assert c_stat.size == 4 - assert mtime_almost_equal(c_stat.mtime, fs.mtime(_c)) + assert isinstance(c_stat.mtime, datetime) + +def test_get_target_stats_with_selector(fs, pathfn): + base_dir = pathfn('selector-dir/') + file_a = pathfn('selector-dir/test_file_a') + file_b = pathfn('selector-dir/test_file_b') + dir_a = pathfn('selector-dir/test_dir_a') -def test_get_target_stats_with_selector(fs): - _base_dir, base_dir = fs.pathpair('selector-dir/') - _file_a, file_a = fs.pathpair('selector-dir/test_file_a') - _file_b, file_b = fs.pathpair('selector-dir/test_file_b') - _dir_a, dir_a = fs.pathpair('selector-dir/test_dir_a') try: - fs.mkdir(_base_dir) - fs.touch(_file_a) - fs.touch(_file_b) - fs.mkdir(_dir_a) + fs.create_dir(base_dir) + with fs.open_output_stream(file_a): + pass + with fs.open_output_stream(file_b): + pass + fs.create_dir(dir_a) selector = Selector(base_dir, allow_non_existent=False, recursive=True) assert selector.base_dir == base_dir - stats = fs.impl.get_target_stats(selector) - expected = list(fs.iterdir(_base_dir)) - assert len(stats) == len(expected) - - left = sorted(stats, key=lambda st: st.path) - right = sorted(expected, key=lambda tpl: tpl[0]) + stats = fs.get_target_stats(selector) + assert len(stats) == 3 - for l, r in zip(left, right): - if r[1] is True: - assert l.type == FileType.Directory + for st in stats: + if st.path.endswith(file_a): + assert st.type == FileType.File + elif st.path.endswith(file_b): + assert st.type == FileType.File + elif st.path.endswith(dir_a): + assert st.type == FileType.Directory else: - assert l.type == FileType.File + raise ValueError('unexpected path {}'.format(st.path)) finally: - fs.unlink(_file_a) - fs.unlink(_file_b) - fs.rmdir(_dir_a) - fs.rmdir(_base_dir) + fs.delete_file(file_a) + fs.delete_file(file_b) + fs.delete_dir(dir_a) + fs.delete_dir(base_dir) -def test_create_dir(fs): - _d, d = fs.pathpair('test-directory/') - assert not fs.exists(_d) - fs.impl.create_dir(d) - assert fs.exists(_d) +def test_create_dir(fs, pathfn): + d = pathfn('test-directory/') - # recursive - _r, r = fs.pathpair('deeply/nested/directory/') - assert not fs.exists(_r) - with pytest.raises(ArrowIOError): - fs.impl.create_dir(r, recursive=False) - fs.impl.create_dir(r) - assert fs.exists(_r) + with pytest.raises(pa.ArrowIOError): + fs.delete_dir(d) + fs.create_dir(d) + fs.delete_dir(d) -def test_delete_dir(fs): - _d, d = fs.pathpair('directory/') - _nd, nd = fs.pathpair('directory/nested/') - fs.mkdir(_nd) + d = pathfn('deeply/nested/test-directory/') + # with pytest.raises(ArrowIOError): + # fs.create_dir(d, recursive=False) - assert fs.exists(_nd) - fs.impl.delete_dir(nd) - assert not fs.exists(_nd) + fs.create_dir(d, recursive=True) + fs.delete_dir(d) - assert fs.exists(_d) - fs.impl.delete_dir(d) - assert not fs.exists(_d) +def test_delete_dir(fs, pathfn): + d = pathfn('directory/') + nd = pathfn('directory/nested/') -def test_copy_file(fs): - _s, s = fs.pathpair('test-copy-source-file') - _t, t = fs.pathpair('test-copy-target-file') - fs.touch(_s) + fs.create_dir(nd) + fs.delete_dir(nd) + fs.delete_dir(d) + with pytest.raises(pa.ArrowIOError): + fs.delete_dir(d) - assert not fs.exists(_t) - fs.impl.copy_file(s, t) - assert fs.exists(_s) - assert fs.exists(_t) +def test_copy_file(fs, pathfn): + s = pathfn('test-copy-source-file') + t = pathfn('test-copy-target-file') -def test_move_directory(fs): + with fs.open_output_stream(s): + pass + + fs.copy_file(s, t) + fs.delete_file(s) + fs.delete_file(t) + + +def test_move_directory(fs, pathfn, allow_move_dir): # move directory (doesn't work with S3) - _s, s = fs.pathpair('source-dir/') - _t, t = fs.pathpair('target-dir/') - fs.mkdir(_s) - - if fs.allow_move_dir: - assert fs.exists(_s) - assert not fs.exists(_t) - fs.impl.move(s, t) - assert not fs.exists(_s) - assert fs.exists(_t) + s = pathfn('source-dir/') + t = pathfn('target-dir/') + + fs.create_dir(s) + + if allow_move_dir: + fs.move(s, t) + with pytest.raises(pa.ArrowIOError): + fs.delete_dir(s) + fs.delete_dir(t) else: with pytest.raises(pa.ArrowIOError): - fs.impl.move(s, t) + fs.move(s, t) -def test_move_file(fs): - _s, s = fs.pathpair('test-move-source-file') - _t, t = fs.pathpair('test-move-target-file') - fs.touch(_s) +def test_move_file(fs, pathfn): + s = pathfn('test-move-source-file') + t = pathfn('test-move-target-file') - assert fs.exists(_s) - assert not fs.exists(_t) - fs.impl.move(s, t) - assert not fs.exists(_s) - assert fs.exists(_t) + with fs.open_output_stream(s): + pass + fs.move(s, t) + with pytest.raises(pa.ArrowIOError): + fs.delete_file(s) + fs.delete_file(t) -def test_delete_file(fs): - _p, p = fs.pathpair('test-delete-target-file') - fs.touch(_p) - assert fs.exists(_p) - fs.impl.delete_file(p) - assert not fs.exists(_p) +def test_delete_file(fs, pathfn): + p = pathfn('test-delete-target-file') + with fs.open_output_stream(p): + pass - _p, p = fs.pathpair('test-delete-nested') - fs.mkdir(_p) + fs.delete_file(p) + with pytest.raises(pa.ArrowIOError): + fs.delete_file(p) - _p, p = fs.pathpair('test-delete-nested/target-file') - fs.touch(_p) + d = pathfn('test-delete-nested') + fs.create_dir(d) + f = pathfn('test-delete-nested/target-file') + with fs.open_output_stream(f) as s: + s.write(b'data') - assert fs.exists(_p) - fs.impl.delete_file(p) - assert not fs.exists(_p) + fs.delete_dir(d) def identity(v): @@ -488,26 +317,28 @@ def identity(v): ('gzip', 256, gzip_compress), ] ) -def test_open_input_stream(fs, compression, buffer_size, compressor): - _p, p = fs.pathpair('open-input-stream') +def test_open_input_stream(fs, pathfn, compression, buffer_size, compressor): + p = pathfn('open-input-stream') data = b'some data for reading' * 1024 - fs.write_bytes(_p, compressor(data)) + with fs.open_output_stream(p) as s: + s.write(compressor(data)) - with fs.impl.open_input_stream(p, compression, buffer_size) as f: - result = f.read(len(data)) + with fs.open_input_stream(p, compression, buffer_size) as s: + result = s.read(len(data)) assert result == data -def test_open_input_file(fs): - _p, p = fs.pathpair('open-input-file') +def test_open_input_file(fs, pathfn): + p = pathfn('open-input-file') data = b'some data' * 1024 - fs.write_bytes(_p, data) + with fs.open_output_stream(p) as s: + s.write(data) read_from = len(b'some data') * 512 - with fs.impl.open_input_file(p) as f: + with fs.open_input_file(p) as f: f.seek(read_from) result = f.read() @@ -523,19 +354,17 @@ def test_open_input_file(fs): ('gzip', 256, gzip_decompress), ] ) -def test_open_output_stream(fs, compression, buffer_size, decompressor): - _p, p = fs.pathpair('open-output-stream') +def test_open_output_stream(fs, pathfn, compression, buffer_size, + decompressor): + p = pathfn('open-output-stream') data = b'some data for writing' * 1024 - with fs.impl.open_output_stream(p, compression, buffer_size) as f: + with fs.open_output_stream(p, compression, buffer_size) as f: f.write(data) - with fs.impl.open_input_stream(p, compression, buffer_size) as f: + with fs.open_input_stream(p, compression, buffer_size) as f: assert f.read(len(data)) == data - result = decompressor(fs.read_bytes(_p)) - assert result == data - @pytest.mark.parametrize( ('compression', 'buffer_size', 'compressor', 'decompressor'), @@ -546,18 +375,24 @@ def test_open_output_stream(fs, compression, buffer_size, decompressor): ('gzip', 256, gzip_compress, gzip_decompress), ] ) -def test_open_append_stream(fs, compression, buffer_size, compressor, - decompressor): - _p, p = fs.pathpair('open-append-stream') +def test_open_append_stream(fs, pathfn, compression, buffer_size, compressor, + decompressor, allow_append_to_file): + p = pathfn('open-append-stream') - data = compressor(b'already existing') - fs.write_bytes(_p, data) + initial = compressor(b'already existing') + with fs.open_output_stream(p) as s: + s.write(initial) - if fs.allow_append_to_file: - with fs.impl.open_append_stream(p, compression, buffer_size) as f: + if allow_append_to_file: + with fs.open_append_stream(p, compression, buffer_size) as f: f.write(b'\nnewly added') - result = decompressor(fs.read_bytes(_p)) + + appended = compressor(b'\nnewly added') + with fs.open_input_stream(p) as f: + result = f.read(len(initial) + len(appended)) + + result = decompressor(result) assert result == b'already existing\nnewly added' else: with pytest.raises(pa.ArrowNotImplementedError): - fs.impl.open_append_stream(p, compression, buffer_size) + fs.open_append_stream(p, compression, buffer_size) From f70f9fbd85279725045f7c0af7db32adb1da806d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 24 Sep 2019 19:49:14 +0200 Subject: [PATCH 32/39] remove minio-client dependency --- ci/conda_env_python.yml | 1 - python/pyarrow/tests/test_parquet.py | 24 ++++++++++++++++++++---- python/requirements-test.txt | 1 - 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ci/conda_env_python.yml b/ci/conda_env_python.yml index 65ab7d8afb8f..0e6e5bf8d5c3 100644 --- a/ci/conda_env_python.yml +++ b/ci/conda_env_python.yml @@ -19,7 +19,6 @@ cython=0.29.7 cloudpickle hypothesis numpy>=1.14 -minio pandas pytest pytest-faulthandler diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index cebb87a9f9bb..8a8f4dddbe5e 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1844,12 +1844,28 @@ def test_filters_read_table(tempdir): @pytest.fixture -def s3_example(minio_server, minio_bucket): - s3fs = pytest.importorskip('s3fs') +def s3_bucket(request, minio_server): + boto3 = pytest.importorskip('boto3') + botocore = pytest.importorskip('botocore') address, access_key, secret_key = minio_server - bucket_name = minio_bucket + s3 = boto3.resource('s3', + endpoint_url='http://{}'.format(address), + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=botocore.client.Config(signature_version='s3v4'), + region_name='us-east-1' + ) + bucket = s3.Bucket('test-s3fs') + bucket.create() + return 'test-s3fs' + +@pytest.fixture +def s3_example(minio_server, s3_bucket): + s3fs = pytest.importorskip('s3fs') + + address, access_key, secret_key = minio_server fs = s3fs.S3FileSystem( key=access_key, secret=secret_key, @@ -1859,7 +1875,7 @@ def s3_example(minio_server, minio_bucket): ) test_dir = guid() - bucket_uri = 's3://{0}/{1}'.format(bucket_name, test_dir) + bucket_uri = 's3://{0}/{1}'.format(s3_bucket, test_dir) fs.mkdir(bucket_uri) yield fs, bucket_uri diff --git a/python/requirements-test.txt b/python/requirements-test.txt index 89921fa29f50..10d445cbc442 100644 --- a/python/requirements-test.txt +++ b/python/requirements-test.txt @@ -1,6 +1,5 @@ cython hypothesis -minio pandas pathlib2; python_version < "3.4" pytest From 192ab6547789a8b7f83dcf70299d9bde496ba285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 24 Sep 2019 19:50:00 +0200 Subject: [PATCH 33/39] flake8 --- python/pyarrow/tests/test_parquet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 8a8f4dddbe5e..566f22f43110 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1849,7 +1849,8 @@ def s3_bucket(request, minio_server): botocore = pytest.importorskip('botocore') address, access_key, secret_key = minio_server - s3 = boto3.resource('s3', + s3 = boto3.resource( + 's3', endpoint_url='http://{}'.format(address), aws_access_key_id=access_key, aws_secret_access_key=secret_key, From c1df10b92caedbee4bcd006047ba3edbcff45b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 25 Sep 2019 10:03:21 +0200 Subject: [PATCH 34/39] initialization in first use --- python/pyarrow/_s3.pyx | 15 ++++++++++++++- python/pyarrow/fs.py | 2 -- python/pyarrow/tests/conftest.py | 19 ------------------- python/pyarrow/tests/test_fs.py | 1 - 4 files changed, 14 insertions(+), 23 deletions(-) diff --git a/python/pyarrow/_s3.pyx b/python/pyarrow/_s3.pyx index 2c537b6c536b..ceacdc429b5d 100644 --- a/python/pyarrow/_s3.pyx +++ b/python/pyarrow/_s3.pyx @@ -47,6 +47,16 @@ def finalize_s3(): check_status(CFinalizeS3()) +cdef bint _initialized = False + + +cdef _ensure_initialized(): + global _initialized + if not _initialized: + initialize_s3() + _initialized = True + + cdef class S3FileSystem(FileSystem): """S3-backed FileSystem implementation @@ -79,9 +89,12 @@ cdef class S3FileSystem(FileSystem): scheme='https', endpoint_override=None, bint background_writes=True): cdef: - CS3Options options = CS3Options.Defaults() + CS3Options options shared_ptr[CS3FileSystem] wrapped + _ensure_initialized() + + options = CS3Options.Defaults() if access_key is not None or secret_key is not None: options.ConfigureAccessKey(tobytes(access_key), tobytes(secret_key)) diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index 0fba639a17e5..23291203818d 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -30,5 +30,3 @@ from pyarrow._s3 import S3FileSystem, initialize_s3, finalize_s3 # noqa except ImportError: pass -else: - initialize_s3() diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index e9fbd86398bc..85297fe81de6 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -274,22 +274,3 @@ def minio_server(): finally: if proc is not None: proc.kill() - - -@pytest.fixture(scope='session') -def minio_client(minio_server): - minio = pytest.importorskip('minio') - address, access_key, secret_key = minio_server - return minio.Minio( - address, - access_key=access_key, - secret_key=secret_key, - secure=False - ) - - -@pytest.fixture(scope='session') -def minio_bucket(minio_client): - bucket_name = 'pyarrow-bucket' - minio_client.make_bucket(bucket_name) - return bucket_name diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index bbdef4f22260..d2c3825d43db 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -54,7 +54,6 @@ def subtree_localfs(request, tempdir, localfs): @pytest.fixture def s3fs(request, minio_server): from pyarrow.fs import S3FileSystem - # initialize_s3() address, access_key, secret_key = minio_server bucket = 'pyarrow-filesystem/' From 44784582ad82f6753cd066d526adb38a2315e433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 25 Sep 2019 12:55:21 +0200 Subject: [PATCH 35/39] fix read() issue --- cpp/src/arrow/filesystem/s3fs.cc | 6 ++++++ python/pyarrow/tests/test_fs.py | 7 +++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index 136d12e20302..0e3c75ea7907 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -340,6 +340,12 @@ class ObjectInputFile : public io::RandomAccessFile { RETURN_NOT_OK(CheckClosed()); RETURN_NOT_OK(CheckPosition(position, "read")); + nbytes = std::min(nbytes, content_length_ - position); + if (nbytes == 0) { + *bytes_read = 0; + return Status::OK(); + } + // Read the desired range of bytes S3Model::GetObjectResult result; RETURN_NOT_OK(GetObjectRange(client_, path_, position, nbytes, &result)); diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index d2c3825d43db..0f47a4a52f31 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -319,12 +319,12 @@ def identity(v): def test_open_input_stream(fs, pathfn, compression, buffer_size, compressor): p = pathfn('open-input-stream') - data = b'some data for reading' * 1024 + data = b'some data for reading\n' * 512 with fs.open_output_stream(p) as s: s.write(compressor(data)) with fs.open_input_stream(p, compression, buffer_size) as s: - result = s.read(len(data)) + result = s.read() assert result == data @@ -386,9 +386,8 @@ def test_open_append_stream(fs, pathfn, compression, buffer_size, compressor, with fs.open_append_stream(p, compression, buffer_size) as f: f.write(b'\nnewly added') - appended = compressor(b'\nnewly added') with fs.open_input_stream(p) as f: - result = f.read(len(initial) + len(appended)) + result = f.read() result = decompressor(result) assert result == b'already existing\nnewly added' From db89859ca262a4002cb54ece2651817b3ea572da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 25 Sep 2019 14:07:04 +0200 Subject: [PATCH 36/39] rename to s3fs --- cpp/src/arrow/filesystem/s3fs_test.cc | 11 +++++++- python/CMakeLists.txt | 2 +- python/pyarrow/{_s3.pyx => _s3fs.pyx} | 14 +--------- python/pyarrow/fs.py | 5 ---- .../{libarrow_s3.pxd => libarrow_s3fs.pxd} | 0 python/pyarrow/s3fs.py | 26 +++++++++++++++++++ python/pyarrow/tests/test_fs.py | 2 +- python/setup.py | 4 +-- 8 files changed, 41 insertions(+), 23 deletions(-) rename python/pyarrow/{_s3.pyx => _s3fs.pyx} (93%) rename python/pyarrow/includes/{libarrow_s3.pxd => libarrow_s3fs.pxd} (100%) create mode 100644 python/pyarrow/s3fs.py diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index b19b18e01638..304ae5a41b93 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -666,7 +666,7 @@ TEST_F(TestS3FS, OpenInputStream) { TEST_F(TestS3FS, OpenInputFile) { std::shared_ptr file; std::shared_ptr buf; - int64_t nbytes = -1, pos = -1; + int64_t nbytes = -1, pos = -1, bytes_read = 0; // Non-existent ASSERT_RAISES(IOError, fs_->OpenInputFile("non-existent-bucket/somefile", &file)); @@ -691,6 +691,15 @@ TEST_F(TestS3FS, OpenInputFile) { AssertBufferEqual(*buf, "data"); ASSERT_OK(file->ReadAt(9, 20, &buf)); AssertBufferEqual(*buf, ""); + + char result[10]; + ASSERT_OK(file->ReadAt(2, 5, &bytes_read, &result)); + ASSERT_EQ(bytes_read, 5); + ASSERT_OK(file->ReadAt(5, 20, &bytes_read, &result)); + ASSERT_EQ(bytes_read, 4); + ASSERT_OK(file->ReadAt(9, 0, &bytes_read, &result)); + ASSERT_EQ(bytes_read, 0); + // Reading past end of file ASSERT_RAISES(IOError, file->ReadAt(10, 20, &buf)); diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 67ce446b5c8f..6925efd2d17e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -386,7 +386,7 @@ set(CYTHON_EXTENSIONS lib _fs _csv _json) set(LINK_LIBS arrow_shared arrow_python_shared) if(PYARROW_BUILD_S3) - set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _s3) + set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _s3fs) endif() if(PYARROW_BUILD_CUDA) diff --git a/python/pyarrow/_s3.pyx b/python/pyarrow/_s3fs.pyx similarity index 93% rename from python/pyarrow/_s3.pyx rename to python/pyarrow/_s3fs.pyx index ceacdc429b5d..532a290607d9 100644 --- a/python/pyarrow/_s3.pyx +++ b/python/pyarrow/_s3fs.pyx @@ -23,7 +23,7 @@ from pyarrow.lib cimport check_status from pyarrow.compat import frombytes, tobytes from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * -from pyarrow.includes.libarrow_s3 cimport * +from pyarrow.includes.libarrow_s3fs cimport * from pyarrow._fs cimport FileSystem @@ -47,16 +47,6 @@ def finalize_s3(): check_status(CFinalizeS3()) -cdef bint _initialized = False - - -cdef _ensure_initialized(): - global _initialized - if not _initialized: - initialize_s3() - _initialized = True - - cdef class S3FileSystem(FileSystem): """S3-backed FileSystem implementation @@ -92,8 +82,6 @@ cdef class S3FileSystem(FileSystem): CS3Options options shared_ptr[CS3FileSystem] wrapped - _ensure_initialized() - options = CS3Options.Defaults() if access_key is not None or secret_key is not None: options.ConfigureAccessKey(tobytes(access_key), diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index 23291203818d..5f257d07f300 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -25,8 +25,3 @@ LocalFileSystem, SubTreeFileSystem ) - -try: - from pyarrow._s3 import S3FileSystem, initialize_s3, finalize_s3 # noqa -except ImportError: - pass diff --git a/python/pyarrow/includes/libarrow_s3.pxd b/python/pyarrow/includes/libarrow_s3fs.pxd similarity index 100% rename from python/pyarrow/includes/libarrow_s3.pxd rename to python/pyarrow/includes/libarrow_s3fs.pxd diff --git a/python/pyarrow/s3fs.py b/python/pyarrow/s3fs.py new file mode 100644 index 000000000000..bc0127ae3c4c --- /dev/null +++ b/python/pyarrow/s3fs.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import + +from pyarrow._s3fs import ( # noqa + initialize_s3, + finalize_s3, + S3FileSystem +) + +initialize_s3() diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 0f47a4a52f31..e01665f9cf3f 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -53,7 +53,7 @@ def subtree_localfs(request, tempdir, localfs): @pytest.fixture def s3fs(request, minio_server): - from pyarrow.fs import S3FileSystem + from pyarrow.s3fs import S3FileSystem address, access_key, secret_key = minio_server bucket = 'pyarrow-filesystem/' diff --git a/python/setup.py b/python/setup.py index 285331190ca4..d7207eedd8f1 100755 --- a/python/setup.py +++ b/python/setup.py @@ -170,7 +170,6 @@ def initialize_options(self): CYTHON_MODULE_NAMES = [ 'lib', - '_s3', '_fs', '_csv', '_json', @@ -179,6 +178,7 @@ def initialize_options(self): '_parquet', '_orc', '_plasma', + '_s3fs', 'gandiva'] def _run_cmake(self): @@ -419,7 +419,7 @@ def _failure_permitted(self, name): return True if name == '_flight' and not self.with_flight: return True - if name == '_s3' and not self.with_s3: + if name == '_s3fs' and not self.with_s3: return True if name == '_cuda' and not self.with_cuda: return True From 98bd91ad1dd5fe13ade7ab59bebe4fa6bba334c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 25 Sep 2019 14:36:27 +0200 Subject: [PATCH 37/39] remove commented tests --- python/pyarrow/tests/test_fs.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index e01665f9cf3f..6e0e1f5e3511 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -159,10 +159,6 @@ def test_get_target_stats(fs, pathfn): assert 'aaa' in repr(aaa_stat) assert aaa_stat.extension == '' assert isinstance(aaa_stat.mtime, datetime) - # assert mtime_almost_equal(aaa_stat.mtime, fs.mtime(_aaa)) - # assert aaa_stat.base_name == 'aaa' - # assert aaa_stat.type == FileType.Directory - # assert aaa_stat is None assert bb_stat.path == str(bb) assert bb_stat.base_name == 'bb' @@ -225,9 +221,6 @@ def test_create_dir(fs, pathfn): fs.delete_dir(d) d = pathfn('deeply/nested/test-directory/') - # with pytest.raises(ArrowIOError): - # fs.create_dir(d, recursive=False) - fs.create_dir(d, recursive=True) fs.delete_dir(d) From 73e6625f9c9f93837b4466e6aeedae4ad463fe04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 26 Sep 2019 16:27:51 +0200 Subject: [PATCH 38/39] S3Options --- python/pyarrow/_s3fs.pyx | 122 ++++++++++++++++++++++++++----- python/pyarrow/s3fs.py | 1 + python/pyarrow/tests/conftest.py | 4 +- python/pyarrow/tests/test_fs.py | 40 +++++++++- 4 files changed, 144 insertions(+), 23 deletions(-) diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx index 532a290607d9..09c1c98ac01a 100644 --- a/python/pyarrow/_s3fs.pyx +++ b/python/pyarrow/_s3fs.pyx @@ -47,6 +47,105 @@ def finalize_s3(): check_status(CFinalizeS3()) +cdef class S3Options: + """Options for S3FileSystem. + + If neither access_key nor secret_key are provided then attempts to + initialize from AWS environment variables, otherwise both access_key and + secret_key must be provided. + + Parameters + ---------- + access_key: str, default None + AWS Access Key ID. Pass None to use the standard AWS environment + variables and/or configuration file. + secret_key: str, default None + AWS Secret Access key. Pass None to use the standard AWS environment + variables and/or configuration file. + region: str, default 'us-east-1' + AWS region to connect to. + scheme: str, default 'https' + S3 connection transport scheme. + endpoint_override: str, default None + Override region with a connect string such as "localhost:9000" + background_writes: boolean, default True + Whether OutputStream writes will be issued in the background, without + blocking. + """ + cdef: + CS3Options options + + # Avoid mistakingly creating attributes + __slots__ = () + + def __init__(self, access_key=None, secret_key=None, region=None, + scheme=None, endpoint_override=None, background_writes=None): + if access_key is not None and secret_key is None: + raise ValueError( + 'In order to initialize with explicit credentials both ' + 'access_key and secret_key must be provided, ' + '`secret_key` is not set.' + ) + elif access_key is None and secret_key is not None: + raise ValueError( + 'In order to initialize with explicit credentials both ' + 'access_key and secret_key must be provided, ' + '`access_key` is not set.' + ) + elif access_key is not None or secret_key is not None: + self.options = CS3Options.FromAccessKey( + tobytes(access_key), + tobytes(secret_key) + ) + else: + self.options = CS3Options.Defaults() + + if region is not None: + self.region = region + if scheme is not None: + self.scheme = scheme + if endpoint_override is not None: + self.endpoint_override = endpoint_override + if background_writes is not None: + self.background_writes = background_writes + + @property + def region(self): + """AWS region to connect to.""" + return frombytes(self.options.region) + + @region.setter + def region(self, value): + self.options.region = tobytes(value) + + @property + def scheme(self): + """S3 connection transport scheme.""" + return frombytes(self.options.scheme) + + @scheme.setter + def scheme(self, value): + self.options.scheme = tobytes(value) + + @property + def endpoint_override(self): + """Override region with a connect string such as localhost:9000""" + return frombytes(self.options.endpoint_override) + + @endpoint_override.setter + def endpoint_override(self, value): + self.options.endpoint_override = tobytes(value) + + @property + def background_writes(self): + """OutputStream writes will be issued in the background""" + return self.options.background_writes + + @background_writes.setter + def background_writes(self, bint value): + self.options.background_writes = value + + cdef class S3FileSystem(FileSystem): """S3-backed FileSystem implementation @@ -75,25 +174,10 @@ cdef class S3FileSystem(FileSystem): cdef: CS3FileSystem* s3fs - def __init__(self, access_key=None, secret_key=None, region='us-east-1', - scheme='https', endpoint_override=None, - bint background_writes=True): - cdef: - CS3Options options - shared_ptr[CS3FileSystem] wrapped - - options = CS3Options.Defaults() - if access_key is not None or secret_key is not None: - options.ConfigureAccessKey(tobytes(access_key), - tobytes(secret_key)) - - options.region = tobytes(region) - options.scheme = tobytes(scheme) - options.background_writes = background_writes - if endpoint_override is not None: - options.endpoint_override = tobytes(endpoint_override) - - check_status(CS3FileSystem.Make(options, &wrapped)) + def __init__(self, S3Options options=None): + cdef shared_ptr[CS3FileSystem] wrapped + options = options or S3Options() + check_status(CS3FileSystem.Make(options.options, &wrapped)) self.init( wrapped) cdef init(self, const shared_ptr[CFileSystem]& wrapped): diff --git a/python/pyarrow/s3fs.py b/python/pyarrow/s3fs.py index bc0127ae3c4c..5619e186f9ea 100644 --- a/python/pyarrow/s3fs.py +++ b/python/pyarrow/s3fs.py @@ -20,6 +20,7 @@ from pyarrow._s3fs import ( # noqa initialize_s3, finalize_s3, + S3Options, S3FileSystem ) diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index 85297fe81de6..68e215b3f422 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -132,7 +132,7 @@ pass try: - from pyarrow.fs import S3FileSystem # noqa + import pyarrow.s3fs # noqa defaults['s3'] = True except ImportError: pass @@ -248,8 +248,8 @@ def __exit__(self, exc_type, exc_value, traceback): shutil.rmtree(self.tmp) -@pytest.fixture(scope='session') @pytest.mark.s3 +@pytest.fixture(scope='session') def minio_server(): host, port = 'localhost', find_free_port() access_key, secret_key = 'arrow', 'apachearrow' diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 6e0e1f5e3511..f6b6bf1d18cd 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -51,18 +51,20 @@ def subtree_localfs(request, tempdir, localfs): ) +@pytest.mark.s3 @pytest.fixture def s3fs(request, minio_server): - from pyarrow.s3fs import S3FileSystem + from pyarrow.s3fs import S3Options, S3FileSystem address, access_key, secret_key = minio_server bucket = 'pyarrow-filesystem/' - fs = S3FileSystem( + options = S3Options( endpoint_override=address, access_key=access_key, secret_key=secret_key, scheme='http' ) + fs = S3FileSystem(options) fs.create_dir(bucket) return dict( @@ -387,3 +389,37 @@ def test_open_append_stream(fs, pathfn, compression, buffer_size, compressor, else: with pytest.raises(pa.ArrowNotImplementedError): fs.open_append_stream(p, compression, buffer_size) + + +@pytest.mark.s3 +def test_s3_options(minio_server): + from pyarrow.s3fs import S3Options + + options = S3Options() + + assert options.region == 'us-east-1' + options.region = 'us-west-1' + assert options.region == 'us-west-1' + + assert options.scheme == 'https' + options.scheme = 'http' + assert options.scheme == 'http' + + assert options.endpoint_override == '' + options.endpoint_override = 'localhost:8999' + assert options.endpoint_override == 'localhost:8999' + + with pytest.raises(ValueError): + S3Options(access_key='access') + with pytest.raises(ValueError): + S3Options(secret_key='secret') + + address, access_key, secret_key = minio_server + options = S3Options( + access_key=access_key, + secret_key=secret_key, + endpoint_override=address, + scheme='http' + ) + assert options.scheme == 'http' + assert options.endpoint_override == address From 384c96052ef0362f25f2a0a3395c61e245002f39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 30 Sep 2019 22:31:57 +0200 Subject: [PATCH 39/39] Resolve review comments --- ci/travis_script_python.sh | 1 - python/pyarrow/_s3fs.pyx | 18 +++--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index b1767dfabc05..89fa4addae52 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -168,7 +168,6 @@ export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_ORC=1 -export PYARROW_WITH_S3=1 if [ "$ARROW_TRAVIS_S3" == "1" ]; then export PYARROW_WITH_S3=1 fi diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx index 09c1c98ac01a..d1f820e51958 100644 --- a/python/pyarrow/_s3fs.pyx +++ b/python/pyarrow/_s3fs.pyx @@ -154,21 +154,9 @@ cdef class S3FileSystem(FileSystem): Parameters ---------- - access_key: str, default None - AWS Access Key ID. Pass None to use the standard AWS environment - variables and/or configuration file. - secret_key: str, default None - AWS Secret Access key. Pass None to use the standard AWS environment - variables and/or configuration file. - region: str, default 'us-east-1' - AWS region to connect to. - scheme: str, default 'https' - S3 connection transport scheme. - endpoint_override: str, default None - Override region with a connect string such as "localhost:9000" - background_writes: boolean, default True - Whether OutputStream writes will be issued in the background, without - blocking. + options: S3Options, default None + Options for connecting to S3. If None is passed then attempts to + initialize the connection from AWS environment variables. """ cdef: