ARROW-751: [Python] Make all Cython modules private. Some code tidying#533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wesm wants to merge 3 commits into from
Closed

ARROW-751: [Python] Make all Cython modules private. Some code tidying #533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1074e7c
Make all Cython modules private. Code cleaning
wesm Apr 12, 2017
0f46116
Fix APIs in Parquet
wesm Apr 12, 2017
63b479b
jemalloc is now private
wesm Apr 13, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ci/travis_script_python.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -115,7 +115,7 @@ python_version_tests() {
python setup.py build_ext --inplace --with-parquet --with-jemalloc

python -c "import pyarrow.parquet"
python -c "import pyarrow.jemalloc"
python -c "import pyarrow._jemalloc"

python -m pytest -vv -r sxX pyarrow

Expand Down
16 changes: 7 additions & 9 deletions python/CMakeLists.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -261,14 +261,12 @@ if (UNIX)
endif()

set(CYTHON_EXTENSIONS
array
config
error
io
memory
scalar
schema
table
_array
_config
_error
_io
_memory
_table
)

set(LINK_LIBS
Expand DownExpand Up@@ -313,7 +311,7 @@ if (PYARROW_BUILD_JEMALLOC)
arrow_jemalloc_shared)
set(CYTHON_EXTENSIONS
${CYTHON_EXTENSIONS}
jemalloc)
_jemalloc)
endif()

############################################################
Expand Down
84 changes: 40 additions & 44 deletions python/pyarrow/__init__.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,60 +25,56 @@
pass


import pyarrow.config
from pyarrow.config import cpu_count, set_cpu_count
import pyarrow._config
from pyarrow._config import cpu_count, set_cpu_count

from pyarrow.array import (Array, Tensor, from_pylist,
NumericArray, IntegerArray, FloatingPointArray,
BooleanArray,
Int8Array, UInt8Array,
Int16Array, UInt16Array,
Int32Array, UInt32Array,
Int64Array, UInt64Array,
ListArray, StringArray,
DictionaryArray)

from pyarrow.error import (ArrowException,
ArrowKeyError,
ArrowInvalid,
ArrowIOError,
ArrowMemoryError,
ArrowNotImplementedError,
ArrowTypeError)

from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
Buffer, BufferReader, InMemoryOutputStream,
MemoryMappedFile, memory_map,
frombuffer, read_tensor, write_tensor,
memory_map, create_memory_map,
get_record_batch_size, get_tensor_size)

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter

from pyarrow.memory import MemoryPool, total_allocated_bytes

from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
BooleanValue,
Int8Value, Int16Value, Int32Value, Int64Value,
UInt8Value, UInt16Value, UInt32Value, UInt64Value,
FloatValue, DoubleValue, ListValue,
BinaryValue, StringValue, FixedSizeBinaryValue)

import pyarrow.schema as _schema

from pyarrow.schema import (null, bool_,
from pyarrow._array import (null, bool_,
int8, int16, int32, int64,
uint8, uint16, uint32, uint64,
timestamp, date32, date64,
float16, float32, float64,
binary, string, decimal,
list_, struct, dictionary, field,
DataType, FixedSizeBinaryType,
Field, Schema, schema)
Field, Schema, schema,
Array, Tensor,
from_pylist,
from_numpy_dtype,
NumericArray, IntegerArray, FloatingPointArray,
BooleanArray,
Int8Array, UInt8Array,
Int16Array, UInt16Array,
Int32Array, UInt32Array,
Int64Array, UInt64Array,
ListArray, StringArray,
DictionaryArray,
ArrayValue, Scalar, NA, NAType,
BooleanValue,
Int8Value, Int16Value, Int32Value, Int64Value,
UInt8Value, UInt16Value, UInt32Value, UInt64Value,
FloatValue, DoubleValue, ListValue,
BinaryValue, StringValue, FixedSizeBinaryValue)

from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
Buffer, BufferReader, InMemoryOutputStream,
OSFile, MemoryMappedFile, memory_map,
frombuffer, read_tensor, write_tensor,
memory_map, create_memory_map,
get_record_batch_size, get_tensor_size)

from pyarrow._memory import MemoryPool, total_allocated_bytes
from pyarrow._table import Column, RecordBatch, Table, concat_tables
from pyarrow._error import (ArrowException,
ArrowKeyError,
ArrowInvalid,
ArrowIOError,
ArrowMemoryError,
ArrowNotImplementedError,
ArrowTypeError)

from pyarrow.table import Column, RecordBatch, Table, concat_tables
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter


localfs = LocalFilesystem.get_instance()
116 changes: 104 additions & 12 deletions python/pyarrow/array.pxd → python/pyarrow/_array.pxd
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,20 +15,109 @@
# specific language governing permissions and limitations
# under the License.

from pyarrow.includes.common cimport shared_ptr, int64_t
from pyarrow.includes.libarrow cimport CArray, CTensor

from pyarrow.scalar import NA

from pyarrow.schema cimport DataType
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *

from cpython cimport PyObject


cdef extern from "Python.h":
int PySlice_Check(object)


cdef class DataType:
cdef:
shared_ptr[CDataType] sp_type
CDataType* type

cdef void init(self, const shared_ptr[CDataType]& type)


cdef class DictionaryType(DataType):
cdef:
const CDictionaryType* dict_type


cdef class TimestampType(DataType):
cdef:
const CTimestampType* ts_type


cdef class FixedSizeBinaryType(DataType):
cdef:
const CFixedSizeBinaryType* fixed_size_binary_type


cdef class DecimalType(FixedSizeBinaryType):
cdef:
const CDecimalType* decimal_type


cdef class Field:
cdef:
shared_ptr[CField] sp_field
CField* field

cdef readonly:
DataType type

cdef init(self, const shared_ptr[CField]& field)


cdef class Schema:
cdef:
shared_ptr[CSchema] sp_schema
CSchema* schema

cdef init(self, const vector[shared_ptr[CField]]& fields)
cdef init_schema(self, const shared_ptr[CSchema]& schema)


cdef class Scalar:
cdef readonly:
DataType type


cdef class NAType(Scalar):
pass


cdef class ArrayValue(Scalar):
cdef:
shared_ptr[CArray] sp_array
int64_t index

cdef void init(self, DataType type,
const shared_ptr[CArray]& sp_array, int64_t index)

cdef void _set_array(self, const shared_ptr[CArray]& sp_array)


cdef class Int8Value(ArrayValue):
pass


cdef class Int64Value(ArrayValue):
pass


cdef class ListValue(ArrayValue):
cdef readonly:
DataType value_type

cdef:
CListArray* ap

cdef getitem(self, int64_t i)


cdef class StringValue(ArrayValue):
pass


cdef class FixedSizeBinaryValue(ArrayValue):
pass


cdef class Array:
cdef:
shared_ptr[CArray] sp_array
Expand All@@ -52,10 +141,6 @@ cdef class Tensor:
cdef init(self, const shared_ptr[CTensor]& sp_tensor)


cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)


cdef class BooleanArray(Array):
pass

Expand DownExpand Up@@ -137,5 +222,12 @@ cdef class DictionaryArray(Array):
object _indices, _dictionary



cdef wrap_array_output(PyObject* output)
cdef DataType box_data_type(const shared_ptr[CDataType]& type)
cdef Field box_field(const shared_ptr[CField]& field)
cdef Schema box_schema(const shared_ptr[CSchema]& schema)
cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
cdef object box_scalar(DataType type,
const shared_ptr[CArray]& sp_array,
int64_t index)
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-751: [Python] Make all Cython modules private. Some code tidying#533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-751: [Python] Make all Cython modules private. Some code tidying
#533
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
1074e7c

Make all Cython modules private. Code cleaning


wesm Apr 12, 2017
0f46116

Fix APIs in Parquet


wesm Apr 12, 2017
63b479b

jemalloc is now private


wesm Apr 13, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
ci/travis_script_python.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -115,7 +115,7 @@ python_version_tests() {
 python setup.py build_ext --inplace --with-parquet --with-jemalloc

 python -c "import pyarrow.parquet"
 python -c "import pyarrow.jemalloc"
 python -c "import pyarrow._jemalloc"

 python -m pytest -vv -r sxX pyarrow

Expand Down

16 changes: 7 additions & 9 deletions
python/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -261,14 +261,12 @@ if (UNIX)
endif()

set(CYTHON_EXTENSIONS
 array
 config
 error
 io
 memory
 scalar
 schema
 table
 _array
 _config
 _error
 _io
 _memory
 _table
)

set(LINK_LIBS
Expand DownExpand Up@@ -313,7 +311,7 @@ if (PYARROW_BUILD_JEMALLOC)
 arrow_jemalloc_shared)
 set(CYTHON_EXTENSIONS
 ${CYTHON_EXTENSIONS}
jemalloc)
_jemalloc)
endif()

############################################################
Expand Down

84 changes: 40 additions & 44 deletions
python/pyarrow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,60 +25,56 @@
 pass


import pyarrow.config
from pyarrow.config import cpu_count, set_cpu_count
import pyarrow._config
from pyarrow._config import cpu_count, set_cpu_count

from pyarrow.array import (Array, Tensor, from_pylist,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray)

from pyarrow.error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter

from pyarrow.memory import MemoryPool, total_allocated_bytes

from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

import pyarrow.schema as _schema

from pyarrow.schema import (null, bool_,
from pyarrow._array import (null, bool_,
 int8, int16, int32, int64,
 uint8, uint16, uint32, uint64,
 timestamp, date32, date64,
 float16, float32, float64,
 binary, string, decimal,
 list_, struct, dictionary, field,
 DataType, FixedSizeBinaryType,
 Field, Schema, schema)
 Field, Schema, schema,
 Array, Tensor,
 from_pylist,
 from_numpy_dtype,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray,
 ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 OSFile, MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow._memory import MemoryPool, total_allocated_bytes
from pyarrow._table import Column, RecordBatch, Table, concat_tables
from pyarrow._error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.table import Column, RecordBatch, Table, concat_tables
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter


localfs = LocalFilesystem.get_instance()

116 changes: 104 additions & 12 deletions
python/pyarrow/array.pxd → python/pyarrow/_array.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,20 +15,109 @@
# specific language governing permissions and limitations
# under the License.

from pyarrow.includes.common cimport shared_ptr, int64_t
from pyarrow.includes.libarrow cimport CArray, CTensor

from pyarrow.scalar import NA

from pyarrow.schema cimport DataType
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *

from cpython cimport PyObject


cdef extern from "Python.h":
 int PySlice_Check(object)


cdef class DataType:
 cdef:
 shared_ptr[CDataType] sp_type
 CDataType* type

 cdef void init(self, const shared_ptr[CDataType]& type)


cdef class DictionaryType(DataType):
 cdef:
 const CDictionaryType* dict_type


cdef class TimestampType(DataType):
 cdef:
 const CTimestampType* ts_type


cdef class FixedSizeBinaryType(DataType):
 cdef:
 const CFixedSizeBinaryType* fixed_size_binary_type


cdef class DecimalType(FixedSizeBinaryType):
 cdef:
 const CDecimalType* decimal_type


cdef class Field:
 cdef:
 shared_ptr[CField] sp_field
 CField* field

 cdef readonly:
 DataType type

 cdef init(self, const shared_ptr[CField]& field)


cdef class Schema:
 cdef:
 shared_ptr[CSchema] sp_schema
 CSchema* schema

 cdef init(self, const vector[shared_ptr[CField]]& fields)
 cdef init_schema(self, const shared_ptr[CSchema]& schema)


cdef class Scalar:
 cdef readonly:
 DataType type


cdef class NAType(Scalar):
 pass


cdef class ArrayValue(Scalar):
 cdef:
 shared_ptr[CArray] sp_array
 int64_t index

 cdef void init(self, DataType type,
 const shared_ptr[CArray]& sp_array, int64_t index)

 cdef void _set_array(self, const shared_ptr[CArray]& sp_array)


cdef class Int8Value(ArrayValue):
 pass


cdef class Int64Value(ArrayValue):
 pass


cdef class ListValue(ArrayValue):
 cdef readonly:
 DataType value_type

 cdef:
 CListArray* ap

 cdef getitem(self, int64_t i)


cdef class StringValue(ArrayValue):
 pass


cdef class FixedSizeBinaryValue(ArrayValue):
 pass


cdef class Array:
 cdef:
 shared_ptr[CArray] sp_array
Expand All@@ -52,10 +141,6 @@ cdef class Tensor:
 cdef init(self, const shared_ptr[CTensor]& sp_tensor)


cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)


cdef class BooleanArray(Array):
 pass

Expand DownExpand Up@@ -137,5 +222,12 @@ cdef class DictionaryArray(Array):
 object _indices, _dictionary



cdef wrap_array_output(PyObject* output)
cdef DataType box_data_type(const shared_ptr[CDataType]& type)
cdef Field box_field(const shared_ptr[CField]& field)
cdef Schema box_schema(const shared_ptr[CSchema]& schema)
cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
cdef object box_scalar(DataType type,
 const shared_ptr[CArray]& sp_array,
 int64_t index)
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-751: [Python] Make all Cython modules private. Some code tidying#533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-751: [Python] Make all Cython modules private. Some code tidying
#533
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
1074e7c

Make all Cython modules private. Code cleaning


wesm Apr 12, 2017
0f46116

Fix APIs in Parquet


wesm Apr 12, 2017
63b479b

jemalloc is now private


wesm Apr 13, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
ci/travis_script_python.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -115,7 +115,7 @@ python_version_tests() {
 python setup.py build_ext --inplace --with-parquet --with-jemalloc

 python -c "import pyarrow.parquet"
 python -c "import pyarrow.jemalloc"
 python -c "import pyarrow._jemalloc"

 python -m pytest -vv -r sxX pyarrow

Expand Down

16 changes: 7 additions & 9 deletions
python/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -261,14 +261,12 @@ if (UNIX)
endif()

set(CYTHON_EXTENSIONS
 array
 config
 error
 io
 memory
 scalar
 schema
 table
 _array
 _config
 _error
 _io
 _memory
 _table
)

set(LINK_LIBS
Expand DownExpand Up@@ -313,7 +311,7 @@ if (PYARROW_BUILD_JEMALLOC)
 arrow_jemalloc_shared)
 set(CYTHON_EXTENSIONS
 ${CYTHON_EXTENSIONS}
jemalloc)
_jemalloc)
endif()

############################################################
Expand Down

84 changes: 40 additions & 44 deletions
python/pyarrow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,60 +25,56 @@
 pass


import pyarrow.config
from pyarrow.config import cpu_count, set_cpu_count
import pyarrow._config
from pyarrow._config import cpu_count, set_cpu_count

from pyarrow.array import (Array, Tensor, from_pylist,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray)

from pyarrow.error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter

from pyarrow.memory import MemoryPool, total_allocated_bytes

from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

import pyarrow.schema as _schema

from pyarrow.schema import (null, bool_,
from pyarrow._array import (null, bool_,
 int8, int16, int32, int64,
 uint8, uint16, uint32, uint64,
 timestamp, date32, date64,
 float16, float32, float64,
 binary, string, decimal,
 list_, struct, dictionary, field,
 DataType, FixedSizeBinaryType,
 Field, Schema, schema)
 Field, Schema, schema,
 Array, Tensor,
 from_pylist,
 from_numpy_dtype,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray,
 ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 OSFile, MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow._memory import MemoryPool, total_allocated_bytes
from pyarrow._table import Column, RecordBatch, Table, concat_tables
from pyarrow._error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.table import Column, RecordBatch, Table, concat_tables
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter


localfs = LocalFilesystem.get_instance()

116 changes: 104 additions & 12 deletions
python/pyarrow/array.pxd → python/pyarrow/_array.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,20 +15,109 @@
# specific language governing permissions and limitations
# under the License.

from pyarrow.includes.common cimport shared_ptr, int64_t
from pyarrow.includes.libarrow cimport CArray, CTensor

from pyarrow.scalar import NA

from pyarrow.schema cimport DataType
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *

from cpython cimport PyObject


cdef extern from "Python.h":
 int PySlice_Check(object)


cdef class DataType:
 cdef:
 shared_ptr[CDataType] sp_type
 CDataType* type

 cdef void init(self, const shared_ptr[CDataType]& type)


cdef class DictionaryType(DataType):
 cdef:
 const CDictionaryType* dict_type


cdef class TimestampType(DataType):
 cdef:
 const CTimestampType* ts_type


cdef class FixedSizeBinaryType(DataType):
 cdef:
 const CFixedSizeBinaryType* fixed_size_binary_type


cdef class DecimalType(FixedSizeBinaryType):
 cdef:
 const CDecimalType* decimal_type


cdef class Field:
 cdef:
 shared_ptr[CField] sp_field
 CField* field

 cdef readonly:
 DataType type

 cdef init(self, const shared_ptr[CField]& field)


cdef class Schema:
 cdef:
 shared_ptr[CSchema] sp_schema
 CSchema* schema

 cdef init(self, const vector[shared_ptr[CField]]& fields)
 cdef init_schema(self, const shared_ptr[CSchema]& schema)


cdef class Scalar:
 cdef readonly:
 DataType type


cdef class NAType(Scalar):
 pass


cdef class ArrayValue(Scalar):
 cdef:
 shared_ptr[CArray] sp_array
 int64_t index

 cdef void init(self, DataType type,
 const shared_ptr[CArray]& sp_array, int64_t index)

 cdef void _set_array(self, const shared_ptr[CArray]& sp_array)


cdef class Int8Value(ArrayValue):
 pass


cdef class Int64Value(ArrayValue):
 pass


cdef class ListValue(ArrayValue):
 cdef readonly:
 DataType value_type

 cdef:
 CListArray* ap

 cdef getitem(self, int64_t i)


cdef class StringValue(ArrayValue):
 pass


cdef class FixedSizeBinaryValue(ArrayValue):
 pass


cdef class Array:
 cdef:
 shared_ptr[CArray] sp_array
Expand All@@ -52,10 +141,6 @@ cdef class Tensor:
 cdef init(self, const shared_ptr[CTensor]& sp_tensor)


cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)


cdef class BooleanArray(Array):
 pass

Expand DownExpand Up@@ -137,5 +222,12 @@ cdef class DictionaryArray(Array):
 object _indices, _dictionary



cdef wrap_array_output(PyObject* output)
cdef DataType box_data_type(const shared_ptr[CDataType]& type)
cdef Field box_field(const shared_ptr[CField]& field)
cdef Schema box_schema(const shared_ptr[CSchema]& schema)
cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
cdef object box_scalar(DataType type,
 const shared_ptr[CArray]& sp_array,
 int64_t index)
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-751: [Python] Make all Cython modules private. Some code tidying#533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-751: [Python] Make all Cython modules private. Some code tidying
#533
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
1074e7c

Make all Cython modules private. Code cleaning


wesm Apr 12, 2017
0f46116

Fix APIs in Parquet


wesm Apr 12, 2017
63b479b

jemalloc is now private


wesm Apr 13, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
ci/travis_script_python.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -115,7 +115,7 @@ python_version_tests() {
 python setup.py build_ext --inplace --with-parquet --with-jemalloc

 python -c "import pyarrow.parquet"
 python -c "import pyarrow.jemalloc"
 python -c "import pyarrow._jemalloc"

 python -m pytest -vv -r sxX pyarrow

Expand Down

16 changes: 7 additions & 9 deletions
python/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -261,14 +261,12 @@ if (UNIX)
endif()

set(CYTHON_EXTENSIONS
 array
 config
 error
 io
 memory
 scalar
 schema
 table
 _array
 _config
 _error
 _io
 _memory
 _table
)

set(LINK_LIBS
Expand DownExpand Up@@ -313,7 +311,7 @@ if (PYARROW_BUILD_JEMALLOC)
 arrow_jemalloc_shared)
 set(CYTHON_EXTENSIONS
 ${CYTHON_EXTENSIONS}
jemalloc)
_jemalloc)
endif()

############################################################
Expand Down

84 changes: 40 additions & 44 deletions
python/pyarrow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,60 +25,56 @@
 pass


import pyarrow.config
from pyarrow.config import cpu_count, set_cpu_count
import pyarrow._config
from pyarrow._config import cpu_count, set_cpu_count

from pyarrow.array import (Array, Tensor, from_pylist,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray)

from pyarrow.error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter

from pyarrow.memory import MemoryPool, total_allocated_bytes

from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

import pyarrow.schema as _schema

from pyarrow.schema import (null, bool_,
from pyarrow._array import (null, bool_,
 int8, int16, int32, int64,
 uint8, uint16, uint32, uint64,
 timestamp, date32, date64,
 float16, float32, float64,
 binary, string, decimal,
 list_, struct, dictionary, field,
 DataType, FixedSizeBinaryType,
 Field, Schema, schema)
 Field, Schema, schema,
 Array, Tensor,
 from_pylist,
 from_numpy_dtype,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray,
 ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 OSFile, MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow._memory import MemoryPool, total_allocated_bytes
from pyarrow._table import Column, RecordBatch, Table, concat_tables
from pyarrow._error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.table import Column, RecordBatch, Table, concat_tables
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter


localfs = LocalFilesystem.get_instance()

116 changes: 104 additions & 12 deletions
python/pyarrow/array.pxd → python/pyarrow/_array.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,20 +15,109 @@
# specific language governing permissions and limitations
# under the License.

from pyarrow.includes.common cimport shared_ptr, int64_t
from pyarrow.includes.libarrow cimport CArray, CTensor

from pyarrow.scalar import NA

from pyarrow.schema cimport DataType
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *

from cpython cimport PyObject


cdef extern from "Python.h":
 int PySlice_Check(object)


cdef class DataType:
 cdef:
 shared_ptr[CDataType] sp_type
 CDataType* type

 cdef void init(self, const shared_ptr[CDataType]& type)


cdef class DictionaryType(DataType):
 cdef:
 const CDictionaryType* dict_type


cdef class TimestampType(DataType):
 cdef:
 const CTimestampType* ts_type


cdef class FixedSizeBinaryType(DataType):
 cdef:
 const CFixedSizeBinaryType* fixed_size_binary_type


cdef class DecimalType(FixedSizeBinaryType):
 cdef:
 const CDecimalType* decimal_type


cdef class Field:
 cdef:
 shared_ptr[CField] sp_field
 CField* field

 cdef readonly:
 DataType type

 cdef init(self, const shared_ptr[CField]& field)


cdef class Schema:
 cdef:
 shared_ptr[CSchema] sp_schema
 CSchema* schema

 cdef init(self, const vector[shared_ptr[CField]]& fields)
 cdef init_schema(self, const shared_ptr[CSchema]& schema)


cdef class Scalar:
 cdef readonly:
 DataType type


cdef class NAType(Scalar):
 pass


cdef class ArrayValue(Scalar):
 cdef:
 shared_ptr[CArray] sp_array
 int64_t index

 cdef void init(self, DataType type,
 const shared_ptr[CArray]& sp_array, int64_t index)

 cdef void _set_array(self, const shared_ptr[CArray]& sp_array)


cdef class Int8Value(ArrayValue):
 pass


cdef class Int64Value(ArrayValue):
 pass


cdef class ListValue(ArrayValue):
 cdef readonly:
 DataType value_type

 cdef:
 CListArray* ap

 cdef getitem(self, int64_t i)


cdef class StringValue(ArrayValue):
 pass


cdef class FixedSizeBinaryValue(ArrayValue):
 pass


cdef class Array:
 cdef:
 shared_ptr[CArray] sp_array
Expand All@@ -52,10 +141,6 @@ cdef class Tensor:
 cdef init(self, const shared_ptr[CTensor]& sp_tensor)


cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)


cdef class BooleanArray(Array):
 pass

Expand DownExpand Up@@ -137,5 +222,12 @@ cdef class DictionaryArray(Array):
 object _indices, _dictionary



cdef wrap_array_output(PyObject* output)
cdef DataType box_data_type(const shared_ptr[CDataType]& type)
cdef Field box_field(const shared_ptr[CField]& field)
cdef Schema box_schema(const shared_ptr[CSchema]& schema)
cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
cdef object box_scalar(DataType type,
 const shared_ptr[CArray]& sp_array,
 int64_t index)
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-751: [Python] Make all Cython modules private. Some code tidying#533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-751: [Python] Make all Cython modules private. Some code tidying
#533
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
1074e7c

Make all Cython modules private. Code cleaning


wesm Apr 12, 2017
0f46116

Fix APIs in Parquet


wesm Apr 12, 2017
63b479b

jemalloc is now private


wesm Apr 13, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
ci/travis_script_python.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -115,7 +115,7 @@ python_version_tests() {
 python setup.py build_ext --inplace --with-parquet --with-jemalloc

 python -c "import pyarrow.parquet"
 python -c "import pyarrow.jemalloc"
 python -c "import pyarrow._jemalloc"

 python -m pytest -vv -r sxX pyarrow

Expand Down

16 changes: 7 additions & 9 deletions
python/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -261,14 +261,12 @@ if (UNIX)
endif()

set(CYTHON_EXTENSIONS
 array
 config
 error
 io
 memory
 scalar
 schema
 table
 _array
 _config
 _error
 _io
 _memory
 _table
)

set(LINK_LIBS
Expand DownExpand Up@@ -313,7 +311,7 @@ if (PYARROW_BUILD_JEMALLOC)
 arrow_jemalloc_shared)
 set(CYTHON_EXTENSIONS
 ${CYTHON_EXTENSIONS}
jemalloc)
_jemalloc)
endif()

############################################################
Expand Down

84 changes: 40 additions & 44 deletions
python/pyarrow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,60 +25,56 @@
 pass


import pyarrow.config
from pyarrow.config import cpu_count, set_cpu_count
import pyarrow._config
from pyarrow._config import cpu_count, set_cpu_count

from pyarrow.array import (Array, Tensor, from_pylist,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray)

from pyarrow.error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter

from pyarrow.memory import MemoryPool, total_allocated_bytes

from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

import pyarrow.schema as _schema

from pyarrow.schema import (null, bool_,
from pyarrow._array import (null, bool_,
 int8, int16, int32, int64,
 uint8, uint16, uint32, uint64,
 timestamp, date32, date64,
 float16, float32, float64,
 binary, string, decimal,
 list_, struct, dictionary, field,
 DataType, FixedSizeBinaryType,
 Field, Schema, schema)
 Field, Schema, schema,
 Array, Tensor,
 from_pylist,
 from_numpy_dtype,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray,
 ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 OSFile, MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow._memory import MemoryPool, total_allocated_bytes
from pyarrow._table import Column, RecordBatch, Table, concat_tables
from pyarrow._error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.table import Column, RecordBatch, Table, concat_tables
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter


localfs = LocalFilesystem.get_instance()

116 changes: 104 additions & 12 deletions
python/pyarrow/array.pxd → python/pyarrow/_array.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,20 +15,109 @@
# specific language governing permissions and limitations
# under the License.

from pyarrow.includes.common cimport shared_ptr, int64_t
from pyarrow.includes.libarrow cimport CArray, CTensor

from pyarrow.scalar import NA

from pyarrow.schema cimport DataType
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *

from cpython cimport PyObject


cdef extern from "Python.h":
 int PySlice_Check(object)


cdef class DataType:
 cdef:
 shared_ptr[CDataType] sp_type
 CDataType* type

 cdef void init(self, const shared_ptr[CDataType]& type)


cdef class DictionaryType(DataType):
 cdef:
 const CDictionaryType* dict_type


cdef class TimestampType(DataType):
 cdef:
 const CTimestampType* ts_type


cdef class FixedSizeBinaryType(DataType):
 cdef:
 const CFixedSizeBinaryType* fixed_size_binary_type


cdef class DecimalType(FixedSizeBinaryType):
 cdef:
 const CDecimalType* decimal_type


cdef class Field:
 cdef:
 shared_ptr[CField] sp_field
 CField* field

 cdef readonly:
 DataType type

 cdef init(self, const shared_ptr[CField]& field)


cdef class Schema:
 cdef:
 shared_ptr[CSchema] sp_schema
 CSchema* schema

 cdef init(self, const vector[shared_ptr[CField]]& fields)
 cdef init_schema(self, const shared_ptr[CSchema]& schema)


cdef class Scalar:
 cdef readonly:
 DataType type


cdef class NAType(Scalar):
 pass


cdef class ArrayValue(Scalar):
 cdef:
 shared_ptr[CArray] sp_array
 int64_t index

 cdef void init(self, DataType type,
 const shared_ptr[CArray]& sp_array, int64_t index)

 cdef void _set_array(self, const shared_ptr[CArray]& sp_array)


cdef class Int8Value(ArrayValue):
 pass


cdef class Int64Value(ArrayValue):
 pass


cdef class ListValue(ArrayValue):
 cdef readonly:
 DataType value_type

 cdef:
 CListArray* ap

 cdef getitem(self, int64_t i)


cdef class StringValue(ArrayValue):
 pass


cdef class FixedSizeBinaryValue(ArrayValue):
 pass


cdef class Array:
 cdef:
 shared_ptr[CArray] sp_array
Expand All@@ -52,10 +141,6 @@ cdef class Tensor:
 cdef init(self, const shared_ptr[CTensor]& sp_tensor)


cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)


cdef class BooleanArray(Array):
 pass

Expand DownExpand Up@@ -137,5 +222,12 @@ cdef class DictionaryArray(Array):
 object _indices, _dictionary



cdef wrap_array_output(PyObject* output)
cdef DataType box_data_type(const shared_ptr[CDataType]& type)
cdef Field box_field(const shared_ptr[CField]& field)
cdef Schema box_schema(const shared_ptr[CSchema]& schema)
cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
cdef object box_scalar(DataType type,
 const shared_ptr[CArray]& sp_array,
 int64_t index)
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-751: [Python] Make all Cython modules private. Some code tidying#533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-751: [Python] Make all Cython modules private. Some code tidying
#533
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
1074e7c

Make all Cython modules private. Code cleaning


wesm Apr 12, 2017
0f46116

Fix APIs in Parquet


wesm Apr 12, 2017
63b479b

jemalloc is now private


wesm Apr 13, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
ci/travis_script_python.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -115,7 +115,7 @@ python_version_tests() {
 python setup.py build_ext --inplace --with-parquet --with-jemalloc

 python -c "import pyarrow.parquet"
 python -c "import pyarrow.jemalloc"
 python -c "import pyarrow._jemalloc"

 python -m pytest -vv -r sxX pyarrow

Expand Down

16 changes: 7 additions & 9 deletions
python/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -261,14 +261,12 @@ if (UNIX)
endif()

set(CYTHON_EXTENSIONS
 array
 config
 error
 io
 memory
 scalar
 schema
 table
 _array
 _config
 _error
 _io
 _memory
 _table
)

set(LINK_LIBS
Expand DownExpand Up@@ -313,7 +311,7 @@ if (PYARROW_BUILD_JEMALLOC)
 arrow_jemalloc_shared)
 set(CYTHON_EXTENSIONS
 ${CYTHON_EXTENSIONS}
jemalloc)
_jemalloc)
endif()

############################################################
Expand Down

84 changes: 40 additions & 44 deletions
python/pyarrow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,60 +25,56 @@
 pass


import pyarrow.config
from pyarrow.config import cpu_count, set_cpu_count
import pyarrow._config
from pyarrow._config import cpu_count, set_cpu_count

from pyarrow.array import (Array, Tensor, from_pylist,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray)

from pyarrow.error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter

from pyarrow.memory import MemoryPool, total_allocated_bytes

from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

import pyarrow.schema as _schema

from pyarrow.schema import (null, bool_,
from pyarrow._array import (null, bool_,
 int8, int16, int32, int64,
 uint8, uint16, uint32, uint64,
 timestamp, date32, date64,
 float16, float32, float64,
 binary, string, decimal,
 list_, struct, dictionary, field,
 DataType, FixedSizeBinaryType,
 Field, Schema, schema)
 Field, Schema, schema,
 Array, Tensor,
 from_pylist,
 from_numpy_dtype,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray,
 ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 OSFile, MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow._memory import MemoryPool, total_allocated_bytes
from pyarrow._table import Column, RecordBatch, Table, concat_tables
from pyarrow._error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.table import Column, RecordBatch, Table, concat_tables
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter


localfs = LocalFilesystem.get_instance()

116 changes: 104 additions & 12 deletions
python/pyarrow/array.pxd → python/pyarrow/_array.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,20 +15,109 @@
# specific language governing permissions and limitations
# under the License.

from pyarrow.includes.common cimport shared_ptr, int64_t
from pyarrow.includes.libarrow cimport CArray, CTensor

from pyarrow.scalar import NA

from pyarrow.schema cimport DataType
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *

from cpython cimport PyObject


cdef extern from "Python.h":
 int PySlice_Check(object)


cdef class DataType:
 cdef:
 shared_ptr[CDataType] sp_type
 CDataType* type

 cdef void init(self, const shared_ptr[CDataType]& type)


cdef class DictionaryType(DataType):
 cdef:
 const CDictionaryType* dict_type


cdef class TimestampType(DataType):
 cdef:
 const CTimestampType* ts_type


cdef class FixedSizeBinaryType(DataType):
 cdef:
 const CFixedSizeBinaryType* fixed_size_binary_type


cdef class DecimalType(FixedSizeBinaryType):
 cdef:
 const CDecimalType* decimal_type


cdef class Field:
 cdef:
 shared_ptr[CField] sp_field
 CField* field

 cdef readonly:
 DataType type

 cdef init(self, const shared_ptr[CField]& field)


cdef class Schema:
 cdef:
 shared_ptr[CSchema] sp_schema
 CSchema* schema

 cdef init(self, const vector[shared_ptr[CField]]& fields)
 cdef init_schema(self, const shared_ptr[CSchema]& schema)


cdef class Scalar:
 cdef readonly:
 DataType type


cdef class NAType(Scalar):
 pass


cdef class ArrayValue(Scalar):
 cdef:
 shared_ptr[CArray] sp_array
 int64_t index

 cdef void init(self, DataType type,
 const shared_ptr[CArray]& sp_array, int64_t index)

 cdef void _set_array(self, const shared_ptr[CArray]& sp_array)


cdef class Int8Value(ArrayValue):
 pass


cdef class Int64Value(ArrayValue):
 pass


cdef class ListValue(ArrayValue):
 cdef readonly:
 DataType value_type

 cdef:
 CListArray* ap

 cdef getitem(self, int64_t i)


cdef class StringValue(ArrayValue):
 pass


cdef class FixedSizeBinaryValue(ArrayValue):
 pass


cdef class Array:
 cdef:
 shared_ptr[CArray] sp_array
Expand All@@ -52,10 +141,6 @@ cdef class Tensor:
 cdef init(self, const shared_ptr[CTensor]& sp_tensor)


cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)


cdef class BooleanArray(Array):
 pass

Expand DownExpand Up@@ -137,5 +222,12 @@ cdef class DictionaryArray(Array):
 object _indices, _dictionary



cdef wrap_array_output(PyObject* output)
cdef DataType box_data_type(const shared_ptr[CDataType]& type)
cdef Field box_field(const shared_ptr[CField]& field)
cdef Schema box_schema(const shared_ptr[CSchema]& schema)
cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
cdef object box_scalar(DataType type,
 const shared_ptr[CArray]& sp_array,
 int64_t index)
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-751: [Python] Make all Cython modules private. Some code tidying#533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-751: [Python] Make all Cython modules private. Some code tidying
#533
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
1074e7c

Make all Cython modules private. Code cleaning


wesm Apr 12, 2017
0f46116

Fix APIs in Parquet


wesm Apr 12, 2017
63b479b

jemalloc is now private


wesm Apr 13, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
ci/travis_script_python.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -115,7 +115,7 @@ python_version_tests() {
 python setup.py build_ext --inplace --with-parquet --with-jemalloc

 python -c "import pyarrow.parquet"
 python -c "import pyarrow.jemalloc"
 python -c "import pyarrow._jemalloc"

 python -m pytest -vv -r sxX pyarrow

Expand Down

16 changes: 7 additions & 9 deletions
python/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -261,14 +261,12 @@ if (UNIX)
endif()

set(CYTHON_EXTENSIONS
 array
 config
 error
 io
 memory
 scalar
 schema
 table
 _array
 _config
 _error
 _io
 _memory
 _table
)

set(LINK_LIBS
Expand DownExpand Up@@ -313,7 +311,7 @@ if (PYARROW_BUILD_JEMALLOC)
 arrow_jemalloc_shared)
 set(CYTHON_EXTENSIONS
 ${CYTHON_EXTENSIONS}
jemalloc)
_jemalloc)
endif()

############################################################
Expand Down

84 changes: 40 additions & 44 deletions
python/pyarrow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,60 +25,56 @@
 pass


import pyarrow.config
from pyarrow.config import cpu_count, set_cpu_count
import pyarrow._config
from pyarrow._config import cpu_count, set_cpu_count

from pyarrow.array import (Array, Tensor, from_pylist,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray)

from pyarrow.error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter

from pyarrow.memory import MemoryPool, total_allocated_bytes

from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

import pyarrow.schema as _schema

from pyarrow.schema import (null, bool_,
from pyarrow._array import (null, bool_,
 int8, int16, int32, int64,
 uint8, uint16, uint32, uint64,
 timestamp, date32, date64,
 float16, float32, float64,
 binary, string, decimal,
 list_, struct, dictionary, field,
 DataType, FixedSizeBinaryType,
 Field, Schema, schema)
 Field, Schema, schema,
 Array, Tensor,
 from_pylist,
 from_numpy_dtype,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray,
 ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 OSFile, MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow._memory import MemoryPool, total_allocated_bytes
from pyarrow._table import Column, RecordBatch, Table, concat_tables
from pyarrow._error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.table import Column, RecordBatch, Table, concat_tables
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter


localfs = LocalFilesystem.get_instance()

116 changes: 104 additions & 12 deletions
python/pyarrow/array.pxd → python/pyarrow/_array.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,20 +15,109 @@
# specific language governing permissions and limitations
# under the License.

from pyarrow.includes.common cimport shared_ptr, int64_t
from pyarrow.includes.libarrow cimport CArray, CTensor

from pyarrow.scalar import NA

from pyarrow.schema cimport DataType
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *

from cpython cimport PyObject


cdef extern from "Python.h":
 int PySlice_Check(object)


cdef class DataType:
 cdef:
 shared_ptr[CDataType] sp_type
 CDataType* type

 cdef void init(self, const shared_ptr[CDataType]& type)


cdef class DictionaryType(DataType):
 cdef:
 const CDictionaryType* dict_type


cdef class TimestampType(DataType):
 cdef:
 const CTimestampType* ts_type


cdef class FixedSizeBinaryType(DataType):
 cdef:
 const CFixedSizeBinaryType* fixed_size_binary_type


cdef class DecimalType(FixedSizeBinaryType):
 cdef:
 const CDecimalType* decimal_type


cdef class Field:
 cdef:
 shared_ptr[CField] sp_field
 CField* field

 cdef readonly:
 DataType type

 cdef init(self, const shared_ptr[CField]& field)


cdef class Schema:
 cdef:
 shared_ptr[CSchema] sp_schema
 CSchema* schema

 cdef init(self, const vector[shared_ptr[CField]]& fields)
 cdef init_schema(self, const shared_ptr[CSchema]& schema)


cdef class Scalar:
 cdef readonly:
 DataType type


cdef class NAType(Scalar):
 pass


cdef class ArrayValue(Scalar):
 cdef:
 shared_ptr[CArray] sp_array
 int64_t index

 cdef void init(self, DataType type,
 const shared_ptr[CArray]& sp_array, int64_t index)

 cdef void _set_array(self, const shared_ptr[CArray]& sp_array)


cdef class Int8Value(ArrayValue):
 pass


cdef class Int64Value(ArrayValue):
 pass


cdef class ListValue(ArrayValue):
 cdef readonly:
 DataType value_type

 cdef:
 CListArray* ap

 cdef getitem(self, int64_t i)


cdef class StringValue(ArrayValue):
 pass


cdef class FixedSizeBinaryValue(ArrayValue):
 pass


cdef class Array:
 cdef:
 shared_ptr[CArray] sp_array
Expand All@@ -52,10 +141,6 @@ cdef class Tensor:
 cdef init(self, const shared_ptr[CTensor]& sp_tensor)


cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)


cdef class BooleanArray(Array):
 pass

Expand DownExpand Up@@ -137,5 +222,12 @@ cdef class DictionaryArray(Array):
 object _indices, _dictionary



cdef wrap_array_output(PyObject* output)
cdef DataType box_data_type(const shared_ptr[CDataType]& type)
cdef Field box_field(const shared_ptr[CField]& field)
cdef Schema box_schema(const shared_ptr[CSchema]& schema)
cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
cdef object box_scalar(DataType type,
 const shared_ptr[CArray]& sp_array,
 int64_t index)
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-751: [Python] Make all Cython modules private. Some code tidying#533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-751: [Python] Make all Cython modules private. Some code tidying
#533
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
1074e7c

Make all Cython modules private. Code cleaning


wesm Apr 12, 2017
0f46116

Fix APIs in Parquet


wesm Apr 12, 2017
63b479b

jemalloc is now private


wesm Apr 13, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
ci/travis_script_python.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -115,7 +115,7 @@ python_version_tests() {
 python setup.py build_ext --inplace --with-parquet --with-jemalloc

 python -c "import pyarrow.parquet"
 python -c "import pyarrow.jemalloc"
 python -c "import pyarrow._jemalloc"

 python -m pytest -vv -r sxX pyarrow

Expand Down

16 changes: 7 additions & 9 deletions
python/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -261,14 +261,12 @@ if (UNIX)
endif()

set(CYTHON_EXTENSIONS
 array
 config
 error
 io
 memory
 scalar
 schema
 table
 _array
 _config
 _error
 _io
 _memory
 _table
)

set(LINK_LIBS
Expand DownExpand Up@@ -313,7 +311,7 @@ if (PYARROW_BUILD_JEMALLOC)
 arrow_jemalloc_shared)
 set(CYTHON_EXTENSIONS
 ${CYTHON_EXTENSIONS}
jemalloc)
_jemalloc)
endif()

############################################################
Expand Down

84 changes: 40 additions & 44 deletions
python/pyarrow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,60 +25,56 @@
 pass


import pyarrow.config
from pyarrow.config import cpu_count, set_cpu_count
import pyarrow._config
from pyarrow._config import cpu_count, set_cpu_count

from pyarrow.array import (Array, Tensor, from_pylist,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray)

from pyarrow.error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter

from pyarrow.memory import MemoryPool, total_allocated_bytes

from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

import pyarrow.schema as _schema

from pyarrow.schema import (null, bool_,
from pyarrow._array import (null, bool_,
 int8, int16, int32, int64,
 uint8, uint16, uint32, uint64,
 timestamp, date32, date64,
 float16, float32, float64,
 binary, string, decimal,
 list_, struct, dictionary, field,
 DataType, FixedSizeBinaryType,
 Field, Schema, schema)
 Field, Schema, schema,
 Array, Tensor,
 from_pylist,
 from_numpy_dtype,
 NumericArray, IntegerArray, FloatingPointArray,
 BooleanArray,
 Int8Array, UInt8Array,
 Int16Array, UInt16Array,
 Int32Array, UInt32Array,
 Int64Array, UInt64Array,
 ListArray, StringArray,
 DictionaryArray,
 ArrayValue, Scalar, NA, NAType,
 BooleanValue,
 Int8Value, Int16Value, Int32Value, Int64Value,
 UInt8Value, UInt16Value, UInt32Value, UInt64Value,
 FloatValue, DoubleValue, ListValue,
 BinaryValue, StringValue, FixedSizeBinaryValue)

from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
 Buffer, BufferReader, InMemoryOutputStream,
 OSFile, MemoryMappedFile, memory_map,
 frombuffer, read_tensor, write_tensor,
 memory_map, create_memory_map,
 get_record_batch_size, get_tensor_size)

from pyarrow._memory import MemoryPool, total_allocated_bytes
from pyarrow._table import Column, RecordBatch, Table, concat_tables
from pyarrow._error import (ArrowException,
 ArrowKeyError,
 ArrowInvalid,
 ArrowIOError,
 ArrowMemoryError,
 ArrowNotImplementedError,
 ArrowTypeError)

from pyarrow.table import Column, RecordBatch, Table, concat_tables
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem

from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter


localfs = LocalFilesystem.get_instance()

116 changes: 104 additions & 12 deletions
python/pyarrow/array.pxd → python/pyarrow/_array.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,20 +15,109 @@
# specific language governing permissions and limitations
# under the License.

from pyarrow.includes.common cimport shared_ptr, int64_t
from pyarrow.includes.libarrow cimport CArray, CTensor

from pyarrow.scalar import NA

from pyarrow.schema cimport DataType
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *

from cpython cimport PyObject


cdef extern from "Python.h":
 int PySlice_Check(object)


cdef class DataType:
 cdef:
 shared_ptr[CDataType] sp_type
 CDataType* type

 cdef void init(self, const shared_ptr[CDataType]& type)


cdef class DictionaryType(DataType):
 cdef:
 const CDictionaryType* dict_type


cdef class TimestampType(DataType):
 cdef:
 const CTimestampType* ts_type


cdef class FixedSizeBinaryType(DataType):
 cdef:
 const CFixedSizeBinaryType* fixed_size_binary_type


cdef class DecimalType(FixedSizeBinaryType):
 cdef:
 const CDecimalType* decimal_type


cdef class Field:
 cdef:
 shared_ptr[CField] sp_field
 CField* field

 cdef readonly:
 DataType type

 cdef init(self, const shared_ptr[CField]& field)


cdef class Schema:
 cdef:
 shared_ptr[CSchema] sp_schema
 CSchema* schema

 cdef init(self, const vector[shared_ptr[CField]]& fields)
 cdef init_schema(self, const shared_ptr[CSchema]& schema)


cdef class Scalar:
 cdef readonly:
 DataType type


cdef class NAType(Scalar):
 pass


cdef class ArrayValue(Scalar):
 cdef:
 shared_ptr[CArray] sp_array
 int64_t index

 cdef void init(self, DataType type,
 const shared_ptr[CArray]& sp_array, int64_t index)

 cdef void _set_array(self, const shared_ptr[CArray]& sp_array)


cdef class Int8Value(ArrayValue):
 pass


cdef class Int64Value(ArrayValue):
 pass


cdef class ListValue(ArrayValue):
 cdef readonly:
 DataType value_type

 cdef:
 CListArray* ap

 cdef getitem(self, int64_t i)


cdef class StringValue(ArrayValue):
 pass


cdef class FixedSizeBinaryValue(ArrayValue):
 pass


cdef class Array:
 cdef:
 shared_ptr[CArray] sp_array
Expand All@@ -52,10 +141,6 @@ cdef class Tensor:
 cdef init(self, const shared_ptr[CTensor]& sp_tensor)


cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)


cdef class BooleanArray(Array):
 pass

Expand DownExpand Up@@ -137,5 +222,12 @@ cdef class DictionaryArray(Array):
 object _indices, _dictionary



cdef wrap_array_output(PyObject* output)
cdef DataType box_data_type(const shared_ptr[CDataType]& type)
cdef Field box_field(const shared_ptr[CField]& field)
cdef Schema box_schema(const shared_ptr[CSchema]& schema)
cdef object box_array(const shared_ptr[CArray]& sp_array)
cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
cdef object box_scalar(DataType type,
 const shared_ptr[CArray]& sp_array,
 int64_t index)
Loading