ARROW-510: Integration tests for all date and time combinations#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wesm wants to merge 2 commits into from
Closed

ARROW-510: Integration tests for all date and time combinations #458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
92cabf6
Integration tests for all date and time combinations
wesm Mar 30, 2017
06c4eed
Set bit width in time metadata, check on read
wesm Apr 1, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions cpp/src/arrow/ipc/json-internal.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -175,6 +175,8 @@ class JsonSchemaWriter {
void WriteTypeMetadata(const TimeType& type) {
writer_->Key("unit");
writer_->String(GetTimeUnitName(type.unit));
writer_->Key("bitWidth");
writer_->Int(type.bit_width());
}

void WriteTypeMetadata(const DateType& type) {
Expand DownExpand Up@@ -608,6 +610,9 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
const auto& json_unit = json_type.FindMember("unit");
RETURN_NOT_STRING("unit", json_unit, json_type);

const auto& json_bit_width = json_type.FindMember("bitWidth");
RETURN_NOT_INT("bitWidth", json_bit_width, json_type);

std::string unit_str = json_unit->value.GetString();

if (unit_str == "SECOND") {
Expand All@@ -623,6 +628,14 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
ss << "Invalid time unit: " << unit_str;
return Status::Invalid(ss.str());
}

const auto& fw_type = static_cast<const FixedWidthType&>(**type);

int bit_width = json_bit_width->value.GetInt();
if (bit_width != fw_type.bit_width()) {
return Status::Invalid("Indicated bit width does not match unit");
}

return Status::OK();
}

Expand Down
11 changes: 9 additions & 2 deletions cpp/src/arrow/ipc/metadata.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -255,12 +255,19 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
case flatbuf::Type_Time: {
auto time_type = static_cast<const flatbuf::Time*>(type_data);
TimeUnit unit = FromFlatbufferUnit(time_type->unit());
int32_t bit_width = time_type->bitWidth();
switch (unit) {
case TimeUnit::SECOND:
case TimeUnit::MILLI:
if (bit_width != 32) {
return Status::Invalid("Time is 32 bits for second/milli unit");
}
*out = time32(unit);
break;
default:
if (bit_width != 64) {
return Status::Invalid("Time is 64 bits for micro/nano unit");
}
*out = time64(unit);
break;
}
Expand DownExpand Up@@ -386,12 +393,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
case Type::TIME32: {
const auto& time_type = static_cast<const Time32Type&>(*type);
*out_type = flatbuf::Type_Time;
*offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
*offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
} break;
case Type::TIME64: {
const auto& time_type = static_cast<const Time64Type&>(*type);
*out_type = flatbuf::Type_Time;
*offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
*offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
} break;
case Type::TIMESTAMP: {
const auto& ts_type = static_cast<const TimestampType&>(*type);
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/arrow/type-test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -207,6 +207,9 @@ TEST(TestTimeType, Equals) {
Time64Type t4(TimeUnit::NANO);
Time64Type t5(TimeUnit::MICRO);

ASSERT_EQ(32, t0.bit_width());
ASSERT_EQ(64, t3.bit_width());

ASSERT_TRUE(t0.Equals(t2));
ASSERT_TRUE(t1.Equals(t1));
ASSERT_FALSE(t1.Equals(t3));
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/type.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -509,7 +509,7 @@ struct ARROW_EXPORT Time32Type : public TimeType {
static constexpr Type::type type_id = Type::TIME32;
using c_type = int32_t;

int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }

explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);

Expand Down
145 changes: 113 additions & 32 deletions integration/integration_test.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -175,10 +175,14 @@ def _get_buffers(self):

class IntegerType(PrimitiveType):

def __init__(self, name, is_signed, bit_width, nullable=True):
def __init__(self, name, is_signed, bit_width, nullable=True,
min_value=TEST_INT_MIN,
max_value=TEST_INT_MAX):
PrimitiveType.__init__(self, name, nullable=nullable)
self.is_signed = is_signed
self.bit_width = bit_width
self.min_value = min_value
self.max_value = max_value

@property
def numpy_type(self):
Expand All@@ -194,14 +198,80 @@ def _get_type(self):
def generate_column(self, size):
iinfo = np.iinfo(self.numpy_type)
values = [int(x) for x in
np.random.randint(max(iinfo.min, TEST_INT_MIN),
min(iinfo.max, TEST_INT_MAX),
np.random.randint(max(iinfo.min, self.min_value),
min(iinfo.max, self.max_value),
size=size)]

is_valid = self._make_is_valid(size)
return PrimitiveColumn(self.name, size, is_valid, values)


class DateType(IntegerType):

DAY = 0
MILLISECOND = 1

def __init__(self, name, unit, nullable=True):
self.unit = unit
bit_width = 32 if unit == self.DAY else 64
IntegerType.__init__(self, name, True, bit_width, nullable=nullable)

def _get_type(self):
return OrderedDict([
('name', 'date'),
('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
])


TIMEUNIT_NAMES = {
's': 'SECOND',
'ms': 'MILLISECOND',
'us': 'MICROSECOND',
'ns': 'NANOSECOND'
}


class TimeType(IntegerType):

BIT_WIDTHS = {
's': 32,
'ms': 32,
'us': 64,
'ns': 64
}

def __init__(self, name, unit='s', nullable=True):
self.unit = unit
IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
nullable=nullable)

def _get_type(self):
return OrderedDict([
('name', 'time'),
('unit', TIMEUNIT_NAMES[self.unit]),
('bitWidth', self.bit_width)
])


class TimestampType(IntegerType):

def __init__(self, name, unit='s', tz=None, nullable=True):
self.unit = unit
self.tz = tz
IntegerType.__init__(self, name, True, 64, nullable=nullable)

def _get_type(self):
fields = [
('name', 'timestamp'),
('unit', TIMEUNIT_NAMES[self.unit])
]

if self.tz is not None:
fields.append(('timezone', self.tz))

return OrderedDict(fields)


class FloatingPointType(PrimitiveType):

def __init__(self, name, bit_width, nullable=True):
Expand DownExpand Up@@ -509,6 +579,20 @@ def get_field(name, type_, nullable=True):
raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
schema = JSONSchema(fields)
batches = []
for size in batch_sizes:
columns = []
for field in fields:
col = field.generate_column(size)
columns.append(col)

batches.append(JSONRecordBatch(size, columns))

return JSONFile(schema, batches)


def generate_primitive_case():
types = ['bool', 'int8', 'int16', 'int32', 'int64',
'uint8', 'uint16', 'uint32', 'uint64',
Expand All@@ -520,19 +604,27 @@ def generate_primitive_case():
fields.append(get_field(type_ + "_nullable", type_, True))
fields.append(get_field(type_ + "_nonnullable", type_, False))

schema = JSONSchema(fields)

batch_sizes = [7, 10]
batches = []
for size in batch_sizes:
columns = []
for field in fields:
col = field.generate_column(size)
columns.append(col)
return _generate_file(fields, batch_sizes)

batches.append(JSONRecordBatch(size, columns))

return JSONFile(schema, batches)
def generate_datetime_case():
fields = [
DateType('f0', DateType.DAY),
DateType('f1', DateType.MILLISECOND),
TimeType('f2', 's'),
TimeType('f3', 'ms'),
TimeType('f4', 'us'),
TimeType('f5', 'ns'),
TimestampType('f6', 's'),
TimestampType('f7', 'ms'),
TimestampType('f8', 'us'),
TimestampType('f9', 'ns'),
TimestampType('f10', 'ms', tz='America/New_York')
]

batch_sizes = [7, 10]
return _generate_file(fields, batch_sizes)


def generate_nested_case():
Expand All@@ -545,19 +637,8 @@ def generate_nested_case():
# ListType('list_nonnullable', get_field('item', 'int32'), False),
]

schema = JSONSchema(fields)

batch_sizes = [7, 10]
batches = []
for size in batch_sizes:
columns = []
for field in fields:
col = field.generate_column(size)
columns.append(col)

batches.append(JSONRecordBatch(size, columns))

return JSONFile(schema, batches)
return _generate_file(fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -566,13 +647,13 @@ def get_generated_json_files():
def _temp_path():
return

file_objs = []

K = 10
for i in range(K):
file_objs.append(generate_primitive_case())

file_objs.append(generate_nested_case())
file_objs = [
generate_primitive_case(),
generate_primitive_case(),
generate_primitive_case(),
# generate_datetime_case(),
generate_nested_case()
]

generated_paths = []
for file_obj in file_objs:
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-510: Integration tests for all date and time combinations#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
2
commits into

from

 Closed


ARROW-510: Integration tests for all date and time combinations
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
92cabf6

Integration tests for all date and time combinations


wesm Mar 30, 2017
06c4eed

Set bit width in time metadata, check on read


wesm Apr 1, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
cpp/src/arrow/ipc/json-internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,6 +175,8 @@ class JsonSchemaWriter {
 void WriteTypeMetadata(const TimeType& type) {
 writer_->Key("unit");
 writer_->String(GetTimeUnitName(type.unit));
 writer_->Key("bitWidth");
 writer_->Int(type.bit_width());
 }

 void WriteTypeMetadata(const DateType& type) {
Expand DownExpand Up@@ -608,6 +610,9 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 const auto& json_unit = json_type.FindMember("unit");
 RETURN_NOT_STRING("unit", json_unit, json_type);

 const auto& json_bit_width = json_type.FindMember("bitWidth");
 RETURN_NOT_INT("bitWidth", json_bit_width, json_type);

 std::string unit_str = json_unit->value.GetString();

 if (unit_str == "SECOND") {
Expand All@@ -623,6 +628,14 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 ss << "Invalid time unit: " << unit_str;
 return Status::Invalid(ss.str());
 }

 const auto& fw_type = static_cast<const FixedWidthType&>(**type);

 int bit_width = json_bit_width->value.GetInt();
 if (bit_width != fw_type.bit_width()) {
 return Status::Invalid("Indicated bit width does not match unit");
 }

 return Status::OK();
}

Expand Down

11 changes: 9 additions & 2 deletions
cpp/src/arrow/ipc/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -255,12 +255,19 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 case flatbuf::Type_Time: {
 auto time_type = static_cast<const flatbuf::Time*>(type_data);
 TimeUnit unit = FromFlatbufferUnit(time_type->unit());
 int32_t bit_width = time_type->bitWidth();
 switch (unit) {
 case TimeUnit::SECOND:
 case TimeUnit::MILLI:
 if (bit_width != 32) {
 return Status::Invalid("Time is 32 bits for second/milli unit");
 }
 *out = time32(unit);
 break;
 default:
 if (bit_width != 64) {
 return Status::Invalid("Time is 64 bits for micro/nano unit");
 }
 *out = time64(unit);
 break;
 }
Expand DownExpand Up@@ -386,12 +393,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 case Type::TIME32: {
 const auto& time_type = static_cast<const Time32Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
 } break;
 case Type::TIME64: {
 const auto& time_type = static_cast<const Time64Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
 } break;
 case Type::TIMESTAMP: {
 const auto& ts_type = static_cast<const TimestampType&>(*type);
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/type-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,6 +207,9 @@ TEST(TestTimeType, Equals) {
 Time64Type t4(TimeUnit::NANO);
 Time64Type t5(TimeUnit::MICRO);

 ASSERT_EQ(32, t0.bit_width());
 ASSERT_EQ(64, t3.bit_width());

 ASSERT_TRUE(t0.Equals(t2));
 ASSERT_TRUE(t1.Equals(t1));
 ASSERT_FALSE(t1.Equals(t3));
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/type.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -509,7 +509,7 @@ struct ARROW_EXPORT Time32Type : public TimeType {
 static constexpr Type::type type_id = Type::TIME32;
 using c_type = int32_t;

 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }

 explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);

Expand Down

145 changes: 113 additions & 32 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,10 +175,14 @@ def _get_buffers(self):

class IntegerType(PrimitiveType):

 def __init__(self, name, is_signed, bit_width, nullable=True):
 def __init__(self, name, is_signed, bit_width, nullable=True,
 min_value=TEST_INT_MIN,
 max_value=TEST_INT_MAX):
 PrimitiveType.__init__(self, name, nullable=nullable)
 self.is_signed = is_signed
 self.bit_width = bit_width
 self.min_value = min_value
 self.max_value = max_value

 @property
 def numpy_type(self):
Expand All@@ -194,14 +198,80 @@ def _get_type(self):
 def generate_column(self, size):
 iinfo = np.iinfo(self.numpy_type)
 values = [int(x) for x in
 np.random.randint(max(iinfo.min, TEST_INT_MIN),
 min(iinfo.max, TEST_INT_MAX),
 np.random.randint(max(iinfo.min, self.min_value),
 min(iinfo.max, self.max_value),
 size=size)]

 is_valid = self._make_is_valid(size)
 return PrimitiveColumn(self.name, size, is_valid, values)


class DateType(IntegerType):

 DAY = 0
 MILLISECOND = 1

 def __init__(self, name, unit, nullable=True):
 self.unit = unit
 bit_width = 32 if unit == self.DAY else 64
 IntegerType.__init__(self, name, True, bit_width, nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'date'),
 ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
 ])


TIMEUNIT_NAMES = {
 's': 'SECOND',
 'ms': 'MILLISECOND',
 'us': 'MICROSECOND',
 'ns': 'NANOSECOND'
}


class TimeType(IntegerType):

 BIT_WIDTHS = {
 's': 32,
 'ms': 32,
 'us': 64,
 'ns': 64
 }

 def __init__(self, name, unit='s', nullable=True):
 self.unit = unit
 IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
 nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'time'),
 ('unit', TIMEUNIT_NAMES[self.unit]),
 ('bitWidth', self.bit_width)
 ])


class TimestampType(IntegerType):

 def __init__(self, name, unit='s', tz=None, nullable=True):
 self.unit = unit
 self.tz = tz
 IntegerType.__init__(self, name, True, 64, nullable=nullable)

 def _get_type(self):
 fields = [
 ('name', 'timestamp'),
 ('unit', TIMEUNIT_NAMES[self.unit])
 ]

 if self.tz is not None:
 fields.append(('timezone', self.tz))

 return OrderedDict(fields)


class FloatingPointType(PrimitiveType):

 def __init__(self, name, bit_width, nullable=True):
Expand DownExpand Up@@ -509,6 +579,20 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)


def generate_primitive_case():
 types = ['bool', 'int8', 'int16', 'int32', 'int64',
 'uint8', 'uint16', 'uint32', 'uint64',
Expand All@@ -520,19 +604,27 @@ def generate_primitive_case():
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)
 return _generate_file(fields, batch_sizes)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
def generate_datetime_case():
 fields = [
 DateType('f0', DateType.DAY),
 DateType('f1', DateType.MILLISECOND),
 TimeType('f2', 's'),
 TimeType('f3', 'ms'),
 TimeType('f4', 'us'),
 TimeType('f5', 'ns'),
 TimestampType('f6', 's'),
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)


def generate_nested_case():
Expand All@@ -545,19 +637,8 @@ def generate_nested_case():
 # ListType('list_nonnullable', get_field('item', 'int32'), False),
 ]

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return _generate_file(fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -566,13 +647,13 @@ def get_generated_json_files():
 def _temp_path():
 return

 file_objs = []

K = 10
for i in range(K):
file_objs.append(generate_primitive_case())

file_objs.append(generate_nested_case())
 file_objs = [
 generate_primitive_case(),
 generate_primitive_case(),
 generate_primitive_case(),
# generate_datetime_case(),
 generate_nested_case()
]

 generated_paths = []
 for file_obj in file_objs:
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-510: Integration tests for all date and time combinations#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
2
commits into

from

 Closed


ARROW-510: Integration tests for all date and time combinations
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
92cabf6

Integration tests for all date and time combinations


wesm Mar 30, 2017
06c4eed

Set bit width in time metadata, check on read


wesm Apr 1, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
cpp/src/arrow/ipc/json-internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,6 +175,8 @@ class JsonSchemaWriter {
 void WriteTypeMetadata(const TimeType& type) {
 writer_->Key("unit");
 writer_->String(GetTimeUnitName(type.unit));
 writer_->Key("bitWidth");
 writer_->Int(type.bit_width());
 }

 void WriteTypeMetadata(const DateType& type) {
Expand DownExpand Up@@ -608,6 +610,9 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 const auto& json_unit = json_type.FindMember("unit");
 RETURN_NOT_STRING("unit", json_unit, json_type);

 const auto& json_bit_width = json_type.FindMember("bitWidth");
 RETURN_NOT_INT("bitWidth", json_bit_width, json_type);

 std::string unit_str = json_unit->value.GetString();

 if (unit_str == "SECOND") {
Expand All@@ -623,6 +628,14 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 ss << "Invalid time unit: " << unit_str;
 return Status::Invalid(ss.str());
 }

 const auto& fw_type = static_cast<const FixedWidthType&>(**type);

 int bit_width = json_bit_width->value.GetInt();
 if (bit_width != fw_type.bit_width()) {
 return Status::Invalid("Indicated bit width does not match unit");
 }

 return Status::OK();
}

Expand Down

11 changes: 9 additions & 2 deletions
cpp/src/arrow/ipc/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -255,12 +255,19 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 case flatbuf::Type_Time: {
 auto time_type = static_cast<const flatbuf::Time*>(type_data);
 TimeUnit unit = FromFlatbufferUnit(time_type->unit());
 int32_t bit_width = time_type->bitWidth();
 switch (unit) {
 case TimeUnit::SECOND:
 case TimeUnit::MILLI:
 if (bit_width != 32) {
 return Status::Invalid("Time is 32 bits for second/milli unit");
 }
 *out = time32(unit);
 break;
 default:
 if (bit_width != 64) {
 return Status::Invalid("Time is 64 bits for micro/nano unit");
 }
 *out = time64(unit);
 break;
 }
Expand DownExpand Up@@ -386,12 +393,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 case Type::TIME32: {
 const auto& time_type = static_cast<const Time32Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
 } break;
 case Type::TIME64: {
 const auto& time_type = static_cast<const Time64Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
 } break;
 case Type::TIMESTAMP: {
 const auto& ts_type = static_cast<const TimestampType&>(*type);
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/type-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,6 +207,9 @@ TEST(TestTimeType, Equals) {
 Time64Type t4(TimeUnit::NANO);
 Time64Type t5(TimeUnit::MICRO);

 ASSERT_EQ(32, t0.bit_width());
 ASSERT_EQ(64, t3.bit_width());

 ASSERT_TRUE(t0.Equals(t2));
 ASSERT_TRUE(t1.Equals(t1));
 ASSERT_FALSE(t1.Equals(t3));
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/type.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -509,7 +509,7 @@ struct ARROW_EXPORT Time32Type : public TimeType {
 static constexpr Type::type type_id = Type::TIME32;
 using c_type = int32_t;

 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }

 explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);

Expand Down

145 changes: 113 additions & 32 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,10 +175,14 @@ def _get_buffers(self):

class IntegerType(PrimitiveType):

 def __init__(self, name, is_signed, bit_width, nullable=True):
 def __init__(self, name, is_signed, bit_width, nullable=True,
 min_value=TEST_INT_MIN,
 max_value=TEST_INT_MAX):
 PrimitiveType.__init__(self, name, nullable=nullable)
 self.is_signed = is_signed
 self.bit_width = bit_width
 self.min_value = min_value
 self.max_value = max_value

 @property
 def numpy_type(self):
Expand All@@ -194,14 +198,80 @@ def _get_type(self):
 def generate_column(self, size):
 iinfo = np.iinfo(self.numpy_type)
 values = [int(x) for x in
 np.random.randint(max(iinfo.min, TEST_INT_MIN),
 min(iinfo.max, TEST_INT_MAX),
 np.random.randint(max(iinfo.min, self.min_value),
 min(iinfo.max, self.max_value),
 size=size)]

 is_valid = self._make_is_valid(size)
 return PrimitiveColumn(self.name, size, is_valid, values)


class DateType(IntegerType):

 DAY = 0
 MILLISECOND = 1

 def __init__(self, name, unit, nullable=True):
 self.unit = unit
 bit_width = 32 if unit == self.DAY else 64
 IntegerType.__init__(self, name, True, bit_width, nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'date'),
 ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
 ])


TIMEUNIT_NAMES = {
 's': 'SECOND',
 'ms': 'MILLISECOND',
 'us': 'MICROSECOND',
 'ns': 'NANOSECOND'
}


class TimeType(IntegerType):

 BIT_WIDTHS = {
 's': 32,
 'ms': 32,
 'us': 64,
 'ns': 64
 }

 def __init__(self, name, unit='s', nullable=True):
 self.unit = unit
 IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
 nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'time'),
 ('unit', TIMEUNIT_NAMES[self.unit]),
 ('bitWidth', self.bit_width)
 ])


class TimestampType(IntegerType):

 def __init__(self, name, unit='s', tz=None, nullable=True):
 self.unit = unit
 self.tz = tz
 IntegerType.__init__(self, name, True, 64, nullable=nullable)

 def _get_type(self):
 fields = [
 ('name', 'timestamp'),
 ('unit', TIMEUNIT_NAMES[self.unit])
 ]

 if self.tz is not None:
 fields.append(('timezone', self.tz))

 return OrderedDict(fields)


class FloatingPointType(PrimitiveType):

 def __init__(self, name, bit_width, nullable=True):
Expand DownExpand Up@@ -509,6 +579,20 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)


def generate_primitive_case():
 types = ['bool', 'int8', 'int16', 'int32', 'int64',
 'uint8', 'uint16', 'uint32', 'uint64',
Expand All@@ -520,19 +604,27 @@ def generate_primitive_case():
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)
 return _generate_file(fields, batch_sizes)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
def generate_datetime_case():
 fields = [
 DateType('f0', DateType.DAY),
 DateType('f1', DateType.MILLISECOND),
 TimeType('f2', 's'),
 TimeType('f3', 'ms'),
 TimeType('f4', 'us'),
 TimeType('f5', 'ns'),
 TimestampType('f6', 's'),
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)


def generate_nested_case():
Expand All@@ -545,19 +637,8 @@ def generate_nested_case():
 # ListType('list_nonnullable', get_field('item', 'int32'), False),
 ]

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return _generate_file(fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -566,13 +647,13 @@ def get_generated_json_files():
 def _temp_path():
 return

 file_objs = []

K = 10
for i in range(K):
file_objs.append(generate_primitive_case())

file_objs.append(generate_nested_case())
 file_objs = [
 generate_primitive_case(),
 generate_primitive_case(),
 generate_primitive_case(),
# generate_datetime_case(),
 generate_nested_case()
]

 generated_paths = []
 for file_obj in file_objs:
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-510: Integration tests for all date and time combinations#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
2
commits into

from

 Closed


ARROW-510: Integration tests for all date and time combinations
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
92cabf6

Integration tests for all date and time combinations


wesm Mar 30, 2017
06c4eed

Set bit width in time metadata, check on read


wesm Apr 1, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
cpp/src/arrow/ipc/json-internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,6 +175,8 @@ class JsonSchemaWriter {
 void WriteTypeMetadata(const TimeType& type) {
 writer_->Key("unit");
 writer_->String(GetTimeUnitName(type.unit));
 writer_->Key("bitWidth");
 writer_->Int(type.bit_width());
 }

 void WriteTypeMetadata(const DateType& type) {
Expand DownExpand Up@@ -608,6 +610,9 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 const auto& json_unit = json_type.FindMember("unit");
 RETURN_NOT_STRING("unit", json_unit, json_type);

 const auto& json_bit_width = json_type.FindMember("bitWidth");
 RETURN_NOT_INT("bitWidth", json_bit_width, json_type);

 std::string unit_str = json_unit->value.GetString();

 if (unit_str == "SECOND") {
Expand All@@ -623,6 +628,14 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 ss << "Invalid time unit: " << unit_str;
 return Status::Invalid(ss.str());
 }

 const auto& fw_type = static_cast<const FixedWidthType&>(**type);

 int bit_width = json_bit_width->value.GetInt();
 if (bit_width != fw_type.bit_width()) {
 return Status::Invalid("Indicated bit width does not match unit");
 }

 return Status::OK();
}

Expand Down

11 changes: 9 additions & 2 deletions
cpp/src/arrow/ipc/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -255,12 +255,19 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 case flatbuf::Type_Time: {
 auto time_type = static_cast<const flatbuf::Time*>(type_data);
 TimeUnit unit = FromFlatbufferUnit(time_type->unit());
 int32_t bit_width = time_type->bitWidth();
 switch (unit) {
 case TimeUnit::SECOND:
 case TimeUnit::MILLI:
 if (bit_width != 32) {
 return Status::Invalid("Time is 32 bits for second/milli unit");
 }
 *out = time32(unit);
 break;
 default:
 if (bit_width != 64) {
 return Status::Invalid("Time is 64 bits for micro/nano unit");
 }
 *out = time64(unit);
 break;
 }
Expand DownExpand Up@@ -386,12 +393,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 case Type::TIME32: {
 const auto& time_type = static_cast<const Time32Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
 } break;
 case Type::TIME64: {
 const auto& time_type = static_cast<const Time64Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
 } break;
 case Type::TIMESTAMP: {
 const auto& ts_type = static_cast<const TimestampType&>(*type);
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/type-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,6 +207,9 @@ TEST(TestTimeType, Equals) {
 Time64Type t4(TimeUnit::NANO);
 Time64Type t5(TimeUnit::MICRO);

 ASSERT_EQ(32, t0.bit_width());
 ASSERT_EQ(64, t3.bit_width());

 ASSERT_TRUE(t0.Equals(t2));
 ASSERT_TRUE(t1.Equals(t1));
 ASSERT_FALSE(t1.Equals(t3));
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/type.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -509,7 +509,7 @@ struct ARROW_EXPORT Time32Type : public TimeType {
 static constexpr Type::type type_id = Type::TIME32;
 using c_type = int32_t;

 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }

 explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);

Expand Down

145 changes: 113 additions & 32 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,10 +175,14 @@ def _get_buffers(self):

class IntegerType(PrimitiveType):

 def __init__(self, name, is_signed, bit_width, nullable=True):
 def __init__(self, name, is_signed, bit_width, nullable=True,
 min_value=TEST_INT_MIN,
 max_value=TEST_INT_MAX):
 PrimitiveType.__init__(self, name, nullable=nullable)
 self.is_signed = is_signed
 self.bit_width = bit_width
 self.min_value = min_value
 self.max_value = max_value

 @property
 def numpy_type(self):
Expand All@@ -194,14 +198,80 @@ def _get_type(self):
 def generate_column(self, size):
 iinfo = np.iinfo(self.numpy_type)
 values = [int(x) for x in
 np.random.randint(max(iinfo.min, TEST_INT_MIN),
 min(iinfo.max, TEST_INT_MAX),
 np.random.randint(max(iinfo.min, self.min_value),
 min(iinfo.max, self.max_value),
 size=size)]

 is_valid = self._make_is_valid(size)
 return PrimitiveColumn(self.name, size, is_valid, values)


class DateType(IntegerType):

 DAY = 0
 MILLISECOND = 1

 def __init__(self, name, unit, nullable=True):
 self.unit = unit
 bit_width = 32 if unit == self.DAY else 64
 IntegerType.__init__(self, name, True, bit_width, nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'date'),
 ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
 ])


TIMEUNIT_NAMES = {
 's': 'SECOND',
 'ms': 'MILLISECOND',
 'us': 'MICROSECOND',
 'ns': 'NANOSECOND'
}


class TimeType(IntegerType):

 BIT_WIDTHS = {
 's': 32,
 'ms': 32,
 'us': 64,
 'ns': 64
 }

 def __init__(self, name, unit='s', nullable=True):
 self.unit = unit
 IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
 nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'time'),
 ('unit', TIMEUNIT_NAMES[self.unit]),
 ('bitWidth', self.bit_width)
 ])


class TimestampType(IntegerType):

 def __init__(self, name, unit='s', tz=None, nullable=True):
 self.unit = unit
 self.tz = tz
 IntegerType.__init__(self, name, True, 64, nullable=nullable)

 def _get_type(self):
 fields = [
 ('name', 'timestamp'),
 ('unit', TIMEUNIT_NAMES[self.unit])
 ]

 if self.tz is not None:
 fields.append(('timezone', self.tz))

 return OrderedDict(fields)


class FloatingPointType(PrimitiveType):

 def __init__(self, name, bit_width, nullable=True):
Expand DownExpand Up@@ -509,6 +579,20 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)


def generate_primitive_case():
 types = ['bool', 'int8', 'int16', 'int32', 'int64',
 'uint8', 'uint16', 'uint32', 'uint64',
Expand All@@ -520,19 +604,27 @@ def generate_primitive_case():
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)
 return _generate_file(fields, batch_sizes)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
def generate_datetime_case():
 fields = [
 DateType('f0', DateType.DAY),
 DateType('f1', DateType.MILLISECOND),
 TimeType('f2', 's'),
 TimeType('f3', 'ms'),
 TimeType('f4', 'us'),
 TimeType('f5', 'ns'),
 TimestampType('f6', 's'),
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)


def generate_nested_case():
Expand All@@ -545,19 +637,8 @@ def generate_nested_case():
 # ListType('list_nonnullable', get_field('item', 'int32'), False),
 ]

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return _generate_file(fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -566,13 +647,13 @@ def get_generated_json_files():
 def _temp_path():
 return

 file_objs = []

K = 10
for i in range(K):
file_objs.append(generate_primitive_case())

file_objs.append(generate_nested_case())
 file_objs = [
 generate_primitive_case(),
 generate_primitive_case(),
 generate_primitive_case(),
# generate_datetime_case(),
 generate_nested_case()
]

 generated_paths = []
 for file_obj in file_objs:
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-510: Integration tests for all date and time combinations#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
2
commits into

from

 Closed


ARROW-510: Integration tests for all date and time combinations
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
92cabf6

Integration tests for all date and time combinations


wesm Mar 30, 2017
06c4eed

Set bit width in time metadata, check on read


wesm Apr 1, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
cpp/src/arrow/ipc/json-internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,6 +175,8 @@ class JsonSchemaWriter {
 void WriteTypeMetadata(const TimeType& type) {
 writer_->Key("unit");
 writer_->String(GetTimeUnitName(type.unit));
 writer_->Key("bitWidth");
 writer_->Int(type.bit_width());
 }

 void WriteTypeMetadata(const DateType& type) {
Expand DownExpand Up@@ -608,6 +610,9 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 const auto& json_unit = json_type.FindMember("unit");
 RETURN_NOT_STRING("unit", json_unit, json_type);

 const auto& json_bit_width = json_type.FindMember("bitWidth");
 RETURN_NOT_INT("bitWidth", json_bit_width, json_type);

 std::string unit_str = json_unit->value.GetString();

 if (unit_str == "SECOND") {
Expand All@@ -623,6 +628,14 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 ss << "Invalid time unit: " << unit_str;
 return Status::Invalid(ss.str());
 }

 const auto& fw_type = static_cast<const FixedWidthType&>(**type);

 int bit_width = json_bit_width->value.GetInt();
 if (bit_width != fw_type.bit_width()) {
 return Status::Invalid("Indicated bit width does not match unit");
 }

 return Status::OK();
}

Expand Down

11 changes: 9 additions & 2 deletions
cpp/src/arrow/ipc/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -255,12 +255,19 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 case flatbuf::Type_Time: {
 auto time_type = static_cast<const flatbuf::Time*>(type_data);
 TimeUnit unit = FromFlatbufferUnit(time_type->unit());
 int32_t bit_width = time_type->bitWidth();
 switch (unit) {
 case TimeUnit::SECOND:
 case TimeUnit::MILLI:
 if (bit_width != 32) {
 return Status::Invalid("Time is 32 bits for second/milli unit");
 }
 *out = time32(unit);
 break;
 default:
 if (bit_width != 64) {
 return Status::Invalid("Time is 64 bits for micro/nano unit");
 }
 *out = time64(unit);
 break;
 }
Expand DownExpand Up@@ -386,12 +393,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 case Type::TIME32: {
 const auto& time_type = static_cast<const Time32Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
 } break;
 case Type::TIME64: {
 const auto& time_type = static_cast<const Time64Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
 } break;
 case Type::TIMESTAMP: {
 const auto& ts_type = static_cast<const TimestampType&>(*type);
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/type-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,6 +207,9 @@ TEST(TestTimeType, Equals) {
 Time64Type t4(TimeUnit::NANO);
 Time64Type t5(TimeUnit::MICRO);

 ASSERT_EQ(32, t0.bit_width());
 ASSERT_EQ(64, t3.bit_width());

 ASSERT_TRUE(t0.Equals(t2));
 ASSERT_TRUE(t1.Equals(t1));
 ASSERT_FALSE(t1.Equals(t3));
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/type.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -509,7 +509,7 @@ struct ARROW_EXPORT Time32Type : public TimeType {
 static constexpr Type::type type_id = Type::TIME32;
 using c_type = int32_t;

 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }

 explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);

Expand Down

145 changes: 113 additions & 32 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,10 +175,14 @@ def _get_buffers(self):

class IntegerType(PrimitiveType):

 def __init__(self, name, is_signed, bit_width, nullable=True):
 def __init__(self, name, is_signed, bit_width, nullable=True,
 min_value=TEST_INT_MIN,
 max_value=TEST_INT_MAX):
 PrimitiveType.__init__(self, name, nullable=nullable)
 self.is_signed = is_signed
 self.bit_width = bit_width
 self.min_value = min_value
 self.max_value = max_value

 @property
 def numpy_type(self):
Expand All@@ -194,14 +198,80 @@ def _get_type(self):
 def generate_column(self, size):
 iinfo = np.iinfo(self.numpy_type)
 values = [int(x) for x in
 np.random.randint(max(iinfo.min, TEST_INT_MIN),
 min(iinfo.max, TEST_INT_MAX),
 np.random.randint(max(iinfo.min, self.min_value),
 min(iinfo.max, self.max_value),
 size=size)]

 is_valid = self._make_is_valid(size)
 return PrimitiveColumn(self.name, size, is_valid, values)


class DateType(IntegerType):

 DAY = 0
 MILLISECOND = 1

 def __init__(self, name, unit, nullable=True):
 self.unit = unit
 bit_width = 32 if unit == self.DAY else 64
 IntegerType.__init__(self, name, True, bit_width, nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'date'),
 ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
 ])


TIMEUNIT_NAMES = {
 's': 'SECOND',
 'ms': 'MILLISECOND',
 'us': 'MICROSECOND',
 'ns': 'NANOSECOND'
}


class TimeType(IntegerType):

 BIT_WIDTHS = {
 's': 32,
 'ms': 32,
 'us': 64,
 'ns': 64
 }

 def __init__(self, name, unit='s', nullable=True):
 self.unit = unit
 IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
 nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'time'),
 ('unit', TIMEUNIT_NAMES[self.unit]),
 ('bitWidth', self.bit_width)
 ])


class TimestampType(IntegerType):

 def __init__(self, name, unit='s', tz=None, nullable=True):
 self.unit = unit
 self.tz = tz
 IntegerType.__init__(self, name, True, 64, nullable=nullable)

 def _get_type(self):
 fields = [
 ('name', 'timestamp'),
 ('unit', TIMEUNIT_NAMES[self.unit])
 ]

 if self.tz is not None:
 fields.append(('timezone', self.tz))

 return OrderedDict(fields)


class FloatingPointType(PrimitiveType):

 def __init__(self, name, bit_width, nullable=True):
Expand DownExpand Up@@ -509,6 +579,20 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)


def generate_primitive_case():
 types = ['bool', 'int8', 'int16', 'int32', 'int64',
 'uint8', 'uint16', 'uint32', 'uint64',
Expand All@@ -520,19 +604,27 @@ def generate_primitive_case():
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)
 return _generate_file(fields, batch_sizes)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
def generate_datetime_case():
 fields = [
 DateType('f0', DateType.DAY),
 DateType('f1', DateType.MILLISECOND),
 TimeType('f2', 's'),
 TimeType('f3', 'ms'),
 TimeType('f4', 'us'),
 TimeType('f5', 'ns'),
 TimestampType('f6', 's'),
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)


def generate_nested_case():
Expand All@@ -545,19 +637,8 @@ def generate_nested_case():
 # ListType('list_nonnullable', get_field('item', 'int32'), False),
 ]

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return _generate_file(fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -566,13 +647,13 @@ def get_generated_json_files():
 def _temp_path():
 return

 file_objs = []

K = 10
for i in range(K):
file_objs.append(generate_primitive_case())

file_objs.append(generate_nested_case())
 file_objs = [
 generate_primitive_case(),
 generate_primitive_case(),
 generate_primitive_case(),
# generate_datetime_case(),
 generate_nested_case()
]

 generated_paths = []
 for file_obj in file_objs:
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-510: Integration tests for all date and time combinations#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
2
commits into

from

 Closed


ARROW-510: Integration tests for all date and time combinations
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
92cabf6

Integration tests for all date and time combinations


wesm Mar 30, 2017
06c4eed

Set bit width in time metadata, check on read


wesm Apr 1, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
cpp/src/arrow/ipc/json-internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,6 +175,8 @@ class JsonSchemaWriter {
 void WriteTypeMetadata(const TimeType& type) {
 writer_->Key("unit");
 writer_->String(GetTimeUnitName(type.unit));
 writer_->Key("bitWidth");
 writer_->Int(type.bit_width());
 }

 void WriteTypeMetadata(const DateType& type) {
Expand DownExpand Up@@ -608,6 +610,9 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 const auto& json_unit = json_type.FindMember("unit");
 RETURN_NOT_STRING("unit", json_unit, json_type);

 const auto& json_bit_width = json_type.FindMember("bitWidth");
 RETURN_NOT_INT("bitWidth", json_bit_width, json_type);

 std::string unit_str = json_unit->value.GetString();

 if (unit_str == "SECOND") {
Expand All@@ -623,6 +628,14 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 ss << "Invalid time unit: " << unit_str;
 return Status::Invalid(ss.str());
 }

 const auto& fw_type = static_cast<const FixedWidthType&>(**type);

 int bit_width = json_bit_width->value.GetInt();
 if (bit_width != fw_type.bit_width()) {
 return Status::Invalid("Indicated bit width does not match unit");
 }

 return Status::OK();
}

Expand Down

11 changes: 9 additions & 2 deletions
cpp/src/arrow/ipc/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -255,12 +255,19 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 case flatbuf::Type_Time: {
 auto time_type = static_cast<const flatbuf::Time*>(type_data);
 TimeUnit unit = FromFlatbufferUnit(time_type->unit());
 int32_t bit_width = time_type->bitWidth();
 switch (unit) {
 case TimeUnit::SECOND:
 case TimeUnit::MILLI:
 if (bit_width != 32) {
 return Status::Invalid("Time is 32 bits for second/milli unit");
 }
 *out = time32(unit);
 break;
 default:
 if (bit_width != 64) {
 return Status::Invalid("Time is 64 bits for micro/nano unit");
 }
 *out = time64(unit);
 break;
 }
Expand DownExpand Up@@ -386,12 +393,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 case Type::TIME32: {
 const auto& time_type = static_cast<const Time32Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
 } break;
 case Type::TIME64: {
 const auto& time_type = static_cast<const Time64Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
 } break;
 case Type::TIMESTAMP: {
 const auto& ts_type = static_cast<const TimestampType&>(*type);
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/type-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,6 +207,9 @@ TEST(TestTimeType, Equals) {
 Time64Type t4(TimeUnit::NANO);
 Time64Type t5(TimeUnit::MICRO);

 ASSERT_EQ(32, t0.bit_width());
 ASSERT_EQ(64, t3.bit_width());

 ASSERT_TRUE(t0.Equals(t2));
 ASSERT_TRUE(t1.Equals(t1));
 ASSERT_FALSE(t1.Equals(t3));
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/type.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -509,7 +509,7 @@ struct ARROW_EXPORT Time32Type : public TimeType {
 static constexpr Type::type type_id = Type::TIME32;
 using c_type = int32_t;

 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }

 explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);

Expand Down

145 changes: 113 additions & 32 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,10 +175,14 @@ def _get_buffers(self):

class IntegerType(PrimitiveType):

 def __init__(self, name, is_signed, bit_width, nullable=True):
 def __init__(self, name, is_signed, bit_width, nullable=True,
 min_value=TEST_INT_MIN,
 max_value=TEST_INT_MAX):
 PrimitiveType.__init__(self, name, nullable=nullable)
 self.is_signed = is_signed
 self.bit_width = bit_width
 self.min_value = min_value
 self.max_value = max_value

 @property
 def numpy_type(self):
Expand All@@ -194,14 +198,80 @@ def _get_type(self):
 def generate_column(self, size):
 iinfo = np.iinfo(self.numpy_type)
 values = [int(x) for x in
 np.random.randint(max(iinfo.min, TEST_INT_MIN),
 min(iinfo.max, TEST_INT_MAX),
 np.random.randint(max(iinfo.min, self.min_value),
 min(iinfo.max, self.max_value),
 size=size)]

 is_valid = self._make_is_valid(size)
 return PrimitiveColumn(self.name, size, is_valid, values)


class DateType(IntegerType):

 DAY = 0
 MILLISECOND = 1

 def __init__(self, name, unit, nullable=True):
 self.unit = unit
 bit_width = 32 if unit == self.DAY else 64
 IntegerType.__init__(self, name, True, bit_width, nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'date'),
 ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
 ])


TIMEUNIT_NAMES = {
 's': 'SECOND',
 'ms': 'MILLISECOND',
 'us': 'MICROSECOND',
 'ns': 'NANOSECOND'
}


class TimeType(IntegerType):

 BIT_WIDTHS = {
 's': 32,
 'ms': 32,
 'us': 64,
 'ns': 64
 }

 def __init__(self, name, unit='s', nullable=True):
 self.unit = unit
 IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
 nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'time'),
 ('unit', TIMEUNIT_NAMES[self.unit]),
 ('bitWidth', self.bit_width)
 ])


class TimestampType(IntegerType):

 def __init__(self, name, unit='s', tz=None, nullable=True):
 self.unit = unit
 self.tz = tz
 IntegerType.__init__(self, name, True, 64, nullable=nullable)

 def _get_type(self):
 fields = [
 ('name', 'timestamp'),
 ('unit', TIMEUNIT_NAMES[self.unit])
 ]

 if self.tz is not None:
 fields.append(('timezone', self.tz))

 return OrderedDict(fields)


class FloatingPointType(PrimitiveType):

 def __init__(self, name, bit_width, nullable=True):
Expand DownExpand Up@@ -509,6 +579,20 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)


def generate_primitive_case():
 types = ['bool', 'int8', 'int16', 'int32', 'int64',
 'uint8', 'uint16', 'uint32', 'uint64',
Expand All@@ -520,19 +604,27 @@ def generate_primitive_case():
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)
 return _generate_file(fields, batch_sizes)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
def generate_datetime_case():
 fields = [
 DateType('f0', DateType.DAY),
 DateType('f1', DateType.MILLISECOND),
 TimeType('f2', 's'),
 TimeType('f3', 'ms'),
 TimeType('f4', 'us'),
 TimeType('f5', 'ns'),
 TimestampType('f6', 's'),
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)


def generate_nested_case():
Expand All@@ -545,19 +637,8 @@ def generate_nested_case():
 # ListType('list_nonnullable', get_field('item', 'int32'), False),
 ]

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return _generate_file(fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -566,13 +647,13 @@ def get_generated_json_files():
 def _temp_path():
 return

 file_objs = []

K = 10
for i in range(K):
file_objs.append(generate_primitive_case())

file_objs.append(generate_nested_case())
 file_objs = [
 generate_primitive_case(),
 generate_primitive_case(),
 generate_primitive_case(),
# generate_datetime_case(),
 generate_nested_case()
]

 generated_paths = []
 for file_obj in file_objs:
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-510: Integration tests for all date and time combinations#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
2
commits into

from

 Closed


ARROW-510: Integration tests for all date and time combinations
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
92cabf6

Integration tests for all date and time combinations


wesm Mar 30, 2017
06c4eed

Set bit width in time metadata, check on read


wesm Apr 1, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
cpp/src/arrow/ipc/json-internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,6 +175,8 @@ class JsonSchemaWriter {
 void WriteTypeMetadata(const TimeType& type) {
 writer_->Key("unit");
 writer_->String(GetTimeUnitName(type.unit));
 writer_->Key("bitWidth");
 writer_->Int(type.bit_width());
 }

 void WriteTypeMetadata(const DateType& type) {
Expand DownExpand Up@@ -608,6 +610,9 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 const auto& json_unit = json_type.FindMember("unit");
 RETURN_NOT_STRING("unit", json_unit, json_type);

 const auto& json_bit_width = json_type.FindMember("bitWidth");
 RETURN_NOT_INT("bitWidth", json_bit_width, json_type);

 std::string unit_str = json_unit->value.GetString();

 if (unit_str == "SECOND") {
Expand All@@ -623,6 +628,14 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 ss << "Invalid time unit: " << unit_str;
 return Status::Invalid(ss.str());
 }

 const auto& fw_type = static_cast<const FixedWidthType&>(**type);

 int bit_width = json_bit_width->value.GetInt();
 if (bit_width != fw_type.bit_width()) {
 return Status::Invalid("Indicated bit width does not match unit");
 }

 return Status::OK();
}

Expand Down

11 changes: 9 additions & 2 deletions
cpp/src/arrow/ipc/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -255,12 +255,19 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 case flatbuf::Type_Time: {
 auto time_type = static_cast<const flatbuf::Time*>(type_data);
 TimeUnit unit = FromFlatbufferUnit(time_type->unit());
 int32_t bit_width = time_type->bitWidth();
 switch (unit) {
 case TimeUnit::SECOND:
 case TimeUnit::MILLI:
 if (bit_width != 32) {
 return Status::Invalid("Time is 32 bits for second/milli unit");
 }
 *out = time32(unit);
 break;
 default:
 if (bit_width != 64) {
 return Status::Invalid("Time is 64 bits for micro/nano unit");
 }
 *out = time64(unit);
 break;
 }
Expand DownExpand Up@@ -386,12 +393,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 case Type::TIME32: {
 const auto& time_type = static_cast<const Time32Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
 } break;
 case Type::TIME64: {
 const auto& time_type = static_cast<const Time64Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
 } break;
 case Type::TIMESTAMP: {
 const auto& ts_type = static_cast<const TimestampType&>(*type);
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/type-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,6 +207,9 @@ TEST(TestTimeType, Equals) {
 Time64Type t4(TimeUnit::NANO);
 Time64Type t5(TimeUnit::MICRO);

 ASSERT_EQ(32, t0.bit_width());
 ASSERT_EQ(64, t3.bit_width());

 ASSERT_TRUE(t0.Equals(t2));
 ASSERT_TRUE(t1.Equals(t1));
 ASSERT_FALSE(t1.Equals(t3));
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/type.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -509,7 +509,7 @@ struct ARROW_EXPORT Time32Type : public TimeType {
 static constexpr Type::type type_id = Type::TIME32;
 using c_type = int32_t;

 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }

 explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);

Expand Down

145 changes: 113 additions & 32 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,10 +175,14 @@ def _get_buffers(self):

class IntegerType(PrimitiveType):

 def __init__(self, name, is_signed, bit_width, nullable=True):
 def __init__(self, name, is_signed, bit_width, nullable=True,
 min_value=TEST_INT_MIN,
 max_value=TEST_INT_MAX):
 PrimitiveType.__init__(self, name, nullable=nullable)
 self.is_signed = is_signed
 self.bit_width = bit_width
 self.min_value = min_value
 self.max_value = max_value

 @property
 def numpy_type(self):
Expand All@@ -194,14 +198,80 @@ def _get_type(self):
 def generate_column(self, size):
 iinfo = np.iinfo(self.numpy_type)
 values = [int(x) for x in
 np.random.randint(max(iinfo.min, TEST_INT_MIN),
 min(iinfo.max, TEST_INT_MAX),
 np.random.randint(max(iinfo.min, self.min_value),
 min(iinfo.max, self.max_value),
 size=size)]

 is_valid = self._make_is_valid(size)
 return PrimitiveColumn(self.name, size, is_valid, values)


class DateType(IntegerType):

 DAY = 0
 MILLISECOND = 1

 def __init__(self, name, unit, nullable=True):
 self.unit = unit
 bit_width = 32 if unit == self.DAY else 64
 IntegerType.__init__(self, name, True, bit_width, nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'date'),
 ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
 ])


TIMEUNIT_NAMES = {
 's': 'SECOND',
 'ms': 'MILLISECOND',
 'us': 'MICROSECOND',
 'ns': 'NANOSECOND'
}


class TimeType(IntegerType):

 BIT_WIDTHS = {
 's': 32,
 'ms': 32,
 'us': 64,
 'ns': 64
 }

 def __init__(self, name, unit='s', nullable=True):
 self.unit = unit
 IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
 nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'time'),
 ('unit', TIMEUNIT_NAMES[self.unit]),
 ('bitWidth', self.bit_width)
 ])


class TimestampType(IntegerType):

 def __init__(self, name, unit='s', tz=None, nullable=True):
 self.unit = unit
 self.tz = tz
 IntegerType.__init__(self, name, True, 64, nullable=nullable)

 def _get_type(self):
 fields = [
 ('name', 'timestamp'),
 ('unit', TIMEUNIT_NAMES[self.unit])
 ]

 if self.tz is not None:
 fields.append(('timezone', self.tz))

 return OrderedDict(fields)


class FloatingPointType(PrimitiveType):

 def __init__(self, name, bit_width, nullable=True):
Expand DownExpand Up@@ -509,6 +579,20 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)


def generate_primitive_case():
 types = ['bool', 'int8', 'int16', 'int32', 'int64',
 'uint8', 'uint16', 'uint32', 'uint64',
Expand All@@ -520,19 +604,27 @@ def generate_primitive_case():
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)
 return _generate_file(fields, batch_sizes)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
def generate_datetime_case():
 fields = [
 DateType('f0', DateType.DAY),
 DateType('f1', DateType.MILLISECOND),
 TimeType('f2', 's'),
 TimeType('f3', 'ms'),
 TimeType('f4', 'us'),
 TimeType('f5', 'ns'),
 TimestampType('f6', 's'),
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)


def generate_nested_case():
Expand All@@ -545,19 +637,8 @@ def generate_nested_case():
 # ListType('list_nonnullable', get_field('item', 'int32'), False),
 ]

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return _generate_file(fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -566,13 +647,13 @@ def get_generated_json_files():
 def _temp_path():
 return

 file_objs = []

K = 10
for i in range(K):
file_objs.append(generate_primitive_case())

file_objs.append(generate_nested_case())
 file_objs = [
 generate_primitive_case(),
 generate_primitive_case(),
 generate_primitive_case(),
# generate_datetime_case(),
 generate_nested_case()
]

 generated_paths = []
 for file_obj in file_objs:
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-510: Integration tests for all date and time combinations#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
2
commits into

from

 Closed


ARROW-510: Integration tests for all date and time combinations
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
92cabf6

Integration tests for all date and time combinations


wesm Mar 30, 2017
06c4eed

Set bit width in time metadata, check on read


wesm Apr 1, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
cpp/src/arrow/ipc/json-internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,6 +175,8 @@ class JsonSchemaWriter {
 void WriteTypeMetadata(const TimeType& type) {
 writer_->Key("unit");
 writer_->String(GetTimeUnitName(type.unit));
 writer_->Key("bitWidth");
 writer_->Int(type.bit_width());
 }

 void WriteTypeMetadata(const DateType& type) {
Expand DownExpand Up@@ -608,6 +610,9 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 const auto& json_unit = json_type.FindMember("unit");
 RETURN_NOT_STRING("unit", json_unit, json_type);

 const auto& json_bit_width = json_type.FindMember("bitWidth");
 RETURN_NOT_INT("bitWidth", json_bit_width, json_type);

 std::string unit_str = json_unit->value.GetString();

 if (unit_str == "SECOND") {
Expand All@@ -623,6 +628,14 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 ss << "Invalid time unit: " << unit_str;
 return Status::Invalid(ss.str());
 }

 const auto& fw_type = static_cast<const FixedWidthType&>(**type);

 int bit_width = json_bit_width->value.GetInt();
 if (bit_width != fw_type.bit_width()) {
 return Status::Invalid("Indicated bit width does not match unit");
 }

 return Status::OK();
}

Expand Down

11 changes: 9 additions & 2 deletions
cpp/src/arrow/ipc/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -255,12 +255,19 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 case flatbuf::Type_Time: {
 auto time_type = static_cast<const flatbuf::Time*>(type_data);
 TimeUnit unit = FromFlatbufferUnit(time_type->unit());
 int32_t bit_width = time_type->bitWidth();
 switch (unit) {
 case TimeUnit::SECOND:
 case TimeUnit::MILLI:
 if (bit_width != 32) {
 return Status::Invalid("Time is 32 bits for second/milli unit");
 }
 *out = time32(unit);
 break;
 default:
 if (bit_width != 64) {
 return Status::Invalid("Time is 64 bits for micro/nano unit");
 }
 *out = time64(unit);
 break;
 }
Expand DownExpand Up@@ -386,12 +393,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 case Type::TIME32: {
 const auto& time_type = static_cast<const Time32Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
 } break;
 case Type::TIME64: {
 const auto& time_type = static_cast<const Time64Type&>(*type);
 *out_type = flatbuf::Type_Time;
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
 *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
 } break;
 case Type::TIMESTAMP: {
 const auto& ts_type = static_cast<const TimestampType&>(*type);
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/type-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,6 +207,9 @@ TEST(TestTimeType, Equals) {
 Time64Type t4(TimeUnit::NANO);
 Time64Type t5(TimeUnit::MICRO);

 ASSERT_EQ(32, t0.bit_width());
 ASSERT_EQ(64, t3.bit_width());

 ASSERT_TRUE(t0.Equals(t2));
 ASSERT_TRUE(t1.Equals(t1));
 ASSERT_FALSE(t1.Equals(t3));
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/type.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -509,7 +509,7 @@ struct ARROW_EXPORT Time32Type : public TimeType {
 static constexpr Type::type type_id = Type::TIME32;
 using c_type = int32_t;

 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }

 explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);

Expand Down

145 changes: 113 additions & 32 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,10 +175,14 @@ def _get_buffers(self):

class IntegerType(PrimitiveType):

 def __init__(self, name, is_signed, bit_width, nullable=True):
 def __init__(self, name, is_signed, bit_width, nullable=True,
 min_value=TEST_INT_MIN,
 max_value=TEST_INT_MAX):
 PrimitiveType.__init__(self, name, nullable=nullable)
 self.is_signed = is_signed
 self.bit_width = bit_width
 self.min_value = min_value
 self.max_value = max_value

 @property
 def numpy_type(self):
Expand All@@ -194,14 +198,80 @@ def _get_type(self):
 def generate_column(self, size):
 iinfo = np.iinfo(self.numpy_type)
 values = [int(x) for x in
 np.random.randint(max(iinfo.min, TEST_INT_MIN),
 min(iinfo.max, TEST_INT_MAX),
 np.random.randint(max(iinfo.min, self.min_value),
 min(iinfo.max, self.max_value),
 size=size)]

 is_valid = self._make_is_valid(size)
 return PrimitiveColumn(self.name, size, is_valid, values)


class DateType(IntegerType):

 DAY = 0
 MILLISECOND = 1

 def __init__(self, name, unit, nullable=True):
 self.unit = unit
 bit_width = 32 if unit == self.DAY else 64
 IntegerType.__init__(self, name, True, bit_width, nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'date'),
 ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
 ])


TIMEUNIT_NAMES = {
 's': 'SECOND',
 'ms': 'MILLISECOND',
 'us': 'MICROSECOND',
 'ns': 'NANOSECOND'
}


class TimeType(IntegerType):

 BIT_WIDTHS = {
 's': 32,
 'ms': 32,
 'us': 64,
 'ns': 64
 }

 def __init__(self, name, unit='s', nullable=True):
 self.unit = unit
 IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
 nullable=nullable)

 def _get_type(self):
 return OrderedDict([
 ('name', 'time'),
 ('unit', TIMEUNIT_NAMES[self.unit]),
 ('bitWidth', self.bit_width)
 ])


class TimestampType(IntegerType):

 def __init__(self, name, unit='s', tz=None, nullable=True):
 self.unit = unit
 self.tz = tz
 IntegerType.__init__(self, name, True, 64, nullable=nullable)

 def _get_type(self):
 fields = [
 ('name', 'timestamp'),
 ('unit', TIMEUNIT_NAMES[self.unit])
 ]

 if self.tz is not None:
 fields.append(('timezone', self.tz))

 return OrderedDict(fields)


class FloatingPointType(PrimitiveType):

 def __init__(self, name, bit_width, nullable=True):
Expand DownExpand Up@@ -509,6 +579,20 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)


def generate_primitive_case():
 types = ['bool', 'int8', 'int16', 'int32', 'int64',
 'uint8', 'uint16', 'uint32', 'uint64',
Expand All@@ -520,19 +604,27 @@ def generate_primitive_case():
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)
 return _generate_file(fields, batch_sizes)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
def generate_datetime_case():
 fields = [
 DateType('f0', DateType.DAY),
 DateType('f1', DateType.MILLISECOND),
 TimeType('f2', 's'),
 TimeType('f3', 'ms'),
 TimeType('f4', 'us'),
 TimeType('f5', 'ns'),
 TimestampType('f6', 's'),
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)


def generate_nested_case():
Expand All@@ -545,19 +637,8 @@ def generate_nested_case():
 # ListType('list_nonnullable', get_field('item', 'int32'), False),
 ]

 schema = JSONSchema(fields)

 batch_sizes = [7, 10]
 batches = []
 for size in batch_sizes:
 columns = []
 for field in fields:
 col = field.generate_column(size)
 columns.append(col)

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return _generate_file(fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -566,13 +647,13 @@ def get_generated_json_files():
 def _temp_path():
 return

 file_objs = []

K = 10
for i in range(K):
file_objs.append(generate_primitive_case())

file_objs.append(generate_nested_case())
 file_objs = [
 generate_primitive_case(),
 generate_primitive_case(),
 generate_primitive_case(),
# generate_datetime_case(),
 generate_nested_case()
]

 generated_paths = []
 for file_obj in file_objs:
Expand Down