ARROW-824: Date and Time Vectors should reflect timezone-less semantics#568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
julienledem wants to merge 6 commits into from
Closed

ARROW-824: Date and Time Vectors should reflect timezone-less semantics #568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ec19e7d
ARROW-824: Date and Time Vectors should reflect timezone-less semantics
julienledem Apr 19, 2017
61518ec
improve travis integration
julienledem May 2, 2017
b0da88c
fix failing integration test
julienledem May 2, 2017
bdac7ff
make integration tests output more readable
julienledem May 2, 2017
e37385c
centralize LocalDateTime.toMillis
julienledem May 3, 2017
3528ad8
add license
julienledem May 3, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ci/travis_script_integration.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh

pushd $ARROW_JAVA_DIR

mvn package
echo "mvn package"
mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)

popd

Expand Down
27 changes: 16 additions & 11 deletions integration/integration_test.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -544,7 +544,8 @@ def get_json(self):

class JSONFile(object):

def __init__(self, schema, batches):
def __init__(self, name, schema, batches):
self.name = name
self.schema = schema
self.batches = batches

Expand DownExpand Up@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
def _generate_file(name, fields, batch_sizes):
schema = JSONSchema(fields)
batches = []
for size in batch_sizes:
Expand All@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):

batches.append(JSONRecordBatch(size, columns))

return JSONFile(schema, batches)
return JSONFile(name, schema, batches)


def generate_primitive_case(batch_sizes):
Expand All@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
fields.append(get_field(type_ + "_nullable", type_, True))
fields.append(get_field(type_ + "_nonnullable", type_, False))

return _generate_file(fields, batch_sizes)
return _generate_file("primitive", fields, batch_sizes)


def generate_datetime_case():
Expand All@@ -619,11 +620,11 @@ def generate_datetime_case():
TimestampType('f7', 'ms'),
TimestampType('f8', 'us'),
TimestampType('f9', 'ns'),
TimestampType('f10', 'ms', tz='America/New_York')
TimestampType('f10', 'ms', tz=None)
]

batch_sizes = [7, 10]
return _generate_file(fields, batch_sizes)
return _generate_file("datetime", fields, batch_sizes)


def generate_nested_case():
Expand All@@ -637,7 +638,7 @@ def generate_nested_case():
]

batch_sizes = [7, 10]
return _generate_file(fields, batch_sizes)
return _generate_file("nested", fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -655,7 +656,7 @@ def _temp_path():

generated_paths = []
for file_obj in file_objs:
out_path = os.path.join(temp_dir, guid() + '.json')
out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
file_obj.write(out_path)
generated_paths.append(out_path)

Expand DownExpand Up@@ -684,20 +685,24 @@ def _compare_implementations(self, producer, consumer):
consumer.name))

for json_path in self.json_files:
print('=====================================================================================')
print('Testing file {0}'.format(json_path))
print('=====================================================================================')

name = os.path.splitext(os.path.basename(json_path))[0]

# Make the random access file
print('-- Creating binary inputs')
producer_file_path = os.path.join(self.temp_dir, guid())
producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
producer.json_to_file(json_path, producer_file_path)

# Validate the file
print('-- Validating file')
consumer.validate(json_path, producer_file_path)

print('-- Validating stream')
producer_stream_path = os.path.join(self.temp_dir, guid())
consumer_file_path = os.path.join(self.temp_dir, guid())
producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
producer.file_to_stream(producer_file_path,
producer_stream_path)
consumer.stream_to_file(producer_stream_path,
Expand Down
14 changes: 7 additions & 7 deletions java/vector/src/main/codegen/data/ValueVectorTypes.tdd
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -59,7 +59,7 @@
{ class: "DateDay" },
{ class: "IntervalYear", javaType: "int", friendlyType: "Period" },
{ class: "TimeSec" },
{ class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
{ class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
]
},
{
Expand All@@ -71,12 +71,12 @@
minor: [
{ class: "BigInt"},
{ class: "UInt8" },
{ class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
{ class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
{ class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
{ class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
{ class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
{ class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
{ class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
{ class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" },
{ class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
{ class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
{ class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
{ class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
{ class: "TimeMicro" },
{ class: "TimeNano" }
]
Expand Down
1 change: 1 addition & 0 deletions java/vector/src/main/codegen/includes/vv_imports.ftl
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;

import org.joda.time.DateTime;
import org.joda.time.LocalDateTime;
import org.joda.time.Period;


Expand Down
6 changes: 3 additions & 3 deletions java/vector/src/main/codegen/templates/AbstractFieldReader.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,8 @@

<#include "/@includes/vv_imports.ftl" />

/*
* This class is generated using freemarker and the ${.template_name} template.
/**
* Source code generated using FreeMarker template ${.template_name}
*/
@SuppressWarnings("unused")
abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
Expand All@@ -51,7 +51,7 @@ public Field getField() {
}

<#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
"Character", "DateTime", "Period", "Double", "Float",
"Character", "LocalDateTime", "Period", "Double", "Float",
"Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
<#assign safeType=friendlyType />
<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down
1 change: 1 addition & 0 deletions java/vector/src/main/codegen/templates/ArrowType.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@

/**
* Arrow types
* Source code generated using FreeMarker template ${.template_name}
**/
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
Expand Down
5 changes: 3 additions & 2 deletions java/vector/src/main/codegen/templates/BaseReader.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,9 @@

<#include "/@includes/vv_imports.ftl" />



/**
* Source code generated using FreeMarker template ${.template_name}
*/
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
Field getField();
Expand Down
7 changes: 7 additions & 0 deletions java/vector/src/main/codegen/templates/ComplexReaders.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,6 +47,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
* Source code generated using FreeMarker template ${.template_name}
*/
@SuppressWarnings("unused")
public class ${name}ReaderImpl extends AbstractFieldReader {

Expand DownExpand Up@@ -123,12 +126,16 @@ public Object readObject(){
package org.apache.arrow.vector.complex.reader;

<#include "/@includes/vv_imports.ftl" />
/**
* Source code generated using FreeMarker template ${.template_name}
*/
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{

public void read(${minor.class?cap_first}Holder h);
public void read(Nullable${minor.class?cap_first}Holder h);
public Object readObject();
// read friendly type
public ${friendlyType} read${safeType}();
public boolean isSet();
public void copyAsValue(${minor.class}Writer writer);
Expand Down
3 changes: 3 additions & 0 deletions java/vector/src/main/codegen/templates/ComplexWriters.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -139,6 +139,9 @@ public void writeNull() {
package org.apache.arrow.vector.complex.writer;

<#include "/@includes/vv_imports.ftl" />
/*
* This class is generated using FreeMarker on the ${.template_name} template.
*/
@SuppressWarnings("unused")
public interface ${eName}Writer extends BaseWriter {
public void write(${minor.class}Holder h);
Expand Down
18 changes: 6 additions & 12 deletions java/vector/src/main/codegen/templates/FixedValueVectors.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -495,33 +495,29 @@ public long getTwoAsLong(int index) {
<#elseif minor.class == "DateMilli">
@Override
public ${friendlyType} getObject(int index) {
org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
return date;
}

<#elseif minor.class == "TimeMilli">
@Override
public ${friendlyType} getObject(int index) {
org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
return time;
}

<#elseif minor.class == "TimeStampSec">
@Override
public ${friendlyType} getObject(int index) {
long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
return date;
}

<#elseif minor.class == "TimeStampMilli">
@Override
public ${friendlyType} getObject(int index) {
org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
return date;
}

Expand All@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
public ${friendlyType} getObject(int index) {
// value is truncated when converting microseconds to milliseconds in order to use DateTime type
long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
return date;
}

Expand All@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
public ${friendlyType} getObject(int index) {
// value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
return date;
}

Expand Down
12 changes: 6 additions & 6 deletions java/vector/src/main/codegen/templates/HolderReaderImpl.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ public boolean isSet() {

}

@Override
@Override
public void read(${name}Holder h) {
<#list fields as field>
h.${field.name} = holder.${field.name};
Expand All@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
h.isSet = isSet() ? 1 : 0;
}


// read friendly type
@Override
public ${friendlyType} read${safeType}(){
<#if nullMode == "Nullable">
Expand All@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
byte[] value = new byte [length];
holder.buffer.getBytes(holder.start, value, 0, length);

<#if minor.class == "VarBinary">
<#if minor.class == "VarBinary">
return value;
<#elseif minor.class == "Var16Char">
<#elseif minor.class == "Var16Char">
return new String(value);
<#elseif minor.class == "VarChar">
<#elseif minor.class == "VarChar">
Text text = new Text();
text.set(value);
return text;
</#if>
</#if>

<#elseif minor.class == "Interval">
Period p = new Period();
Expand Down
6 changes: 4 additions & 2 deletions java/vector/src/main/codegen/templates/NullReader.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,7 +29,9 @@

<#include "/@includes/vv_imports.ftl" />


/**
* Source code generated using FreeMarker template ${.template_name}
*/
@SuppressWarnings("unused")
public class NullReader extends AbstractBaseReader implements FieldReader{

Expand DownExpand Up@@ -127,7 +129,7 @@ private void fail(String name){
}

<#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
"Character", "DateTime", "Period", "Double", "Float",
"Character", "LocalDateTime", "Period", "Double", "Float",
"Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
<#assign safeType=friendlyType />
<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down
6 changes: 4 additions & 2 deletions java/vector/src/main/codegen/templates/UnionReader.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,9 @@
package org.apache.arrow.vector.complex.impl;

<#include "/@includes/vv_imports.ftl" />

/**
* Source code generated using FreeMarker template ${.template_name}
*/
@SuppressWarnings("unused")
public class UnionReader extends AbstractFieldReader {

Expand DownExpand Up@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
}

<#list ["Object", "Integer", "Long", "Boolean",
"Character", "DateTime", "Double", "Float",
"Character", "LocalDateTime", "Double", "Float",
"Text", "Byte", "Short", "byte[]"] as friendlyType>
<#assign safeType=friendlyType />
<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down
1 change: 1 addition & 0 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@
*
* For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
* each time the vector is accessed.
* Source code generated using FreeMarker template ${.template_name}
*/
public class UnionVector implements FieldVector {

Expand Down
3 changes: 3 additions & 0 deletions java/vector/src/main/codegen/templates/ValueHolders.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
* Source code generated using FreeMarker template ${.template_name}
*/
public final class ${className} implements ValueHolder{

<#if mode.name == "Repeated">
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-824: Date and Time Vectors should reflect timezone-less semantics#568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

julienledem
wants to merge
6
commits into

from

 Closed


ARROW-824: Date and Time Vectors should reflect timezone-less semantics
#568
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
ec19e7d

ARROW-824: Date and Time Vectors should reflect timezone-less semantics


julienledem Apr 19, 2017
61518ec

improve travis integration


julienledem May 2, 2017
b0da88c

fix failing integration test


julienledem May 2, 2017
bdac7ff

make integration tests output more readable


julienledem May 2, 2017
e37385c

centralize LocalDateTime.toMillis


julienledem May 3, 2017
3528ad8

add license


julienledem May 3, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


3 changes: 2 additions & 1 deletion
ci/travis_script_integration.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh

pushd $ARROW_JAVA_DIR

mvn package
echo "mvn package"
mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)

popd

Expand Down

27 changes: 16 additions & 11 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -544,7 +544,8 @@ def get_json(self):

class JSONFile(object):

 def __init__(self, schema, batches):
 def __init__(self, name, schema, batches):
 self.name = name
 self.schema = schema
 self.batches = batches

Expand DownExpand Up@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
def _generate_file(name, fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
Expand All@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return JSONFile(name, schema, batches)


def generate_primitive_case(batch_sizes):
Expand All@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 return _generate_file(fields, batch_sizes)
 return _generate_file("primitive", fields, batch_sizes)


def generate_datetime_case():
Expand All@@ -619,11 +620,11 @@ def generate_datetime_case():
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 TimestampType('f10', 'ms', tz=None)
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("datetime", fields, batch_sizes)


def generate_nested_case():
Expand All@@ -637,7 +638,7 @@ def generate_nested_case():
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("nested", fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -655,7 +656,7 @@ def _temp_path():

 generated_paths = []
 for file_obj in file_objs:
 out_path = os.path.join(temp_dir, guid() + '.json')
 out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
 file_obj.write(out_path)
 generated_paths.append(out_path)

Expand DownExpand Up@@ -684,20 +685,24 @@ def _compare_implementations(self, producer, consumer):
 consumer.name))

 for json_path in self.json_files:
 print('=====================================================================================')
 print('Testing file {0}'.format(json_path))
 print('=====================================================================================')

 name = os.path.splitext(os.path.basename(json_path))[0]

 # Make the random access file
 print('-- Creating binary inputs')
 producer_file_path = os.path.join(self.temp_dir, guid())
 producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
 producer.json_to_file(json_path, producer_file_path)

 # Validate the file
 print('-- Validating file')
 consumer.validate(json_path, producer_file_path)

 print('-- Validating stream')
 producer_stream_path = os.path.join(self.temp_dir, guid())
 consumer_file_path = os.path.join(self.temp_dir, guid())
 producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
 consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
 producer.file_to_stream(producer_file_path,
 producer_stream_path)
 consumer.stream_to_file(producer_stream_path,
Expand Down

14 changes: 7 additions & 7 deletions
java/vector/src/main/codegen/data/ValueVectorTypes.tdd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,7 +59,7 @@
 { class: "DateDay" },
 { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
 { class: "TimeSec" },
 { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
 { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
 ]
 },
 {
Expand All@@ -71,12 +71,12 @@
 minor: [
 { class: "BigInt"},
 { class: "UInt8" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeMicro" },
 { class: "TimeNano" }
 ]
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/includes/vv_imports.ftl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;

import org.joda.time.DateTime;
import org.joda.time.LocalDateTime;
import org.joda.time.Period;


Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/AbstractFieldReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,8 @@

<#include "/@includes/vv_imports.ftl" />

/*
 * This class is generated using freemarker and the ${.template_name} template.
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
Expand All@@ -51,7 +51,7 @@ public Field getField() {
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/ArrowType.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@

/**
 * Arrow types
 * Source code generated using FreeMarker template ${.template_name}
 **/
@JsonTypeInfo(
 use = JsonTypeInfo.Id.NAME,
Expand Down

5 changes: 3 additions & 2 deletions
java/vector/src/main/codegen/templates/BaseReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,9 @@

<#include "/@includes/vv_imports.ftl" />



/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
 Field getField();
Expand Down

7 changes: 7 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexReaders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,6 +47,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class ${name}ReaderImpl extends AbstractFieldReader {

Expand DownExpand Up@@ -123,12 +126,16 @@ public Object readObject(){
package org.apache.arrow.vector.complex.reader;

<#include "/@includes/vv_imports.ftl" />
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{

 public void read(${minor.class?cap_first}Holder h);
 public void read(Nullable${minor.class?cap_first}Holder h);
 public Object readObject();
 // read friendly type
 public ${friendlyType} read${safeType}();
 public boolean isSet();
 public void copyAsValue(${minor.class}Writer writer);
Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexWriters.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -139,6 +139,9 @@ public void writeNull() {
package org.apache.arrow.vector.complex.writer;

<#include "/@includes/vv_imports.ftl" />
/*
 * This class is generated using FreeMarker on the ${.template_name} template.
 */
@SuppressWarnings("unused")
public interface ${eName}Writer extends BaseWriter {
 public void write(${minor.class}Holder h);
Expand Down

18 changes: 6 additions & 12 deletions
java/vector/src/main/codegen/templates/FixedValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -495,33 +495,29 @@ public long getTwoAsLong(int index) {
 <#elseif minor.class == "DateMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return time;
 }

 <#elseif minor.class == "TimeStampSec">
 @Override
 public ${friendlyType} getObject(int index) {
 long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeStampMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting microseconds to milliseconds in order to use DateTime type
 long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
 long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand Down

12 changes: 6 additions & 6 deletions
java/vector/src/main/codegen/templates/HolderReaderImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ public boolean isSet() {

 }

@Override
@Override
 public void read(${name}Holder h) {
 <#list fields as field>
 h.${field.name} = holder.${field.name};
Expand All@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
 h.isSet = isSet() ? 1 : 0;
 }


 // read friendly type
 @Override
 public ${friendlyType} read${safeType}(){
<#if nullMode == "Nullable">
Expand All@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
 byte[] value = new byte [length];
 holder.buffer.getBytes(holder.start, value, 0, length);

<#if minor.class == "VarBinary">
<#if minor.class == "VarBinary">
 return value;
<#elseif minor.class == "Var16Char">
<#elseif minor.class == "Var16Char">
 return new String(value);
<#elseif minor.class == "VarChar">
<#elseif minor.class == "VarChar">
 Text text = new Text();
 text.set(value);
 return text;
</#if>
</#if>

<#elseif minor.class == "Interval">
 Period p = new Period();
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,7 +29,9 @@

<#include "/@includes/vv_imports.ftl" />


/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class NullReader extends AbstractBaseReader implements FieldReader{

Expand DownExpand Up@@ -127,7 +129,7 @@ private void fail(String name){
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/UnionReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,9 @@
package org.apache.arrow.vector.complex.impl;

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class UnionReader extends AbstractFieldReader {

Expand DownExpand Up@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
 }

 <#list ["Object", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Double", "Float",
 "Character", "LocalDateTime", "Double", "Float",
 "Text", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@
 *
 * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
 * each time the vector is accessed.
 * Source code generated using FreeMarker template ${.template_name}
 */
public class UnionVector implements FieldVector {

Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ValueHolders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
public final class ${className} implements ValueHolder{

 <#if mode.name == "Repeated">
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-824: Date and Time Vectors should reflect timezone-less semantics#568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

julienledem
wants to merge
6
commits into

from

 Closed


ARROW-824: Date and Time Vectors should reflect timezone-less semantics
#568
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
ec19e7d

ARROW-824: Date and Time Vectors should reflect timezone-less semantics


julienledem Apr 19, 2017
61518ec

improve travis integration


julienledem May 2, 2017
b0da88c

fix failing integration test


julienledem May 2, 2017
bdac7ff

make integration tests output more readable


julienledem May 2, 2017
e37385c

centralize LocalDateTime.toMillis


julienledem May 3, 2017
3528ad8

add license


julienledem May 3, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


3 changes: 2 additions & 1 deletion
ci/travis_script_integration.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh

pushd $ARROW_JAVA_DIR

mvn package
echo "mvn package"
mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)

popd

Expand Down

27 changes: 16 additions & 11 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -544,7 +544,8 @@ def get_json(self):

class JSONFile(object):

 def __init__(self, schema, batches):
 def __init__(self, name, schema, batches):
 self.name = name
 self.schema = schema
 self.batches = batches

Expand DownExpand Up@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
def _generate_file(name, fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
Expand All@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return JSONFile(name, schema, batches)


def generate_primitive_case(batch_sizes):
Expand All@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 return _generate_file(fields, batch_sizes)
 return _generate_file("primitive", fields, batch_sizes)


def generate_datetime_case():
Expand All@@ -619,11 +620,11 @@ def generate_datetime_case():
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 TimestampType('f10', 'ms', tz=None)
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("datetime", fields, batch_sizes)


def generate_nested_case():
Expand All@@ -637,7 +638,7 @@ def generate_nested_case():
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("nested", fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -655,7 +656,7 @@ def _temp_path():

 generated_paths = []
 for file_obj in file_objs:
 out_path = os.path.join(temp_dir, guid() + '.json')
 out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
 file_obj.write(out_path)
 generated_paths.append(out_path)

Expand DownExpand Up@@ -684,20 +685,24 @@ def _compare_implementations(self, producer, consumer):
 consumer.name))

 for json_path in self.json_files:
 print('=====================================================================================')
 print('Testing file {0}'.format(json_path))
 print('=====================================================================================')

 name = os.path.splitext(os.path.basename(json_path))[0]

 # Make the random access file
 print('-- Creating binary inputs')
 producer_file_path = os.path.join(self.temp_dir, guid())
 producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
 producer.json_to_file(json_path, producer_file_path)

 # Validate the file
 print('-- Validating file')
 consumer.validate(json_path, producer_file_path)

 print('-- Validating stream')
 producer_stream_path = os.path.join(self.temp_dir, guid())
 consumer_file_path = os.path.join(self.temp_dir, guid())
 producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
 consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
 producer.file_to_stream(producer_file_path,
 producer_stream_path)
 consumer.stream_to_file(producer_stream_path,
Expand Down

14 changes: 7 additions & 7 deletions
java/vector/src/main/codegen/data/ValueVectorTypes.tdd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,7 +59,7 @@
 { class: "DateDay" },
 { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
 { class: "TimeSec" },
 { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
 { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
 ]
 },
 {
Expand All@@ -71,12 +71,12 @@
 minor: [
 { class: "BigInt"},
 { class: "UInt8" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeMicro" },
 { class: "TimeNano" }
 ]
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/includes/vv_imports.ftl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;

import org.joda.time.DateTime;
import org.joda.time.LocalDateTime;
import org.joda.time.Period;


Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/AbstractFieldReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,8 @@

<#include "/@includes/vv_imports.ftl" />

/*
 * This class is generated using freemarker and the ${.template_name} template.
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
Expand All@@ -51,7 +51,7 @@ public Field getField() {
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/ArrowType.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@

/**
 * Arrow types
 * Source code generated using FreeMarker template ${.template_name}
 **/
@JsonTypeInfo(
 use = JsonTypeInfo.Id.NAME,
Expand Down

5 changes: 3 additions & 2 deletions
java/vector/src/main/codegen/templates/BaseReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,9 @@

<#include "/@includes/vv_imports.ftl" />



/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
 Field getField();
Expand Down

7 changes: 7 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexReaders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,6 +47,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class ${name}ReaderImpl extends AbstractFieldReader {

Expand DownExpand Up@@ -123,12 +126,16 @@ public Object readObject(){
package org.apache.arrow.vector.complex.reader;

<#include "/@includes/vv_imports.ftl" />
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{

 public void read(${minor.class?cap_first}Holder h);
 public void read(Nullable${minor.class?cap_first}Holder h);
 public Object readObject();
 // read friendly type
 public ${friendlyType} read${safeType}();
 public boolean isSet();
 public void copyAsValue(${minor.class}Writer writer);
Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexWriters.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -139,6 +139,9 @@ public void writeNull() {
package org.apache.arrow.vector.complex.writer;

<#include "/@includes/vv_imports.ftl" />
/*
 * This class is generated using FreeMarker on the ${.template_name} template.
 */
@SuppressWarnings("unused")
public interface ${eName}Writer extends BaseWriter {
 public void write(${minor.class}Holder h);
Expand Down

18 changes: 6 additions & 12 deletions
java/vector/src/main/codegen/templates/FixedValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -495,33 +495,29 @@ public long getTwoAsLong(int index) {
 <#elseif minor.class == "DateMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return time;
 }

 <#elseif minor.class == "TimeStampSec">
 @Override
 public ${friendlyType} getObject(int index) {
 long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeStampMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting microseconds to milliseconds in order to use DateTime type
 long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
 long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand Down

12 changes: 6 additions & 6 deletions
java/vector/src/main/codegen/templates/HolderReaderImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ public boolean isSet() {

 }

@Override
@Override
 public void read(${name}Holder h) {
 <#list fields as field>
 h.${field.name} = holder.${field.name};
Expand All@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
 h.isSet = isSet() ? 1 : 0;
 }


 // read friendly type
 @Override
 public ${friendlyType} read${safeType}(){
<#if nullMode == "Nullable">
Expand All@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
 byte[] value = new byte [length];
 holder.buffer.getBytes(holder.start, value, 0, length);

<#if minor.class == "VarBinary">
<#if minor.class == "VarBinary">
 return value;
<#elseif minor.class == "Var16Char">
<#elseif minor.class == "Var16Char">
 return new String(value);
<#elseif minor.class == "VarChar">
<#elseif minor.class == "VarChar">
 Text text = new Text();
 text.set(value);
 return text;
</#if>
</#if>

<#elseif minor.class == "Interval">
 Period p = new Period();
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,7 +29,9 @@

<#include "/@includes/vv_imports.ftl" />


/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class NullReader extends AbstractBaseReader implements FieldReader{

Expand DownExpand Up@@ -127,7 +129,7 @@ private void fail(String name){
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/UnionReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,9 @@
package org.apache.arrow.vector.complex.impl;

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class UnionReader extends AbstractFieldReader {

Expand DownExpand Up@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
 }

 <#list ["Object", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Double", "Float",
 "Character", "LocalDateTime", "Double", "Float",
 "Text", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@
 *
 * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
 * each time the vector is accessed.
 * Source code generated using FreeMarker template ${.template_name}
 */
public class UnionVector implements FieldVector {

Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ValueHolders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
public final class ${className} implements ValueHolder{

 <#if mode.name == "Repeated">
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-824: Date and Time Vectors should reflect timezone-less semantics#568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

julienledem
wants to merge
6
commits into

from

 Closed


ARROW-824: Date and Time Vectors should reflect timezone-less semantics
#568
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
ec19e7d

ARROW-824: Date and Time Vectors should reflect timezone-less semantics


julienledem Apr 19, 2017
61518ec

improve travis integration


julienledem May 2, 2017
b0da88c

fix failing integration test


julienledem May 2, 2017
bdac7ff

make integration tests output more readable


julienledem May 2, 2017
e37385c

centralize LocalDateTime.toMillis


julienledem May 3, 2017
3528ad8

add license


julienledem May 3, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


3 changes: 2 additions & 1 deletion
ci/travis_script_integration.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh

pushd $ARROW_JAVA_DIR

mvn package
echo "mvn package"
mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)

popd

Expand Down

27 changes: 16 additions & 11 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -544,7 +544,8 @@ def get_json(self):

class JSONFile(object):

 def __init__(self, schema, batches):
 def __init__(self, name, schema, batches):
 self.name = name
 self.schema = schema
 self.batches = batches

Expand DownExpand Up@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
def _generate_file(name, fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
Expand All@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return JSONFile(name, schema, batches)


def generate_primitive_case(batch_sizes):
Expand All@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 return _generate_file(fields, batch_sizes)
 return _generate_file("primitive", fields, batch_sizes)


def generate_datetime_case():
Expand All@@ -619,11 +620,11 @@ def generate_datetime_case():
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 TimestampType('f10', 'ms', tz=None)
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("datetime", fields, batch_sizes)


def generate_nested_case():
Expand All@@ -637,7 +638,7 @@ def generate_nested_case():
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("nested", fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -655,7 +656,7 @@ def _temp_path():

 generated_paths = []
 for file_obj in file_objs:
 out_path = os.path.join(temp_dir, guid() + '.json')
 out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
 file_obj.write(out_path)
 generated_paths.append(out_path)

Expand DownExpand Up@@ -684,20 +685,24 @@ def _compare_implementations(self, producer, consumer):
 consumer.name))

 for json_path in self.json_files:
 print('=====================================================================================')
 print('Testing file {0}'.format(json_path))
 print('=====================================================================================')

 name = os.path.splitext(os.path.basename(json_path))[0]

 # Make the random access file
 print('-- Creating binary inputs')
 producer_file_path = os.path.join(self.temp_dir, guid())
 producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
 producer.json_to_file(json_path, producer_file_path)

 # Validate the file
 print('-- Validating file')
 consumer.validate(json_path, producer_file_path)

 print('-- Validating stream')
 producer_stream_path = os.path.join(self.temp_dir, guid())
 consumer_file_path = os.path.join(self.temp_dir, guid())
 producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
 consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
 producer.file_to_stream(producer_file_path,
 producer_stream_path)
 consumer.stream_to_file(producer_stream_path,
Expand Down

14 changes: 7 additions & 7 deletions
java/vector/src/main/codegen/data/ValueVectorTypes.tdd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,7 +59,7 @@
 { class: "DateDay" },
 { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
 { class: "TimeSec" },
 { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
 { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
 ]
 },
 {
Expand All@@ -71,12 +71,12 @@
 minor: [
 { class: "BigInt"},
 { class: "UInt8" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeMicro" },
 { class: "TimeNano" }
 ]
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/includes/vv_imports.ftl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;

import org.joda.time.DateTime;
import org.joda.time.LocalDateTime;
import org.joda.time.Period;


Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/AbstractFieldReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,8 @@

<#include "/@includes/vv_imports.ftl" />

/*
 * This class is generated using freemarker and the ${.template_name} template.
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
Expand All@@ -51,7 +51,7 @@ public Field getField() {
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/ArrowType.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@

/**
 * Arrow types
 * Source code generated using FreeMarker template ${.template_name}
 **/
@JsonTypeInfo(
 use = JsonTypeInfo.Id.NAME,
Expand Down

5 changes: 3 additions & 2 deletions
java/vector/src/main/codegen/templates/BaseReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,9 @@

<#include "/@includes/vv_imports.ftl" />



/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
 Field getField();
Expand Down

7 changes: 7 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexReaders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,6 +47,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class ${name}ReaderImpl extends AbstractFieldReader {

Expand DownExpand Up@@ -123,12 +126,16 @@ public Object readObject(){
package org.apache.arrow.vector.complex.reader;

<#include "/@includes/vv_imports.ftl" />
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{

 public void read(${minor.class?cap_first}Holder h);
 public void read(Nullable${minor.class?cap_first}Holder h);
 public Object readObject();
 // read friendly type
 public ${friendlyType} read${safeType}();
 public boolean isSet();
 public void copyAsValue(${minor.class}Writer writer);
Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexWriters.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -139,6 +139,9 @@ public void writeNull() {
package org.apache.arrow.vector.complex.writer;

<#include "/@includes/vv_imports.ftl" />
/*
 * This class is generated using FreeMarker on the ${.template_name} template.
 */
@SuppressWarnings("unused")
public interface ${eName}Writer extends BaseWriter {
 public void write(${minor.class}Holder h);
Expand Down

18 changes: 6 additions & 12 deletions
java/vector/src/main/codegen/templates/FixedValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -495,33 +495,29 @@ public long getTwoAsLong(int index) {
 <#elseif minor.class == "DateMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return time;
 }

 <#elseif minor.class == "TimeStampSec">
 @Override
 public ${friendlyType} getObject(int index) {
 long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeStampMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting microseconds to milliseconds in order to use DateTime type
 long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
 long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand Down

12 changes: 6 additions & 6 deletions
java/vector/src/main/codegen/templates/HolderReaderImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ public boolean isSet() {

 }

@Override
@Override
 public void read(${name}Holder h) {
 <#list fields as field>
 h.${field.name} = holder.${field.name};
Expand All@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
 h.isSet = isSet() ? 1 : 0;
 }


 // read friendly type
 @Override
 public ${friendlyType} read${safeType}(){
<#if nullMode == "Nullable">
Expand All@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
 byte[] value = new byte [length];
 holder.buffer.getBytes(holder.start, value, 0, length);

<#if minor.class == "VarBinary">
<#if minor.class == "VarBinary">
 return value;
<#elseif minor.class == "Var16Char">
<#elseif minor.class == "Var16Char">
 return new String(value);
<#elseif minor.class == "VarChar">
<#elseif minor.class == "VarChar">
 Text text = new Text();
 text.set(value);
 return text;
</#if>
</#if>

<#elseif minor.class == "Interval">
 Period p = new Period();
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,7 +29,9 @@

<#include "/@includes/vv_imports.ftl" />


/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class NullReader extends AbstractBaseReader implements FieldReader{

Expand DownExpand Up@@ -127,7 +129,7 @@ private void fail(String name){
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/UnionReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,9 @@
package org.apache.arrow.vector.complex.impl;

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class UnionReader extends AbstractFieldReader {

Expand DownExpand Up@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
 }

 <#list ["Object", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Double", "Float",
 "Character", "LocalDateTime", "Double", "Float",
 "Text", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@
 *
 * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
 * each time the vector is accessed.
 * Source code generated using FreeMarker template ${.template_name}
 */
public class UnionVector implements FieldVector {

Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ValueHolders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
public final class ${className} implements ValueHolder{

 <#if mode.name == "Repeated">
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-824: Date and Time Vectors should reflect timezone-less semantics#568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

julienledem
wants to merge
6
commits into

from

 Closed


ARROW-824: Date and Time Vectors should reflect timezone-less semantics
#568
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
ec19e7d

ARROW-824: Date and Time Vectors should reflect timezone-less semantics


julienledem Apr 19, 2017
61518ec

improve travis integration


julienledem May 2, 2017
b0da88c

fix failing integration test


julienledem May 2, 2017
bdac7ff

make integration tests output more readable


julienledem May 2, 2017
e37385c

centralize LocalDateTime.toMillis


julienledem May 3, 2017
3528ad8

add license


julienledem May 3, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


3 changes: 2 additions & 1 deletion
ci/travis_script_integration.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh

pushd $ARROW_JAVA_DIR

mvn package
echo "mvn package"
mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)

popd

Expand Down

27 changes: 16 additions & 11 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -544,7 +544,8 @@ def get_json(self):

class JSONFile(object):

 def __init__(self, schema, batches):
 def __init__(self, name, schema, batches):
 self.name = name
 self.schema = schema
 self.batches = batches

Expand DownExpand Up@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
def _generate_file(name, fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
Expand All@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return JSONFile(name, schema, batches)


def generate_primitive_case(batch_sizes):
Expand All@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 return _generate_file(fields, batch_sizes)
 return _generate_file("primitive", fields, batch_sizes)


def generate_datetime_case():
Expand All@@ -619,11 +620,11 @@ def generate_datetime_case():
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 TimestampType('f10', 'ms', tz=None)
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("datetime", fields, batch_sizes)


def generate_nested_case():
Expand All@@ -637,7 +638,7 @@ def generate_nested_case():
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("nested", fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -655,7 +656,7 @@ def _temp_path():

 generated_paths = []
 for file_obj in file_objs:
 out_path = os.path.join(temp_dir, guid() + '.json')
 out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
 file_obj.write(out_path)
 generated_paths.append(out_path)

Expand DownExpand Up@@ -684,20 +685,24 @@ def _compare_implementations(self, producer, consumer):
 consumer.name))

 for json_path in self.json_files:
 print('=====================================================================================')
 print('Testing file {0}'.format(json_path))
 print('=====================================================================================')

 name = os.path.splitext(os.path.basename(json_path))[0]

 # Make the random access file
 print('-- Creating binary inputs')
 producer_file_path = os.path.join(self.temp_dir, guid())
 producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
 producer.json_to_file(json_path, producer_file_path)

 # Validate the file
 print('-- Validating file')
 consumer.validate(json_path, producer_file_path)

 print('-- Validating stream')
 producer_stream_path = os.path.join(self.temp_dir, guid())
 consumer_file_path = os.path.join(self.temp_dir, guid())
 producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
 consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
 producer.file_to_stream(producer_file_path,
 producer_stream_path)
 consumer.stream_to_file(producer_stream_path,
Expand Down

14 changes: 7 additions & 7 deletions
java/vector/src/main/codegen/data/ValueVectorTypes.tdd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,7 +59,7 @@
 { class: "DateDay" },
 { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
 { class: "TimeSec" },
 { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
 { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
 ]
 },
 {
Expand All@@ -71,12 +71,12 @@
 minor: [
 { class: "BigInt"},
 { class: "UInt8" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeMicro" },
 { class: "TimeNano" }
 ]
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/includes/vv_imports.ftl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;

import org.joda.time.DateTime;
import org.joda.time.LocalDateTime;
import org.joda.time.Period;


Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/AbstractFieldReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,8 @@

<#include "/@includes/vv_imports.ftl" />

/*
 * This class is generated using freemarker and the ${.template_name} template.
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
Expand All@@ -51,7 +51,7 @@ public Field getField() {
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/ArrowType.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@

/**
 * Arrow types
 * Source code generated using FreeMarker template ${.template_name}
 **/
@JsonTypeInfo(
 use = JsonTypeInfo.Id.NAME,
Expand Down

5 changes: 3 additions & 2 deletions
java/vector/src/main/codegen/templates/BaseReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,9 @@

<#include "/@includes/vv_imports.ftl" />



/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
 Field getField();
Expand Down

7 changes: 7 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexReaders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,6 +47,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class ${name}ReaderImpl extends AbstractFieldReader {

Expand DownExpand Up@@ -123,12 +126,16 @@ public Object readObject(){
package org.apache.arrow.vector.complex.reader;

<#include "/@includes/vv_imports.ftl" />
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{

 public void read(${minor.class?cap_first}Holder h);
 public void read(Nullable${minor.class?cap_first}Holder h);
 public Object readObject();
 // read friendly type
 public ${friendlyType} read${safeType}();
 public boolean isSet();
 public void copyAsValue(${minor.class}Writer writer);
Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexWriters.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -139,6 +139,9 @@ public void writeNull() {
package org.apache.arrow.vector.complex.writer;

<#include "/@includes/vv_imports.ftl" />
/*
 * This class is generated using FreeMarker on the ${.template_name} template.
 */
@SuppressWarnings("unused")
public interface ${eName}Writer extends BaseWriter {
 public void write(${minor.class}Holder h);
Expand Down

18 changes: 6 additions & 12 deletions
java/vector/src/main/codegen/templates/FixedValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -495,33 +495,29 @@ public long getTwoAsLong(int index) {
 <#elseif minor.class == "DateMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return time;
 }

 <#elseif minor.class == "TimeStampSec">
 @Override
 public ${friendlyType} getObject(int index) {
 long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeStampMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting microseconds to milliseconds in order to use DateTime type
 long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
 long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand Down

12 changes: 6 additions & 6 deletions
java/vector/src/main/codegen/templates/HolderReaderImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ public boolean isSet() {

 }

@Override
@Override
 public void read(${name}Holder h) {
 <#list fields as field>
 h.${field.name} = holder.${field.name};
Expand All@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
 h.isSet = isSet() ? 1 : 0;
 }


 // read friendly type
 @Override
 public ${friendlyType} read${safeType}(){
<#if nullMode == "Nullable">
Expand All@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
 byte[] value = new byte [length];
 holder.buffer.getBytes(holder.start, value, 0, length);

<#if minor.class == "VarBinary">
<#if minor.class == "VarBinary">
 return value;
<#elseif minor.class == "Var16Char">
<#elseif minor.class == "Var16Char">
 return new String(value);
<#elseif minor.class == "VarChar">
<#elseif minor.class == "VarChar">
 Text text = new Text();
 text.set(value);
 return text;
</#if>
</#if>

<#elseif minor.class == "Interval">
 Period p = new Period();
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,7 +29,9 @@

<#include "/@includes/vv_imports.ftl" />


/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class NullReader extends AbstractBaseReader implements FieldReader{

Expand DownExpand Up@@ -127,7 +129,7 @@ private void fail(String name){
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/UnionReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,9 @@
package org.apache.arrow.vector.complex.impl;

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class UnionReader extends AbstractFieldReader {

Expand DownExpand Up@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
 }

 <#list ["Object", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Double", "Float",
 "Character", "LocalDateTime", "Double", "Float",
 "Text", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@
 *
 * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
 * each time the vector is accessed.
 * Source code generated using FreeMarker template ${.template_name}
 */
public class UnionVector implements FieldVector {

Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ValueHolders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
public final class ${className} implements ValueHolder{

 <#if mode.name == "Repeated">
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-824: Date and Time Vectors should reflect timezone-less semantics#568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

julienledem
wants to merge
6
commits into

from

 Closed


ARROW-824: Date and Time Vectors should reflect timezone-less semantics
#568
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
ec19e7d

ARROW-824: Date and Time Vectors should reflect timezone-less semantics


julienledem Apr 19, 2017
61518ec

improve travis integration


julienledem May 2, 2017
b0da88c

fix failing integration test


julienledem May 2, 2017
bdac7ff

make integration tests output more readable


julienledem May 2, 2017
e37385c

centralize LocalDateTime.toMillis


julienledem May 3, 2017
3528ad8

add license


julienledem May 3, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


3 changes: 2 additions & 1 deletion
ci/travis_script_integration.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh

pushd $ARROW_JAVA_DIR

mvn package
echo "mvn package"
mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)

popd

Expand Down

27 changes: 16 additions & 11 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -544,7 +544,8 @@ def get_json(self):

class JSONFile(object):

 def __init__(self, schema, batches):
 def __init__(self, name, schema, batches):
 self.name = name
 self.schema = schema
 self.batches = batches

Expand DownExpand Up@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
def _generate_file(name, fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
Expand All@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return JSONFile(name, schema, batches)


def generate_primitive_case(batch_sizes):
Expand All@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 return _generate_file(fields, batch_sizes)
 return _generate_file("primitive", fields, batch_sizes)


def generate_datetime_case():
Expand All@@ -619,11 +620,11 @@ def generate_datetime_case():
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 TimestampType('f10', 'ms', tz=None)
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("datetime", fields, batch_sizes)


def generate_nested_case():
Expand All@@ -637,7 +638,7 @@ def generate_nested_case():
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("nested", fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -655,7 +656,7 @@ def _temp_path():

 generated_paths = []
 for file_obj in file_objs:
 out_path = os.path.join(temp_dir, guid() + '.json')
 out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
 file_obj.write(out_path)
 generated_paths.append(out_path)

Expand DownExpand Up@@ -684,20 +685,24 @@ def _compare_implementations(self, producer, consumer):
 consumer.name))

 for json_path in self.json_files:
 print('=====================================================================================')
 print('Testing file {0}'.format(json_path))
 print('=====================================================================================')

 name = os.path.splitext(os.path.basename(json_path))[0]

 # Make the random access file
 print('-- Creating binary inputs')
 producer_file_path = os.path.join(self.temp_dir, guid())
 producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
 producer.json_to_file(json_path, producer_file_path)

 # Validate the file
 print('-- Validating file')
 consumer.validate(json_path, producer_file_path)

 print('-- Validating stream')
 producer_stream_path = os.path.join(self.temp_dir, guid())
 consumer_file_path = os.path.join(self.temp_dir, guid())
 producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
 consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
 producer.file_to_stream(producer_file_path,
 producer_stream_path)
 consumer.stream_to_file(producer_stream_path,
Expand Down

14 changes: 7 additions & 7 deletions
java/vector/src/main/codegen/data/ValueVectorTypes.tdd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,7 +59,7 @@
 { class: "DateDay" },
 { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
 { class: "TimeSec" },
 { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
 { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
 ]
 },
 {
Expand All@@ -71,12 +71,12 @@
 minor: [
 { class: "BigInt"},
 { class: "UInt8" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeMicro" },
 { class: "TimeNano" }
 ]
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/includes/vv_imports.ftl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;

import org.joda.time.DateTime;
import org.joda.time.LocalDateTime;
import org.joda.time.Period;


Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/AbstractFieldReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,8 @@

<#include "/@includes/vv_imports.ftl" />

/*
 * This class is generated using freemarker and the ${.template_name} template.
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
Expand All@@ -51,7 +51,7 @@ public Field getField() {
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/ArrowType.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@

/**
 * Arrow types
 * Source code generated using FreeMarker template ${.template_name}
 **/
@JsonTypeInfo(
 use = JsonTypeInfo.Id.NAME,
Expand Down

5 changes: 3 additions & 2 deletions
java/vector/src/main/codegen/templates/BaseReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,9 @@

<#include "/@includes/vv_imports.ftl" />



/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
 Field getField();
Expand Down

7 changes: 7 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexReaders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,6 +47,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class ${name}ReaderImpl extends AbstractFieldReader {

Expand DownExpand Up@@ -123,12 +126,16 @@ public Object readObject(){
package org.apache.arrow.vector.complex.reader;

<#include "/@includes/vv_imports.ftl" />
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{

 public void read(${minor.class?cap_first}Holder h);
 public void read(Nullable${minor.class?cap_first}Holder h);
 public Object readObject();
 // read friendly type
 public ${friendlyType} read${safeType}();
 public boolean isSet();
 public void copyAsValue(${minor.class}Writer writer);
Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexWriters.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -139,6 +139,9 @@ public void writeNull() {
package org.apache.arrow.vector.complex.writer;

<#include "/@includes/vv_imports.ftl" />
/*
 * This class is generated using FreeMarker on the ${.template_name} template.
 */
@SuppressWarnings("unused")
public interface ${eName}Writer extends BaseWriter {
 public void write(${minor.class}Holder h);
Expand Down

18 changes: 6 additions & 12 deletions
java/vector/src/main/codegen/templates/FixedValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -495,33 +495,29 @@ public long getTwoAsLong(int index) {
 <#elseif minor.class == "DateMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return time;
 }

 <#elseif minor.class == "TimeStampSec">
 @Override
 public ${friendlyType} getObject(int index) {
 long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeStampMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting microseconds to milliseconds in order to use DateTime type
 long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
 long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand Down

12 changes: 6 additions & 6 deletions
java/vector/src/main/codegen/templates/HolderReaderImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ public boolean isSet() {

 }

@Override
@Override
 public void read(${name}Holder h) {
 <#list fields as field>
 h.${field.name} = holder.${field.name};
Expand All@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
 h.isSet = isSet() ? 1 : 0;
 }


 // read friendly type
 @Override
 public ${friendlyType} read${safeType}(){
<#if nullMode == "Nullable">
Expand All@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
 byte[] value = new byte [length];
 holder.buffer.getBytes(holder.start, value, 0, length);

<#if minor.class == "VarBinary">
<#if minor.class == "VarBinary">
 return value;
<#elseif minor.class == "Var16Char">
<#elseif minor.class == "Var16Char">
 return new String(value);
<#elseif minor.class == "VarChar">
<#elseif minor.class == "VarChar">
 Text text = new Text();
 text.set(value);
 return text;
</#if>
</#if>

<#elseif minor.class == "Interval">
 Period p = new Period();
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,7 +29,9 @@

<#include "/@includes/vv_imports.ftl" />


/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class NullReader extends AbstractBaseReader implements FieldReader{

Expand DownExpand Up@@ -127,7 +129,7 @@ private void fail(String name){
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/UnionReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,9 @@
package org.apache.arrow.vector.complex.impl;

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class UnionReader extends AbstractFieldReader {

Expand DownExpand Up@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
 }

 <#list ["Object", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Double", "Float",
 "Character", "LocalDateTime", "Double", "Float",
 "Text", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@
 *
 * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
 * each time the vector is accessed.
 * Source code generated using FreeMarker template ${.template_name}
 */
public class UnionVector implements FieldVector {

Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ValueHolders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
public final class ${className} implements ValueHolder{

 <#if mode.name == "Repeated">
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-824: Date and Time Vectors should reflect timezone-less semantics#568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

julienledem
wants to merge
6
commits into

from

 Closed


ARROW-824: Date and Time Vectors should reflect timezone-less semantics
#568
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
ec19e7d

ARROW-824: Date and Time Vectors should reflect timezone-less semantics


julienledem Apr 19, 2017
61518ec

improve travis integration


julienledem May 2, 2017
b0da88c

fix failing integration test


julienledem May 2, 2017
bdac7ff

make integration tests output more readable


julienledem May 2, 2017
e37385c

centralize LocalDateTime.toMillis


julienledem May 3, 2017
3528ad8

add license


julienledem May 3, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


3 changes: 2 additions & 1 deletion
ci/travis_script_integration.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh

pushd $ARROW_JAVA_DIR

mvn package
echo "mvn package"
mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)

popd

Expand Down

27 changes: 16 additions & 11 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -544,7 +544,8 @@ def get_json(self):

class JSONFile(object):

 def __init__(self, schema, batches):
 def __init__(self, name, schema, batches):
 self.name = name
 self.schema = schema
 self.batches = batches

Expand DownExpand Up@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
def _generate_file(name, fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
Expand All@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return JSONFile(name, schema, batches)


def generate_primitive_case(batch_sizes):
Expand All@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 return _generate_file(fields, batch_sizes)
 return _generate_file("primitive", fields, batch_sizes)


def generate_datetime_case():
Expand All@@ -619,11 +620,11 @@ def generate_datetime_case():
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 TimestampType('f10', 'ms', tz=None)
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("datetime", fields, batch_sizes)


def generate_nested_case():
Expand All@@ -637,7 +638,7 @@ def generate_nested_case():
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("nested", fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -655,7 +656,7 @@ def _temp_path():

 generated_paths = []
 for file_obj in file_objs:
 out_path = os.path.join(temp_dir, guid() + '.json')
 out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
 file_obj.write(out_path)
 generated_paths.append(out_path)

Expand DownExpand Up@@ -684,20 +685,24 @@ def _compare_implementations(self, producer, consumer):
 consumer.name))

 for json_path in self.json_files:
 print('=====================================================================================')
 print('Testing file {0}'.format(json_path))
 print('=====================================================================================')

 name = os.path.splitext(os.path.basename(json_path))[0]

 # Make the random access file
 print('-- Creating binary inputs')
 producer_file_path = os.path.join(self.temp_dir, guid())
 producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
 producer.json_to_file(json_path, producer_file_path)

 # Validate the file
 print('-- Validating file')
 consumer.validate(json_path, producer_file_path)

 print('-- Validating stream')
 producer_stream_path = os.path.join(self.temp_dir, guid())
 consumer_file_path = os.path.join(self.temp_dir, guid())
 producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
 consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
 producer.file_to_stream(producer_file_path,
 producer_stream_path)
 consumer.stream_to_file(producer_stream_path,
Expand Down

14 changes: 7 additions & 7 deletions
java/vector/src/main/codegen/data/ValueVectorTypes.tdd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,7 +59,7 @@
 { class: "DateDay" },
 { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
 { class: "TimeSec" },
 { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
 { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
 ]
 },
 {
Expand All@@ -71,12 +71,12 @@
 minor: [
 { class: "BigInt"},
 { class: "UInt8" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeMicro" },
 { class: "TimeNano" }
 ]
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/includes/vv_imports.ftl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;

import org.joda.time.DateTime;
import org.joda.time.LocalDateTime;
import org.joda.time.Period;


Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/AbstractFieldReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,8 @@

<#include "/@includes/vv_imports.ftl" />

/*
 * This class is generated using freemarker and the ${.template_name} template.
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
Expand All@@ -51,7 +51,7 @@ public Field getField() {
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/ArrowType.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@

/**
 * Arrow types
 * Source code generated using FreeMarker template ${.template_name}
 **/
@JsonTypeInfo(
 use = JsonTypeInfo.Id.NAME,
Expand Down

5 changes: 3 additions & 2 deletions
java/vector/src/main/codegen/templates/BaseReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,9 @@

<#include "/@includes/vv_imports.ftl" />



/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
 Field getField();
Expand Down

7 changes: 7 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexReaders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,6 +47,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class ${name}ReaderImpl extends AbstractFieldReader {

Expand DownExpand Up@@ -123,12 +126,16 @@ public Object readObject(){
package org.apache.arrow.vector.complex.reader;

<#include "/@includes/vv_imports.ftl" />
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{

 public void read(${minor.class?cap_first}Holder h);
 public void read(Nullable${minor.class?cap_first}Holder h);
 public Object readObject();
 // read friendly type
 public ${friendlyType} read${safeType}();
 public boolean isSet();
 public void copyAsValue(${minor.class}Writer writer);
Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexWriters.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -139,6 +139,9 @@ public void writeNull() {
package org.apache.arrow.vector.complex.writer;

<#include "/@includes/vv_imports.ftl" />
/*
 * This class is generated using FreeMarker on the ${.template_name} template.
 */
@SuppressWarnings("unused")
public interface ${eName}Writer extends BaseWriter {
 public void write(${minor.class}Holder h);
Expand Down

18 changes: 6 additions & 12 deletions
java/vector/src/main/codegen/templates/FixedValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -495,33 +495,29 @@ public long getTwoAsLong(int index) {
 <#elseif minor.class == "DateMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return time;
 }

 <#elseif minor.class == "TimeStampSec">
 @Override
 public ${friendlyType} getObject(int index) {
 long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeStampMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting microseconds to milliseconds in order to use DateTime type
 long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
 long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand Down

12 changes: 6 additions & 6 deletions
java/vector/src/main/codegen/templates/HolderReaderImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ public boolean isSet() {

 }

@Override
@Override
 public void read(${name}Holder h) {
 <#list fields as field>
 h.${field.name} = holder.${field.name};
Expand All@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
 h.isSet = isSet() ? 1 : 0;
 }


 // read friendly type
 @Override
 public ${friendlyType} read${safeType}(){
<#if nullMode == "Nullable">
Expand All@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
 byte[] value = new byte [length];
 holder.buffer.getBytes(holder.start, value, 0, length);

<#if minor.class == "VarBinary">
<#if minor.class == "VarBinary">
 return value;
<#elseif minor.class == "Var16Char">
<#elseif minor.class == "Var16Char">
 return new String(value);
<#elseif minor.class == "VarChar">
<#elseif minor.class == "VarChar">
 Text text = new Text();
 text.set(value);
 return text;
</#if>
</#if>

<#elseif minor.class == "Interval">
 Period p = new Period();
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,7 +29,9 @@

<#include "/@includes/vv_imports.ftl" />


/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class NullReader extends AbstractBaseReader implements FieldReader{

Expand DownExpand Up@@ -127,7 +129,7 @@ private void fail(String name){
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/UnionReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,9 @@
package org.apache.arrow.vector.complex.impl;

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class UnionReader extends AbstractFieldReader {

Expand DownExpand Up@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
 }

 <#list ["Object", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Double", "Float",
 "Character", "LocalDateTime", "Double", "Float",
 "Text", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@
 *
 * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
 * each time the vector is accessed.
 * Source code generated using FreeMarker template ${.template_name}
 */
public class UnionVector implements FieldVector {

Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ValueHolders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
public final class ${className} implements ValueHolder{

 <#if mode.name == "Repeated">
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-824: Date and Time Vectors should reflect timezone-less semantics#568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

julienledem
wants to merge
6
commits into

from

 Closed


ARROW-824: Date and Time Vectors should reflect timezone-less semantics
#568
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
ec19e7d

ARROW-824: Date and Time Vectors should reflect timezone-less semantics


julienledem Apr 19, 2017
61518ec

improve travis integration


julienledem May 2, 2017
b0da88c

fix failing integration test


julienledem May 2, 2017
bdac7ff

make integration tests output more readable


julienledem May 2, 2017
e37385c

centralize LocalDateTime.toMillis


julienledem May 3, 2017
3528ad8

add license


julienledem May 3, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


3 changes: 2 additions & 1 deletion
ci/travis_script_integration.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh

pushd $ARROW_JAVA_DIR

mvn package
echo "mvn package"
mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)

popd

Expand Down

27 changes: 16 additions & 11 deletions
integration/integration_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -544,7 +544,8 @@ def get_json(self):

class JSONFile(object):

 def __init__(self, schema, batches):
 def __init__(self, name, schema, batches):
 self.name = name
 self.schema = schema
 self.batches = batches

Expand DownExpand Up@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
 raise TypeError(dtype)


def _generate_file(fields, batch_sizes):
def _generate_file(name, fields, batch_sizes):
 schema = JSONSchema(fields)
 batches = []
 for size in batch_sizes:
Expand All@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):

 batches.append(JSONRecordBatch(size, columns))

 return JSONFile(schema, batches)
 return JSONFile(name, schema, batches)


def generate_primitive_case(batch_sizes):
Expand All@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
 fields.append(get_field(type_ + "_nullable", type_, True))
 fields.append(get_field(type_ + "_nonnullable", type_, False))

 return _generate_file(fields, batch_sizes)
 return _generate_file("primitive", fields, batch_sizes)


def generate_datetime_case():
Expand All@@ -619,11 +620,11 @@ def generate_datetime_case():
 TimestampType('f7', 'ms'),
 TimestampType('f8', 'us'),
 TimestampType('f9', 'ns'),
 TimestampType('f10', 'ms', tz='America/New_York')
 TimestampType('f10', 'ms', tz=None)
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("datetime", fields, batch_sizes)


def generate_nested_case():
Expand All@@ -637,7 +638,7 @@ def generate_nested_case():
 ]

 batch_sizes = [7, 10]
 return _generate_file(fields, batch_sizes)
 return _generate_file("nested", fields, batch_sizes)


def get_generated_json_files():
Expand All@@ -655,7 +656,7 @@ def _temp_path():

 generated_paths = []
 for file_obj in file_objs:
 out_path = os.path.join(temp_dir, guid() + '.json')
 out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
 file_obj.write(out_path)
 generated_paths.append(out_path)

Expand DownExpand Up@@ -684,20 +685,24 @@ def _compare_implementations(self, producer, consumer):
 consumer.name))

 for json_path in self.json_files:
 print('=====================================================================================')
 print('Testing file {0}'.format(json_path))
 print('=====================================================================================')

 name = os.path.splitext(os.path.basename(json_path))[0]

 # Make the random access file
 print('-- Creating binary inputs')
 producer_file_path = os.path.join(self.temp_dir, guid())
 producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
 producer.json_to_file(json_path, producer_file_path)

 # Validate the file
 print('-- Validating file')
 consumer.validate(json_path, producer_file_path)

 print('-- Validating stream')
 producer_stream_path = os.path.join(self.temp_dir, guid())
 consumer_file_path = os.path.join(self.temp_dir, guid())
 producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
 consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
 producer.file_to_stream(producer_file_path,
 producer_stream_path)
 consumer.stream_to_file(producer_stream_path,
Expand Down

14 changes: 7 additions & 7 deletions
java/vector/src/main/codegen/data/ValueVectorTypes.tdd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,7 +59,7 @@
 { class: "DateDay" },
 { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
 { class: "TimeSec" },
 { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
 { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
 ]
 },
 {
Expand All@@ -71,12 +71,12 @@
 minor: [
 { class: "BigInt"},
 { class: "UInt8" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
 { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], },
 { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" },
 { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" },
 { class: "TimeMicro" },
 { class: "TimeNano" }
 ]
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/includes/vv_imports.ftl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;

import org.joda.time.DateTime;
import org.joda.time.LocalDateTime;
import org.joda.time.Period;


Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/AbstractFieldReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,8 @@

<#include "/@includes/vv_imports.ftl" />

/*
 * This class is generated using freemarker and the ${.template_name} template.
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
Expand All@@ -51,7 +51,7 @@ public Field getField() {
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/ArrowType.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@

/**
 * Arrow types
 * Source code generated using FreeMarker template ${.template_name}
 **/
@JsonTypeInfo(
 use = JsonTypeInfo.Id.NAME,
Expand Down

5 changes: 3 additions & 2 deletions
java/vector/src/main/codegen/templates/BaseReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,8 +26,9 @@

<#include "/@includes/vv_imports.ftl" />



/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
 Field getField();
Expand Down

7 changes: 7 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexReaders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -47,6 +47,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class ${name}ReaderImpl extends AbstractFieldReader {

Expand DownExpand Up@@ -123,12 +126,16 @@ public Object readObject(){
package org.apache.arrow.vector.complex.reader;

<#include "/@includes/vv_imports.ftl" />
/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{

 public void read(${minor.class?cap_first}Holder h);
 public void read(Nullable${minor.class?cap_first}Holder h);
 public Object readObject();
 // read friendly type
 public ${friendlyType} read${safeType}();
 public boolean isSet();
 public void copyAsValue(${minor.class}Writer writer);
Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ComplexWriters.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -139,6 +139,9 @@ public void writeNull() {
package org.apache.arrow.vector.complex.writer;

<#include "/@includes/vv_imports.ftl" />
/*
 * This class is generated using FreeMarker on the ${.template_name} template.
 */
@SuppressWarnings("unused")
public interface ${eName}Writer extends BaseWriter {
 public void write(${minor.class}Holder h);
Expand Down

18 changes: 6 additions & 12 deletions
java/vector/src/main/codegen/templates/FixedValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -495,33 +495,29 @@ public long getTwoAsLong(int index) {
 <#elseif minor.class == "DateMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return time;
 }

 <#elseif minor.class == "TimeStampSec">
 @Override
 public ${friendlyType} getObject(int index) {
 long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
 return date;
 }

 <#elseif minor.class == "TimeStampMilli">
 @Override
 public ${friendlyType} getObject(int index) {
 org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting microseconds to milliseconds in order to use DateTime type
 long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand All@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
 public ${friendlyType} getObject(int index) {
 // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
 long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
 org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
 date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
 org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
 return date;
 }

Expand Down

12 changes: 6 additions & 6 deletions
java/vector/src/main/codegen/templates/HolderReaderImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,7 @@ public boolean isSet() {

 }

@Override
@Override
 public void read(${name}Holder h) {
 <#list fields as field>
 h.${field.name} = holder.${field.name};
Expand All@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
 h.isSet = isSet() ? 1 : 0;
 }


 // read friendly type
 @Override
 public ${friendlyType} read${safeType}(){
<#if nullMode == "Nullable">
Expand All@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
 byte[] value = new byte [length];
 holder.buffer.getBytes(holder.start, value, 0, length);

<#if minor.class == "VarBinary">
<#if minor.class == "VarBinary">
 return value;
<#elseif minor.class == "Var16Char">
<#elseif minor.class == "Var16Char">
 return new String(value);
<#elseif minor.class == "VarChar">
<#elseif minor.class == "VarChar">
 Text text = new Text();
 text.set(value);
 return text;
</#if>
</#if>

<#elseif minor.class == "Interval">
 Period p = new Period();
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,7 +29,9 @@

<#include "/@includes/vv_imports.ftl" />


/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class NullReader extends AbstractBaseReader implements FieldReader{

Expand DownExpand Up@@ -127,7 +129,7 @@ private void fail(String name){
 }

 <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
 "Character", "DateTime", "Period", "Double", "Float",
 "Character", "LocalDateTime", "Period", "Double", "Float",
 "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

6 changes: 4 additions & 2 deletions
java/vector/src/main/codegen/templates/UnionReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,9 @@
package org.apache.arrow.vector.complex.impl;

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
@SuppressWarnings("unused")
public class UnionReader extends AbstractFieldReader {

Expand DownExpand Up@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
 }

 <#list ["Object", "Integer", "Long", "Boolean",
 "Character", "DateTime", "Double", "Float",
 "Character", "LocalDateTime", "Double", "Float",
 "Text", "Byte", "Short", "byte[]"] as friendlyType>
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
Expand Down

1 change: 1 addition & 0 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,7 @@
 *
 * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
 * each time the vector is accessed.
 * Source code generated using FreeMarker template ${.template_name}
 */
public class UnionVector implements FieldVector {

Expand Down

3 changes: 3 additions & 0 deletions
java/vector/src/main/codegen/templates/ValueHolders.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@

<#include "/@includes/vv_imports.ftl" />

/**
 * Source code generated using FreeMarker template ${.template_name}
 */
public final class ${className} implements ValueHolder{

 <#if mode.name == "Repeated">
Expand Down
Loading