ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference#1618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cpcloud wants to merge 7 commits into from
Closed

ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference #1618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3bf399c
ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference
cpcloud Feb 15, 2018
4045f9b
Cleanup
cpcloud Feb 21, 2018
bb3959e
Remove comment
cpcloud Feb 21, 2018
30989dd
Use max scale
cpcloud Feb 21, 2018
ad82d13
More tests
cpcloud Feb 21, 2018
b03a0c2
Skip leading zeros in arrow decimal128
cpcloud Feb 21, 2018
e66e415
Adjust tests for skipping leading zeros
cpcloud Feb 21, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions cpp/CMakeLists.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -577,11 +577,13 @@ set(ARROW_LINK_LIBS

set(ARROW_SHARED_PRIVATE_LINK_LIBS
${BOOST_SYSTEM_LIBRARY}
${BOOST_FILESYSTEM_LIBRARY})
${BOOST_FILESYSTEM_LIBRARY}
${BOOST_REGEX_LIBRARY})

set(ARROW_STATIC_PRIVATE_LINK_LIBS
${BOOST_SYSTEM_LIBRARY}
${BOOST_FILESYSTEM_LIBRARY})
${BOOST_FILESYSTEM_LIBRARY}
${BOOST_REGEX_LIBRARY})

if (NOT MSVC)
set(ARROW_LINK_LIBS
Expand Down
22 changes: 18 additions & 4 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -157,16 +157,20 @@ if (ARROW_BOOST_VENDORED)
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(BOOST_STATIC_FILESYSTEM_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(BOOST_STATIC_REGEX_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
if (ARROW_BOOST_HEADER_ONLY)
set(BOOST_BUILD_PRODUCTS)
set(BOOST_CONFIGURE_COMMAND "")
set(BOOST_BUILD_COMMAND "")
else()
set(BOOST_BUILD_PRODUCTS
${BOOST_SYSTEM_LIBRARY}
${BOOST_FILESYSTEM_LIBRARY})
${BOOST_FILESYSTEM_LIBRARY}
${BOOST_REGEX_LIBRARY})
set(BOOST_CONFIGURE_COMMAND
"./bootstrap.sh"
"--prefix=${BOOST_PREFIX}"
Expand DownExpand Up@@ -210,16 +214,19 @@ else()
if (ARROW_BOOST_HEADER_ONLY)
find_package(Boost REQUIRED)
else()
find_package(Boost COMPONENTS system filesystem REQUIRED)
find_package(Boost COMPONENTS system filesystem regex REQUIRED)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
else()
set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
endif()
set(BOOST_SYSTEM_LIBRARY boost_system_shared)
set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
set(BOOST_REGEX_LIBRARY boost_regex_shared)
endif()
else()
# Find static boost headers and libs
Expand All@@ -228,16 +235,19 @@ else()
if (ARROW_BOOST_HEADER_ONLY)
find_package(Boost REQUIRED)
else()
find_package(Boost COMPONENTS system filesystem REQUIRED)
find_package(Boost COMPONENTS system filesystem regex REQUIRED)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
else()
set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
endif()
set(BOOST_SYSTEM_LIBRARY boost_system_static)
set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
set(BOOST_REGEX_LIBRARY boost_regex_static)
endif()
endif()
endif()
Expand All@@ -254,7 +264,11 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")

SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
ADD_THIRDPARTY_LIB(boost_regex
STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")

SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
endif()

include_directories(SYSTEM ${Boost_INCLUDE_DIR})
Expand Down
19 changes: 17 additions & 2 deletions cpp/src/arrow/python/helpers.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,8 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
DCHECK_NE(precision, NULLPTR);
DCHECK_NE(scale, NULLPTR);

OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
// TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK

@cpcloudcpcloudFeb 16, 2018•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires ARROW-2145 to address.

@wesmwesmFeb 19, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to wait for ARROW-2145 or merge this independently?

@cpcloudcpcloudFeb 19, 2018

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a couple more tests to add here

OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
RETURN_IF_PYERROR();
DCHECK(PyTuple_Check(as_tuple.obj()));

Expand All@@ -117,7 +118,21 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
RETURN_IF_PYERROR();

*precision = num_digits;
const int32_t abs_exponent = std::abs(exponent);

int32_t num_additional_zeros;

if (num_digits < abs_exponent) {
DCHECK_NE(exponent, 0) << "exponent should never be zero here";

// we have leading/trailing zeros, leading if exponent is negative
num_additional_zeros = exponent < 0 ? abs_exponent - num_digits : exponent;
} else {
// we can use the number of digits as the precision
num_additional_zeros = 0;
}

*precision = num_digits + num_additional_zeros;
*scale = -exponent;
return Status::OK();
}
Expand Down
5 changes: 1 addition & 4 deletions cpp/src/arrow/python/numpy_to_arrow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -766,10 +766,7 @@ Status NumPyConverter::ConvertDecimals() {
RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
&tmp_scale));
precision = std::max(precision, tmp_precision);

if (std::abs(desired_scale) < std::abs(tmp_scale)) {
desired_scale = tmp_scale;
}
desired_scale = std::max(desired_scale, tmp_scale);
}

type_ = ::arrow::decimal(precision, desired_scale);
Expand Down
89 changes: 89 additions & 0 deletions cpp/src/arrow/python/python-test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,57 @@ TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZeros) {
std::string decimal_string("0.001");
OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

int32_t precision;
int32_t scale;

ASSERT_OK(
internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

const int32_t expected_precision = 3;
const int32_t expected_scale = 3;

ASSERT_EQ(expected_precision, precision);
ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) {
std::string decimal_string("0.01E5");
OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

int32_t precision;
int32_t scale;

ASSERT_OK(
internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

const int32_t expected_precision = 4;
const int32_t expected_scale = -3;

ASSERT_EQ(expected_precision, precision);
ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) {
std::string decimal_string("0.01E3");
OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

int32_t precision;
int32_t scale;

ASSERT_OK(
internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

const int32_t expected_precision = 1;
const int32_t expected_scale = -1;

ASSERT_EQ(expected_precision, precision);
ASSERT_EQ(expected_scale, scale);
}

TEST(PandasConversionTest, TestObjectBlockWriteFails) {
StringBuilder builder;
const char value[] = {'\xf1', '\0'};
Expand DownExpand Up@@ -241,5 +292,43 @@ TEST_F(DecimalTest, TestOverflowFails) {
decimal_type, &value));
}


TEST_F(DecimalTest, SimpleInference) {
OwnedRef value(this->CreatePythonDecimal("0.01"));
ASSERT_NE(value.obj(), nullptr);
int32_t precision;
int32_t scale;
ASSERT_OK(internal::InferDecimalPrecisionAndScale(value.obj(), &precision, &scale));
ASSERT_EQ(2, precision);
ASSERT_EQ(2, scale);
}


TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {

PyAcquireGIL lock;
MemoryPool* pool = default_memory_pool();
std::shared_ptr<Array> arr;

OwnedRef list_ref(PyList_New(2));
PyObject* list = list_ref.obj();

ASSERT_NE(list, nullptr);

PyObject* value1 = this->CreatePythonDecimal("0.01").detach();
ASSERT_NE(value1, nullptr);

PyObject* value2 = this->CreatePythonDecimal("0.001").detach();
ASSERT_NE(value2, nullptr);

// This steals a reference to each object, so we don't need to decref them later
// just the list

ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);

ASSERT_OK(ConvertPySequence(list, pool, &arr));
}

} // namespace py
} // namespace arrow
56 changes: 32 additions & 24 deletions cpp/src/arrow/util/decimal-test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -37,15 +37,15 @@ class DecimalTestFixture : public ::testing::Test {

TEST_F(DecimalTestFixture, TestToString) {
Decimal128 decimal(this->integer_value_);
int scale = 5;
int32_t scale = 5;
std::string result = decimal.ToString(scale);
ASSERT_EQ(result, this->string_value_);
}

TEST_F(DecimalTestFixture, TestFromString) {
Decimal128 expected(this->integer_value_);
Decimal128 result;
int precision, scale;
int32_t precision, scale;
ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
ASSERT_EQ(result, expected);
ASSERT_EQ(precision, 8);
Expand All@@ -55,8 +55,8 @@ TEST_F(DecimalTestFixture, TestFromString) {
TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
std::string plus_value("+234.234");
Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
ASSERT_EQ(234234, out);
ASSERT_EQ(6, precision);
Expand All@@ -67,8 +67,8 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
std::string plus_value("+2342394230592.232349023094");
Decimal128 expected_value("2342394230592232349023094");
Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
ASSERT_EQ(expected_value, out);
ASSERT_EQ(25, precision);
Expand All@@ -90,9 +90,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
Decimal128 result;
ASSERT_OK(Decimal128::FromString(string_value, &result));
Decimal128 expected(static_cast<int64_t>(-230492239423435324));
expected *= 100;
expected -= 12;
ASSERT_EQ(result, expected);
ASSERT_EQ(result, expected * 100 - 12);

// Sanity check that our number is actually using more than 64 bits
ASSERT_NE(result.high_bits(), 0);
Expand DownExpand Up@@ -194,36 +192,36 @@ TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
std::string string_value("0000000");
Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
ASSERT_EQ(precision, 7);
ASSERT_EQ(scale, 0);
ASSERT_EQ(d, 0);
ASSERT_EQ(0, precision);
ASSERT_EQ(0, scale);
ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
std::string string_value("000.0000");
Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
// We explicitly do not support this for now, otherwise this would be ASSERT_EQ
ASSERT_NE(precision, 7);
ASSERT_EQ(4, precision);

ASSERT_EQ(scale, 4);
ASSERT_EQ(d, 0);
ASSERT_EQ(4, scale);
ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
std::string string_value(".00000");
Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
ASSERT_EQ(precision, 5);
ASSERT_EQ(scale, 5);
ASSERT_EQ(d, 0);
ASSERT_EQ(5, precision);
ASSERT_EQ(5, scale);
ASSERT_EQ(0, d);
}

template <typename T>
Expand DownExpand Up@@ -375,4 +373,14 @@ TEST(Decimal128Test, TestSmallNumberFormat) {
ASSERT_EQ(expected, result);
}

TEST(Decimal128Test, TestNoDecimalPointExponential) {
Decimal128 value;
int32_t precision;
int32_t scale;
ASSERT_OK(Decimal128::FromString("1E1", &value, &precision, &scale));
ASSERT_EQ(1, value.low_bits());
ASSERT_EQ(1, precision);
ASSERT_EQ(-1, scale);
}

} // namespace arrow
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference#1618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

cpcloud
wants to merge
7
commits into

from

 Closed


ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference
#1618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3bf399c

ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference


cpcloud Feb 15, 2018
4045f9b

Cleanup


cpcloud Feb 21, 2018
bb3959e

Remove comment


cpcloud Feb 21, 2018
30989dd

Use max scale


cpcloud Feb 21, 2018
ad82d13

More tests


cpcloud Feb 21, 2018
b03a0c2

Skip leading zeros in arrow decimal128


cpcloud Feb 21, 2018
e66e415

Adjust tests for skipping leading zeros


cpcloud Feb 21, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 4 additions & 2 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -577,11 +577,13 @@ set(ARROW_LINK_LIBS

set(ARROW_SHARED_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

set(ARROW_STATIC_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

if (NOT MSVC)
 set(ARROW_LINK_LIBS
Expand Down

22 changes: 18 additions & 4 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,16 +157,20 @@ if (ARROW_BOOST_VENDORED)
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_FILESYSTEM_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_REGEX_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
 set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
 set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
 if (ARROW_BOOST_HEADER_ONLY)
 set(BOOST_BUILD_PRODUCTS)
 set(BOOST_CONFIGURE_COMMAND "")
 set(BOOST_BUILD_COMMAND "")
 else()
 set(BOOST_BUILD_PRODUCTS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})
 set(BOOST_CONFIGURE_COMMAND
 "./bootstrap.sh"
 "--prefix=${BOOST_PREFIX}"
Expand DownExpand Up@@ -210,16 +214,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_shared)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
 set(BOOST_REGEX_LIBRARY boost_regex_shared)
 endif()
 else()
 # Find static boost headers and libs
Expand All@@ -228,16 +235,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_static)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
 set(BOOST_REGEX_LIBRARY boost_regex_static)
 endif()
 endif()
endif()
Expand All@@ -254,7 +264,11 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
 STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 ADD_THIRDPARTY_LIB(boost_regex
 STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
endif()

include_directories(SYSTEM ${Boost_INCLUDE_DIR})
Expand Down

19 changes: 17 additions & 2 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,8 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 DCHECK_NE(precision, NULLPTR);
 DCHECK_NE(scale, NULLPTR);

 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
 // TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK
@cpcloudcpcloudFeb 16, 2018•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires ARROW-2145 to address.
@wesmwesmFeb 19, 2018

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to wait for ARROW-2145 or merge this independently?
@cpcloudcpcloudFeb 19, 2018

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a couple more tests to add here
 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
 RETURN_IF_PYERROR();
 DCHECK(PyTuple_Check(as_tuple.obj()));

Expand All@@ -117,7 +118,21 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
 RETURN_IF_PYERROR();

 *precision = num_digits;
 const int32_t abs_exponent = std::abs(exponent);

 int32_t num_additional_zeros;

 if (num_digits < abs_exponent) {
 DCHECK_NE(exponent, 0) << "exponent should never be zero here";

 // we have leading/trailing zeros, leading if exponent is negative
 num_additional_zeros = exponent < 0 ? abs_exponent - num_digits : exponent;
 } else {
 // we can use the number of digits as the precision
 num_additional_zeros = 0;
 }

 *precision = num_digits + num_additional_zeros;
 *scale = -exponent;
 return Status::OK();
}
Expand Down

5 changes: 1 addition & 4 deletions
cpp/src/arrow/python/numpy_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -766,10 +766,7 @@ Status NumPyConverter::ConvertDecimals() {
 RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
 &tmp_scale));
 precision = std::max(precision, tmp_precision);

 if (std::abs(desired_scale) < std::abs(tmp_scale)) {
 desired_scale = tmp_scale;
 }
 desired_scale = std::max(desired_scale, tmp_scale);
 }

 type_ = ::arrow::decimal(precision, desired_scale);
Expand Down

89 changes: 89 additions & 0 deletions
cpp/src/arrow/python/python-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,57 @@ TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZeros) {
 std::string decimal_string("0.001");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 3;
 const int32_t expected_scale = 3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) {
 std::string decimal_string("0.01E5");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 4;
 const int32_t expected_scale = -3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) {
 std::string decimal_string("0.01E3");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 1;
 const int32_t expected_scale = -1;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST(PandasConversionTest, TestObjectBlockWriteFails) {
 StringBuilder builder;
 const char value[] = {'\xf1', '\0'};
Expand DownExpand Up@@ -241,5 +292,43 @@ TEST_F(DecimalTest, TestOverflowFails) {
 decimal_type, &value));
}


TEST_F(DecimalTest, SimpleInference) {
 OwnedRef value(this->CreatePythonDecimal("0.01"));
 ASSERT_NE(value.obj(), nullptr);
 int32_t precision;
 int32_t scale;
 ASSERT_OK(internal::InferDecimalPrecisionAndScale(value.obj(), &precision, &scale));
 ASSERT_EQ(2, precision);
 ASSERT_EQ(2, scale);
}


TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {

 PyAcquireGIL lock;
 MemoryPool* pool = default_memory_pool();
 std::shared_ptr<Array> arr;

 OwnedRef list_ref(PyList_New(2));
 PyObject* list = list_ref.obj();

 ASSERT_NE(list, nullptr);

 PyObject* value1 = this->CreatePythonDecimal("0.01").detach();
 ASSERT_NE(value1, nullptr);

 PyObject* value2 = this->CreatePythonDecimal("0.001").detach();
 ASSERT_NE(value2, nullptr);

 // This steals a reference to each object, so we don't need to decref them later
 // just the list

 ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
 ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);

 ASSERT_OK(ConvertPySequence(list, pool, &arr));
}

} // namespace py
} // namespace arrow

56 changes: 32 additions & 24 deletions
cpp/src/arrow/util/decimal-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,15 +37,15 @@ class DecimalTestFixture : public ::testing::Test {

TEST_F(DecimalTestFixture, TestToString) {
 Decimal128 decimal(this->integer_value_);
int scale = 5;
int32_t scale = 5;
 std::string result = decimal.ToString(scale);
 ASSERT_EQ(result, this->string_value_);
}

TEST_F(DecimalTestFixture, TestFromString) {
 Decimal128 expected(this->integer_value_);
 Decimal128 result;
int precision, scale;
int32_t precision, scale;
 ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
 ASSERT_EQ(result, expected);
 ASSERT_EQ(precision, 8);
Expand All@@ -55,8 +55,8 @@ TEST_F(DecimalTestFixture, TestFromString) {
TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
 std::string plus_value("+234.234");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(234234, out);
 ASSERT_EQ(6, precision);
Expand All@@ -67,8 +67,8 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
 std::string plus_value("+2342394230592.232349023094");
 Decimal128 expected_value("2342394230592232349023094");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(expected_value, out);
 ASSERT_EQ(25, precision);
Expand All@@ -90,9 +90,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
 Decimal128 result;
 ASSERT_OK(Decimal128::FromString(string_value, &result));
 Decimal128 expected(static_cast<int64_t>(-230492239423435324));
 expected *= 100;
 expected -= 12;
 ASSERT_EQ(result, expected);
 ASSERT_EQ(result, expected * 100 - 12);

 // Sanity check that our number is actually using more than 64 bits
 ASSERT_NE(result.high_bits(), 0);
Expand DownExpand Up@@ -194,36 +192,36 @@ TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
 std::string string_value("0000000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 7);
 ASSERT_EQ(scale, 0);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(0, precision);
 ASSERT_EQ(0, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
 std::string string_value("000.0000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
ASSERT_NE(precision, 7);
ASSERT_EQ(4, precision);

 ASSERT_EQ(scale, 4);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(4, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
 std::string string_value(".00000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 5);
 ASSERT_EQ(scale, 5);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(5, precision);
 ASSERT_EQ(5, scale);
 ASSERT_EQ(0, d);
}

template <typename T>
Expand DownExpand Up@@ -375,4 +373,14 @@ TEST(Decimal128Test, TestSmallNumberFormat) {
 ASSERT_EQ(expected, result);
}

TEST(Decimal128Test, TestNoDecimalPointExponential) {
 Decimal128 value;
 int32_t precision;
 int32_t scale;
 ASSERT_OK(Decimal128::FromString("1E1", &value, &precision, &scale));
 ASSERT_EQ(1, value.low_bits());
 ASSERT_EQ(1, precision);
 ASSERT_EQ(-1, scale);
}

} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference#1618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

cpcloud
wants to merge
7
commits into

from

 Closed


ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference
#1618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3bf399c

ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference


cpcloud Feb 15, 2018
4045f9b

Cleanup


cpcloud Feb 21, 2018
bb3959e

Remove comment


cpcloud Feb 21, 2018
30989dd

Use max scale


cpcloud Feb 21, 2018
ad82d13

More tests


cpcloud Feb 21, 2018
b03a0c2

Skip leading zeros in arrow decimal128


cpcloud Feb 21, 2018
e66e415

Adjust tests for skipping leading zeros


cpcloud Feb 21, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 4 additions & 2 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -577,11 +577,13 @@ set(ARROW_LINK_LIBS

set(ARROW_SHARED_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

set(ARROW_STATIC_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

if (NOT MSVC)
 set(ARROW_LINK_LIBS
Expand Down

22 changes: 18 additions & 4 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,16 +157,20 @@ if (ARROW_BOOST_VENDORED)
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_FILESYSTEM_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_REGEX_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
 set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
 set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
 if (ARROW_BOOST_HEADER_ONLY)
 set(BOOST_BUILD_PRODUCTS)
 set(BOOST_CONFIGURE_COMMAND "")
 set(BOOST_BUILD_COMMAND "")
 else()
 set(BOOST_BUILD_PRODUCTS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})
 set(BOOST_CONFIGURE_COMMAND
 "./bootstrap.sh"
 "--prefix=${BOOST_PREFIX}"
Expand DownExpand Up@@ -210,16 +214,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_shared)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
 set(BOOST_REGEX_LIBRARY boost_regex_shared)
 endif()
 else()
 # Find static boost headers and libs
Expand All@@ -228,16 +235,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_static)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
 set(BOOST_REGEX_LIBRARY boost_regex_static)
 endif()
 endif()
endif()
Expand All@@ -254,7 +264,11 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
 STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 ADD_THIRDPARTY_LIB(boost_regex
 STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
endif()

include_directories(SYSTEM ${Boost_INCLUDE_DIR})
Expand Down

19 changes: 17 additions & 2 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,8 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 DCHECK_NE(precision, NULLPTR);
 DCHECK_NE(scale, NULLPTR);

 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
 // TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK
@cpcloudcpcloudFeb 16, 2018•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires ARROW-2145 to address.
@wesmwesmFeb 19, 2018

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to wait for ARROW-2145 or merge this independently?
@cpcloudcpcloudFeb 19, 2018

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a couple more tests to add here
 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
 RETURN_IF_PYERROR();
 DCHECK(PyTuple_Check(as_tuple.obj()));

Expand All@@ -117,7 +118,21 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
 RETURN_IF_PYERROR();

 *precision = num_digits;
 const int32_t abs_exponent = std::abs(exponent);

 int32_t num_additional_zeros;

 if (num_digits < abs_exponent) {
 DCHECK_NE(exponent, 0) << "exponent should never be zero here";

 // we have leading/trailing zeros, leading if exponent is negative
 num_additional_zeros = exponent < 0 ? abs_exponent - num_digits : exponent;
 } else {
 // we can use the number of digits as the precision
 num_additional_zeros = 0;
 }

 *precision = num_digits + num_additional_zeros;
 *scale = -exponent;
 return Status::OK();
}
Expand Down

5 changes: 1 addition & 4 deletions
cpp/src/arrow/python/numpy_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -766,10 +766,7 @@ Status NumPyConverter::ConvertDecimals() {
 RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
 &tmp_scale));
 precision = std::max(precision, tmp_precision);

 if (std::abs(desired_scale) < std::abs(tmp_scale)) {
 desired_scale = tmp_scale;
 }
 desired_scale = std::max(desired_scale, tmp_scale);
 }

 type_ = ::arrow::decimal(precision, desired_scale);
Expand Down

89 changes: 89 additions & 0 deletions
cpp/src/arrow/python/python-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,57 @@ TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZeros) {
 std::string decimal_string("0.001");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 3;
 const int32_t expected_scale = 3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) {
 std::string decimal_string("0.01E5");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 4;
 const int32_t expected_scale = -3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) {
 std::string decimal_string("0.01E3");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 1;
 const int32_t expected_scale = -1;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST(PandasConversionTest, TestObjectBlockWriteFails) {
 StringBuilder builder;
 const char value[] = {'\xf1', '\0'};
Expand DownExpand Up@@ -241,5 +292,43 @@ TEST_F(DecimalTest, TestOverflowFails) {
 decimal_type, &value));
}


TEST_F(DecimalTest, SimpleInference) {
 OwnedRef value(this->CreatePythonDecimal("0.01"));
 ASSERT_NE(value.obj(), nullptr);
 int32_t precision;
 int32_t scale;
 ASSERT_OK(internal::InferDecimalPrecisionAndScale(value.obj(), &precision, &scale));
 ASSERT_EQ(2, precision);
 ASSERT_EQ(2, scale);
}


TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {

 PyAcquireGIL lock;
 MemoryPool* pool = default_memory_pool();
 std::shared_ptr<Array> arr;

 OwnedRef list_ref(PyList_New(2));
 PyObject* list = list_ref.obj();

 ASSERT_NE(list, nullptr);

 PyObject* value1 = this->CreatePythonDecimal("0.01").detach();
 ASSERT_NE(value1, nullptr);

 PyObject* value2 = this->CreatePythonDecimal("0.001").detach();
 ASSERT_NE(value2, nullptr);

 // This steals a reference to each object, so we don't need to decref them later
 // just the list

 ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
 ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);

 ASSERT_OK(ConvertPySequence(list, pool, &arr));
}

} // namespace py
} // namespace arrow

56 changes: 32 additions & 24 deletions
cpp/src/arrow/util/decimal-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,15 +37,15 @@ class DecimalTestFixture : public ::testing::Test {

TEST_F(DecimalTestFixture, TestToString) {
 Decimal128 decimal(this->integer_value_);
int scale = 5;
int32_t scale = 5;
 std::string result = decimal.ToString(scale);
 ASSERT_EQ(result, this->string_value_);
}

TEST_F(DecimalTestFixture, TestFromString) {
 Decimal128 expected(this->integer_value_);
 Decimal128 result;
int precision, scale;
int32_t precision, scale;
 ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
 ASSERT_EQ(result, expected);
 ASSERT_EQ(precision, 8);
Expand All@@ -55,8 +55,8 @@ TEST_F(DecimalTestFixture, TestFromString) {
TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
 std::string plus_value("+234.234");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(234234, out);
 ASSERT_EQ(6, precision);
Expand All@@ -67,8 +67,8 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
 std::string plus_value("+2342394230592.232349023094");
 Decimal128 expected_value("2342394230592232349023094");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(expected_value, out);
 ASSERT_EQ(25, precision);
Expand All@@ -90,9 +90,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
 Decimal128 result;
 ASSERT_OK(Decimal128::FromString(string_value, &result));
 Decimal128 expected(static_cast<int64_t>(-230492239423435324));
 expected *= 100;
 expected -= 12;
 ASSERT_EQ(result, expected);
 ASSERT_EQ(result, expected * 100 - 12);

 // Sanity check that our number is actually using more than 64 bits
 ASSERT_NE(result.high_bits(), 0);
Expand DownExpand Up@@ -194,36 +192,36 @@ TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
 std::string string_value("0000000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 7);
 ASSERT_EQ(scale, 0);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(0, precision);
 ASSERT_EQ(0, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
 std::string string_value("000.0000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
ASSERT_NE(precision, 7);
ASSERT_EQ(4, precision);

 ASSERT_EQ(scale, 4);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(4, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
 std::string string_value(".00000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 5);
 ASSERT_EQ(scale, 5);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(5, precision);
 ASSERT_EQ(5, scale);
 ASSERT_EQ(0, d);
}

template <typename T>
Expand DownExpand Up@@ -375,4 +373,14 @@ TEST(Decimal128Test, TestSmallNumberFormat) {
 ASSERT_EQ(expected, result);
}

TEST(Decimal128Test, TestNoDecimalPointExponential) {
 Decimal128 value;
 int32_t precision;
 int32_t scale;
 ASSERT_OK(Decimal128::FromString("1E1", &value, &precision, &scale));
 ASSERT_EQ(1, value.low_bits());
 ASSERT_EQ(1, precision);
 ASSERT_EQ(-1, scale);
}

} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference#1618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

cpcloud
wants to merge
7
commits into

from

 Closed


ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference
#1618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3bf399c

ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference


cpcloud Feb 15, 2018
4045f9b

Cleanup


cpcloud Feb 21, 2018
bb3959e

Remove comment


cpcloud Feb 21, 2018
30989dd

Use max scale


cpcloud Feb 21, 2018
ad82d13

More tests


cpcloud Feb 21, 2018
b03a0c2

Skip leading zeros in arrow decimal128


cpcloud Feb 21, 2018
e66e415

Adjust tests for skipping leading zeros


cpcloud Feb 21, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 4 additions & 2 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -577,11 +577,13 @@ set(ARROW_LINK_LIBS

set(ARROW_SHARED_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

set(ARROW_STATIC_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

if (NOT MSVC)
 set(ARROW_LINK_LIBS
Expand Down

22 changes: 18 additions & 4 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,16 +157,20 @@ if (ARROW_BOOST_VENDORED)
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_FILESYSTEM_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_REGEX_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
 set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
 set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
 if (ARROW_BOOST_HEADER_ONLY)
 set(BOOST_BUILD_PRODUCTS)
 set(BOOST_CONFIGURE_COMMAND "")
 set(BOOST_BUILD_COMMAND "")
 else()
 set(BOOST_BUILD_PRODUCTS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})
 set(BOOST_CONFIGURE_COMMAND
 "./bootstrap.sh"
 "--prefix=${BOOST_PREFIX}"
Expand DownExpand Up@@ -210,16 +214,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_shared)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
 set(BOOST_REGEX_LIBRARY boost_regex_shared)
 endif()
 else()
 # Find static boost headers and libs
Expand All@@ -228,16 +235,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_static)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
 set(BOOST_REGEX_LIBRARY boost_regex_static)
 endif()
 endif()
endif()
Expand All@@ -254,7 +264,11 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
 STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 ADD_THIRDPARTY_LIB(boost_regex
 STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
endif()

include_directories(SYSTEM ${Boost_INCLUDE_DIR})
Expand Down

19 changes: 17 additions & 2 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,8 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 DCHECK_NE(precision, NULLPTR);
 DCHECK_NE(scale, NULLPTR);

 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
 // TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK
@cpcloudcpcloudFeb 16, 2018•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires ARROW-2145 to address.
@wesmwesmFeb 19, 2018

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to wait for ARROW-2145 or merge this independently?
@cpcloudcpcloudFeb 19, 2018

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a couple more tests to add here
 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
 RETURN_IF_PYERROR();
 DCHECK(PyTuple_Check(as_tuple.obj()));

Expand All@@ -117,7 +118,21 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
 RETURN_IF_PYERROR();

 *precision = num_digits;
 const int32_t abs_exponent = std::abs(exponent);

 int32_t num_additional_zeros;

 if (num_digits < abs_exponent) {
 DCHECK_NE(exponent, 0) << "exponent should never be zero here";

 // we have leading/trailing zeros, leading if exponent is negative
 num_additional_zeros = exponent < 0 ? abs_exponent - num_digits : exponent;
 } else {
 // we can use the number of digits as the precision
 num_additional_zeros = 0;
 }

 *precision = num_digits + num_additional_zeros;
 *scale = -exponent;
 return Status::OK();
}
Expand Down

5 changes: 1 addition & 4 deletions
cpp/src/arrow/python/numpy_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -766,10 +766,7 @@ Status NumPyConverter::ConvertDecimals() {
 RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
 &tmp_scale));
 precision = std::max(precision, tmp_precision);

 if (std::abs(desired_scale) < std::abs(tmp_scale)) {
 desired_scale = tmp_scale;
 }
 desired_scale = std::max(desired_scale, tmp_scale);
 }

 type_ = ::arrow::decimal(precision, desired_scale);
Expand Down

89 changes: 89 additions & 0 deletions
cpp/src/arrow/python/python-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,57 @@ TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZeros) {
 std::string decimal_string("0.001");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 3;
 const int32_t expected_scale = 3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) {
 std::string decimal_string("0.01E5");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 4;
 const int32_t expected_scale = -3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) {
 std::string decimal_string("0.01E3");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 1;
 const int32_t expected_scale = -1;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST(PandasConversionTest, TestObjectBlockWriteFails) {
 StringBuilder builder;
 const char value[] = {'\xf1', '\0'};
Expand DownExpand Up@@ -241,5 +292,43 @@ TEST_F(DecimalTest, TestOverflowFails) {
 decimal_type, &value));
}


TEST_F(DecimalTest, SimpleInference) {
 OwnedRef value(this->CreatePythonDecimal("0.01"));
 ASSERT_NE(value.obj(), nullptr);
 int32_t precision;
 int32_t scale;
 ASSERT_OK(internal::InferDecimalPrecisionAndScale(value.obj(), &precision, &scale));
 ASSERT_EQ(2, precision);
 ASSERT_EQ(2, scale);
}


TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {

 PyAcquireGIL lock;
 MemoryPool* pool = default_memory_pool();
 std::shared_ptr<Array> arr;

 OwnedRef list_ref(PyList_New(2));
 PyObject* list = list_ref.obj();

 ASSERT_NE(list, nullptr);

 PyObject* value1 = this->CreatePythonDecimal("0.01").detach();
 ASSERT_NE(value1, nullptr);

 PyObject* value2 = this->CreatePythonDecimal("0.001").detach();
 ASSERT_NE(value2, nullptr);

 // This steals a reference to each object, so we don't need to decref them later
 // just the list

 ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
 ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);

 ASSERT_OK(ConvertPySequence(list, pool, &arr));
}

} // namespace py
} // namespace arrow

56 changes: 32 additions & 24 deletions
cpp/src/arrow/util/decimal-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,15 +37,15 @@ class DecimalTestFixture : public ::testing::Test {

TEST_F(DecimalTestFixture, TestToString) {
 Decimal128 decimal(this->integer_value_);
int scale = 5;
int32_t scale = 5;
 std::string result = decimal.ToString(scale);
 ASSERT_EQ(result, this->string_value_);
}

TEST_F(DecimalTestFixture, TestFromString) {
 Decimal128 expected(this->integer_value_);
 Decimal128 result;
int precision, scale;
int32_t precision, scale;
 ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
 ASSERT_EQ(result, expected);
 ASSERT_EQ(precision, 8);
Expand All@@ -55,8 +55,8 @@ TEST_F(DecimalTestFixture, TestFromString) {
TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
 std::string plus_value("+234.234");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(234234, out);
 ASSERT_EQ(6, precision);
Expand All@@ -67,8 +67,8 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
 std::string plus_value("+2342394230592.232349023094");
 Decimal128 expected_value("2342394230592232349023094");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(expected_value, out);
 ASSERT_EQ(25, precision);
Expand All@@ -90,9 +90,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
 Decimal128 result;
 ASSERT_OK(Decimal128::FromString(string_value, &result));
 Decimal128 expected(static_cast<int64_t>(-230492239423435324));
 expected *= 100;
 expected -= 12;
 ASSERT_EQ(result, expected);
 ASSERT_EQ(result, expected * 100 - 12);

 // Sanity check that our number is actually using more than 64 bits
 ASSERT_NE(result.high_bits(), 0);
Expand DownExpand Up@@ -194,36 +192,36 @@ TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
 std::string string_value("0000000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 7);
 ASSERT_EQ(scale, 0);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(0, precision);
 ASSERT_EQ(0, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
 std::string string_value("000.0000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
ASSERT_NE(precision, 7);
ASSERT_EQ(4, precision);

 ASSERT_EQ(scale, 4);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(4, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
 std::string string_value(".00000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 5);
 ASSERT_EQ(scale, 5);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(5, precision);
 ASSERT_EQ(5, scale);
 ASSERT_EQ(0, d);
}

template <typename T>
Expand DownExpand Up@@ -375,4 +373,14 @@ TEST(Decimal128Test, TestSmallNumberFormat) {
 ASSERT_EQ(expected, result);
}

TEST(Decimal128Test, TestNoDecimalPointExponential) {
 Decimal128 value;
 int32_t precision;
 int32_t scale;
 ASSERT_OK(Decimal128::FromString("1E1", &value, &precision, &scale));
 ASSERT_EQ(1, value.low_bits());
 ASSERT_EQ(1, precision);
 ASSERT_EQ(-1, scale);
}

} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference#1618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

cpcloud
wants to merge
7
commits into

from

 Closed


ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference
#1618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3bf399c

ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference


cpcloud Feb 15, 2018
4045f9b

Cleanup


cpcloud Feb 21, 2018
bb3959e

Remove comment


cpcloud Feb 21, 2018
30989dd

Use max scale


cpcloud Feb 21, 2018
ad82d13

More tests


cpcloud Feb 21, 2018
b03a0c2

Skip leading zeros in arrow decimal128


cpcloud Feb 21, 2018
e66e415

Adjust tests for skipping leading zeros


cpcloud Feb 21, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 4 additions & 2 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -577,11 +577,13 @@ set(ARROW_LINK_LIBS

set(ARROW_SHARED_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

set(ARROW_STATIC_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

if (NOT MSVC)
 set(ARROW_LINK_LIBS
Expand Down

22 changes: 18 additions & 4 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,16 +157,20 @@ if (ARROW_BOOST_VENDORED)
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_FILESYSTEM_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_REGEX_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
 set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
 set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
 if (ARROW_BOOST_HEADER_ONLY)
 set(BOOST_BUILD_PRODUCTS)
 set(BOOST_CONFIGURE_COMMAND "")
 set(BOOST_BUILD_COMMAND "")
 else()
 set(BOOST_BUILD_PRODUCTS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})
 set(BOOST_CONFIGURE_COMMAND
 "./bootstrap.sh"
 "--prefix=${BOOST_PREFIX}"
Expand DownExpand Up@@ -210,16 +214,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_shared)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
 set(BOOST_REGEX_LIBRARY boost_regex_shared)
 endif()
 else()
 # Find static boost headers and libs
Expand All@@ -228,16 +235,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_static)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
 set(BOOST_REGEX_LIBRARY boost_regex_static)
 endif()
 endif()
endif()
Expand All@@ -254,7 +264,11 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
 STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 ADD_THIRDPARTY_LIB(boost_regex
 STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
endif()

include_directories(SYSTEM ${Boost_INCLUDE_DIR})
Expand Down

19 changes: 17 additions & 2 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,8 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 DCHECK_NE(precision, NULLPTR);
 DCHECK_NE(scale, NULLPTR);

 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
 // TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK
@cpcloudcpcloudFeb 16, 2018•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires ARROW-2145 to address.
@wesmwesmFeb 19, 2018

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to wait for ARROW-2145 or merge this independently?
@cpcloudcpcloudFeb 19, 2018

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a couple more tests to add here
 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
 RETURN_IF_PYERROR();
 DCHECK(PyTuple_Check(as_tuple.obj()));

Expand All@@ -117,7 +118,21 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
 RETURN_IF_PYERROR();

 *precision = num_digits;
 const int32_t abs_exponent = std::abs(exponent);

 int32_t num_additional_zeros;

 if (num_digits < abs_exponent) {
 DCHECK_NE(exponent, 0) << "exponent should never be zero here";

 // we have leading/trailing zeros, leading if exponent is negative
 num_additional_zeros = exponent < 0 ? abs_exponent - num_digits : exponent;
 } else {
 // we can use the number of digits as the precision
 num_additional_zeros = 0;
 }

 *precision = num_digits + num_additional_zeros;
 *scale = -exponent;
 return Status::OK();
}
Expand Down

5 changes: 1 addition & 4 deletions
cpp/src/arrow/python/numpy_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -766,10 +766,7 @@ Status NumPyConverter::ConvertDecimals() {
 RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
 &tmp_scale));
 precision = std::max(precision, tmp_precision);

 if (std::abs(desired_scale) < std::abs(tmp_scale)) {
 desired_scale = tmp_scale;
 }
 desired_scale = std::max(desired_scale, tmp_scale);
 }

 type_ = ::arrow::decimal(precision, desired_scale);
Expand Down

89 changes: 89 additions & 0 deletions
cpp/src/arrow/python/python-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,57 @@ TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZeros) {
 std::string decimal_string("0.001");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 3;
 const int32_t expected_scale = 3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) {
 std::string decimal_string("0.01E5");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 4;
 const int32_t expected_scale = -3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) {
 std::string decimal_string("0.01E3");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 1;
 const int32_t expected_scale = -1;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST(PandasConversionTest, TestObjectBlockWriteFails) {
 StringBuilder builder;
 const char value[] = {'\xf1', '\0'};
Expand DownExpand Up@@ -241,5 +292,43 @@ TEST_F(DecimalTest, TestOverflowFails) {
 decimal_type, &value));
}


TEST_F(DecimalTest, SimpleInference) {
 OwnedRef value(this->CreatePythonDecimal("0.01"));
 ASSERT_NE(value.obj(), nullptr);
 int32_t precision;
 int32_t scale;
 ASSERT_OK(internal::InferDecimalPrecisionAndScale(value.obj(), &precision, &scale));
 ASSERT_EQ(2, precision);
 ASSERT_EQ(2, scale);
}


TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {

 PyAcquireGIL lock;
 MemoryPool* pool = default_memory_pool();
 std::shared_ptr<Array> arr;

 OwnedRef list_ref(PyList_New(2));
 PyObject* list = list_ref.obj();

 ASSERT_NE(list, nullptr);

 PyObject* value1 = this->CreatePythonDecimal("0.01").detach();
 ASSERT_NE(value1, nullptr);

 PyObject* value2 = this->CreatePythonDecimal("0.001").detach();
 ASSERT_NE(value2, nullptr);

 // This steals a reference to each object, so we don't need to decref them later
 // just the list

 ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
 ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);

 ASSERT_OK(ConvertPySequence(list, pool, &arr));
}

} // namespace py
} // namespace arrow

56 changes: 32 additions & 24 deletions
cpp/src/arrow/util/decimal-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,15 +37,15 @@ class DecimalTestFixture : public ::testing::Test {

TEST_F(DecimalTestFixture, TestToString) {
 Decimal128 decimal(this->integer_value_);
int scale = 5;
int32_t scale = 5;
 std::string result = decimal.ToString(scale);
 ASSERT_EQ(result, this->string_value_);
}

TEST_F(DecimalTestFixture, TestFromString) {
 Decimal128 expected(this->integer_value_);
 Decimal128 result;
int precision, scale;
int32_t precision, scale;
 ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
 ASSERT_EQ(result, expected);
 ASSERT_EQ(precision, 8);
Expand All@@ -55,8 +55,8 @@ TEST_F(DecimalTestFixture, TestFromString) {
TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
 std::string plus_value("+234.234");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(234234, out);
 ASSERT_EQ(6, precision);
Expand All@@ -67,8 +67,8 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
 std::string plus_value("+2342394230592.232349023094");
 Decimal128 expected_value("2342394230592232349023094");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(expected_value, out);
 ASSERT_EQ(25, precision);
Expand All@@ -90,9 +90,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
 Decimal128 result;
 ASSERT_OK(Decimal128::FromString(string_value, &result));
 Decimal128 expected(static_cast<int64_t>(-230492239423435324));
 expected *= 100;
 expected -= 12;
 ASSERT_EQ(result, expected);
 ASSERT_EQ(result, expected * 100 - 12);

 // Sanity check that our number is actually using more than 64 bits
 ASSERT_NE(result.high_bits(), 0);
Expand DownExpand Up@@ -194,36 +192,36 @@ TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
 std::string string_value("0000000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 7);
 ASSERT_EQ(scale, 0);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(0, precision);
 ASSERT_EQ(0, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
 std::string string_value("000.0000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
ASSERT_NE(precision, 7);
ASSERT_EQ(4, precision);

 ASSERT_EQ(scale, 4);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(4, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
 std::string string_value(".00000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 5);
 ASSERT_EQ(scale, 5);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(5, precision);
 ASSERT_EQ(5, scale);
 ASSERT_EQ(0, d);
}

template <typename T>
Expand DownExpand Up@@ -375,4 +373,14 @@ TEST(Decimal128Test, TestSmallNumberFormat) {
 ASSERT_EQ(expected, result);
}

TEST(Decimal128Test, TestNoDecimalPointExponential) {
 Decimal128 value;
 int32_t precision;
 int32_t scale;
 ASSERT_OK(Decimal128::FromString("1E1", &value, &precision, &scale));
 ASSERT_EQ(1, value.low_bits());
 ASSERT_EQ(1, precision);
 ASSERT_EQ(-1, scale);
}

} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference#1618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

cpcloud
wants to merge
7
commits into

from

 Closed


ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference
#1618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3bf399c

ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference


cpcloud Feb 15, 2018
4045f9b

Cleanup


cpcloud Feb 21, 2018
bb3959e

Remove comment


cpcloud Feb 21, 2018
30989dd

Use max scale


cpcloud Feb 21, 2018
ad82d13

More tests


cpcloud Feb 21, 2018
b03a0c2

Skip leading zeros in arrow decimal128


cpcloud Feb 21, 2018
e66e415

Adjust tests for skipping leading zeros


cpcloud Feb 21, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 4 additions & 2 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -577,11 +577,13 @@ set(ARROW_LINK_LIBS

set(ARROW_SHARED_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

set(ARROW_STATIC_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

if (NOT MSVC)
 set(ARROW_LINK_LIBS
Expand Down

22 changes: 18 additions & 4 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,16 +157,20 @@ if (ARROW_BOOST_VENDORED)
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_FILESYSTEM_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_REGEX_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
 set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
 set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
 if (ARROW_BOOST_HEADER_ONLY)
 set(BOOST_BUILD_PRODUCTS)
 set(BOOST_CONFIGURE_COMMAND "")
 set(BOOST_BUILD_COMMAND "")
 else()
 set(BOOST_BUILD_PRODUCTS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})
 set(BOOST_CONFIGURE_COMMAND
 "./bootstrap.sh"
 "--prefix=${BOOST_PREFIX}"
Expand DownExpand Up@@ -210,16 +214,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_shared)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
 set(BOOST_REGEX_LIBRARY boost_regex_shared)
 endif()
 else()
 # Find static boost headers and libs
Expand All@@ -228,16 +235,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_static)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
 set(BOOST_REGEX_LIBRARY boost_regex_static)
 endif()
 endif()
endif()
Expand All@@ -254,7 +264,11 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
 STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 ADD_THIRDPARTY_LIB(boost_regex
 STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
endif()

include_directories(SYSTEM ${Boost_INCLUDE_DIR})
Expand Down

19 changes: 17 additions & 2 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,8 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 DCHECK_NE(precision, NULLPTR);
 DCHECK_NE(scale, NULLPTR);

 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
 // TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK
@cpcloudcpcloudFeb 16, 2018•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires ARROW-2145 to address.
@wesmwesmFeb 19, 2018

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to wait for ARROW-2145 or merge this independently?
@cpcloudcpcloudFeb 19, 2018

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a couple more tests to add here
 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
 RETURN_IF_PYERROR();
 DCHECK(PyTuple_Check(as_tuple.obj()));

Expand All@@ -117,7 +118,21 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
 RETURN_IF_PYERROR();

 *precision = num_digits;
 const int32_t abs_exponent = std::abs(exponent);

 int32_t num_additional_zeros;

 if (num_digits < abs_exponent) {
 DCHECK_NE(exponent, 0) << "exponent should never be zero here";

 // we have leading/trailing zeros, leading if exponent is negative
 num_additional_zeros = exponent < 0 ? abs_exponent - num_digits : exponent;
 } else {
 // we can use the number of digits as the precision
 num_additional_zeros = 0;
 }

 *precision = num_digits + num_additional_zeros;
 *scale = -exponent;
 return Status::OK();
}
Expand Down

5 changes: 1 addition & 4 deletions
cpp/src/arrow/python/numpy_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -766,10 +766,7 @@ Status NumPyConverter::ConvertDecimals() {
 RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
 &tmp_scale));
 precision = std::max(precision, tmp_precision);

 if (std::abs(desired_scale) < std::abs(tmp_scale)) {
 desired_scale = tmp_scale;
 }
 desired_scale = std::max(desired_scale, tmp_scale);
 }

 type_ = ::arrow::decimal(precision, desired_scale);
Expand Down

89 changes: 89 additions & 0 deletions
cpp/src/arrow/python/python-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,57 @@ TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZeros) {
 std::string decimal_string("0.001");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 3;
 const int32_t expected_scale = 3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) {
 std::string decimal_string("0.01E5");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 4;
 const int32_t expected_scale = -3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) {
 std::string decimal_string("0.01E3");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 1;
 const int32_t expected_scale = -1;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST(PandasConversionTest, TestObjectBlockWriteFails) {
 StringBuilder builder;
 const char value[] = {'\xf1', '\0'};
Expand DownExpand Up@@ -241,5 +292,43 @@ TEST_F(DecimalTest, TestOverflowFails) {
 decimal_type, &value));
}


TEST_F(DecimalTest, SimpleInference) {
 OwnedRef value(this->CreatePythonDecimal("0.01"));
 ASSERT_NE(value.obj(), nullptr);
 int32_t precision;
 int32_t scale;
 ASSERT_OK(internal::InferDecimalPrecisionAndScale(value.obj(), &precision, &scale));
 ASSERT_EQ(2, precision);
 ASSERT_EQ(2, scale);
}


TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {

 PyAcquireGIL lock;
 MemoryPool* pool = default_memory_pool();
 std::shared_ptr<Array> arr;

 OwnedRef list_ref(PyList_New(2));
 PyObject* list = list_ref.obj();

 ASSERT_NE(list, nullptr);

 PyObject* value1 = this->CreatePythonDecimal("0.01").detach();
 ASSERT_NE(value1, nullptr);

 PyObject* value2 = this->CreatePythonDecimal("0.001").detach();
 ASSERT_NE(value2, nullptr);

 // This steals a reference to each object, so we don't need to decref them later
 // just the list

 ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
 ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);

 ASSERT_OK(ConvertPySequence(list, pool, &arr));
}

} // namespace py
} // namespace arrow

56 changes: 32 additions & 24 deletions
cpp/src/arrow/util/decimal-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,15 +37,15 @@ class DecimalTestFixture : public ::testing::Test {

TEST_F(DecimalTestFixture, TestToString) {
 Decimal128 decimal(this->integer_value_);
int scale = 5;
int32_t scale = 5;
 std::string result = decimal.ToString(scale);
 ASSERT_EQ(result, this->string_value_);
}

TEST_F(DecimalTestFixture, TestFromString) {
 Decimal128 expected(this->integer_value_);
 Decimal128 result;
int precision, scale;
int32_t precision, scale;
 ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
 ASSERT_EQ(result, expected);
 ASSERT_EQ(precision, 8);
Expand All@@ -55,8 +55,8 @@ TEST_F(DecimalTestFixture, TestFromString) {
TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
 std::string plus_value("+234.234");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(234234, out);
 ASSERT_EQ(6, precision);
Expand All@@ -67,8 +67,8 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
 std::string plus_value("+2342394230592.232349023094");
 Decimal128 expected_value("2342394230592232349023094");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(expected_value, out);
 ASSERT_EQ(25, precision);
Expand All@@ -90,9 +90,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
 Decimal128 result;
 ASSERT_OK(Decimal128::FromString(string_value, &result));
 Decimal128 expected(static_cast<int64_t>(-230492239423435324));
 expected *= 100;
 expected -= 12;
 ASSERT_EQ(result, expected);
 ASSERT_EQ(result, expected * 100 - 12);

 // Sanity check that our number is actually using more than 64 bits
 ASSERT_NE(result.high_bits(), 0);
Expand DownExpand Up@@ -194,36 +192,36 @@ TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
 std::string string_value("0000000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 7);
 ASSERT_EQ(scale, 0);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(0, precision);
 ASSERT_EQ(0, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
 std::string string_value("000.0000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
ASSERT_NE(precision, 7);
ASSERT_EQ(4, precision);

 ASSERT_EQ(scale, 4);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(4, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
 std::string string_value(".00000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 5);
 ASSERT_EQ(scale, 5);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(5, precision);
 ASSERT_EQ(5, scale);
 ASSERT_EQ(0, d);
}

template <typename T>
Expand DownExpand Up@@ -375,4 +373,14 @@ TEST(Decimal128Test, TestSmallNumberFormat) {
 ASSERT_EQ(expected, result);
}

TEST(Decimal128Test, TestNoDecimalPointExponential) {
 Decimal128 value;
 int32_t precision;
 int32_t scale;
 ASSERT_OK(Decimal128::FromString("1E1", &value, &precision, &scale));
 ASSERT_EQ(1, value.low_bits());
 ASSERT_EQ(1, precision);
 ASSERT_EQ(-1, scale);
}

} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference#1618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

cpcloud
wants to merge
7
commits into

from

 Closed


ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference
#1618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3bf399c

ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference


cpcloud Feb 15, 2018
4045f9b

Cleanup


cpcloud Feb 21, 2018
bb3959e

Remove comment


cpcloud Feb 21, 2018
30989dd

Use max scale


cpcloud Feb 21, 2018
ad82d13

More tests


cpcloud Feb 21, 2018
b03a0c2

Skip leading zeros in arrow decimal128


cpcloud Feb 21, 2018
e66e415

Adjust tests for skipping leading zeros


cpcloud Feb 21, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 4 additions & 2 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -577,11 +577,13 @@ set(ARROW_LINK_LIBS

set(ARROW_SHARED_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

set(ARROW_STATIC_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

if (NOT MSVC)
 set(ARROW_LINK_LIBS
Expand Down

22 changes: 18 additions & 4 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,16 +157,20 @@ if (ARROW_BOOST_VENDORED)
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_FILESYSTEM_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_REGEX_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
 set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
 set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
 if (ARROW_BOOST_HEADER_ONLY)
 set(BOOST_BUILD_PRODUCTS)
 set(BOOST_CONFIGURE_COMMAND "")
 set(BOOST_BUILD_COMMAND "")
 else()
 set(BOOST_BUILD_PRODUCTS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})
 set(BOOST_CONFIGURE_COMMAND
 "./bootstrap.sh"
 "--prefix=${BOOST_PREFIX}"
Expand DownExpand Up@@ -210,16 +214,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_shared)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
 set(BOOST_REGEX_LIBRARY boost_regex_shared)
 endif()
 else()
 # Find static boost headers and libs
Expand All@@ -228,16 +235,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_static)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
 set(BOOST_REGEX_LIBRARY boost_regex_static)
 endif()
 endif()
endif()
Expand All@@ -254,7 +264,11 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
 STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 ADD_THIRDPARTY_LIB(boost_regex
 STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
endif()

include_directories(SYSTEM ${Boost_INCLUDE_DIR})
Expand Down

19 changes: 17 additions & 2 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,8 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 DCHECK_NE(precision, NULLPTR);
 DCHECK_NE(scale, NULLPTR);

 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
 // TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK
@cpcloudcpcloudFeb 16, 2018•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires ARROW-2145 to address.
@wesmwesmFeb 19, 2018

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to wait for ARROW-2145 or merge this independently?
@cpcloudcpcloudFeb 19, 2018

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a couple more tests to add here
 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
 RETURN_IF_PYERROR();
 DCHECK(PyTuple_Check(as_tuple.obj()));

Expand All@@ -117,7 +118,21 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
 RETURN_IF_PYERROR();

 *precision = num_digits;
 const int32_t abs_exponent = std::abs(exponent);

 int32_t num_additional_zeros;

 if (num_digits < abs_exponent) {
 DCHECK_NE(exponent, 0) << "exponent should never be zero here";

 // we have leading/trailing zeros, leading if exponent is negative
 num_additional_zeros = exponent < 0 ? abs_exponent - num_digits : exponent;
 } else {
 // we can use the number of digits as the precision
 num_additional_zeros = 0;
 }

 *precision = num_digits + num_additional_zeros;
 *scale = -exponent;
 return Status::OK();
}
Expand Down

5 changes: 1 addition & 4 deletions
cpp/src/arrow/python/numpy_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -766,10 +766,7 @@ Status NumPyConverter::ConvertDecimals() {
 RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
 &tmp_scale));
 precision = std::max(precision, tmp_precision);

 if (std::abs(desired_scale) < std::abs(tmp_scale)) {
 desired_scale = tmp_scale;
 }
 desired_scale = std::max(desired_scale, tmp_scale);
 }

 type_ = ::arrow::decimal(precision, desired_scale);
Expand Down

89 changes: 89 additions & 0 deletions
cpp/src/arrow/python/python-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,57 @@ TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZeros) {
 std::string decimal_string("0.001");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 3;
 const int32_t expected_scale = 3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) {
 std::string decimal_string("0.01E5");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 4;
 const int32_t expected_scale = -3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) {
 std::string decimal_string("0.01E3");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 1;
 const int32_t expected_scale = -1;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST(PandasConversionTest, TestObjectBlockWriteFails) {
 StringBuilder builder;
 const char value[] = {'\xf1', '\0'};
Expand DownExpand Up@@ -241,5 +292,43 @@ TEST_F(DecimalTest, TestOverflowFails) {
 decimal_type, &value));
}


TEST_F(DecimalTest, SimpleInference) {
 OwnedRef value(this->CreatePythonDecimal("0.01"));
 ASSERT_NE(value.obj(), nullptr);
 int32_t precision;
 int32_t scale;
 ASSERT_OK(internal::InferDecimalPrecisionAndScale(value.obj(), &precision, &scale));
 ASSERT_EQ(2, precision);
 ASSERT_EQ(2, scale);
}


TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {

 PyAcquireGIL lock;
 MemoryPool* pool = default_memory_pool();
 std::shared_ptr<Array> arr;

 OwnedRef list_ref(PyList_New(2));
 PyObject* list = list_ref.obj();

 ASSERT_NE(list, nullptr);

 PyObject* value1 = this->CreatePythonDecimal("0.01").detach();
 ASSERT_NE(value1, nullptr);

 PyObject* value2 = this->CreatePythonDecimal("0.001").detach();
 ASSERT_NE(value2, nullptr);

 // This steals a reference to each object, so we don't need to decref them later
 // just the list

 ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
 ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);

 ASSERT_OK(ConvertPySequence(list, pool, &arr));
}

} // namespace py
} // namespace arrow

56 changes: 32 additions & 24 deletions
cpp/src/arrow/util/decimal-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,15 +37,15 @@ class DecimalTestFixture : public ::testing::Test {

TEST_F(DecimalTestFixture, TestToString) {
 Decimal128 decimal(this->integer_value_);
int scale = 5;
int32_t scale = 5;
 std::string result = decimal.ToString(scale);
 ASSERT_EQ(result, this->string_value_);
}

TEST_F(DecimalTestFixture, TestFromString) {
 Decimal128 expected(this->integer_value_);
 Decimal128 result;
int precision, scale;
int32_t precision, scale;
 ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
 ASSERT_EQ(result, expected);
 ASSERT_EQ(precision, 8);
Expand All@@ -55,8 +55,8 @@ TEST_F(DecimalTestFixture, TestFromString) {
TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
 std::string plus_value("+234.234");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(234234, out);
 ASSERT_EQ(6, precision);
Expand All@@ -67,8 +67,8 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
 std::string plus_value("+2342394230592.232349023094");
 Decimal128 expected_value("2342394230592232349023094");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(expected_value, out);
 ASSERT_EQ(25, precision);
Expand All@@ -90,9 +90,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
 Decimal128 result;
 ASSERT_OK(Decimal128::FromString(string_value, &result));
 Decimal128 expected(static_cast<int64_t>(-230492239423435324));
 expected *= 100;
 expected -= 12;
 ASSERT_EQ(result, expected);
 ASSERT_EQ(result, expected * 100 - 12);

 // Sanity check that our number is actually using more than 64 bits
 ASSERT_NE(result.high_bits(), 0);
Expand DownExpand Up@@ -194,36 +192,36 @@ TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
 std::string string_value("0000000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 7);
 ASSERT_EQ(scale, 0);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(0, precision);
 ASSERT_EQ(0, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
 std::string string_value("000.0000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
ASSERT_NE(precision, 7);
ASSERT_EQ(4, precision);

 ASSERT_EQ(scale, 4);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(4, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
 std::string string_value(".00000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 5);
 ASSERT_EQ(scale, 5);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(5, precision);
 ASSERT_EQ(5, scale);
 ASSERT_EQ(0, d);
}

template <typename T>
Expand DownExpand Up@@ -375,4 +373,14 @@ TEST(Decimal128Test, TestSmallNumberFormat) {
 ASSERT_EQ(expected, result);
}

TEST(Decimal128Test, TestNoDecimalPointExponential) {
 Decimal128 value;
 int32_t precision;
 int32_t scale;
 ASSERT_OK(Decimal128::FromString("1E1", &value, &precision, &scale));
 ASSERT_EQ(1, value.low_bits());
 ASSERT_EQ(1, precision);
 ASSERT_EQ(-1, scale);
}

} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference#1618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

cpcloud
wants to merge
7
commits into

from

 Closed


ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference
#1618
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3bf399c

ARROW-2153/ARROW-2160: [C++/Python] Fix decimal precision inference


cpcloud Feb 15, 2018
4045f9b

Cleanup


cpcloud Feb 21, 2018
bb3959e

Remove comment


cpcloud Feb 21, 2018
30989dd

Use max scale


cpcloud Feb 21, 2018
ad82d13

More tests


cpcloud Feb 21, 2018
b03a0c2

Skip leading zeros in arrow decimal128


cpcloud Feb 21, 2018
e66e415

Adjust tests for skipping leading zeros


cpcloud Feb 21, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 4 additions & 2 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -577,11 +577,13 @@ set(ARROW_LINK_LIBS

set(ARROW_SHARED_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

set(ARROW_STATIC_PRIVATE_LINK_LIBS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})

if (NOT MSVC)
 set(ARROW_LINK_LIBS
Expand Down

22 changes: 18 additions & 4 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -157,16 +157,20 @@ if (ARROW_BOOST_VENDORED)
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_FILESYSTEM_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_STATIC_REGEX_LIBRARY
 "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
 set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
 set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
 set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
 if (ARROW_BOOST_HEADER_ONLY)
 set(BOOST_BUILD_PRODUCTS)
 set(BOOST_CONFIGURE_COMMAND "")
 set(BOOST_BUILD_COMMAND "")
 else()
 set(BOOST_BUILD_PRODUCTS
 ${BOOST_SYSTEM_LIBRARY}
 ${BOOST_FILESYSTEM_LIBRARY})
 ${BOOST_FILESYSTEM_LIBRARY}
 ${BOOST_REGEX_LIBRARY})
 set(BOOST_CONFIGURE_COMMAND
 "./bootstrap.sh"
 "--prefix=${BOOST_PREFIX}"
Expand DownExpand Up@@ -210,16 +214,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_shared)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
 set(BOOST_REGEX_LIBRARY boost_regex_shared)
 endif()
 else()
 # Find static boost headers and libs
Expand All@@ -228,16 +235,19 @@ else()
 if (ARROW_BOOST_HEADER_ONLY)
 find_package(Boost REQUIRED)
 else()
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(Boost COMPONENTS system filesystem regex REQUIRED)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
 else()
 set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
 endif()
 set(BOOST_SYSTEM_LIBRARY boost_system_static)
 set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
 set(BOOST_REGEX_LIBRARY boost_regex_static)
 endif()
 endif()
endif()
Expand All@@ -254,7 +264,11 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
 STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 ADD_THIRDPARTY_LIB(boost_regex
 STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
 SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")

 SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
endif()

include_directories(SYSTEM ${Boost_INCLUDE_DIR})
Expand Down

19 changes: 17 additions & 2 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,8 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 DCHECK_NE(precision, NULLPTR);
 DCHECK_NE(scale, NULLPTR);

 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
 // TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK
@cpcloudcpcloudFeb 16, 2018•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires ARROW-2145 to address.
@wesmwesmFeb 19, 2018

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to wait for ARROW-2145 or merge this independently?
@cpcloudcpcloudFeb 19, 2018

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a couple more tests to add here
 OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
 RETURN_IF_PYERROR();
 DCHECK(PyTuple_Check(as_tuple.obj()));

Expand All@@ -117,7 +118,21 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
 const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
 RETURN_IF_PYERROR();

 *precision = num_digits;
 const int32_t abs_exponent = std::abs(exponent);

 int32_t num_additional_zeros;

 if (num_digits < abs_exponent) {
 DCHECK_NE(exponent, 0) << "exponent should never be zero here";

 // we have leading/trailing zeros, leading if exponent is negative
 num_additional_zeros = exponent < 0 ? abs_exponent - num_digits : exponent;
 } else {
 // we can use the number of digits as the precision
 num_additional_zeros = 0;
 }

 *precision = num_digits + num_additional_zeros;
 *scale = -exponent;
 return Status::OK();
}
Expand Down

5 changes: 1 addition & 4 deletions
cpp/src/arrow/python/numpy_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -766,10 +766,7 @@ Status NumPyConverter::ConvertDecimals() {
 RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
 &tmp_scale));
 precision = std::max(precision, tmp_precision);

 if (std::abs(desired_scale) < std::abs(tmp_scale)) {
 desired_scale = tmp_scale;
 }
 desired_scale = std::max(desired_scale, tmp_scale);
 }

 type_ = ::arrow::decimal(precision, desired_scale);
Expand Down

89 changes: 89 additions & 0 deletions
cpp/src/arrow/python/python-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,57 @@ TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZeros) {
 std::string decimal_string("0.001");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 3;
 const int32_t expected_scale = 3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) {
 std::string decimal_string("0.01E5");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 4;
 const int32_t expected_scale = -3;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) {
 std::string decimal_string("0.01E3");
 OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));

 int32_t precision;
 int32_t scale;

 ASSERT_OK(
 internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));

 const int32_t expected_precision = 1;
 const int32_t expected_scale = -1;

 ASSERT_EQ(expected_precision, precision);
 ASSERT_EQ(expected_scale, scale);
}

TEST(PandasConversionTest, TestObjectBlockWriteFails) {
 StringBuilder builder;
 const char value[] = {'\xf1', '\0'};
Expand DownExpand Up@@ -241,5 +292,43 @@ TEST_F(DecimalTest, TestOverflowFails) {
 decimal_type, &value));
}


TEST_F(DecimalTest, SimpleInference) {
 OwnedRef value(this->CreatePythonDecimal("0.01"));
 ASSERT_NE(value.obj(), nullptr);
 int32_t precision;
 int32_t scale;
 ASSERT_OK(internal::InferDecimalPrecisionAndScale(value.obj(), &precision, &scale));
 ASSERT_EQ(2, precision);
 ASSERT_EQ(2, scale);
}


TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {

 PyAcquireGIL lock;
 MemoryPool* pool = default_memory_pool();
 std::shared_ptr<Array> arr;

 OwnedRef list_ref(PyList_New(2));
 PyObject* list = list_ref.obj();

 ASSERT_NE(list, nullptr);

 PyObject* value1 = this->CreatePythonDecimal("0.01").detach();
 ASSERT_NE(value1, nullptr);

 PyObject* value2 = this->CreatePythonDecimal("0.001").detach();
 ASSERT_NE(value2, nullptr);

 // This steals a reference to each object, so we don't need to decref them later
 // just the list

 ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
 ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);

 ASSERT_OK(ConvertPySequence(list, pool, &arr));
}

} // namespace py
} // namespace arrow

56 changes: 32 additions & 24 deletions
cpp/src/arrow/util/decimal-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,15 +37,15 @@ class DecimalTestFixture : public ::testing::Test {

TEST_F(DecimalTestFixture, TestToString) {
 Decimal128 decimal(this->integer_value_);
int scale = 5;
int32_t scale = 5;
 std::string result = decimal.ToString(scale);
 ASSERT_EQ(result, this->string_value_);
}

TEST_F(DecimalTestFixture, TestFromString) {
 Decimal128 expected(this->integer_value_);
 Decimal128 result;
int precision, scale;
int32_t precision, scale;
 ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
 ASSERT_EQ(result, expected);
 ASSERT_EQ(precision, 8);
Expand All@@ -55,8 +55,8 @@ TEST_F(DecimalTestFixture, TestFromString) {
TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
 std::string plus_value("+234.234");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(234234, out);
 ASSERT_EQ(6, precision);
Expand All@@ -67,8 +67,8 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
 std::string plus_value("+2342394230592.232349023094");
 Decimal128 expected_value("2342394230592232349023094");
 Decimal128 out;
int scale;
int precision;
int32_t scale;
int32_t precision;
 ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
 ASSERT_EQ(expected_value, out);
 ASSERT_EQ(25, precision);
Expand All@@ -90,9 +90,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
 Decimal128 result;
 ASSERT_OK(Decimal128::FromString(string_value, &result));
 Decimal128 expected(static_cast<int64_t>(-230492239423435324));
 expected *= 100;
 expected -= 12;
 ASSERT_EQ(result, expected);
 ASSERT_EQ(result, expected * 100 - 12);

 // Sanity check that our number is actually using more than 64 bits
 ASSERT_NE(result.high_bits(), 0);
Expand DownExpand Up@@ -194,36 +192,36 @@ TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
 std::string string_value("0000000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 7);
 ASSERT_EQ(scale, 0);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(0, precision);
 ASSERT_EQ(0, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
 std::string string_value("000.0000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
ASSERT_NE(precision, 7);
ASSERT_EQ(4, precision);

 ASSERT_EQ(scale, 4);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(4, scale);
 ASSERT_EQ(0, d);
}

TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
 std::string string_value(".00000");
 Decimal128 d;
int precision;
int scale;
int32_t precision;
int32_t scale;
 ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
 ASSERT_EQ(precision, 5);
 ASSERT_EQ(scale, 5);
 ASSERT_EQ(d, 0);
 ASSERT_EQ(5, precision);
 ASSERT_EQ(5, scale);
 ASSERT_EQ(0, d);
}

template <typename T>
Expand DownExpand Up@@ -375,4 +373,14 @@ TEST(Decimal128Test, TestSmallNumberFormat) {
 ASSERT_EQ(expected, result);
}

TEST(Decimal128Test, TestNoDecimalPointExponential) {
 Decimal128 value;
 int32_t precision;
 int32_t scale;
 ASSERT_OK(Decimal128::FromString("1E1", &value, &precision, &scale));
 ASSERT_EQ(1, value.low_bits());
 ASSERT_EQ(1, precision);
 ASSERT_EQ(-1, scale);
}

} // namespace arrow
Loading