PARQUET-1990: [C++] Refuse to write ConvertedType::NA#9863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 1 commit into from
Closed

PARQUET-1990: [C++] Refuse to write ConvertedType::NA #9863

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions cpp/src/parquet/printer.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -87,11 +87,15 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selecte
const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
<< TypeToString(descr->physical_type());
if (descr->converted_type() != ConvertedType::NONE) {
stream << "/" << ConvertedTypeToString(descr->converted_type());
const auto& logical_type = descr->logical_type();
if (!logical_type->is_none()) {
stream << " / " << logical_type->ToString();
}
if (descr->converted_type() == ConvertedType::DECIMAL) {
stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
if (descr->converted_type() != ConvertedType::NONE) {
stream << " / " << ConvertedTypeToString(descr->converted_type());
if (descr->converted_type() == ConvertedType::DECIMAL) {
stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
}
}
stream << ")" << std::endl;
}
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/parquet/reader_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -317,7 +317,7 @@ Number of RowGroups: 1
Number of Real Columns: 2
Number of Columns: 2
Number of Selected Columns: 2
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY/UTF8)
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY / String / UTF8)
Column 1: b (INT32)
--- Row Group: 0 ---
--- Total Bytes: 155 ---
Expand Down
13 changes: 11 additions & 2 deletions cpp/src/parquet/schema.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -448,7 +448,7 @@ std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
LogicalType::FromThrift(element->logicalType),
LoadEnumSafe(&element->type), element->type_length, field_id));
} else if (element->__isset.converted_type) {
// legacy writer with logical type present
// legacy writer with converted type present
primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
element->name, LoadEnumSafe(&element->repetition_type),
LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type),
Expand DownExpand Up@@ -500,7 +500,16 @@ void PrimitiveNode::ToParquet(void* opaque_element) const {
element->__set_name(name_);
element->__set_repetition_type(ToThrift(repetition_));
if (converted_type_ != ConvertedType::NONE) {
element->__set_converted_type(ToThrift(converted_type_));
if (converted_type_ != ConvertedType::NA) {
element->__set_converted_type(ToThrift(converted_type_));
} else {
// ConvertedType::NA is an unreleased, obsolete synonym for LogicalType::Null.
// Never emit it (see PARQUET-1990 for discussion).
if (!logical_type_ || !logical_type_->is_null()) {
throw ParquetException(
"ConvertedType::NA is obsolete, please use LogicalType::Null instead");
}
}
}
if (field_id_ >= 0) {
element->__set_field_id(field_id_);
Expand Down
14 changes: 5 additions & 9 deletions cpp/src/parquet/schema_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1101,12 +1101,12 @@ TEST(TestLogicalTypeConstruction, ConvertedTypeCompatibility) {
ASSERT_TRUE(reconstructed->is_valid());
ASSERT_TRUE(reconstructed->Equals(*original));

// Unknown
original = LogicalType::Unknown();
// Undefined
original = UndefinedLogicalType::Make();
ASSERT_TRUE(original->is_invalid());
ASSERT_FALSE(original->is_valid());
converted_type = original->ToConvertedType(&converted_decimal_metadata);
ASSERT_EQ(converted_type, ConvertedType::NA);
ASSERT_EQ(converted_type, ConvertedType::UNDEFINED);
ASSERT_FALSE(converted_decimal_metadata.isset);
ASSERT_TRUE(original->is_compatible(converted_type, converted_decimal_metadata));
ASSERT_TRUE(original->is_compatible(converted_type));
Expand DownExpand Up@@ -1243,7 +1243,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeProperties) {
{BSONLogicalType::Make(), false, true, true},
{UUIDLogicalType::Make(), false, true, true},
{NoLogicalType::Make(), false, false, true},
{UnknownLogicalType::Make(), false, false, false},
};

for (const ExpectedProperties& c : cases) {
Expand DownExpand Up@@ -1339,7 +1338,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) {
}

std::vector<std::shared_ptr<const LogicalType>> any_type_cases = {
LogicalType::Null(), LogicalType::None(), LogicalType::Unknown()};
LogicalType::Null(), LogicalType::None(), UndefinedLogicalType::Make()};

for (auto c : any_type_cases) {
ConfirmAnyPrimitiveTypeApplicability(c);
Expand DownExpand Up@@ -1453,7 +1452,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) {
};

std::vector<ExpectedRepresentation> cases = {
{LogicalType::Unknown(), "Unknown", R"({"Type": "Unknown"})"},
{UndefinedLogicalType::Make(), "Undefined", R"({"Type": "Undefined"})"},
{LogicalType::String(), "String", R"({"Type": "String"})"},
{LogicalType::Map(), "Map", R"({"Type": "Map"})"},
{LogicalType::List(), "List", R"({"Type": "List"})"},
Expand DownExpand Up@@ -1550,7 +1549,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeSortOrder) {
};

std::vector<ExpectedSortOrder> cases = {
{LogicalType::Unknown(), SortOrder::UNKNOWN},
{LogicalType::String(), SortOrder::UNSIGNED},
{LogicalType::Map(), SortOrder::UNKNOWN},
{LogicalType::List(), SortOrder::UNKNOWN},
Expand DownExpand Up@@ -1888,8 +1886,6 @@ TEST_F(TestSchemaElementConstruction, SimpleCases) {
{"uuid", LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16, false,
ConvertedType::NA, true, [this]() { return element_->logicalType.__isset.UUID; }},
{"none", LogicalType::None(), Type::INT64, -1, false, ConvertedType::NA, false,
check_nothing},
{"unknown", LogicalType::Unknown(), Type::INT64, -1, true, ConvertedType::NA, false,
check_nothing}};

for (const SchemaElementConstructionArguments& c : cases) {
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/parquet/thrift_internal.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -256,6 +256,9 @@ static inline format::Type::type ToThrift(Type::type type) {
static inline format::ConvertedType::type ToThrift(ConvertedType::type type) {
// item 0 is NONE
DCHECK_NE(type, ConvertedType::NONE);
// it is forbidden to emit "NA" (PARQUET-1990)
DCHECK_NE(type, ConvertedType::NA);

@emkornfieldemkornfieldMar 31, 2021

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DCHECK_Lt might be better here.

@pitroupitrouMar 31, 2021

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it would change anything. Or we would have to do a full-blown check for valid values, which is a bit out of scope.

@emkornfieldemkornfieldMar 31, 2021•
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. I think it prevents accidental writes of UNDEFINED as well.

@pitroupitrouMar 31, 2021

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I can add that as well :-)

DCHECK_NE(type, ConvertedType::UNDEFINED);
return static_cast<format::ConvertedType::type>(static_cast<int>(type) - 1);
}

Expand Down
37 changes: 18 additions & 19 deletions cpp/src/parquet/types.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -366,13 +366,14 @@ std::shared_ptr<const LogicalType> LogicalType::FromConvertedType(
return JSONLogicalType::Make();
case ConvertedType::BSON:
return BSONLogicalType::Make();
case ConvertedType::NA:
return NullLogicalType::Make();
case ConvertedType::NONE:
return NoLogicalType::Make();
case ConvertedType::NA:
case ConvertedType::UNDEFINED:
return UnknownLogicalType::Make();
return UndefinedLogicalType::Make();
}
return UnknownLogicalType::Make();
return UndefinedLogicalType::Make();
}

std::shared_ptr<const LogicalType> LogicalType::FromThrift(
Expand DownExpand Up@@ -483,10 +484,6 @@ std::shared_ptr<const LogicalType> LogicalType::UUID() { return UUIDLogicalType:

std::shared_ptr<const LogicalType> LogicalType::None() { return NoLogicalType::Make(); }

std::shared_ptr<const LogicalType> LogicalType::Unknown() {
return UnknownLogicalType::Make();
}

/*
* The logical type implementation classes are built in four layers: (1) the base
* layer, which establishes the interface and provides generally reusable implementations
Expand DownExpand Up@@ -516,7 +513,7 @@ class LogicalType::Impl {
virtual std::string ToString() const = 0;

virtual bool is_serialized() const {
return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNKNOWN);
return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNDEFINED);
}

virtual std::string ToJSON() const {
Expand DownExpand Up@@ -567,14 +564,14 @@ class LogicalType::Impl {
class BSON;
class UUID;
class No;
class Unknown;
class Undefined;

protected:
Impl(LogicalType::Type::type t, SortOrder::type o) : type_(t), order_(o) {}
Impl() = default;

private:
LogicalType::Type::type type_ = LogicalType::Type::UNKNOWN;
LogicalType::Type::type type_ = LogicalType::Type::UNDEFINED;
SortOrder::type order_ = SortOrder::UNKNOWN;
};

Expand DownExpand Up@@ -636,7 +633,9 @@ bool LogicalType::is_JSON() const { return impl_->type() == LogicalType::Type::J
bool LogicalType::is_BSON() const { return impl_->type() == LogicalType::Type::BSON; }
bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; }
bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; }
bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNKNOWN; }
bool LogicalType::is_valid() const {
return impl_->type() != LogicalType::Type::UNDEFINED;
}
bool LogicalType::is_invalid() const { return !is_valid(); }
bool LogicalType::is_nested() const {
return (impl_->type() == LogicalType::Type::LIST) ||
Expand DownExpand Up@@ -1555,19 +1554,19 @@ class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible,

GENERATE_MAKE(No)

class LogicalType::Impl::Unknown final : public LogicalType::Impl::SimpleCompatible,
public LogicalType::Impl::UniversalApplicable {
class LogicalType::Impl::Undefined final : public LogicalType::Impl::SimpleCompatible,
public LogicalType::Impl::UniversalApplicable {
public:
friend class UnknownLogicalType;
friend class UndefinedLogicalType;

OVERRIDE_TOSTRING(Unknown)
OVERRIDE_TOSTRING(Undefined)

private:
Unknown()
: LogicalType::Impl(LogicalType::Type::UNKNOWN, SortOrder::UNKNOWN),
LogicalType::Impl::SimpleCompatible(ConvertedType::NA) {}
Undefined()
: LogicalType::Impl(LogicalType::Type::UNDEFINED, SortOrder::UNKNOWN),
LogicalType::Impl::SimpleCompatible(ConvertedType::UNDEFINED) {}
};

GENERATE_MAKE(Unknown)
GENERATE_MAKE(Undefined)

} // namespace parquet
29 changes: 19 additions & 10 deletions cpp/src/parquet/types.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ struct Type {
// Mirrors parquet::ConvertedType
struct ConvertedType {
enum type {
NONE,
NONE, // Not a real converted type, but means no converted type is specified
UTF8,
MAP,
MAP_KEY_VALUE,
Expand All@@ -94,9 +94,12 @@ struct ConvertedType {
JSON,
BSON,
INTERVAL,
// DEPRECATED INVALID ConvertedType for all-null data.
// Only useful for reading legacy files written out by interim Parquet C++ releases.
// For writing, always emit LogicalType::Null instead.
// See PARQUET-1990.
NA = 25,
// Should always be last element.
UNDEFINED = 26
UNDEFINED = 26 // Not a real converted type; should always be last element
};
};

Expand DownExpand Up@@ -140,7 +143,7 @@ class PARQUET_EXPORT LogicalType {
public:
struct Type {
enum type {
UNKNOWN = 0,
UNDEFINED = 0, // Not a real logical type
STRING = 1,
MAP,
LIST,
Expand All@@ -151,11 +154,11 @@ class PARQUET_EXPORT LogicalType {
TIMESTAMP,
INTERVAL,
INT,
NIL, // Thrift NullType
NIL, // Thrift NullType: annotates data that is always null
JSON,
BSON,
UUID,
NONE
NONE // Not a real logical type; should always be last element
};
};

Expand DownExpand Up@@ -199,12 +202,18 @@ class PARQUET_EXPORT LogicalType {

static std::shared_ptr<const LogicalType> Interval();
static std::shared_ptr<const LogicalType> Int(int bit_width, bool is_signed);

/// \brief Create a logical type for data that's always null
///
/// Any physical type can be annotated with this logical type.
static std::shared_ptr<const LogicalType> Null();

static std::shared_ptr<const LogicalType> JSON();
static std::shared_ptr<const LogicalType> BSON();
static std::shared_ptr<const LogicalType> UUID();

/// \brief Create a placeholder for when no logical type is specified
static std::shared_ptr<const LogicalType> None();
static std::shared_ptr<const LogicalType> Unknown();

/// \brief Return true if this logical type is consistent with the given underlying
/// physical type.
Expand DownExpand Up@@ -434,13 +443,13 @@ class PARQUET_EXPORT NoLogicalType : public LogicalType {
NoLogicalType() = default;
};

/// \brief Allowed for any type.
class PARQUET_EXPORT UnknownLogicalType : public LogicalType {
// Internal API, for unrecognized logical types
class PARQUET_EXPORT UndefinedLogicalType : public LogicalType {
public:
static std::shared_ptr<const LogicalType> Make();

private:
UnknownLogicalType() = default;
UndefinedLogicalType() = default;
};

// Data encodings. Mirrors parquet::Encoding
Expand Down
2 changes: 1 addition & 1 deletion python/pyarrow/_parquet.pxd
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -54,7 +54,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY"

enum ParquetLogicalTypeId" parquet::LogicalType::Type::type":
ParquetLogicalType_UNKNOWN" parquet::LogicalType::Type::UNKNOWN"
ParquetLogicalType_UNDEFINED" parquet::LogicalType::Type::UNDEFINED"
ParquetLogicalType_STRING" parquet::LogicalType::Type::STRING"
ParquetLogicalType_MAP" parquet::LogicalType::Type::MAP"
ParquetLogicalType_LIST" parquet::LogicalType::Type::LIST"
Expand Down
2 changes: 1 addition & 1 deletion python/pyarrow/_parquet.pyx
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -809,7 +809,7 @@ cdef physical_type_name_from_enum(ParquetType type_):

cdef logical_type_name_from_enum(ParquetLogicalTypeId type_):
return {
ParquetLogicalType_UNKNOWN: 'UNKNOWN',
ParquetLogicalType_UNDEFINED: 'UNDEFINED',
ParquetLogicalType_STRING: 'STRING',
ParquetLogicalType_MAP: 'MAP',
ParquetLogicalType_LIST: 'LIST',
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
PARQUET-1990: [C++] Refuse to write ConvertedType::NA#9863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
1
commit into

from

 Closed


PARQUET-1990: [C++] Refuse to write ConvertedType::NA
#9863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 8 additions & 4 deletions
cpp/src/parquet/printer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -87,11 +87,15 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selecte
 const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
 stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
 << TypeToString(descr->physical_type());
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << "/" << ConvertedTypeToString(descr->converted_type());
 const auto& logical_type = descr->logical_type();
 if (!logical_type->is_none()) {
 stream << " / " << logical_type->ToString();
 }
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << " / " << ConvertedTypeToString(descr->converted_type());
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 }
 }
 stream << ")" << std::endl;
 }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/parquet/reader_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,7 +317,7 @@ Number of RowGroups: 1
Number of Real Columns: 2
Number of Columns: 2
Number of Selected Columns: 2
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY/UTF8)
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY / String / UTF8)
Column 1: b (INT32)
--- Row Group: 0 ---
--- Total Bytes: 155 ---
Expand Down

13 changes: 11 additions & 2 deletions
cpp/src/parquet/schema.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -448,7 +448,7 @@ std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
 LogicalType::FromThrift(element->logicalType),
 LoadEnumSafe(&element->type), element->type_length, field_id));
 } else if (element->__isset.converted_type) {
 // legacy writer with logical type present
 // legacy writer with converted type present
 primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
 element->name, LoadEnumSafe(&element->repetition_type),
 LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type),
Expand DownExpand Up@@ -500,7 +500,16 @@ void PrimitiveNode::ToParquet(void* opaque_element) const {
 element->__set_name(name_);
 element->__set_repetition_type(ToThrift(repetition_));
 if (converted_type_ != ConvertedType::NONE) {
 element->__set_converted_type(ToThrift(converted_type_));
 if (converted_type_ != ConvertedType::NA) {
 element->__set_converted_type(ToThrift(converted_type_));
 } else {
 // ConvertedType::NA is an unreleased, obsolete synonym for LogicalType::Null.
 // Never emit it (see PARQUET-1990 for discussion).
 if (!logical_type_ || !logical_type_->is_null()) {
 throw ParquetException(
 "ConvertedType::NA is obsolete, please use LogicalType::Null instead");
 }
 }
 }
 if (field_id_ >= 0) {
 element->__set_field_id(field_id_);
Expand Down

14 changes: 5 additions & 9 deletions
cpp/src/parquet/schema_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1101,12 +1101,12 @@ TEST(TestLogicalTypeConstruction, ConvertedTypeCompatibility) {
 ASSERT_TRUE(reconstructed->is_valid());
 ASSERT_TRUE(reconstructed->Equals(*original));

 // Unknown
 original = LogicalType::Unknown();
 // Undefined
 original = UndefinedLogicalType::Make();
 ASSERT_TRUE(original->is_invalid());
 ASSERT_FALSE(original->is_valid());
 converted_type = original->ToConvertedType(&converted_decimal_metadata);
 ASSERT_EQ(converted_type, ConvertedType::NA);
 ASSERT_EQ(converted_type, ConvertedType::UNDEFINED);
 ASSERT_FALSE(converted_decimal_metadata.isset);
 ASSERT_TRUE(original->is_compatible(converted_type, converted_decimal_metadata));
 ASSERT_TRUE(original->is_compatible(converted_type));
Expand DownExpand Up@@ -1243,7 +1243,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeProperties) {
 {BSONLogicalType::Make(), false, true, true},
 {UUIDLogicalType::Make(), false, true, true},
 {NoLogicalType::Make(), false, false, true},
 {UnknownLogicalType::Make(), false, false, false},
 };

 for (const ExpectedProperties& c : cases) {
Expand DownExpand Up@@ -1339,7 +1338,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) {
 }

 std::vector<std::shared_ptr<const LogicalType>> any_type_cases = {
 LogicalType::Null(), LogicalType::None(), LogicalType::Unknown()};
 LogicalType::Null(), LogicalType::None(), UndefinedLogicalType::Make()};

 for (auto c : any_type_cases) {
 ConfirmAnyPrimitiveTypeApplicability(c);
Expand DownExpand Up@@ -1453,7 +1452,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) {
 };

 std::vector<ExpectedRepresentation> cases = {
 {LogicalType::Unknown(), "Unknown", R"({"Type": "Unknown"})"},
 {UndefinedLogicalType::Make(), "Undefined", R"({"Type": "Undefined"})"},
 {LogicalType::String(), "String", R"({"Type": "String"})"},
 {LogicalType::Map(), "Map", R"({"Type": "Map"})"},
 {LogicalType::List(), "List", R"({"Type": "List"})"},
Expand DownExpand Up@@ -1550,7 +1549,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeSortOrder) {
 };

 std::vector<ExpectedSortOrder> cases = {
 {LogicalType::Unknown(), SortOrder::UNKNOWN},
 {LogicalType::String(), SortOrder::UNSIGNED},
 {LogicalType::Map(), SortOrder::UNKNOWN},
 {LogicalType::List(), SortOrder::UNKNOWN},
Expand DownExpand Up@@ -1888,8 +1886,6 @@ TEST_F(TestSchemaElementConstruction, SimpleCases) {
 {"uuid", LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16, false,
 ConvertedType::NA, true, [this]() { return element_->logicalType.__isset.UUID; }},
 {"none", LogicalType::None(), Type::INT64, -1, false, ConvertedType::NA, false,
 check_nothing},
 {"unknown", LogicalType::Unknown(), Type::INT64, -1, true, ConvertedType::NA, false,
 check_nothing}};

 for (const SchemaElementConstructionArguments& c : cases) {
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/parquet/thrift_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -256,6 +256,9 @@ static inline format::Type::type ToThrift(Type::type type) {
static inline format::ConvertedType::type ToThrift(ConvertedType::type type) {
 // item 0 is NONE
 DCHECK_NE(type, ConvertedType::NONE);
 // it is forbidden to emit "NA" (PARQUET-1990)
 DCHECK_NE(type, ConvertedType::NA);
@emkornfieldemkornfieldMar 31, 2021

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DCHECK_Lt might be better here.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it would change anything. Or we would have to do a full-blown check for valid values, which is a bit out of scope.
@emkornfieldemkornfieldMar 31, 2021•

edited

Loading

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. I think it prevents accidental writes of UNDEFINED as well.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I can add that as well :-)
 DCHECK_NE(type, ConvertedType::UNDEFINED);
 return static_cast<format::ConvertedType::type>(static_cast<int>(type) - 1);
}

Expand Down

37 changes: 18 additions & 19 deletions
cpp/src/parquet/types.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -366,13 +366,14 @@ std::shared_ptr<const LogicalType> LogicalType::FromConvertedType(
 return JSONLogicalType::Make();
 case ConvertedType::BSON:
 return BSONLogicalType::Make();
 case ConvertedType::NA:
 return NullLogicalType::Make();
 case ConvertedType::NONE:
 return NoLogicalType::Make();
 case ConvertedType::NA:
 case ConvertedType::UNDEFINED:
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
 }
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
}

std::shared_ptr<const LogicalType> LogicalType::FromThrift(
Expand DownExpand Up@@ -483,10 +484,6 @@ std::shared_ptr<const LogicalType> LogicalType::UUID() { return UUIDLogicalType:

std::shared_ptr<const LogicalType> LogicalType::None() { return NoLogicalType::Make(); }

std::shared_ptr<const LogicalType> LogicalType::Unknown() {
 return UnknownLogicalType::Make();
}

/*
 * The logical type implementation classes are built in four layers: (1) the base
 * layer, which establishes the interface and provides generally reusable implementations
Expand DownExpand Up@@ -516,7 +513,7 @@ class LogicalType::Impl {
 virtual std::string ToString() const = 0;

 virtual bool is_serialized() const {
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNKNOWN);
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNDEFINED);
 }

 virtual std::string ToJSON() const {
Expand DownExpand Up@@ -567,14 +564,14 @@ class LogicalType::Impl {
 class BSON;
 class UUID;
 class No;
 class Unknown;
 class Undefined;

 protected:
 Impl(LogicalType::Type::type t, SortOrder::type o) : type_(t), order_(o) {}
 Impl() = default;

 private:
 LogicalType::Type::type type_ = LogicalType::Type::UNKNOWN;
 LogicalType::Type::type type_ = LogicalType::Type::UNDEFINED;
 SortOrder::type order_ = SortOrder::UNKNOWN;
};

Expand DownExpand Up@@ -636,7 +633,9 @@ bool LogicalType::is_JSON() const { return impl_->type() == LogicalType::Type::J
bool LogicalType::is_BSON() const { return impl_->type() == LogicalType::Type::BSON; }
bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; }
bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; }
bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNKNOWN; }
bool LogicalType::is_valid() const {
 return impl_->type() != LogicalType::Type::UNDEFINED;
}
bool LogicalType::is_invalid() const { return !is_valid(); }
bool LogicalType::is_nested() const {
 return (impl_->type() == LogicalType::Type::LIST) ||
Expand DownExpand Up@@ -1555,19 +1554,19 @@ class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible,

GENERATE_MAKE(No)

class LogicalType::Impl::Unknown final : public LogicalType::Impl::SimpleCompatible,
 public LogicalType::Impl::UniversalApplicable {
class LogicalType::Impl::Undefined final : public LogicalType::Impl::SimpleCompatible,
public LogicalType::Impl::UniversalApplicable {
 public:
 friend class UnknownLogicalType;
 friend class UndefinedLogicalType;

 OVERRIDE_TOSTRING(Unknown)
 OVERRIDE_TOSTRING(Undefined)

 private:
Unknown()
 : LogicalType::Impl(LogicalType::Type::UNKNOWN, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::NA) {}
Undefined()
 : LogicalType::Impl(LogicalType::Type::UNDEFINED, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::UNDEFINED) {}
};

GENERATE_MAKE(Unknown)
GENERATE_MAKE(Undefined)

} // namespace parquet

29 changes: 19 additions & 10 deletions
cpp/src/parquet/types.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ struct Type {
// Mirrors parquet::ConvertedType
struct ConvertedType {
 enum type {
 NONE,
 NONE, // Not a real converted type, but means no converted type is specified
 UTF8,
 MAP,
 MAP_KEY_VALUE,
Expand All@@ -94,9 +94,12 @@ struct ConvertedType {
 JSON,
 BSON,
 INTERVAL,
 // DEPRECATED INVALID ConvertedType for all-null data.
 // Only useful for reading legacy files written out by interim Parquet C++ releases.
 // For writing, always emit LogicalType::Null instead.
 // See PARQUET-1990.
 NA = 25,
 // Should always be last element.
 UNDEFINED = 26
 UNDEFINED = 26 // Not a real converted type; should always be last element
 };
};

Expand DownExpand Up@@ -140,7 +143,7 @@ class PARQUET_EXPORT LogicalType {
 public:
 struct Type {
 enum type {
UNKNOWN = 0,
UNDEFINED = 0, // Not a real logical type
 STRING = 1,
 MAP,
 LIST,
Expand All@@ -151,11 +154,11 @@ class PARQUET_EXPORT LogicalType {
 TIMESTAMP,
 INTERVAL,
 INT,
 NIL, // Thrift NullType
 NIL, // Thrift NullType: annotates data that is always null
 JSON,
 BSON,
 UUID,
 NONE
 NONE // Not a real logical type; should always be last element
 };
 };

Expand DownExpand Up@@ -199,12 +202,18 @@ class PARQUET_EXPORT LogicalType {

 static std::shared_ptr<const LogicalType> Interval();
 static std::shared_ptr<const LogicalType> Int(int bit_width, bool is_signed);

 /// \brief Create a logical type for data that's always null
 ///
 /// Any physical type can be annotated with this logical type.
 static std::shared_ptr<const LogicalType> Null();

 static std::shared_ptr<const LogicalType> JSON();
 static std::shared_ptr<const LogicalType> BSON();
 static std::shared_ptr<const LogicalType> UUID();

 /// \brief Create a placeholder for when no logical type is specified
 static std::shared_ptr<const LogicalType> None();
 static std::shared_ptr<const LogicalType> Unknown();

 /// \brief Return true if this logical type is consistent with the given underlying
 /// physical type.
Expand DownExpand Up@@ -434,13 +443,13 @@ class PARQUET_EXPORT NoLogicalType : public LogicalType {
 NoLogicalType() = default;
};

/// \brief Allowed for any type.
class PARQUET_EXPORT UnknownLogicalType : public LogicalType {
// Internal API, for unrecognized logical types
class PARQUET_EXPORT UndefinedLogicalType : public LogicalType {
 public:
 static std::shared_ptr<const LogicalType> Make();

 private:
UnknownLogicalType() = default;
UndefinedLogicalType() = default;
};

// Data encodings. Mirrors parquet::Encoding
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,7 +54,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
 ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY"

 enum ParquetLogicalTypeId" parquet::LogicalType::Type::type":
ParquetLogicalType_UNKNOWN" parquet::LogicalType::Type::UNKNOWN"
ParquetLogicalType_UNDEFINED" parquet::LogicalType::Type::UNDEFINED"
 ParquetLogicalType_STRING" parquet::LogicalType::Type::STRING"
 ParquetLogicalType_MAP" parquet::LogicalType::Type::MAP"
 ParquetLogicalType_LIST" parquet::LogicalType::Type::LIST"
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pyx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -809,7 +809,7 @@ cdef physical_type_name_from_enum(ParquetType type_):

cdef logical_type_name_from_enum(ParquetLogicalTypeId type_):
 return {
ParquetLogicalType_UNKNOWN: 'UNKNOWN',
ParquetLogicalType_UNDEFINED: 'UNDEFINED',
 ParquetLogicalType_STRING: 'STRING',
 ParquetLogicalType_MAP: 'MAP',
 ParquetLogicalType_LIST: 'LIST',
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
PARQUET-1990: [C++] Refuse to write ConvertedType::NA#9863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
1
commit into

from

 Closed


PARQUET-1990: [C++] Refuse to write ConvertedType::NA
#9863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 8 additions & 4 deletions
cpp/src/parquet/printer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -87,11 +87,15 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selecte
 const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
 stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
 << TypeToString(descr->physical_type());
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << "/" << ConvertedTypeToString(descr->converted_type());
 const auto& logical_type = descr->logical_type();
 if (!logical_type->is_none()) {
 stream << " / " << logical_type->ToString();
 }
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << " / " << ConvertedTypeToString(descr->converted_type());
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 }
 }
 stream << ")" << std::endl;
 }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/parquet/reader_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,7 +317,7 @@ Number of RowGroups: 1
Number of Real Columns: 2
Number of Columns: 2
Number of Selected Columns: 2
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY/UTF8)
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY / String / UTF8)
Column 1: b (INT32)
--- Row Group: 0 ---
--- Total Bytes: 155 ---
Expand Down

13 changes: 11 additions & 2 deletions
cpp/src/parquet/schema.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -448,7 +448,7 @@ std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
 LogicalType::FromThrift(element->logicalType),
 LoadEnumSafe(&element->type), element->type_length, field_id));
 } else if (element->__isset.converted_type) {
 // legacy writer with logical type present
 // legacy writer with converted type present
 primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
 element->name, LoadEnumSafe(&element->repetition_type),
 LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type),
Expand DownExpand Up@@ -500,7 +500,16 @@ void PrimitiveNode::ToParquet(void* opaque_element) const {
 element->__set_name(name_);
 element->__set_repetition_type(ToThrift(repetition_));
 if (converted_type_ != ConvertedType::NONE) {
 element->__set_converted_type(ToThrift(converted_type_));
 if (converted_type_ != ConvertedType::NA) {
 element->__set_converted_type(ToThrift(converted_type_));
 } else {
 // ConvertedType::NA is an unreleased, obsolete synonym for LogicalType::Null.
 // Never emit it (see PARQUET-1990 for discussion).
 if (!logical_type_ || !logical_type_->is_null()) {
 throw ParquetException(
 "ConvertedType::NA is obsolete, please use LogicalType::Null instead");
 }
 }
 }
 if (field_id_ >= 0) {
 element->__set_field_id(field_id_);
Expand Down

14 changes: 5 additions & 9 deletions
cpp/src/parquet/schema_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1101,12 +1101,12 @@ TEST(TestLogicalTypeConstruction, ConvertedTypeCompatibility) {
 ASSERT_TRUE(reconstructed->is_valid());
 ASSERT_TRUE(reconstructed->Equals(*original));

 // Unknown
 original = LogicalType::Unknown();
 // Undefined
 original = UndefinedLogicalType::Make();
 ASSERT_TRUE(original->is_invalid());
 ASSERT_FALSE(original->is_valid());
 converted_type = original->ToConvertedType(&converted_decimal_metadata);
 ASSERT_EQ(converted_type, ConvertedType::NA);
 ASSERT_EQ(converted_type, ConvertedType::UNDEFINED);
 ASSERT_FALSE(converted_decimal_metadata.isset);
 ASSERT_TRUE(original->is_compatible(converted_type, converted_decimal_metadata));
 ASSERT_TRUE(original->is_compatible(converted_type));
Expand DownExpand Up@@ -1243,7 +1243,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeProperties) {
 {BSONLogicalType::Make(), false, true, true},
 {UUIDLogicalType::Make(), false, true, true},
 {NoLogicalType::Make(), false, false, true},
 {UnknownLogicalType::Make(), false, false, false},
 };

 for (const ExpectedProperties& c : cases) {
Expand DownExpand Up@@ -1339,7 +1338,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) {
 }

 std::vector<std::shared_ptr<const LogicalType>> any_type_cases = {
 LogicalType::Null(), LogicalType::None(), LogicalType::Unknown()};
 LogicalType::Null(), LogicalType::None(), UndefinedLogicalType::Make()};

 for (auto c : any_type_cases) {
 ConfirmAnyPrimitiveTypeApplicability(c);
Expand DownExpand Up@@ -1453,7 +1452,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) {
 };

 std::vector<ExpectedRepresentation> cases = {
 {LogicalType::Unknown(), "Unknown", R"({"Type": "Unknown"})"},
 {UndefinedLogicalType::Make(), "Undefined", R"({"Type": "Undefined"})"},
 {LogicalType::String(), "String", R"({"Type": "String"})"},
 {LogicalType::Map(), "Map", R"({"Type": "Map"})"},
 {LogicalType::List(), "List", R"({"Type": "List"})"},
Expand DownExpand Up@@ -1550,7 +1549,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeSortOrder) {
 };

 std::vector<ExpectedSortOrder> cases = {
 {LogicalType::Unknown(), SortOrder::UNKNOWN},
 {LogicalType::String(), SortOrder::UNSIGNED},
 {LogicalType::Map(), SortOrder::UNKNOWN},
 {LogicalType::List(), SortOrder::UNKNOWN},
Expand DownExpand Up@@ -1888,8 +1886,6 @@ TEST_F(TestSchemaElementConstruction, SimpleCases) {
 {"uuid", LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16, false,
 ConvertedType::NA, true, [this]() { return element_->logicalType.__isset.UUID; }},
 {"none", LogicalType::None(), Type::INT64, -1, false, ConvertedType::NA, false,
 check_nothing},
 {"unknown", LogicalType::Unknown(), Type::INT64, -1, true, ConvertedType::NA, false,
 check_nothing}};

 for (const SchemaElementConstructionArguments& c : cases) {
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/parquet/thrift_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -256,6 +256,9 @@ static inline format::Type::type ToThrift(Type::type type) {
static inline format::ConvertedType::type ToThrift(ConvertedType::type type) {
 // item 0 is NONE
 DCHECK_NE(type, ConvertedType::NONE);
 // it is forbidden to emit "NA" (PARQUET-1990)
 DCHECK_NE(type, ConvertedType::NA);
@emkornfieldemkornfieldMar 31, 2021

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DCHECK_Lt might be better here.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it would change anything. Or we would have to do a full-blown check for valid values, which is a bit out of scope.
@emkornfieldemkornfieldMar 31, 2021•

edited

Loading

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. I think it prevents accidental writes of UNDEFINED as well.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I can add that as well :-)
 DCHECK_NE(type, ConvertedType::UNDEFINED);
 return static_cast<format::ConvertedType::type>(static_cast<int>(type) - 1);
}

Expand Down

37 changes: 18 additions & 19 deletions
cpp/src/parquet/types.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -366,13 +366,14 @@ std::shared_ptr<const LogicalType> LogicalType::FromConvertedType(
 return JSONLogicalType::Make();
 case ConvertedType::BSON:
 return BSONLogicalType::Make();
 case ConvertedType::NA:
 return NullLogicalType::Make();
 case ConvertedType::NONE:
 return NoLogicalType::Make();
 case ConvertedType::NA:
 case ConvertedType::UNDEFINED:
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
 }
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
}

std::shared_ptr<const LogicalType> LogicalType::FromThrift(
Expand DownExpand Up@@ -483,10 +484,6 @@ std::shared_ptr<const LogicalType> LogicalType::UUID() { return UUIDLogicalType:

std::shared_ptr<const LogicalType> LogicalType::None() { return NoLogicalType::Make(); }

std::shared_ptr<const LogicalType> LogicalType::Unknown() {
 return UnknownLogicalType::Make();
}

/*
 * The logical type implementation classes are built in four layers: (1) the base
 * layer, which establishes the interface and provides generally reusable implementations
Expand DownExpand Up@@ -516,7 +513,7 @@ class LogicalType::Impl {
 virtual std::string ToString() const = 0;

 virtual bool is_serialized() const {
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNKNOWN);
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNDEFINED);
 }

 virtual std::string ToJSON() const {
Expand DownExpand Up@@ -567,14 +564,14 @@ class LogicalType::Impl {
 class BSON;
 class UUID;
 class No;
 class Unknown;
 class Undefined;

 protected:
 Impl(LogicalType::Type::type t, SortOrder::type o) : type_(t), order_(o) {}
 Impl() = default;

 private:
 LogicalType::Type::type type_ = LogicalType::Type::UNKNOWN;
 LogicalType::Type::type type_ = LogicalType::Type::UNDEFINED;
 SortOrder::type order_ = SortOrder::UNKNOWN;
};

Expand DownExpand Up@@ -636,7 +633,9 @@ bool LogicalType::is_JSON() const { return impl_->type() == LogicalType::Type::J
bool LogicalType::is_BSON() const { return impl_->type() == LogicalType::Type::BSON; }
bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; }
bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; }
bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNKNOWN; }
bool LogicalType::is_valid() const {
 return impl_->type() != LogicalType::Type::UNDEFINED;
}
bool LogicalType::is_invalid() const { return !is_valid(); }
bool LogicalType::is_nested() const {
 return (impl_->type() == LogicalType::Type::LIST) ||
Expand DownExpand Up@@ -1555,19 +1554,19 @@ class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible,

GENERATE_MAKE(No)

class LogicalType::Impl::Unknown final : public LogicalType::Impl::SimpleCompatible,
 public LogicalType::Impl::UniversalApplicable {
class LogicalType::Impl::Undefined final : public LogicalType::Impl::SimpleCompatible,
public LogicalType::Impl::UniversalApplicable {
 public:
 friend class UnknownLogicalType;
 friend class UndefinedLogicalType;

 OVERRIDE_TOSTRING(Unknown)
 OVERRIDE_TOSTRING(Undefined)

 private:
Unknown()
 : LogicalType::Impl(LogicalType::Type::UNKNOWN, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::NA) {}
Undefined()
 : LogicalType::Impl(LogicalType::Type::UNDEFINED, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::UNDEFINED) {}
};

GENERATE_MAKE(Unknown)
GENERATE_MAKE(Undefined)

} // namespace parquet

29 changes: 19 additions & 10 deletions
cpp/src/parquet/types.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ struct Type {
// Mirrors parquet::ConvertedType
struct ConvertedType {
 enum type {
 NONE,
 NONE, // Not a real converted type, but means no converted type is specified
 UTF8,
 MAP,
 MAP_KEY_VALUE,
Expand All@@ -94,9 +94,12 @@ struct ConvertedType {
 JSON,
 BSON,
 INTERVAL,
 // DEPRECATED INVALID ConvertedType for all-null data.
 // Only useful for reading legacy files written out by interim Parquet C++ releases.
 // For writing, always emit LogicalType::Null instead.
 // See PARQUET-1990.
 NA = 25,
 // Should always be last element.
 UNDEFINED = 26
 UNDEFINED = 26 // Not a real converted type; should always be last element
 };
};

Expand DownExpand Up@@ -140,7 +143,7 @@ class PARQUET_EXPORT LogicalType {
 public:
 struct Type {
 enum type {
UNKNOWN = 0,
UNDEFINED = 0, // Not a real logical type
 STRING = 1,
 MAP,
 LIST,
Expand All@@ -151,11 +154,11 @@ class PARQUET_EXPORT LogicalType {
 TIMESTAMP,
 INTERVAL,
 INT,
 NIL, // Thrift NullType
 NIL, // Thrift NullType: annotates data that is always null
 JSON,
 BSON,
 UUID,
 NONE
 NONE // Not a real logical type; should always be last element
 };
 };

Expand DownExpand Up@@ -199,12 +202,18 @@ class PARQUET_EXPORT LogicalType {

 static std::shared_ptr<const LogicalType> Interval();
 static std::shared_ptr<const LogicalType> Int(int bit_width, bool is_signed);

 /// \brief Create a logical type for data that's always null
 ///
 /// Any physical type can be annotated with this logical type.
 static std::shared_ptr<const LogicalType> Null();

 static std::shared_ptr<const LogicalType> JSON();
 static std::shared_ptr<const LogicalType> BSON();
 static std::shared_ptr<const LogicalType> UUID();

 /// \brief Create a placeholder for when no logical type is specified
 static std::shared_ptr<const LogicalType> None();
 static std::shared_ptr<const LogicalType> Unknown();

 /// \brief Return true if this logical type is consistent with the given underlying
 /// physical type.
Expand DownExpand Up@@ -434,13 +443,13 @@ class PARQUET_EXPORT NoLogicalType : public LogicalType {
 NoLogicalType() = default;
};

/// \brief Allowed for any type.
class PARQUET_EXPORT UnknownLogicalType : public LogicalType {
// Internal API, for unrecognized logical types
class PARQUET_EXPORT UndefinedLogicalType : public LogicalType {
 public:
 static std::shared_ptr<const LogicalType> Make();

 private:
UnknownLogicalType() = default;
UndefinedLogicalType() = default;
};

// Data encodings. Mirrors parquet::Encoding
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,7 +54,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
 ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY"

 enum ParquetLogicalTypeId" parquet::LogicalType::Type::type":
ParquetLogicalType_UNKNOWN" parquet::LogicalType::Type::UNKNOWN"
ParquetLogicalType_UNDEFINED" parquet::LogicalType::Type::UNDEFINED"
 ParquetLogicalType_STRING" parquet::LogicalType::Type::STRING"
 ParquetLogicalType_MAP" parquet::LogicalType::Type::MAP"
 ParquetLogicalType_LIST" parquet::LogicalType::Type::LIST"
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pyx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -809,7 +809,7 @@ cdef physical_type_name_from_enum(ParquetType type_):

cdef logical_type_name_from_enum(ParquetLogicalTypeId type_):
 return {
ParquetLogicalType_UNKNOWN: 'UNKNOWN',
ParquetLogicalType_UNDEFINED: 'UNDEFINED',
 ParquetLogicalType_STRING: 'STRING',
 ParquetLogicalType_MAP: 'MAP',
 ParquetLogicalType_LIST: 'LIST',
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
PARQUET-1990: [C++] Refuse to write ConvertedType::NA#9863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
1
commit into

from

 Closed


PARQUET-1990: [C++] Refuse to write ConvertedType::NA
#9863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 8 additions & 4 deletions
cpp/src/parquet/printer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -87,11 +87,15 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selecte
 const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
 stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
 << TypeToString(descr->physical_type());
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << "/" << ConvertedTypeToString(descr->converted_type());
 const auto& logical_type = descr->logical_type();
 if (!logical_type->is_none()) {
 stream << " / " << logical_type->ToString();
 }
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << " / " << ConvertedTypeToString(descr->converted_type());
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 }
 }
 stream << ")" << std::endl;
 }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/parquet/reader_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,7 +317,7 @@ Number of RowGroups: 1
Number of Real Columns: 2
Number of Columns: 2
Number of Selected Columns: 2
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY/UTF8)
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY / String / UTF8)
Column 1: b (INT32)
--- Row Group: 0 ---
--- Total Bytes: 155 ---
Expand Down

13 changes: 11 additions & 2 deletions
cpp/src/parquet/schema.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -448,7 +448,7 @@ std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
 LogicalType::FromThrift(element->logicalType),
 LoadEnumSafe(&element->type), element->type_length, field_id));
 } else if (element->__isset.converted_type) {
 // legacy writer with logical type present
 // legacy writer with converted type present
 primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
 element->name, LoadEnumSafe(&element->repetition_type),
 LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type),
Expand DownExpand Up@@ -500,7 +500,16 @@ void PrimitiveNode::ToParquet(void* opaque_element) const {
 element->__set_name(name_);
 element->__set_repetition_type(ToThrift(repetition_));
 if (converted_type_ != ConvertedType::NONE) {
 element->__set_converted_type(ToThrift(converted_type_));
 if (converted_type_ != ConvertedType::NA) {
 element->__set_converted_type(ToThrift(converted_type_));
 } else {
 // ConvertedType::NA is an unreleased, obsolete synonym for LogicalType::Null.
 // Never emit it (see PARQUET-1990 for discussion).
 if (!logical_type_ || !logical_type_->is_null()) {
 throw ParquetException(
 "ConvertedType::NA is obsolete, please use LogicalType::Null instead");
 }
 }
 }
 if (field_id_ >= 0) {
 element->__set_field_id(field_id_);
Expand Down

14 changes: 5 additions & 9 deletions
cpp/src/parquet/schema_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1101,12 +1101,12 @@ TEST(TestLogicalTypeConstruction, ConvertedTypeCompatibility) {
 ASSERT_TRUE(reconstructed->is_valid());
 ASSERT_TRUE(reconstructed->Equals(*original));

 // Unknown
 original = LogicalType::Unknown();
 // Undefined
 original = UndefinedLogicalType::Make();
 ASSERT_TRUE(original->is_invalid());
 ASSERT_FALSE(original->is_valid());
 converted_type = original->ToConvertedType(&converted_decimal_metadata);
 ASSERT_EQ(converted_type, ConvertedType::NA);
 ASSERT_EQ(converted_type, ConvertedType::UNDEFINED);
 ASSERT_FALSE(converted_decimal_metadata.isset);
 ASSERT_TRUE(original->is_compatible(converted_type, converted_decimal_metadata));
 ASSERT_TRUE(original->is_compatible(converted_type));
Expand DownExpand Up@@ -1243,7 +1243,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeProperties) {
 {BSONLogicalType::Make(), false, true, true},
 {UUIDLogicalType::Make(), false, true, true},
 {NoLogicalType::Make(), false, false, true},
 {UnknownLogicalType::Make(), false, false, false},
 };

 for (const ExpectedProperties& c : cases) {
Expand DownExpand Up@@ -1339,7 +1338,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) {
 }

 std::vector<std::shared_ptr<const LogicalType>> any_type_cases = {
 LogicalType::Null(), LogicalType::None(), LogicalType::Unknown()};
 LogicalType::Null(), LogicalType::None(), UndefinedLogicalType::Make()};

 for (auto c : any_type_cases) {
 ConfirmAnyPrimitiveTypeApplicability(c);
Expand DownExpand Up@@ -1453,7 +1452,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) {
 };

 std::vector<ExpectedRepresentation> cases = {
 {LogicalType::Unknown(), "Unknown", R"({"Type": "Unknown"})"},
 {UndefinedLogicalType::Make(), "Undefined", R"({"Type": "Undefined"})"},
 {LogicalType::String(), "String", R"({"Type": "String"})"},
 {LogicalType::Map(), "Map", R"({"Type": "Map"})"},
 {LogicalType::List(), "List", R"({"Type": "List"})"},
Expand DownExpand Up@@ -1550,7 +1549,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeSortOrder) {
 };

 std::vector<ExpectedSortOrder> cases = {
 {LogicalType::Unknown(), SortOrder::UNKNOWN},
 {LogicalType::String(), SortOrder::UNSIGNED},
 {LogicalType::Map(), SortOrder::UNKNOWN},
 {LogicalType::List(), SortOrder::UNKNOWN},
Expand DownExpand Up@@ -1888,8 +1886,6 @@ TEST_F(TestSchemaElementConstruction, SimpleCases) {
 {"uuid", LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16, false,
 ConvertedType::NA, true, [this]() { return element_->logicalType.__isset.UUID; }},
 {"none", LogicalType::None(), Type::INT64, -1, false, ConvertedType::NA, false,
 check_nothing},
 {"unknown", LogicalType::Unknown(), Type::INT64, -1, true, ConvertedType::NA, false,
 check_nothing}};

 for (const SchemaElementConstructionArguments& c : cases) {
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/parquet/thrift_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -256,6 +256,9 @@ static inline format::Type::type ToThrift(Type::type type) {
static inline format::ConvertedType::type ToThrift(ConvertedType::type type) {
 // item 0 is NONE
 DCHECK_NE(type, ConvertedType::NONE);
 // it is forbidden to emit "NA" (PARQUET-1990)
 DCHECK_NE(type, ConvertedType::NA);
@emkornfieldemkornfieldMar 31, 2021

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DCHECK_Lt might be better here.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it would change anything. Or we would have to do a full-blown check for valid values, which is a bit out of scope.
@emkornfieldemkornfieldMar 31, 2021•

edited

Loading

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. I think it prevents accidental writes of UNDEFINED as well.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I can add that as well :-)
 DCHECK_NE(type, ConvertedType::UNDEFINED);
 return static_cast<format::ConvertedType::type>(static_cast<int>(type) - 1);
}

Expand Down

37 changes: 18 additions & 19 deletions
cpp/src/parquet/types.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -366,13 +366,14 @@ std::shared_ptr<const LogicalType> LogicalType::FromConvertedType(
 return JSONLogicalType::Make();
 case ConvertedType::BSON:
 return BSONLogicalType::Make();
 case ConvertedType::NA:
 return NullLogicalType::Make();
 case ConvertedType::NONE:
 return NoLogicalType::Make();
 case ConvertedType::NA:
 case ConvertedType::UNDEFINED:
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
 }
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
}

std::shared_ptr<const LogicalType> LogicalType::FromThrift(
Expand DownExpand Up@@ -483,10 +484,6 @@ std::shared_ptr<const LogicalType> LogicalType::UUID() { return UUIDLogicalType:

std::shared_ptr<const LogicalType> LogicalType::None() { return NoLogicalType::Make(); }

std::shared_ptr<const LogicalType> LogicalType::Unknown() {
 return UnknownLogicalType::Make();
}

/*
 * The logical type implementation classes are built in four layers: (1) the base
 * layer, which establishes the interface and provides generally reusable implementations
Expand DownExpand Up@@ -516,7 +513,7 @@ class LogicalType::Impl {
 virtual std::string ToString() const = 0;

 virtual bool is_serialized() const {
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNKNOWN);
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNDEFINED);
 }

 virtual std::string ToJSON() const {
Expand DownExpand Up@@ -567,14 +564,14 @@ class LogicalType::Impl {
 class BSON;
 class UUID;
 class No;
 class Unknown;
 class Undefined;

 protected:
 Impl(LogicalType::Type::type t, SortOrder::type o) : type_(t), order_(o) {}
 Impl() = default;

 private:
 LogicalType::Type::type type_ = LogicalType::Type::UNKNOWN;
 LogicalType::Type::type type_ = LogicalType::Type::UNDEFINED;
 SortOrder::type order_ = SortOrder::UNKNOWN;
};

Expand DownExpand Up@@ -636,7 +633,9 @@ bool LogicalType::is_JSON() const { return impl_->type() == LogicalType::Type::J
bool LogicalType::is_BSON() const { return impl_->type() == LogicalType::Type::BSON; }
bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; }
bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; }
bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNKNOWN; }
bool LogicalType::is_valid() const {
 return impl_->type() != LogicalType::Type::UNDEFINED;
}
bool LogicalType::is_invalid() const { return !is_valid(); }
bool LogicalType::is_nested() const {
 return (impl_->type() == LogicalType::Type::LIST) ||
Expand DownExpand Up@@ -1555,19 +1554,19 @@ class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible,

GENERATE_MAKE(No)

class LogicalType::Impl::Unknown final : public LogicalType::Impl::SimpleCompatible,
 public LogicalType::Impl::UniversalApplicable {
class LogicalType::Impl::Undefined final : public LogicalType::Impl::SimpleCompatible,
public LogicalType::Impl::UniversalApplicable {
 public:
 friend class UnknownLogicalType;
 friend class UndefinedLogicalType;

 OVERRIDE_TOSTRING(Unknown)
 OVERRIDE_TOSTRING(Undefined)

 private:
Unknown()
 : LogicalType::Impl(LogicalType::Type::UNKNOWN, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::NA) {}
Undefined()
 : LogicalType::Impl(LogicalType::Type::UNDEFINED, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::UNDEFINED) {}
};

GENERATE_MAKE(Unknown)
GENERATE_MAKE(Undefined)

} // namespace parquet

29 changes: 19 additions & 10 deletions
cpp/src/parquet/types.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ struct Type {
// Mirrors parquet::ConvertedType
struct ConvertedType {
 enum type {
 NONE,
 NONE, // Not a real converted type, but means no converted type is specified
 UTF8,
 MAP,
 MAP_KEY_VALUE,
Expand All@@ -94,9 +94,12 @@ struct ConvertedType {
 JSON,
 BSON,
 INTERVAL,
 // DEPRECATED INVALID ConvertedType for all-null data.
 // Only useful for reading legacy files written out by interim Parquet C++ releases.
 // For writing, always emit LogicalType::Null instead.
 // See PARQUET-1990.
 NA = 25,
 // Should always be last element.
 UNDEFINED = 26
 UNDEFINED = 26 // Not a real converted type; should always be last element
 };
};

Expand DownExpand Up@@ -140,7 +143,7 @@ class PARQUET_EXPORT LogicalType {
 public:
 struct Type {
 enum type {
UNKNOWN = 0,
UNDEFINED = 0, // Not a real logical type
 STRING = 1,
 MAP,
 LIST,
Expand All@@ -151,11 +154,11 @@ class PARQUET_EXPORT LogicalType {
 TIMESTAMP,
 INTERVAL,
 INT,
 NIL, // Thrift NullType
 NIL, // Thrift NullType: annotates data that is always null
 JSON,
 BSON,
 UUID,
 NONE
 NONE // Not a real logical type; should always be last element
 };
 };

Expand DownExpand Up@@ -199,12 +202,18 @@ class PARQUET_EXPORT LogicalType {

 static std::shared_ptr<const LogicalType> Interval();
 static std::shared_ptr<const LogicalType> Int(int bit_width, bool is_signed);

 /// \brief Create a logical type for data that's always null
 ///
 /// Any physical type can be annotated with this logical type.
 static std::shared_ptr<const LogicalType> Null();

 static std::shared_ptr<const LogicalType> JSON();
 static std::shared_ptr<const LogicalType> BSON();
 static std::shared_ptr<const LogicalType> UUID();

 /// \brief Create a placeholder for when no logical type is specified
 static std::shared_ptr<const LogicalType> None();
 static std::shared_ptr<const LogicalType> Unknown();

 /// \brief Return true if this logical type is consistent with the given underlying
 /// physical type.
Expand DownExpand Up@@ -434,13 +443,13 @@ class PARQUET_EXPORT NoLogicalType : public LogicalType {
 NoLogicalType() = default;
};

/// \brief Allowed for any type.
class PARQUET_EXPORT UnknownLogicalType : public LogicalType {
// Internal API, for unrecognized logical types
class PARQUET_EXPORT UndefinedLogicalType : public LogicalType {
 public:
 static std::shared_ptr<const LogicalType> Make();

 private:
UnknownLogicalType() = default;
UndefinedLogicalType() = default;
};

// Data encodings. Mirrors parquet::Encoding
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,7 +54,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
 ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY"

 enum ParquetLogicalTypeId" parquet::LogicalType::Type::type":
ParquetLogicalType_UNKNOWN" parquet::LogicalType::Type::UNKNOWN"
ParquetLogicalType_UNDEFINED" parquet::LogicalType::Type::UNDEFINED"
 ParquetLogicalType_STRING" parquet::LogicalType::Type::STRING"
 ParquetLogicalType_MAP" parquet::LogicalType::Type::MAP"
 ParquetLogicalType_LIST" parquet::LogicalType::Type::LIST"
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pyx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -809,7 +809,7 @@ cdef physical_type_name_from_enum(ParquetType type_):

cdef logical_type_name_from_enum(ParquetLogicalTypeId type_):
 return {
ParquetLogicalType_UNKNOWN: 'UNKNOWN',
ParquetLogicalType_UNDEFINED: 'UNDEFINED',
 ParquetLogicalType_STRING: 'STRING',
 ParquetLogicalType_MAP: 'MAP',
 ParquetLogicalType_LIST: 'LIST',
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
PARQUET-1990: [C++] Refuse to write ConvertedType::NA#9863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
1
commit into

from

 Closed


PARQUET-1990: [C++] Refuse to write ConvertedType::NA
#9863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 8 additions & 4 deletions
cpp/src/parquet/printer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -87,11 +87,15 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selecte
 const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
 stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
 << TypeToString(descr->physical_type());
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << "/" << ConvertedTypeToString(descr->converted_type());
 const auto& logical_type = descr->logical_type();
 if (!logical_type->is_none()) {
 stream << " / " << logical_type->ToString();
 }
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << " / " << ConvertedTypeToString(descr->converted_type());
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 }
 }
 stream << ")" << std::endl;
 }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/parquet/reader_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,7 +317,7 @@ Number of RowGroups: 1
Number of Real Columns: 2
Number of Columns: 2
Number of Selected Columns: 2
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY/UTF8)
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY / String / UTF8)
Column 1: b (INT32)
--- Row Group: 0 ---
--- Total Bytes: 155 ---
Expand Down

13 changes: 11 additions & 2 deletions
cpp/src/parquet/schema.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -448,7 +448,7 @@ std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
 LogicalType::FromThrift(element->logicalType),
 LoadEnumSafe(&element->type), element->type_length, field_id));
 } else if (element->__isset.converted_type) {
 // legacy writer with logical type present
 // legacy writer with converted type present
 primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
 element->name, LoadEnumSafe(&element->repetition_type),
 LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type),
Expand DownExpand Up@@ -500,7 +500,16 @@ void PrimitiveNode::ToParquet(void* opaque_element) const {
 element->__set_name(name_);
 element->__set_repetition_type(ToThrift(repetition_));
 if (converted_type_ != ConvertedType::NONE) {
 element->__set_converted_type(ToThrift(converted_type_));
 if (converted_type_ != ConvertedType::NA) {
 element->__set_converted_type(ToThrift(converted_type_));
 } else {
 // ConvertedType::NA is an unreleased, obsolete synonym for LogicalType::Null.
 // Never emit it (see PARQUET-1990 for discussion).
 if (!logical_type_ || !logical_type_->is_null()) {
 throw ParquetException(
 "ConvertedType::NA is obsolete, please use LogicalType::Null instead");
 }
 }
 }
 if (field_id_ >= 0) {
 element->__set_field_id(field_id_);
Expand Down

14 changes: 5 additions & 9 deletions
cpp/src/parquet/schema_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1101,12 +1101,12 @@ TEST(TestLogicalTypeConstruction, ConvertedTypeCompatibility) {
 ASSERT_TRUE(reconstructed->is_valid());
 ASSERT_TRUE(reconstructed->Equals(*original));

 // Unknown
 original = LogicalType::Unknown();
 // Undefined
 original = UndefinedLogicalType::Make();
 ASSERT_TRUE(original->is_invalid());
 ASSERT_FALSE(original->is_valid());
 converted_type = original->ToConvertedType(&converted_decimal_metadata);
 ASSERT_EQ(converted_type, ConvertedType::NA);
 ASSERT_EQ(converted_type, ConvertedType::UNDEFINED);
 ASSERT_FALSE(converted_decimal_metadata.isset);
 ASSERT_TRUE(original->is_compatible(converted_type, converted_decimal_metadata));
 ASSERT_TRUE(original->is_compatible(converted_type));
Expand DownExpand Up@@ -1243,7 +1243,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeProperties) {
 {BSONLogicalType::Make(), false, true, true},
 {UUIDLogicalType::Make(), false, true, true},
 {NoLogicalType::Make(), false, false, true},
 {UnknownLogicalType::Make(), false, false, false},
 };

 for (const ExpectedProperties& c : cases) {
Expand DownExpand Up@@ -1339,7 +1338,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) {
 }

 std::vector<std::shared_ptr<const LogicalType>> any_type_cases = {
 LogicalType::Null(), LogicalType::None(), LogicalType::Unknown()};
 LogicalType::Null(), LogicalType::None(), UndefinedLogicalType::Make()};

 for (auto c : any_type_cases) {
 ConfirmAnyPrimitiveTypeApplicability(c);
Expand DownExpand Up@@ -1453,7 +1452,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) {
 };

 std::vector<ExpectedRepresentation> cases = {
 {LogicalType::Unknown(), "Unknown", R"({"Type": "Unknown"})"},
 {UndefinedLogicalType::Make(), "Undefined", R"({"Type": "Undefined"})"},
 {LogicalType::String(), "String", R"({"Type": "String"})"},
 {LogicalType::Map(), "Map", R"({"Type": "Map"})"},
 {LogicalType::List(), "List", R"({"Type": "List"})"},
Expand DownExpand Up@@ -1550,7 +1549,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeSortOrder) {
 };

 std::vector<ExpectedSortOrder> cases = {
 {LogicalType::Unknown(), SortOrder::UNKNOWN},
 {LogicalType::String(), SortOrder::UNSIGNED},
 {LogicalType::Map(), SortOrder::UNKNOWN},
 {LogicalType::List(), SortOrder::UNKNOWN},
Expand DownExpand Up@@ -1888,8 +1886,6 @@ TEST_F(TestSchemaElementConstruction, SimpleCases) {
 {"uuid", LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16, false,
 ConvertedType::NA, true, [this]() { return element_->logicalType.__isset.UUID; }},
 {"none", LogicalType::None(), Type::INT64, -1, false, ConvertedType::NA, false,
 check_nothing},
 {"unknown", LogicalType::Unknown(), Type::INT64, -1, true, ConvertedType::NA, false,
 check_nothing}};

 for (const SchemaElementConstructionArguments& c : cases) {
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/parquet/thrift_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -256,6 +256,9 @@ static inline format::Type::type ToThrift(Type::type type) {
static inline format::ConvertedType::type ToThrift(ConvertedType::type type) {
 // item 0 is NONE
 DCHECK_NE(type, ConvertedType::NONE);
 // it is forbidden to emit "NA" (PARQUET-1990)
 DCHECK_NE(type, ConvertedType::NA);
@emkornfieldemkornfieldMar 31, 2021

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DCHECK_Lt might be better here.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it would change anything. Or we would have to do a full-blown check for valid values, which is a bit out of scope.
@emkornfieldemkornfieldMar 31, 2021•

edited

Loading

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. I think it prevents accidental writes of UNDEFINED as well.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I can add that as well :-)
 DCHECK_NE(type, ConvertedType::UNDEFINED);
 return static_cast<format::ConvertedType::type>(static_cast<int>(type) - 1);
}

Expand Down

37 changes: 18 additions & 19 deletions
cpp/src/parquet/types.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -366,13 +366,14 @@ std::shared_ptr<const LogicalType> LogicalType::FromConvertedType(
 return JSONLogicalType::Make();
 case ConvertedType::BSON:
 return BSONLogicalType::Make();
 case ConvertedType::NA:
 return NullLogicalType::Make();
 case ConvertedType::NONE:
 return NoLogicalType::Make();
 case ConvertedType::NA:
 case ConvertedType::UNDEFINED:
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
 }
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
}

std::shared_ptr<const LogicalType> LogicalType::FromThrift(
Expand DownExpand Up@@ -483,10 +484,6 @@ std::shared_ptr<const LogicalType> LogicalType::UUID() { return UUIDLogicalType:

std::shared_ptr<const LogicalType> LogicalType::None() { return NoLogicalType::Make(); }

std::shared_ptr<const LogicalType> LogicalType::Unknown() {
 return UnknownLogicalType::Make();
}

/*
 * The logical type implementation classes are built in four layers: (1) the base
 * layer, which establishes the interface and provides generally reusable implementations
Expand DownExpand Up@@ -516,7 +513,7 @@ class LogicalType::Impl {
 virtual std::string ToString() const = 0;

 virtual bool is_serialized() const {
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNKNOWN);
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNDEFINED);
 }

 virtual std::string ToJSON() const {
Expand DownExpand Up@@ -567,14 +564,14 @@ class LogicalType::Impl {
 class BSON;
 class UUID;
 class No;
 class Unknown;
 class Undefined;

 protected:
 Impl(LogicalType::Type::type t, SortOrder::type o) : type_(t), order_(o) {}
 Impl() = default;

 private:
 LogicalType::Type::type type_ = LogicalType::Type::UNKNOWN;
 LogicalType::Type::type type_ = LogicalType::Type::UNDEFINED;
 SortOrder::type order_ = SortOrder::UNKNOWN;
};

Expand DownExpand Up@@ -636,7 +633,9 @@ bool LogicalType::is_JSON() const { return impl_->type() == LogicalType::Type::J
bool LogicalType::is_BSON() const { return impl_->type() == LogicalType::Type::BSON; }
bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; }
bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; }
bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNKNOWN; }
bool LogicalType::is_valid() const {
 return impl_->type() != LogicalType::Type::UNDEFINED;
}
bool LogicalType::is_invalid() const { return !is_valid(); }
bool LogicalType::is_nested() const {
 return (impl_->type() == LogicalType::Type::LIST) ||
Expand DownExpand Up@@ -1555,19 +1554,19 @@ class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible,

GENERATE_MAKE(No)

class LogicalType::Impl::Unknown final : public LogicalType::Impl::SimpleCompatible,
 public LogicalType::Impl::UniversalApplicable {
class LogicalType::Impl::Undefined final : public LogicalType::Impl::SimpleCompatible,
public LogicalType::Impl::UniversalApplicable {
 public:
 friend class UnknownLogicalType;
 friend class UndefinedLogicalType;

 OVERRIDE_TOSTRING(Unknown)
 OVERRIDE_TOSTRING(Undefined)

 private:
Unknown()
 : LogicalType::Impl(LogicalType::Type::UNKNOWN, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::NA) {}
Undefined()
 : LogicalType::Impl(LogicalType::Type::UNDEFINED, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::UNDEFINED) {}
};

GENERATE_MAKE(Unknown)
GENERATE_MAKE(Undefined)

} // namespace parquet

29 changes: 19 additions & 10 deletions
cpp/src/parquet/types.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ struct Type {
// Mirrors parquet::ConvertedType
struct ConvertedType {
 enum type {
 NONE,
 NONE, // Not a real converted type, but means no converted type is specified
 UTF8,
 MAP,
 MAP_KEY_VALUE,
Expand All@@ -94,9 +94,12 @@ struct ConvertedType {
 JSON,
 BSON,
 INTERVAL,
 // DEPRECATED INVALID ConvertedType for all-null data.
 // Only useful for reading legacy files written out by interim Parquet C++ releases.
 // For writing, always emit LogicalType::Null instead.
 // See PARQUET-1990.
 NA = 25,
 // Should always be last element.
 UNDEFINED = 26
 UNDEFINED = 26 // Not a real converted type; should always be last element
 };
};

Expand DownExpand Up@@ -140,7 +143,7 @@ class PARQUET_EXPORT LogicalType {
 public:
 struct Type {
 enum type {
UNKNOWN = 0,
UNDEFINED = 0, // Not a real logical type
 STRING = 1,
 MAP,
 LIST,
Expand All@@ -151,11 +154,11 @@ class PARQUET_EXPORT LogicalType {
 TIMESTAMP,
 INTERVAL,
 INT,
 NIL, // Thrift NullType
 NIL, // Thrift NullType: annotates data that is always null
 JSON,
 BSON,
 UUID,
 NONE
 NONE // Not a real logical type; should always be last element
 };
 };

Expand DownExpand Up@@ -199,12 +202,18 @@ class PARQUET_EXPORT LogicalType {

 static std::shared_ptr<const LogicalType> Interval();
 static std::shared_ptr<const LogicalType> Int(int bit_width, bool is_signed);

 /// \brief Create a logical type for data that's always null
 ///
 /// Any physical type can be annotated with this logical type.
 static std::shared_ptr<const LogicalType> Null();

 static std::shared_ptr<const LogicalType> JSON();
 static std::shared_ptr<const LogicalType> BSON();
 static std::shared_ptr<const LogicalType> UUID();

 /// \brief Create a placeholder for when no logical type is specified
 static std::shared_ptr<const LogicalType> None();
 static std::shared_ptr<const LogicalType> Unknown();

 /// \brief Return true if this logical type is consistent with the given underlying
 /// physical type.
Expand DownExpand Up@@ -434,13 +443,13 @@ class PARQUET_EXPORT NoLogicalType : public LogicalType {
 NoLogicalType() = default;
};

/// \brief Allowed for any type.
class PARQUET_EXPORT UnknownLogicalType : public LogicalType {
// Internal API, for unrecognized logical types
class PARQUET_EXPORT UndefinedLogicalType : public LogicalType {
 public:
 static std::shared_ptr<const LogicalType> Make();

 private:
UnknownLogicalType() = default;
UndefinedLogicalType() = default;
};

// Data encodings. Mirrors parquet::Encoding
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,7 +54,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
 ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY"

 enum ParquetLogicalTypeId" parquet::LogicalType::Type::type":
ParquetLogicalType_UNKNOWN" parquet::LogicalType::Type::UNKNOWN"
ParquetLogicalType_UNDEFINED" parquet::LogicalType::Type::UNDEFINED"
 ParquetLogicalType_STRING" parquet::LogicalType::Type::STRING"
 ParquetLogicalType_MAP" parquet::LogicalType::Type::MAP"
 ParquetLogicalType_LIST" parquet::LogicalType::Type::LIST"
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pyx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -809,7 +809,7 @@ cdef physical_type_name_from_enum(ParquetType type_):

cdef logical_type_name_from_enum(ParquetLogicalTypeId type_):
 return {
ParquetLogicalType_UNKNOWN: 'UNKNOWN',
ParquetLogicalType_UNDEFINED: 'UNDEFINED',
 ParquetLogicalType_STRING: 'STRING',
 ParquetLogicalType_MAP: 'MAP',
 ParquetLogicalType_LIST: 'LIST',
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
PARQUET-1990: [C++] Refuse to write ConvertedType::NA#9863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
1
commit into

from

 Closed


PARQUET-1990: [C++] Refuse to write ConvertedType::NA
#9863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 8 additions & 4 deletions
cpp/src/parquet/printer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -87,11 +87,15 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selecte
 const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
 stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
 << TypeToString(descr->physical_type());
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << "/" << ConvertedTypeToString(descr->converted_type());
 const auto& logical_type = descr->logical_type();
 if (!logical_type->is_none()) {
 stream << " / " << logical_type->ToString();
 }
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << " / " << ConvertedTypeToString(descr->converted_type());
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 }
 }
 stream << ")" << std::endl;
 }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/parquet/reader_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,7 +317,7 @@ Number of RowGroups: 1
Number of Real Columns: 2
Number of Columns: 2
Number of Selected Columns: 2
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY/UTF8)
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY / String / UTF8)
Column 1: b (INT32)
--- Row Group: 0 ---
--- Total Bytes: 155 ---
Expand Down

13 changes: 11 additions & 2 deletions
cpp/src/parquet/schema.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -448,7 +448,7 @@ std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
 LogicalType::FromThrift(element->logicalType),
 LoadEnumSafe(&element->type), element->type_length, field_id));
 } else if (element->__isset.converted_type) {
 // legacy writer with logical type present
 // legacy writer with converted type present
 primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
 element->name, LoadEnumSafe(&element->repetition_type),
 LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type),
Expand DownExpand Up@@ -500,7 +500,16 @@ void PrimitiveNode::ToParquet(void* opaque_element) const {
 element->__set_name(name_);
 element->__set_repetition_type(ToThrift(repetition_));
 if (converted_type_ != ConvertedType::NONE) {
 element->__set_converted_type(ToThrift(converted_type_));
 if (converted_type_ != ConvertedType::NA) {
 element->__set_converted_type(ToThrift(converted_type_));
 } else {
 // ConvertedType::NA is an unreleased, obsolete synonym for LogicalType::Null.
 // Never emit it (see PARQUET-1990 for discussion).
 if (!logical_type_ || !logical_type_->is_null()) {
 throw ParquetException(
 "ConvertedType::NA is obsolete, please use LogicalType::Null instead");
 }
 }
 }
 if (field_id_ >= 0) {
 element->__set_field_id(field_id_);
Expand Down

14 changes: 5 additions & 9 deletions
cpp/src/parquet/schema_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1101,12 +1101,12 @@ TEST(TestLogicalTypeConstruction, ConvertedTypeCompatibility) {
 ASSERT_TRUE(reconstructed->is_valid());
 ASSERT_TRUE(reconstructed->Equals(*original));

 // Unknown
 original = LogicalType::Unknown();
 // Undefined
 original = UndefinedLogicalType::Make();
 ASSERT_TRUE(original->is_invalid());
 ASSERT_FALSE(original->is_valid());
 converted_type = original->ToConvertedType(&converted_decimal_metadata);
 ASSERT_EQ(converted_type, ConvertedType::NA);
 ASSERT_EQ(converted_type, ConvertedType::UNDEFINED);
 ASSERT_FALSE(converted_decimal_metadata.isset);
 ASSERT_TRUE(original->is_compatible(converted_type, converted_decimal_metadata));
 ASSERT_TRUE(original->is_compatible(converted_type));
Expand DownExpand Up@@ -1243,7 +1243,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeProperties) {
 {BSONLogicalType::Make(), false, true, true},
 {UUIDLogicalType::Make(), false, true, true},
 {NoLogicalType::Make(), false, false, true},
 {UnknownLogicalType::Make(), false, false, false},
 };

 for (const ExpectedProperties& c : cases) {
Expand DownExpand Up@@ -1339,7 +1338,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) {
 }

 std::vector<std::shared_ptr<const LogicalType>> any_type_cases = {
 LogicalType::Null(), LogicalType::None(), LogicalType::Unknown()};
 LogicalType::Null(), LogicalType::None(), UndefinedLogicalType::Make()};

 for (auto c : any_type_cases) {
 ConfirmAnyPrimitiveTypeApplicability(c);
Expand DownExpand Up@@ -1453,7 +1452,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) {
 };

 std::vector<ExpectedRepresentation> cases = {
 {LogicalType::Unknown(), "Unknown", R"({"Type": "Unknown"})"},
 {UndefinedLogicalType::Make(), "Undefined", R"({"Type": "Undefined"})"},
 {LogicalType::String(), "String", R"({"Type": "String"})"},
 {LogicalType::Map(), "Map", R"({"Type": "Map"})"},
 {LogicalType::List(), "List", R"({"Type": "List"})"},
Expand DownExpand Up@@ -1550,7 +1549,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeSortOrder) {
 };

 std::vector<ExpectedSortOrder> cases = {
 {LogicalType::Unknown(), SortOrder::UNKNOWN},
 {LogicalType::String(), SortOrder::UNSIGNED},
 {LogicalType::Map(), SortOrder::UNKNOWN},
 {LogicalType::List(), SortOrder::UNKNOWN},
Expand DownExpand Up@@ -1888,8 +1886,6 @@ TEST_F(TestSchemaElementConstruction, SimpleCases) {
 {"uuid", LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16, false,
 ConvertedType::NA, true, [this]() { return element_->logicalType.__isset.UUID; }},
 {"none", LogicalType::None(), Type::INT64, -1, false, ConvertedType::NA, false,
 check_nothing},
 {"unknown", LogicalType::Unknown(), Type::INT64, -1, true, ConvertedType::NA, false,
 check_nothing}};

 for (const SchemaElementConstructionArguments& c : cases) {
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/parquet/thrift_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -256,6 +256,9 @@ static inline format::Type::type ToThrift(Type::type type) {
static inline format::ConvertedType::type ToThrift(ConvertedType::type type) {
 // item 0 is NONE
 DCHECK_NE(type, ConvertedType::NONE);
 // it is forbidden to emit "NA" (PARQUET-1990)
 DCHECK_NE(type, ConvertedType::NA);
@emkornfieldemkornfieldMar 31, 2021

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DCHECK_Lt might be better here.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it would change anything. Or we would have to do a full-blown check for valid values, which is a bit out of scope.
@emkornfieldemkornfieldMar 31, 2021•

edited

Loading

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. I think it prevents accidental writes of UNDEFINED as well.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I can add that as well :-)
 DCHECK_NE(type, ConvertedType::UNDEFINED);
 return static_cast<format::ConvertedType::type>(static_cast<int>(type) - 1);
}

Expand Down

37 changes: 18 additions & 19 deletions
cpp/src/parquet/types.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -366,13 +366,14 @@ std::shared_ptr<const LogicalType> LogicalType::FromConvertedType(
 return JSONLogicalType::Make();
 case ConvertedType::BSON:
 return BSONLogicalType::Make();
 case ConvertedType::NA:
 return NullLogicalType::Make();
 case ConvertedType::NONE:
 return NoLogicalType::Make();
 case ConvertedType::NA:
 case ConvertedType::UNDEFINED:
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
 }
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
}

std::shared_ptr<const LogicalType> LogicalType::FromThrift(
Expand DownExpand Up@@ -483,10 +484,6 @@ std::shared_ptr<const LogicalType> LogicalType::UUID() { return UUIDLogicalType:

std::shared_ptr<const LogicalType> LogicalType::None() { return NoLogicalType::Make(); }

std::shared_ptr<const LogicalType> LogicalType::Unknown() {
 return UnknownLogicalType::Make();
}

/*
 * The logical type implementation classes are built in four layers: (1) the base
 * layer, which establishes the interface and provides generally reusable implementations
Expand DownExpand Up@@ -516,7 +513,7 @@ class LogicalType::Impl {
 virtual std::string ToString() const = 0;

 virtual bool is_serialized() const {
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNKNOWN);
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNDEFINED);
 }

 virtual std::string ToJSON() const {
Expand DownExpand Up@@ -567,14 +564,14 @@ class LogicalType::Impl {
 class BSON;
 class UUID;
 class No;
 class Unknown;
 class Undefined;

 protected:
 Impl(LogicalType::Type::type t, SortOrder::type o) : type_(t), order_(o) {}
 Impl() = default;

 private:
 LogicalType::Type::type type_ = LogicalType::Type::UNKNOWN;
 LogicalType::Type::type type_ = LogicalType::Type::UNDEFINED;
 SortOrder::type order_ = SortOrder::UNKNOWN;
};

Expand DownExpand Up@@ -636,7 +633,9 @@ bool LogicalType::is_JSON() const { return impl_->type() == LogicalType::Type::J
bool LogicalType::is_BSON() const { return impl_->type() == LogicalType::Type::BSON; }
bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; }
bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; }
bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNKNOWN; }
bool LogicalType::is_valid() const {
 return impl_->type() != LogicalType::Type::UNDEFINED;
}
bool LogicalType::is_invalid() const { return !is_valid(); }
bool LogicalType::is_nested() const {
 return (impl_->type() == LogicalType::Type::LIST) ||
Expand DownExpand Up@@ -1555,19 +1554,19 @@ class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible,

GENERATE_MAKE(No)

class LogicalType::Impl::Unknown final : public LogicalType::Impl::SimpleCompatible,
 public LogicalType::Impl::UniversalApplicable {
class LogicalType::Impl::Undefined final : public LogicalType::Impl::SimpleCompatible,
public LogicalType::Impl::UniversalApplicable {
 public:
 friend class UnknownLogicalType;
 friend class UndefinedLogicalType;

 OVERRIDE_TOSTRING(Unknown)
 OVERRIDE_TOSTRING(Undefined)

 private:
Unknown()
 : LogicalType::Impl(LogicalType::Type::UNKNOWN, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::NA) {}
Undefined()
 : LogicalType::Impl(LogicalType::Type::UNDEFINED, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::UNDEFINED) {}
};

GENERATE_MAKE(Unknown)
GENERATE_MAKE(Undefined)

} // namespace parquet

29 changes: 19 additions & 10 deletions
cpp/src/parquet/types.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ struct Type {
// Mirrors parquet::ConvertedType
struct ConvertedType {
 enum type {
 NONE,
 NONE, // Not a real converted type, but means no converted type is specified
 UTF8,
 MAP,
 MAP_KEY_VALUE,
Expand All@@ -94,9 +94,12 @@ struct ConvertedType {
 JSON,
 BSON,
 INTERVAL,
 // DEPRECATED INVALID ConvertedType for all-null data.
 // Only useful for reading legacy files written out by interim Parquet C++ releases.
 // For writing, always emit LogicalType::Null instead.
 // See PARQUET-1990.
 NA = 25,
 // Should always be last element.
 UNDEFINED = 26
 UNDEFINED = 26 // Not a real converted type; should always be last element
 };
};

Expand DownExpand Up@@ -140,7 +143,7 @@ class PARQUET_EXPORT LogicalType {
 public:
 struct Type {
 enum type {
UNKNOWN = 0,
UNDEFINED = 0, // Not a real logical type
 STRING = 1,
 MAP,
 LIST,
Expand All@@ -151,11 +154,11 @@ class PARQUET_EXPORT LogicalType {
 TIMESTAMP,
 INTERVAL,
 INT,
 NIL, // Thrift NullType
 NIL, // Thrift NullType: annotates data that is always null
 JSON,
 BSON,
 UUID,
 NONE
 NONE // Not a real logical type; should always be last element
 };
 };

Expand DownExpand Up@@ -199,12 +202,18 @@ class PARQUET_EXPORT LogicalType {

 static std::shared_ptr<const LogicalType> Interval();
 static std::shared_ptr<const LogicalType> Int(int bit_width, bool is_signed);

 /// \brief Create a logical type for data that's always null
 ///
 /// Any physical type can be annotated with this logical type.
 static std::shared_ptr<const LogicalType> Null();

 static std::shared_ptr<const LogicalType> JSON();
 static std::shared_ptr<const LogicalType> BSON();
 static std::shared_ptr<const LogicalType> UUID();

 /// \brief Create a placeholder for when no logical type is specified
 static std::shared_ptr<const LogicalType> None();
 static std::shared_ptr<const LogicalType> Unknown();

 /// \brief Return true if this logical type is consistent with the given underlying
 /// physical type.
Expand DownExpand Up@@ -434,13 +443,13 @@ class PARQUET_EXPORT NoLogicalType : public LogicalType {
 NoLogicalType() = default;
};

/// \brief Allowed for any type.
class PARQUET_EXPORT UnknownLogicalType : public LogicalType {
// Internal API, for unrecognized logical types
class PARQUET_EXPORT UndefinedLogicalType : public LogicalType {
 public:
 static std::shared_ptr<const LogicalType> Make();

 private:
UnknownLogicalType() = default;
UndefinedLogicalType() = default;
};

// Data encodings. Mirrors parquet::Encoding
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,7 +54,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
 ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY"

 enum ParquetLogicalTypeId" parquet::LogicalType::Type::type":
ParquetLogicalType_UNKNOWN" parquet::LogicalType::Type::UNKNOWN"
ParquetLogicalType_UNDEFINED" parquet::LogicalType::Type::UNDEFINED"
 ParquetLogicalType_STRING" parquet::LogicalType::Type::STRING"
 ParquetLogicalType_MAP" parquet::LogicalType::Type::MAP"
 ParquetLogicalType_LIST" parquet::LogicalType::Type::LIST"
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pyx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -809,7 +809,7 @@ cdef physical_type_name_from_enum(ParquetType type_):

cdef logical_type_name_from_enum(ParquetLogicalTypeId type_):
 return {
ParquetLogicalType_UNKNOWN: 'UNKNOWN',
ParquetLogicalType_UNDEFINED: 'UNDEFINED',
 ParquetLogicalType_STRING: 'STRING',
 ParquetLogicalType_MAP: 'MAP',
 ParquetLogicalType_LIST: 'LIST',
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
PARQUET-1990: [C++] Refuse to write ConvertedType::NA#9863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
1
commit into

from

 Closed


PARQUET-1990: [C++] Refuse to write ConvertedType::NA
#9863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 8 additions & 4 deletions
cpp/src/parquet/printer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -87,11 +87,15 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selecte
 const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
 stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
 << TypeToString(descr->physical_type());
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << "/" << ConvertedTypeToString(descr->converted_type());
 const auto& logical_type = descr->logical_type();
 if (!logical_type->is_none()) {
 stream << " / " << logical_type->ToString();
 }
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << " / " << ConvertedTypeToString(descr->converted_type());
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 }
 }
 stream << ")" << std::endl;
 }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/parquet/reader_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,7 +317,7 @@ Number of RowGroups: 1
Number of Real Columns: 2
Number of Columns: 2
Number of Selected Columns: 2
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY/UTF8)
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY / String / UTF8)
Column 1: b (INT32)
--- Row Group: 0 ---
--- Total Bytes: 155 ---
Expand Down

13 changes: 11 additions & 2 deletions
cpp/src/parquet/schema.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -448,7 +448,7 @@ std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
 LogicalType::FromThrift(element->logicalType),
 LoadEnumSafe(&element->type), element->type_length, field_id));
 } else if (element->__isset.converted_type) {
 // legacy writer with logical type present
 // legacy writer with converted type present
 primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
 element->name, LoadEnumSafe(&element->repetition_type),
 LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type),
Expand DownExpand Up@@ -500,7 +500,16 @@ void PrimitiveNode::ToParquet(void* opaque_element) const {
 element->__set_name(name_);
 element->__set_repetition_type(ToThrift(repetition_));
 if (converted_type_ != ConvertedType::NONE) {
 element->__set_converted_type(ToThrift(converted_type_));
 if (converted_type_ != ConvertedType::NA) {
 element->__set_converted_type(ToThrift(converted_type_));
 } else {
 // ConvertedType::NA is an unreleased, obsolete synonym for LogicalType::Null.
 // Never emit it (see PARQUET-1990 for discussion).
 if (!logical_type_ || !logical_type_->is_null()) {
 throw ParquetException(
 "ConvertedType::NA is obsolete, please use LogicalType::Null instead");
 }
 }
 }
 if (field_id_ >= 0) {
 element->__set_field_id(field_id_);
Expand Down

14 changes: 5 additions & 9 deletions
cpp/src/parquet/schema_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1101,12 +1101,12 @@ TEST(TestLogicalTypeConstruction, ConvertedTypeCompatibility) {
 ASSERT_TRUE(reconstructed->is_valid());
 ASSERT_TRUE(reconstructed->Equals(*original));

 // Unknown
 original = LogicalType::Unknown();
 // Undefined
 original = UndefinedLogicalType::Make();
 ASSERT_TRUE(original->is_invalid());
 ASSERT_FALSE(original->is_valid());
 converted_type = original->ToConvertedType(&converted_decimal_metadata);
 ASSERT_EQ(converted_type, ConvertedType::NA);
 ASSERT_EQ(converted_type, ConvertedType::UNDEFINED);
 ASSERT_FALSE(converted_decimal_metadata.isset);
 ASSERT_TRUE(original->is_compatible(converted_type, converted_decimal_metadata));
 ASSERT_TRUE(original->is_compatible(converted_type));
Expand DownExpand Up@@ -1243,7 +1243,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeProperties) {
 {BSONLogicalType::Make(), false, true, true},
 {UUIDLogicalType::Make(), false, true, true},
 {NoLogicalType::Make(), false, false, true},
 {UnknownLogicalType::Make(), false, false, false},
 };

 for (const ExpectedProperties& c : cases) {
Expand DownExpand Up@@ -1339,7 +1338,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) {
 }

 std::vector<std::shared_ptr<const LogicalType>> any_type_cases = {
 LogicalType::Null(), LogicalType::None(), LogicalType::Unknown()};
 LogicalType::Null(), LogicalType::None(), UndefinedLogicalType::Make()};

 for (auto c : any_type_cases) {
 ConfirmAnyPrimitiveTypeApplicability(c);
Expand DownExpand Up@@ -1453,7 +1452,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) {
 };

 std::vector<ExpectedRepresentation> cases = {
 {LogicalType::Unknown(), "Unknown", R"({"Type": "Unknown"})"},
 {UndefinedLogicalType::Make(), "Undefined", R"({"Type": "Undefined"})"},
 {LogicalType::String(), "String", R"({"Type": "String"})"},
 {LogicalType::Map(), "Map", R"({"Type": "Map"})"},
 {LogicalType::List(), "List", R"({"Type": "List"})"},
Expand DownExpand Up@@ -1550,7 +1549,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeSortOrder) {
 };

 std::vector<ExpectedSortOrder> cases = {
 {LogicalType::Unknown(), SortOrder::UNKNOWN},
 {LogicalType::String(), SortOrder::UNSIGNED},
 {LogicalType::Map(), SortOrder::UNKNOWN},
 {LogicalType::List(), SortOrder::UNKNOWN},
Expand DownExpand Up@@ -1888,8 +1886,6 @@ TEST_F(TestSchemaElementConstruction, SimpleCases) {
 {"uuid", LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16, false,
 ConvertedType::NA, true, [this]() { return element_->logicalType.__isset.UUID; }},
 {"none", LogicalType::None(), Type::INT64, -1, false, ConvertedType::NA, false,
 check_nothing},
 {"unknown", LogicalType::Unknown(), Type::INT64, -1, true, ConvertedType::NA, false,
 check_nothing}};

 for (const SchemaElementConstructionArguments& c : cases) {
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/parquet/thrift_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -256,6 +256,9 @@ static inline format::Type::type ToThrift(Type::type type) {
static inline format::ConvertedType::type ToThrift(ConvertedType::type type) {
 // item 0 is NONE
 DCHECK_NE(type, ConvertedType::NONE);
 // it is forbidden to emit "NA" (PARQUET-1990)
 DCHECK_NE(type, ConvertedType::NA);
@emkornfieldemkornfieldMar 31, 2021

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DCHECK_Lt might be better here.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it would change anything. Or we would have to do a full-blown check for valid values, which is a bit out of scope.
@emkornfieldemkornfieldMar 31, 2021•

edited

Loading

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. I think it prevents accidental writes of UNDEFINED as well.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I can add that as well :-)
 DCHECK_NE(type, ConvertedType::UNDEFINED);
 return static_cast<format::ConvertedType::type>(static_cast<int>(type) - 1);
}

Expand Down

37 changes: 18 additions & 19 deletions
cpp/src/parquet/types.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -366,13 +366,14 @@ std::shared_ptr<const LogicalType> LogicalType::FromConvertedType(
 return JSONLogicalType::Make();
 case ConvertedType::BSON:
 return BSONLogicalType::Make();
 case ConvertedType::NA:
 return NullLogicalType::Make();
 case ConvertedType::NONE:
 return NoLogicalType::Make();
 case ConvertedType::NA:
 case ConvertedType::UNDEFINED:
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
 }
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
}

std::shared_ptr<const LogicalType> LogicalType::FromThrift(
Expand DownExpand Up@@ -483,10 +484,6 @@ std::shared_ptr<const LogicalType> LogicalType::UUID() { return UUIDLogicalType:

std::shared_ptr<const LogicalType> LogicalType::None() { return NoLogicalType::Make(); }

std::shared_ptr<const LogicalType> LogicalType::Unknown() {
 return UnknownLogicalType::Make();
}

/*
 * The logical type implementation classes are built in four layers: (1) the base
 * layer, which establishes the interface and provides generally reusable implementations
Expand DownExpand Up@@ -516,7 +513,7 @@ class LogicalType::Impl {
 virtual std::string ToString() const = 0;

 virtual bool is_serialized() const {
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNKNOWN);
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNDEFINED);
 }

 virtual std::string ToJSON() const {
Expand DownExpand Up@@ -567,14 +564,14 @@ class LogicalType::Impl {
 class BSON;
 class UUID;
 class No;
 class Unknown;
 class Undefined;

 protected:
 Impl(LogicalType::Type::type t, SortOrder::type o) : type_(t), order_(o) {}
 Impl() = default;

 private:
 LogicalType::Type::type type_ = LogicalType::Type::UNKNOWN;
 LogicalType::Type::type type_ = LogicalType::Type::UNDEFINED;
 SortOrder::type order_ = SortOrder::UNKNOWN;
};

Expand DownExpand Up@@ -636,7 +633,9 @@ bool LogicalType::is_JSON() const { return impl_->type() == LogicalType::Type::J
bool LogicalType::is_BSON() const { return impl_->type() == LogicalType::Type::BSON; }
bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; }
bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; }
bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNKNOWN; }
bool LogicalType::is_valid() const {
 return impl_->type() != LogicalType::Type::UNDEFINED;
}
bool LogicalType::is_invalid() const { return !is_valid(); }
bool LogicalType::is_nested() const {
 return (impl_->type() == LogicalType::Type::LIST) ||
Expand DownExpand Up@@ -1555,19 +1554,19 @@ class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible,

GENERATE_MAKE(No)

class LogicalType::Impl::Unknown final : public LogicalType::Impl::SimpleCompatible,
 public LogicalType::Impl::UniversalApplicable {
class LogicalType::Impl::Undefined final : public LogicalType::Impl::SimpleCompatible,
public LogicalType::Impl::UniversalApplicable {
 public:
 friend class UnknownLogicalType;
 friend class UndefinedLogicalType;

 OVERRIDE_TOSTRING(Unknown)
 OVERRIDE_TOSTRING(Undefined)

 private:
Unknown()
 : LogicalType::Impl(LogicalType::Type::UNKNOWN, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::NA) {}
Undefined()
 : LogicalType::Impl(LogicalType::Type::UNDEFINED, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::UNDEFINED) {}
};

GENERATE_MAKE(Unknown)
GENERATE_MAKE(Undefined)

} // namespace parquet

29 changes: 19 additions & 10 deletions
cpp/src/parquet/types.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ struct Type {
// Mirrors parquet::ConvertedType
struct ConvertedType {
 enum type {
 NONE,
 NONE, // Not a real converted type, but means no converted type is specified
 UTF8,
 MAP,
 MAP_KEY_VALUE,
Expand All@@ -94,9 +94,12 @@ struct ConvertedType {
 JSON,
 BSON,
 INTERVAL,
 // DEPRECATED INVALID ConvertedType for all-null data.
 // Only useful for reading legacy files written out by interim Parquet C++ releases.
 // For writing, always emit LogicalType::Null instead.
 // See PARQUET-1990.
 NA = 25,
 // Should always be last element.
 UNDEFINED = 26
 UNDEFINED = 26 // Not a real converted type; should always be last element
 };
};

Expand DownExpand Up@@ -140,7 +143,7 @@ class PARQUET_EXPORT LogicalType {
 public:
 struct Type {
 enum type {
UNKNOWN = 0,
UNDEFINED = 0, // Not a real logical type
 STRING = 1,
 MAP,
 LIST,
Expand All@@ -151,11 +154,11 @@ class PARQUET_EXPORT LogicalType {
 TIMESTAMP,
 INTERVAL,
 INT,
 NIL, // Thrift NullType
 NIL, // Thrift NullType: annotates data that is always null
 JSON,
 BSON,
 UUID,
 NONE
 NONE // Not a real logical type; should always be last element
 };
 };

Expand DownExpand Up@@ -199,12 +202,18 @@ class PARQUET_EXPORT LogicalType {

 static std::shared_ptr<const LogicalType> Interval();
 static std::shared_ptr<const LogicalType> Int(int bit_width, bool is_signed);

 /// \brief Create a logical type for data that's always null
 ///
 /// Any physical type can be annotated with this logical type.
 static std::shared_ptr<const LogicalType> Null();

 static std::shared_ptr<const LogicalType> JSON();
 static std::shared_ptr<const LogicalType> BSON();
 static std::shared_ptr<const LogicalType> UUID();

 /// \brief Create a placeholder for when no logical type is specified
 static std::shared_ptr<const LogicalType> None();
 static std::shared_ptr<const LogicalType> Unknown();

 /// \brief Return true if this logical type is consistent with the given underlying
 /// physical type.
Expand DownExpand Up@@ -434,13 +443,13 @@ class PARQUET_EXPORT NoLogicalType : public LogicalType {
 NoLogicalType() = default;
};

/// \brief Allowed for any type.
class PARQUET_EXPORT UnknownLogicalType : public LogicalType {
// Internal API, for unrecognized logical types
class PARQUET_EXPORT UndefinedLogicalType : public LogicalType {
 public:
 static std::shared_ptr<const LogicalType> Make();

 private:
UnknownLogicalType() = default;
UndefinedLogicalType() = default;
};

// Data encodings. Mirrors parquet::Encoding
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,7 +54,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
 ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY"

 enum ParquetLogicalTypeId" parquet::LogicalType::Type::type":
ParquetLogicalType_UNKNOWN" parquet::LogicalType::Type::UNKNOWN"
ParquetLogicalType_UNDEFINED" parquet::LogicalType::Type::UNDEFINED"
 ParquetLogicalType_STRING" parquet::LogicalType::Type::STRING"
 ParquetLogicalType_MAP" parquet::LogicalType::Type::MAP"
 ParquetLogicalType_LIST" parquet::LogicalType::Type::LIST"
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pyx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -809,7 +809,7 @@ cdef physical_type_name_from_enum(ParquetType type_):

cdef logical_type_name_from_enum(ParquetLogicalTypeId type_):
 return {
ParquetLogicalType_UNKNOWN: 'UNKNOWN',
ParquetLogicalType_UNDEFINED: 'UNDEFINED',
 ParquetLogicalType_STRING: 'STRING',
 ParquetLogicalType_MAP: 'MAP',
 ParquetLogicalType_LIST: 'LIST',
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
PARQUET-1990: [C++] Refuse to write ConvertedType::NA#9863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
1
commit into

from

 Closed


PARQUET-1990: [C++] Refuse to write ConvertedType::NA
#9863
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 8 additions & 4 deletions
cpp/src/parquet/printer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -87,11 +87,15 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selecte
 const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
 stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
 << TypeToString(descr->physical_type());
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << "/" << ConvertedTypeToString(descr->converted_type());
 const auto& logical_type = descr->logical_type();
 if (!logical_type->is_none()) {
 stream << " / " << logical_type->ToString();
 }
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 if (descr->converted_type() != ConvertedType::NONE) {
 stream << " / " << ConvertedTypeToString(descr->converted_type());
 if (descr->converted_type() == ConvertedType::DECIMAL) {
 stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
 }
 }
 stream << ")" << std::endl;
 }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/parquet/reader_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,7 +317,7 @@ Number of RowGroups: 1
Number of Real Columns: 2
Number of Columns: 2
Number of Selected Columns: 2
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY/UTF8)
Column 0: a.list.element.list.element.list.element (BYTE_ARRAY / String / UTF8)
Column 1: b (INT32)
--- Row Group: 0 ---
--- Total Bytes: 155 ---
Expand Down

13 changes: 11 additions & 2 deletions
cpp/src/parquet/schema.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -448,7 +448,7 @@ std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
 LogicalType::FromThrift(element->logicalType),
 LoadEnumSafe(&element->type), element->type_length, field_id));
 } else if (element->__isset.converted_type) {
 // legacy writer with logical type present
 // legacy writer with converted type present
 primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
 element->name, LoadEnumSafe(&element->repetition_type),
 LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type),
Expand DownExpand Up@@ -500,7 +500,16 @@ void PrimitiveNode::ToParquet(void* opaque_element) const {
 element->__set_name(name_);
 element->__set_repetition_type(ToThrift(repetition_));
 if (converted_type_ != ConvertedType::NONE) {
 element->__set_converted_type(ToThrift(converted_type_));
 if (converted_type_ != ConvertedType::NA) {
 element->__set_converted_type(ToThrift(converted_type_));
 } else {
 // ConvertedType::NA is an unreleased, obsolete synonym for LogicalType::Null.
 // Never emit it (see PARQUET-1990 for discussion).
 if (!logical_type_ || !logical_type_->is_null()) {
 throw ParquetException(
 "ConvertedType::NA is obsolete, please use LogicalType::Null instead");
 }
 }
 }
 if (field_id_ >= 0) {
 element->__set_field_id(field_id_);
Expand Down

14 changes: 5 additions & 9 deletions
cpp/src/parquet/schema_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1101,12 +1101,12 @@ TEST(TestLogicalTypeConstruction, ConvertedTypeCompatibility) {
 ASSERT_TRUE(reconstructed->is_valid());
 ASSERT_TRUE(reconstructed->Equals(*original));

 // Unknown
 original = LogicalType::Unknown();
 // Undefined
 original = UndefinedLogicalType::Make();
 ASSERT_TRUE(original->is_invalid());
 ASSERT_FALSE(original->is_valid());
 converted_type = original->ToConvertedType(&converted_decimal_metadata);
 ASSERT_EQ(converted_type, ConvertedType::NA);
 ASSERT_EQ(converted_type, ConvertedType::UNDEFINED);
 ASSERT_FALSE(converted_decimal_metadata.isset);
 ASSERT_TRUE(original->is_compatible(converted_type, converted_decimal_metadata));
 ASSERT_TRUE(original->is_compatible(converted_type));
Expand DownExpand Up@@ -1243,7 +1243,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeProperties) {
 {BSONLogicalType::Make(), false, true, true},
 {UUIDLogicalType::Make(), false, true, true},
 {NoLogicalType::Make(), false, false, true},
 {UnknownLogicalType::Make(), false, false, false},
 };

 for (const ExpectedProperties& c : cases) {
Expand DownExpand Up@@ -1339,7 +1338,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) {
 }

 std::vector<std::shared_ptr<const LogicalType>> any_type_cases = {
 LogicalType::Null(), LogicalType::None(), LogicalType::Unknown()};
 LogicalType::Null(), LogicalType::None(), UndefinedLogicalType::Make()};

 for (auto c : any_type_cases) {
 ConfirmAnyPrimitiveTypeApplicability(c);
Expand DownExpand Up@@ -1453,7 +1452,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) {
 };

 std::vector<ExpectedRepresentation> cases = {
 {LogicalType::Unknown(), "Unknown", R"({"Type": "Unknown"})"},
 {UndefinedLogicalType::Make(), "Undefined", R"({"Type": "Undefined"})"},
 {LogicalType::String(), "String", R"({"Type": "String"})"},
 {LogicalType::Map(), "Map", R"({"Type": "Map"})"},
 {LogicalType::List(), "List", R"({"Type": "List"})"},
Expand DownExpand Up@@ -1550,7 +1549,6 @@ TEST(TestLogicalTypeOperation, LogicalTypeSortOrder) {
 };

 std::vector<ExpectedSortOrder> cases = {
 {LogicalType::Unknown(), SortOrder::UNKNOWN},
 {LogicalType::String(), SortOrder::UNSIGNED},
 {LogicalType::Map(), SortOrder::UNKNOWN},
 {LogicalType::List(), SortOrder::UNKNOWN},
Expand DownExpand Up@@ -1888,8 +1886,6 @@ TEST_F(TestSchemaElementConstruction, SimpleCases) {
 {"uuid", LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16, false,
 ConvertedType::NA, true, [this]() { return element_->logicalType.__isset.UUID; }},
 {"none", LogicalType::None(), Type::INT64, -1, false, ConvertedType::NA, false,
 check_nothing},
 {"unknown", LogicalType::Unknown(), Type::INT64, -1, true, ConvertedType::NA, false,
 check_nothing}};

 for (const SchemaElementConstructionArguments& c : cases) {
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/parquet/thrift_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -256,6 +256,9 @@ static inline format::Type::type ToThrift(Type::type type) {
static inline format::ConvertedType::type ToThrift(ConvertedType::type type) {
 // item 0 is NONE
 DCHECK_NE(type, ConvertedType::NONE);
 // it is forbidden to emit "NA" (PARQUET-1990)
 DCHECK_NE(type, ConvertedType::NA);
@emkornfieldemkornfieldMar 31, 2021

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DCHECK_Lt might be better here.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it would change anything. Or we would have to do a full-blown check for valid values, which is a bit out of scope.
@emkornfieldemkornfieldMar 31, 2021•

edited

Loading

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. I think it prevents accidental writes of UNDEFINED as well.
@pitroupitrouMar 31, 2021

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I can add that as well :-)
 DCHECK_NE(type, ConvertedType::UNDEFINED);
 return static_cast<format::ConvertedType::type>(static_cast<int>(type) - 1);
}

Expand Down

37 changes: 18 additions & 19 deletions
cpp/src/parquet/types.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -366,13 +366,14 @@ std::shared_ptr<const LogicalType> LogicalType::FromConvertedType(
 return JSONLogicalType::Make();
 case ConvertedType::BSON:
 return BSONLogicalType::Make();
 case ConvertedType::NA:
 return NullLogicalType::Make();
 case ConvertedType::NONE:
 return NoLogicalType::Make();
 case ConvertedType::NA:
 case ConvertedType::UNDEFINED:
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
 }
 return UnknownLogicalType::Make();
 return UndefinedLogicalType::Make();
}

std::shared_ptr<const LogicalType> LogicalType::FromThrift(
Expand DownExpand Up@@ -483,10 +484,6 @@ std::shared_ptr<const LogicalType> LogicalType::UUID() { return UUIDLogicalType:

std::shared_ptr<const LogicalType> LogicalType::None() { return NoLogicalType::Make(); }

std::shared_ptr<const LogicalType> LogicalType::Unknown() {
 return UnknownLogicalType::Make();
}

/*
 * The logical type implementation classes are built in four layers: (1) the base
 * layer, which establishes the interface and provides generally reusable implementations
Expand DownExpand Up@@ -516,7 +513,7 @@ class LogicalType::Impl {
 virtual std::string ToString() const = 0;

 virtual bool is_serialized() const {
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNKNOWN);
 return !(type_ == LogicalType::Type::NONE || type_ == LogicalType::Type::UNDEFINED);
 }

 virtual std::string ToJSON() const {
Expand DownExpand Up@@ -567,14 +564,14 @@ class LogicalType::Impl {
 class BSON;
 class UUID;
 class No;
 class Unknown;
 class Undefined;

 protected:
 Impl(LogicalType::Type::type t, SortOrder::type o) : type_(t), order_(o) {}
 Impl() = default;

 private:
 LogicalType::Type::type type_ = LogicalType::Type::UNKNOWN;
 LogicalType::Type::type type_ = LogicalType::Type::UNDEFINED;
 SortOrder::type order_ = SortOrder::UNKNOWN;
};

Expand DownExpand Up@@ -636,7 +633,9 @@ bool LogicalType::is_JSON() const { return impl_->type() == LogicalType::Type::J
bool LogicalType::is_BSON() const { return impl_->type() == LogicalType::Type::BSON; }
bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; }
bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; }
bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNKNOWN; }
bool LogicalType::is_valid() const {
 return impl_->type() != LogicalType::Type::UNDEFINED;
}
bool LogicalType::is_invalid() const { return !is_valid(); }
bool LogicalType::is_nested() const {
 return (impl_->type() == LogicalType::Type::LIST) ||
Expand DownExpand Up@@ -1555,19 +1554,19 @@ class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible,

GENERATE_MAKE(No)

class LogicalType::Impl::Unknown final : public LogicalType::Impl::SimpleCompatible,
 public LogicalType::Impl::UniversalApplicable {
class LogicalType::Impl::Undefined final : public LogicalType::Impl::SimpleCompatible,
public LogicalType::Impl::UniversalApplicable {
 public:
 friend class UnknownLogicalType;
 friend class UndefinedLogicalType;

 OVERRIDE_TOSTRING(Unknown)
 OVERRIDE_TOSTRING(Undefined)

 private:
Unknown()
 : LogicalType::Impl(LogicalType::Type::UNKNOWN, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::NA) {}
Undefined()
 : LogicalType::Impl(LogicalType::Type::UNDEFINED, SortOrder::UNKNOWN),
 LogicalType::Impl::SimpleCompatible(ConvertedType::UNDEFINED) {}
};

GENERATE_MAKE(Unknown)
GENERATE_MAKE(Undefined)

} // namespace parquet

29 changes: 19 additions & 10 deletions
cpp/src/parquet/types.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ struct Type {
// Mirrors parquet::ConvertedType
struct ConvertedType {
 enum type {
 NONE,
 NONE, // Not a real converted type, but means no converted type is specified
 UTF8,
 MAP,
 MAP_KEY_VALUE,
Expand All@@ -94,9 +94,12 @@ struct ConvertedType {
 JSON,
 BSON,
 INTERVAL,
 // DEPRECATED INVALID ConvertedType for all-null data.
 // Only useful for reading legacy files written out by interim Parquet C++ releases.
 // For writing, always emit LogicalType::Null instead.
 // See PARQUET-1990.
 NA = 25,
 // Should always be last element.
 UNDEFINED = 26
 UNDEFINED = 26 // Not a real converted type; should always be last element
 };
};

Expand DownExpand Up@@ -140,7 +143,7 @@ class PARQUET_EXPORT LogicalType {
 public:
 struct Type {
 enum type {
UNKNOWN = 0,
UNDEFINED = 0, // Not a real logical type
 STRING = 1,
 MAP,
 LIST,
Expand All@@ -151,11 +154,11 @@ class PARQUET_EXPORT LogicalType {
 TIMESTAMP,
 INTERVAL,
 INT,
 NIL, // Thrift NullType
 NIL, // Thrift NullType: annotates data that is always null
 JSON,
 BSON,
 UUID,
 NONE
 NONE // Not a real logical type; should always be last element
 };
 };

Expand DownExpand Up@@ -199,12 +202,18 @@ class PARQUET_EXPORT LogicalType {

 static std::shared_ptr<const LogicalType> Interval();
 static std::shared_ptr<const LogicalType> Int(int bit_width, bool is_signed);

 /// \brief Create a logical type for data that's always null
 ///
 /// Any physical type can be annotated with this logical type.
 static std::shared_ptr<const LogicalType> Null();

 static std::shared_ptr<const LogicalType> JSON();
 static std::shared_ptr<const LogicalType> BSON();
 static std::shared_ptr<const LogicalType> UUID();

 /// \brief Create a placeholder for when no logical type is specified
 static std::shared_ptr<const LogicalType> None();
 static std::shared_ptr<const LogicalType> Unknown();

 /// \brief Return true if this logical type is consistent with the given underlying
 /// physical type.
Expand DownExpand Up@@ -434,13 +443,13 @@ class PARQUET_EXPORT NoLogicalType : public LogicalType {
 NoLogicalType() = default;
};

/// \brief Allowed for any type.
class PARQUET_EXPORT UnknownLogicalType : public LogicalType {
// Internal API, for unrecognized logical types
class PARQUET_EXPORT UndefinedLogicalType : public LogicalType {
 public:
 static std::shared_ptr<const LogicalType> Make();

 private:
UnknownLogicalType() = default;
UndefinedLogicalType() = default;
};

// Data encodings. Mirrors parquet::Encoding
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,7 +54,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
 ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY"

 enum ParquetLogicalTypeId" parquet::LogicalType::Type::type":
ParquetLogicalType_UNKNOWN" parquet::LogicalType::Type::UNKNOWN"
ParquetLogicalType_UNDEFINED" parquet::LogicalType::Type::UNDEFINED"
 ParquetLogicalType_STRING" parquet::LogicalType::Type::STRING"
 ParquetLogicalType_MAP" parquet::LogicalType::Type::MAP"
 ParquetLogicalType_LIST" parquet::LogicalType::Type::LIST"
Expand Down

2 changes: 1 addition & 1 deletion
python/pyarrow/_parquet.pyx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -809,7 +809,7 @@ cdef physical_type_name_from_enum(ParquetType type_):

cdef logical_type_name_from_enum(ParquetLogicalTypeId type_):
 return {
ParquetLogicalType_UNKNOWN: 'UNKNOWN',
ParquetLogicalType_UNDEFINED: 'UNDEFINED',
 ParquetLogicalType_STRING: 'STRING',
 ParquetLogicalType_MAP: 'MAP',
 ParquetLogicalType_LIST: 'LIST',
Expand Down