GH-34888: [C++][Parquet] Writer supports adding extra kv meta#34889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wjones127 merged 5 commits into from
Apr 13, 2023
Merged

GH-34888: [C++][Parquet] Writer supports adding extra kv meta #34889

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b84556c
GH-34888: [C++][Parquet] Writer supports adding extra kv meta
wgtmac Apr 4, 2023
bed0661
address comments
wgtmac Apr 9, 2023
6f6c41c
make it backward compatible
wgtmac Apr 10, 2023
9b0d022
try to fix doxygen issue
wgtmac Apr 11, 2023
5e5a39c
Update cpp/src/parquet/file_writer.cc
wgtmac Apr 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 24 additions & 5 deletions cpp/src/parquet/file_writer.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

#include "parquet/file_writer.h"

#include <cstddef>
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>

#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "parquet/column_writer.h"
#include "parquet/encryption/encryption_internal.h"
#include "parquet/encryption/internal_file_encryptor.h"
Expand DownExpand Up@@ -338,7 +339,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
auto file_encryption_properties = properties_->file_encryption_properties();

if (file_encryption_properties == nullptr) { // Non encrypted file.
file_metadata_ = metadata_->Finish();
file_metadata_ = metadata_->Finish(key_value_metadata_);
WriteFileMetaData(*file_metadata_, sink_.get());
} else { // Encrypted file
CloseEncryptedFile(file_encryption_properties);
Expand DownExpand Up@@ -376,6 +377,15 @@ class FileSerializer : public ParquetFileWriter::Contents {

RowGroupWriter* AppendBufferedRowGroup() override { return AppendRowGroup(true); }

void AddKeyValueMetadata(
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) override {
if (key_value_metadata_ == nullptr) {
key_value_metadata_ = std::move(key_value_metadata);
} else if (key_value_metadata != nullptr) {
key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
Comment thread
wgtmac marked this conversation as resolved.
Outdated
}
}

~FileSerializer() override {
try {
Close();
Expand All@@ -394,7 +404,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
properties_(std::move(properties)),
num_row_groups_(0),
num_rows_(0),
metadata_(FileMetaDataBuilder::Make(&schema_, properties_, key_value_metadata_)) {
metadata_(FileMetaDataBuilder::Make(&schema_, properties_)) {
PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
if (position == 0) {
StartFile();
Expand All@@ -407,7 +417,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
// Encrypted file with encrypted footer
if (file_encryption_properties->encrypted_footer()) {
// encrypted footer
file_metadata_ = metadata_->Finish();
file_metadata_ = metadata_->Finish(key_value_metadata_);

PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
uint64_t metadata_start = static_cast<uint64_t>(position);
Expand All@@ -422,7 +432,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
PARQUET_THROW_NOT_OK(sink_->Write(kParquetEMagic, 4));
} else { // Encrypted file with plaintext footer
file_metadata_ = metadata_->Finish();
file_metadata_ = metadata_->Finish(key_value_metadata_);
auto footer_signing_encryptor = file_encryptor_->GetFooterSigningEncryptor();
WriteEncryptedFileMetadata(*file_metadata_, sink_.get(), footer_signing_encryptor,
false);
Expand DownExpand Up@@ -613,6 +623,15 @@ RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) {
return AppendRowGroup();
}

void ParquetFileWriter::AddKeyValueMetadata(
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
if (contents_) {
contents_->AddKeyValueMetadata(key_value_metadata);
} else {
throw ParquetException("Cannot add key-value metadata to closed file");
}
}

const std::shared_ptr<WriterProperties>& ParquetFileWriter::properties() const {
return contents_->properties();
}
Expand Down
10 changes: 10 additions & 0 deletions cpp/src/parquet/file_writer.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -163,6 +163,9 @@ class PARQUET_EXPORT ParquetFileWriter {
return key_value_metadata_;
}

virtual void AddKeyValueMetadata(
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;

// Return const-pointer to make it clear that this object is not to be copied
const SchemaDescriptor* schema() const { return &schema_; }

Expand DownExpand Up@@ -209,6 +212,13 @@ class PARQUET_EXPORT ParquetFileWriter {
/// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
RowGroupWriter* AppendBufferedRowGroup();

/// \brief Add key-value metadata to the file.
/// \param[in] key_value_metadata the metadata to add.
/// \note This will overwrite any existing metadata with the same key.
/// \throw ParquetException if Close() has been called.
void AddKeyValueMetadata(
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);

/// Number of columns.
///
/// This number is fixed during the lifetime of the writer as it is determined via
Expand Down
24 changes: 19 additions & 5 deletions cpp/src/parquet/metadata.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1740,7 +1740,6 @@ void RowGroupMetaDataBuilder::Finish(int64_t total_bytes_written,
}

// file metadata
// TODO(PARQUET-595) Support key_value_metadata
class FileMetaDataBuilder::FileMetaDataBuilderImpl {
public:
explicit FileMetaDataBuilderImpl(
Expand DownExpand Up@@ -1797,15 +1796,21 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
}
}

std::unique_ptr<FileMetaData> Finish() {
std::unique_ptr<FileMetaData> Finish(
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
int64_t total_rows = 0;
for (auto row_group : row_groups_) {
total_rows += row_group.num_rows;
}
metadata_->__set_num_rows(total_rows);
metadata_->__set_row_groups(row_groups_);

if (key_value_metadata_) {
if (key_value_metadata_ || key_value_metadata) {
if (!key_value_metadata_) {
key_value_metadata_ = key_value_metadata;
} else if (key_value_metadata) {
key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
}
metadata_->key_value_metadata.clear();
metadata_->key_value_metadata.reserve(key_value_metadata_->size());
for (int64_t i = 0; i < key_value_metadata_->size(); ++i) {
Expand All@@ -1829,7 +1834,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
metadata_->__set_version(file_version);
metadata_->__set_created_by(properties_->created_by());

// Users cannot set the `ColumnOrder` since we donot not have user defined sort order
// Users cannot set the `ColumnOrder` since we do not have user defined sort order
// in the spec yet.
// We always default to `TYPE_DEFINED_ORDER`. We can expose it in
// the API once we have user defined sort orders in the Parquet format.
Expand DownExpand Up@@ -1910,6 +1915,12 @@ std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
new FileMetaDataBuilder(schema, std::move(props), std::move(key_value_metadata)));
}

std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props) {
return std::unique_ptr<FileMetaDataBuilder>(
new FileMetaDataBuilder(schema, std::move(props)));
}

FileMetaDataBuilder::FileMetaDataBuilder(
const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
std::shared_ptr<const KeyValueMetadata> key_value_metadata)
Expand All@@ -1926,7 +1937,10 @@ void FileMetaDataBuilder::SetPageIndexLocation(const PageIndexLocation& location
impl_->SetPageIndexLocation(location);
}

std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish() { return impl_->Finish(); }
std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish(
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
return impl_->Finish(key_value_metadata);
}

std::unique_ptr<FileCryptoMetaData> FileMetaDataBuilder::GetCryptoMetaData() {
return impl_->BuildFileCryptoMetaData();
Expand Down
11 changes: 8 additions & 3 deletions cpp/src/parquet/metadata.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -523,10 +523,14 @@ struct PageIndexLocation {

class PARQUET_EXPORT FileMetaDataBuilder {
public:
// API convenience to get a MetaData reader
ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")

@wgtmacwgtmacApr 12, 2023

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
ARROW_DEPRECATED("Deprecated in 13.0.0. Use overload without KeyValueMetadata instead.")

Should I change this since it may not be included in the 12.0.0 release? @wjones127

static std::unique_ptr<FileMetaDataBuilder> Make(
const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
std::shared_ptr<const KeyValueMetadata> key_value_metadata);

// API convenience to get a MetaData builder
static std::unique_ptr<FileMetaDataBuilder> Make(
const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);

~FileMetaDataBuilder();

Expand All@@ -537,7 +541,8 @@ class PARQUET_EXPORT FileMetaDataBuilder {
void SetPageIndexLocation(const PageIndexLocation& location);

// Complete the Thrift structure
std::unique_ptr<FileMetaData> Finish();
std::unique_ptr<FileMetaData> Finish(
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);

// crypto metadata
std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
Expand Down
51 changes: 49 additions & 2 deletions cpp/src/parquet/metadata_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

#include "arrow/util/key_value_metadata.h"
#include "parquet/file_reader.h"
#include "parquet/file_writer.h"
#include "parquet/schema.h"
#include "parquet/statistics.h"
#include "parquet/test_util.h"
Expand DownExpand Up@@ -284,16 +285,62 @@ TEST(Metadata, TestKeyValueMetadata) {
auto kvmeta = std::make_shared<KeyValueMetadata>();
kvmeta->Append("test_key", "test_value");

auto f_builder = FileMetaDataBuilder::Make(&schema, props, kvmeta);
auto f_builder = FileMetaDataBuilder::Make(&schema, props);

// Read the metadata
auto f_accessor = f_builder->Finish();
auto f_accessor = f_builder->Finish(kvmeta);

// Key value metadata
ASSERT_TRUE(f_accessor->key_value_metadata());
EXPECT_TRUE(f_accessor->key_value_metadata()->Equals(*kvmeta));
}

TEST(Metadata, TestAddKeyValueMetadata) {
schema::NodeVector fields;
fields.push_back(schema::Int32("int_col", Repetition::REQUIRED));
auto schema = std::static_pointer_cast<schema::GroupNode>(
schema::GroupNode::Make("schema", Repetition::REQUIRED, fields));

auto kv_meta = std::make_shared<KeyValueMetadata>();
kv_meta->Append("test_key_1", "test_value_1");
kv_meta->Append("test_key_2", "test_value_2_");

auto sink = CreateOutputStream();
auto writer_props = parquet::WriterProperties::Builder().disable_dictionary()->build();
auto file_writer =
parquet::ParquetFileWriter::Open(sink, schema, writer_props, kv_meta);

// Key value metadata that will be added to the file.
auto kv_meta_added = std::make_shared<KeyValueMetadata>();
kv_meta_added->Append("test_key_2", "test_value_2");
kv_meta_added->Append("test_key_3", "test_value_3");

file_writer->AddKeyValueMetadata(kv_meta_added);
file_writer->Close();

// Throw if appending key value metadata to closed file.
auto kv_meta_ignored = std::make_shared<KeyValueMetadata>();
kv_meta_ignored->Append("test_key_4", "test_value_4");
EXPECT_THROW(file_writer->AddKeyValueMetadata(kv_meta_ignored), ParquetException);

PARQUET_ASSIGN_OR_THROW(auto buffer, sink->Finish());
auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
auto file_reader = ParquetFileReader::Open(source);

ASSERT_NE(nullptr, file_reader->metadata());
ASSERT_NE(nullptr, file_reader->metadata()->key_value_metadata());
auto read_kv_meta = file_reader->metadata()->key_value_metadata();

// Verify keys that were added before file writer was closed are present.
for (int i = 1; i <= 3; ++i) {
auto index = std::to_string(i);
PARQUET_ASSIGN_OR_THROW(auto value, read_kv_meta->Get("test_key_" + index));
EXPECT_EQ("test_value_" + index, value);
}
// Verify keys that were added after file writer was closed are not present.
EXPECT_FALSE(read_kv_meta->Contains("test_key_4"));
}

TEST(Metadata, TestHasBloomFilter) {
std::string dir_string(parquet::test::get_data_dir());
std::string path = dir_string + "/data_index_bloom_encoding_stats.parquet";
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
GH-34888: [C++][Parquet] Writer supports adding extra kv meta#34889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

wjones127
merged 5 commits into

from

Apr 13, 2023
 Merged


GH-34888: [C++][Parquet] Writer supports adding extra kv meta
#34889
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b84556c

GH-34888: [C++][Parquet] Writer supports adding extra kv meta


wgtmac Apr 4, 2023
bed0661

address comments


wgtmac Apr 9, 2023
6f6c41c

make it backward compatible


wgtmac Apr 10, 2023
9b0d022

try to fix doxygen issue


wgtmac Apr 11, 2023
5e5a39c

Update cpp/src/parquet/file_writer.cc


wgtmac Apr 12, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 24 additions & 5 deletions
cpp/src/parquet/file_writer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

#include "parquet/file_writer.h"

#include <cstddef>
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>

#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "parquet/column_writer.h"
#include "parquet/encryption/encryption_internal.h"
#include "parquet/encryption/internal_file_encryptor.h"
Expand DownExpand Up@@ -338,7 +339,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 auto file_encryption_properties = properties_->file_encryption_properties();

 if (file_encryption_properties == nullptr) { // Non encrypted file.
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 WriteFileMetaData(*file_metadata_, sink_.get());
 } else { // Encrypted file
 CloseEncryptedFile(file_encryption_properties);
Expand DownExpand Up@@ -376,6 +377,15 @@ class FileSerializer : public ParquetFileWriter::Contents {

 RowGroupWriter* AppendBufferedRowGroup() override { return AppendRowGroup(true); }

 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) override {
 if (key_value_metadata_ == nullptr) {
 key_value_metadata_ = std::move(key_value_metadata);
 } else if (key_value_metadata != nullptr) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
Comment thread
wgtmac marked this conversation as resolved.


Outdated

 }
 }

 ~FileSerializer() override {
 try {
 Close();
Expand All@@ -394,7 +404,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 properties_(std::move(properties)),
 num_row_groups_(0),
 num_rows_(0),
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_, key_value_metadata_)) {
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_)) {
 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 if (position == 0) {
 StartFile();
Expand All@@ -407,7 +417,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 // Encrypted file with encrypted footer
 if (file_encryption_properties->encrypted_footer()) {
 // encrypted footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);

 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 uint64_t metadata_start = static_cast<uint64_t>(position);
Expand All@@ -422,7 +432,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
 PARQUET_THROW_NOT_OK(sink_->Write(kParquetEMagic, 4));
 } else { // Encrypted file with plaintext footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 auto footer_signing_encryptor = file_encryptor_->GetFooterSigningEncryptor();
 WriteEncryptedFileMetadata(*file_metadata_, sink_.get(), footer_signing_encryptor,
 false);
Expand DownExpand Up@@ -613,6 +623,15 @@ RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) {
 return AppendRowGroup();
}

void ParquetFileWriter::AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 if (contents_) {
 contents_->AddKeyValueMetadata(key_value_metadata);
 } else {
 throw ParquetException("Cannot add key-value metadata to closed file");
 }
}

const std::shared_ptr<WriterProperties>& ParquetFileWriter::properties() const {
 return contents_->properties();
}
Expand Down

10 changes: 10 additions & 0 deletions
cpp/src/parquet/file_writer.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -163,6 +163,9 @@ class PARQUET_EXPORT ParquetFileWriter {
 return key_value_metadata_;
 }

 virtual void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;

 // Return const-pointer to make it clear that this object is not to be copied
 const SchemaDescriptor* schema() const { return &schema_; }

Expand DownExpand Up@@ -209,6 +212,13 @@ class PARQUET_EXPORT ParquetFileWriter {
 /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
 RowGroupWriter* AppendBufferedRowGroup();

 /// \brief Add key-value metadata to the file.
 /// \param[in] key_value_metadata the metadata to add.
 /// \note This will overwrite any existing metadata with the same key.
 /// \throw ParquetException if Close() has been called.
 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);

 /// Number of columns.
 ///
 /// This number is fixed during the lifetime of the writer as it is determined via
Expand Down

24 changes: 19 additions & 5 deletions
cpp/src/parquet/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1740,7 +1740,6 @@ void RowGroupMetaDataBuilder::Finish(int64_t total_bytes_written,
}

// file metadata
// TODO(PARQUET-595) Support key_value_metadata
class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 public:
 explicit FileMetaDataBuilderImpl(
Expand DownExpand Up@@ -1797,15 +1796,21 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 }
 }

 std::unique_ptr<FileMetaData> Finish() {
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 int64_t total_rows = 0;
 for (auto row_group : row_groups_) {
 total_rows += row_group.num_rows;
 }
 metadata_->__set_num_rows(total_rows);
 metadata_->__set_row_groups(row_groups_);

 if (key_value_metadata_) {
 if (key_value_metadata_ || key_value_metadata) {
 if (!key_value_metadata_) {
 key_value_metadata_ = key_value_metadata;
 } else if (key_value_metadata) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
 }
 metadata_->key_value_metadata.clear();
 metadata_->key_value_metadata.reserve(key_value_metadata_->size());
 for (int64_t i = 0; i < key_value_metadata_->size(); ++i) {
Expand All@@ -1829,7 +1834,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 metadata_->__set_version(file_version);
 metadata_->__set_created_by(properties_->created_by());

 // Users cannot set the `ColumnOrder` since we donot not have user defined sort order
 // Users cannot set the `ColumnOrder` since we do not have user defined sort order
 // in the spec yet.
 // We always default to `TYPE_DEFINED_ORDER`. We can expose it in
 // the API once we have user defined sort orders in the Parquet format.
Expand DownExpand Up@@ -1910,6 +1915,12 @@ std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 new FileMetaDataBuilder(schema, std::move(props), std::move(key_value_metadata)));
}

std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props) {
 return std::unique_ptr<FileMetaDataBuilder>(
 new FileMetaDataBuilder(schema, std::move(props)));
}

FileMetaDataBuilder::FileMetaDataBuilder(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata)
Expand All@@ -1926,7 +1937,10 @@ void FileMetaDataBuilder::SetPageIndexLocation(const PageIndexLocation& location
 impl_->SetPageIndexLocation(location);
}

std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish() { return impl_->Finish(); }
std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 return impl_->Finish(key_value_metadata);
}

std::unique_ptr<FileCryptoMetaData> FileMetaDataBuilder::GetCryptoMetaData() {
 return impl_->BuildFileCryptoMetaData();
Expand Down

11 changes: 8 additions & 3 deletions
cpp/src/parquet/metadata.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -523,10 +523,14 @@ struct PageIndexLocation {

class PARQUET_EXPORT FileMetaDataBuilder {
 public:
// API convenience to get a MetaData reader
ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
@wgtmacwgtmacApr 12, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


Suggested change

ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
ARROW_DEPRECATED("Deprecated in 13.0.0. Use overload without KeyValueMetadata instead.")
Should I change this since it may not be included in the 12.0.0 release? @wjones127
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
 std::shared_ptr<const KeyValueMetadata> key_value_metadata);

 // API convenience to get a MetaData builder
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);

 ~FileMetaDataBuilder();

Expand All@@ -537,7 +541,8 @@ class PARQUET_EXPORT FileMetaDataBuilder {
 void SetPageIndexLocation(const PageIndexLocation& location);

 // Complete the Thrift structure
 std::unique_ptr<FileMetaData> Finish();
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);

 // crypto metadata
 std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
Expand Down

51 changes: 49 additions & 2 deletions
cpp/src/parquet/metadata_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

#include "arrow/util/key_value_metadata.h"
#include "parquet/file_reader.h"
#include "parquet/file_writer.h"
#include "parquet/schema.h"
#include "parquet/statistics.h"
#include "parquet/test_util.h"
Expand DownExpand Up@@ -284,16 +285,62 @@ TEST(Metadata, TestKeyValueMetadata) {
 auto kvmeta = std::make_shared<KeyValueMetadata>();
 kvmeta->Append("test_key", "test_value");

 auto f_builder = FileMetaDataBuilder::Make(&schema, props, kvmeta);
 auto f_builder = FileMetaDataBuilder::Make(&schema, props);

 // Read the metadata
 auto f_accessor = f_builder->Finish();
 auto f_accessor = f_builder->Finish(kvmeta);

 // Key value metadata
 ASSERT_TRUE(f_accessor->key_value_metadata());
 EXPECT_TRUE(f_accessor->key_value_metadata()->Equals(*kvmeta));
}

TEST(Metadata, TestAddKeyValueMetadata) {
 schema::NodeVector fields;
 fields.push_back(schema::Int32("int_col", Repetition::REQUIRED));
 auto schema = std::static_pointer_cast<schema::GroupNode>(
 schema::GroupNode::Make("schema", Repetition::REQUIRED, fields));

 auto kv_meta = std::make_shared<KeyValueMetadata>();
 kv_meta->Append("test_key_1", "test_value_1");
 kv_meta->Append("test_key_2", "test_value_2_");

 auto sink = CreateOutputStream();
 auto writer_props = parquet::WriterProperties::Builder().disable_dictionary()->build();
 auto file_writer =
 parquet::ParquetFileWriter::Open(sink, schema, writer_props, kv_meta);

 // Key value metadata that will be added to the file.
 auto kv_meta_added = std::make_shared<KeyValueMetadata>();
 kv_meta_added->Append("test_key_2", "test_value_2");
 kv_meta_added->Append("test_key_3", "test_value_3");

 file_writer->AddKeyValueMetadata(kv_meta_added);
 file_writer->Close();

 // Throw if appending key value metadata to closed file.
 auto kv_meta_ignored = std::make_shared<KeyValueMetadata>();
 kv_meta_ignored->Append("test_key_4", "test_value_4");
 EXPECT_THROW(file_writer->AddKeyValueMetadata(kv_meta_ignored), ParquetException);

 PARQUET_ASSIGN_OR_THROW(auto buffer, sink->Finish());
 auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
 auto file_reader = ParquetFileReader::Open(source);

 ASSERT_NE(nullptr, file_reader->metadata());
 ASSERT_NE(nullptr, file_reader->metadata()->key_value_metadata());
 auto read_kv_meta = file_reader->metadata()->key_value_metadata();

 // Verify keys that were added before file writer was closed are present.
 for (int i = 1; i <= 3; ++i) {
 auto index = std::to_string(i);
 PARQUET_ASSIGN_OR_THROW(auto value, read_kv_meta->Get("test_key_" + index));
 EXPECT_EQ("test_value_" + index, value);
 }
 // Verify keys that were added after file writer was closed are not present.
 EXPECT_FALSE(read_kv_meta->Contains("test_key_4"));
}

TEST(Metadata, TestHasBloomFilter) {
 std::string dir_string(parquet::test::get_data_dir());
 std::string path = dir_string + "/data_index_bloom_encoding_stats.parquet";
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-34888: [C++][Parquet] Writer supports adding extra kv meta#34889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

wjones127
merged 5 commits into

from

Apr 13, 2023
 Merged


GH-34888: [C++][Parquet] Writer supports adding extra kv meta
#34889
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b84556c

GH-34888: [C++][Parquet] Writer supports adding extra kv meta


wgtmac Apr 4, 2023
bed0661

address comments


wgtmac Apr 9, 2023
6f6c41c

make it backward compatible


wgtmac Apr 10, 2023
9b0d022

try to fix doxygen issue


wgtmac Apr 11, 2023
5e5a39c

Update cpp/src/parquet/file_writer.cc


wgtmac Apr 12, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 24 additions & 5 deletions
cpp/src/parquet/file_writer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

#include "parquet/file_writer.h"

#include <cstddef>
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>

#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "parquet/column_writer.h"
#include "parquet/encryption/encryption_internal.h"
#include "parquet/encryption/internal_file_encryptor.h"
Expand DownExpand Up@@ -338,7 +339,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 auto file_encryption_properties = properties_->file_encryption_properties();

 if (file_encryption_properties == nullptr) { // Non encrypted file.
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 WriteFileMetaData(*file_metadata_, sink_.get());
 } else { // Encrypted file
 CloseEncryptedFile(file_encryption_properties);
Expand DownExpand Up@@ -376,6 +377,15 @@ class FileSerializer : public ParquetFileWriter::Contents {

 RowGroupWriter* AppendBufferedRowGroup() override { return AppendRowGroup(true); }

 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) override {
 if (key_value_metadata_ == nullptr) {
 key_value_metadata_ = std::move(key_value_metadata);
 } else if (key_value_metadata != nullptr) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
Comment thread
wgtmac marked this conversation as resolved.


Outdated

 }
 }

 ~FileSerializer() override {
 try {
 Close();
Expand All@@ -394,7 +404,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 properties_(std::move(properties)),
 num_row_groups_(0),
 num_rows_(0),
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_, key_value_metadata_)) {
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_)) {
 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 if (position == 0) {
 StartFile();
Expand All@@ -407,7 +417,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 // Encrypted file with encrypted footer
 if (file_encryption_properties->encrypted_footer()) {
 // encrypted footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);

 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 uint64_t metadata_start = static_cast<uint64_t>(position);
Expand All@@ -422,7 +432,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
 PARQUET_THROW_NOT_OK(sink_->Write(kParquetEMagic, 4));
 } else { // Encrypted file with plaintext footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 auto footer_signing_encryptor = file_encryptor_->GetFooterSigningEncryptor();
 WriteEncryptedFileMetadata(*file_metadata_, sink_.get(), footer_signing_encryptor,
 false);
Expand DownExpand Up@@ -613,6 +623,15 @@ RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) {
 return AppendRowGroup();
}

void ParquetFileWriter::AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 if (contents_) {
 contents_->AddKeyValueMetadata(key_value_metadata);
 } else {
 throw ParquetException("Cannot add key-value metadata to closed file");
 }
}

const std::shared_ptr<WriterProperties>& ParquetFileWriter::properties() const {
 return contents_->properties();
}
Expand Down

10 changes: 10 additions & 0 deletions
cpp/src/parquet/file_writer.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -163,6 +163,9 @@ class PARQUET_EXPORT ParquetFileWriter {
 return key_value_metadata_;
 }

 virtual void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;

 // Return const-pointer to make it clear that this object is not to be copied
 const SchemaDescriptor* schema() const { return &schema_; }

Expand DownExpand Up@@ -209,6 +212,13 @@ class PARQUET_EXPORT ParquetFileWriter {
 /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
 RowGroupWriter* AppendBufferedRowGroup();

 /// \brief Add key-value metadata to the file.
 /// \param[in] key_value_metadata the metadata to add.
 /// \note This will overwrite any existing metadata with the same key.
 /// \throw ParquetException if Close() has been called.
 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);

 /// Number of columns.
 ///
 /// This number is fixed during the lifetime of the writer as it is determined via
Expand Down

24 changes: 19 additions & 5 deletions
cpp/src/parquet/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1740,7 +1740,6 @@ void RowGroupMetaDataBuilder::Finish(int64_t total_bytes_written,
}

// file metadata
// TODO(PARQUET-595) Support key_value_metadata
class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 public:
 explicit FileMetaDataBuilderImpl(
Expand DownExpand Up@@ -1797,15 +1796,21 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 }
 }

 std::unique_ptr<FileMetaData> Finish() {
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 int64_t total_rows = 0;
 for (auto row_group : row_groups_) {
 total_rows += row_group.num_rows;
 }
 metadata_->__set_num_rows(total_rows);
 metadata_->__set_row_groups(row_groups_);

 if (key_value_metadata_) {
 if (key_value_metadata_ || key_value_metadata) {
 if (!key_value_metadata_) {
 key_value_metadata_ = key_value_metadata;
 } else if (key_value_metadata) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
 }
 metadata_->key_value_metadata.clear();
 metadata_->key_value_metadata.reserve(key_value_metadata_->size());
 for (int64_t i = 0; i < key_value_metadata_->size(); ++i) {
Expand All@@ -1829,7 +1834,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 metadata_->__set_version(file_version);
 metadata_->__set_created_by(properties_->created_by());

 // Users cannot set the `ColumnOrder` since we donot not have user defined sort order
 // Users cannot set the `ColumnOrder` since we do not have user defined sort order
 // in the spec yet.
 // We always default to `TYPE_DEFINED_ORDER`. We can expose it in
 // the API once we have user defined sort orders in the Parquet format.
Expand DownExpand Up@@ -1910,6 +1915,12 @@ std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 new FileMetaDataBuilder(schema, std::move(props), std::move(key_value_metadata)));
}

std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props) {
 return std::unique_ptr<FileMetaDataBuilder>(
 new FileMetaDataBuilder(schema, std::move(props)));
}

FileMetaDataBuilder::FileMetaDataBuilder(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata)
Expand All@@ -1926,7 +1937,10 @@ void FileMetaDataBuilder::SetPageIndexLocation(const PageIndexLocation& location
 impl_->SetPageIndexLocation(location);
}

std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish() { return impl_->Finish(); }
std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 return impl_->Finish(key_value_metadata);
}

std::unique_ptr<FileCryptoMetaData> FileMetaDataBuilder::GetCryptoMetaData() {
 return impl_->BuildFileCryptoMetaData();
Expand Down

11 changes: 8 additions & 3 deletions
cpp/src/parquet/metadata.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -523,10 +523,14 @@ struct PageIndexLocation {

class PARQUET_EXPORT FileMetaDataBuilder {
 public:
// API convenience to get a MetaData reader
ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
@wgtmacwgtmacApr 12, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


Suggested change

ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
ARROW_DEPRECATED("Deprecated in 13.0.0. Use overload without KeyValueMetadata instead.")
Should I change this since it may not be included in the 12.0.0 release? @wjones127
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
 std::shared_ptr<const KeyValueMetadata> key_value_metadata);

 // API convenience to get a MetaData builder
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);

 ~FileMetaDataBuilder();

Expand All@@ -537,7 +541,8 @@ class PARQUET_EXPORT FileMetaDataBuilder {
 void SetPageIndexLocation(const PageIndexLocation& location);

 // Complete the Thrift structure
 std::unique_ptr<FileMetaData> Finish();
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);

 // crypto metadata
 std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
Expand Down

51 changes: 49 additions & 2 deletions
cpp/src/parquet/metadata_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

#include "arrow/util/key_value_metadata.h"
#include "parquet/file_reader.h"
#include "parquet/file_writer.h"
#include "parquet/schema.h"
#include "parquet/statistics.h"
#include "parquet/test_util.h"
Expand DownExpand Up@@ -284,16 +285,62 @@ TEST(Metadata, TestKeyValueMetadata) {
 auto kvmeta = std::make_shared<KeyValueMetadata>();
 kvmeta->Append("test_key", "test_value");

 auto f_builder = FileMetaDataBuilder::Make(&schema, props, kvmeta);
 auto f_builder = FileMetaDataBuilder::Make(&schema, props);

 // Read the metadata
 auto f_accessor = f_builder->Finish();
 auto f_accessor = f_builder->Finish(kvmeta);

 // Key value metadata
 ASSERT_TRUE(f_accessor->key_value_metadata());
 EXPECT_TRUE(f_accessor->key_value_metadata()->Equals(*kvmeta));
}

TEST(Metadata, TestAddKeyValueMetadata) {
 schema::NodeVector fields;
 fields.push_back(schema::Int32("int_col", Repetition::REQUIRED));
 auto schema = std::static_pointer_cast<schema::GroupNode>(
 schema::GroupNode::Make("schema", Repetition::REQUIRED, fields));

 auto kv_meta = std::make_shared<KeyValueMetadata>();
 kv_meta->Append("test_key_1", "test_value_1");
 kv_meta->Append("test_key_2", "test_value_2_");

 auto sink = CreateOutputStream();
 auto writer_props = parquet::WriterProperties::Builder().disable_dictionary()->build();
 auto file_writer =
 parquet::ParquetFileWriter::Open(sink, schema, writer_props, kv_meta);

 // Key value metadata that will be added to the file.
 auto kv_meta_added = std::make_shared<KeyValueMetadata>();
 kv_meta_added->Append("test_key_2", "test_value_2");
 kv_meta_added->Append("test_key_3", "test_value_3");

 file_writer->AddKeyValueMetadata(kv_meta_added);
 file_writer->Close();

 // Throw if appending key value metadata to closed file.
 auto kv_meta_ignored = std::make_shared<KeyValueMetadata>();
 kv_meta_ignored->Append("test_key_4", "test_value_4");
 EXPECT_THROW(file_writer->AddKeyValueMetadata(kv_meta_ignored), ParquetException);

 PARQUET_ASSIGN_OR_THROW(auto buffer, sink->Finish());
 auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
 auto file_reader = ParquetFileReader::Open(source);

 ASSERT_NE(nullptr, file_reader->metadata());
 ASSERT_NE(nullptr, file_reader->metadata()->key_value_metadata());
 auto read_kv_meta = file_reader->metadata()->key_value_metadata();

 // Verify keys that were added before file writer was closed are present.
 for (int i = 1; i <= 3; ++i) {
 auto index = std::to_string(i);
 PARQUET_ASSIGN_OR_THROW(auto value, read_kv_meta->Get("test_key_" + index));
 EXPECT_EQ("test_value_" + index, value);
 }
 // Verify keys that were added after file writer was closed are not present.
 EXPECT_FALSE(read_kv_meta->Contains("test_key_4"));
}

TEST(Metadata, TestHasBloomFilter) {
 std::string dir_string(parquet::test::get_data_dir());
 std::string path = dir_string + "/data_index_bloom_encoding_stats.parquet";
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-34888: [C++][Parquet] Writer supports adding extra kv meta#34889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

wjones127
merged 5 commits into

from

Apr 13, 2023
 Merged


GH-34888: [C++][Parquet] Writer supports adding extra kv meta
#34889
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b84556c

GH-34888: [C++][Parquet] Writer supports adding extra kv meta


wgtmac Apr 4, 2023
bed0661

address comments


wgtmac Apr 9, 2023
6f6c41c

make it backward compatible


wgtmac Apr 10, 2023
9b0d022

try to fix doxygen issue


wgtmac Apr 11, 2023
5e5a39c

Update cpp/src/parquet/file_writer.cc


wgtmac Apr 12, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 24 additions & 5 deletions
cpp/src/parquet/file_writer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

#include "parquet/file_writer.h"

#include <cstddef>
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>

#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "parquet/column_writer.h"
#include "parquet/encryption/encryption_internal.h"
#include "parquet/encryption/internal_file_encryptor.h"
Expand DownExpand Up@@ -338,7 +339,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 auto file_encryption_properties = properties_->file_encryption_properties();

 if (file_encryption_properties == nullptr) { // Non encrypted file.
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 WriteFileMetaData(*file_metadata_, sink_.get());
 } else { // Encrypted file
 CloseEncryptedFile(file_encryption_properties);
Expand DownExpand Up@@ -376,6 +377,15 @@ class FileSerializer : public ParquetFileWriter::Contents {

 RowGroupWriter* AppendBufferedRowGroup() override { return AppendRowGroup(true); }

 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) override {
 if (key_value_metadata_ == nullptr) {
 key_value_metadata_ = std::move(key_value_metadata);
 } else if (key_value_metadata != nullptr) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
Comment thread
wgtmac marked this conversation as resolved.


Outdated

 }
 }

 ~FileSerializer() override {
 try {
 Close();
Expand All@@ -394,7 +404,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 properties_(std::move(properties)),
 num_row_groups_(0),
 num_rows_(0),
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_, key_value_metadata_)) {
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_)) {
 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 if (position == 0) {
 StartFile();
Expand All@@ -407,7 +417,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 // Encrypted file with encrypted footer
 if (file_encryption_properties->encrypted_footer()) {
 // encrypted footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);

 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 uint64_t metadata_start = static_cast<uint64_t>(position);
Expand All@@ -422,7 +432,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
 PARQUET_THROW_NOT_OK(sink_->Write(kParquetEMagic, 4));
 } else { // Encrypted file with plaintext footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 auto footer_signing_encryptor = file_encryptor_->GetFooterSigningEncryptor();
 WriteEncryptedFileMetadata(*file_metadata_, sink_.get(), footer_signing_encryptor,
 false);
Expand DownExpand Up@@ -613,6 +623,15 @@ RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) {
 return AppendRowGroup();
}

void ParquetFileWriter::AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 if (contents_) {
 contents_->AddKeyValueMetadata(key_value_metadata);
 } else {
 throw ParquetException("Cannot add key-value metadata to closed file");
 }
}

const std::shared_ptr<WriterProperties>& ParquetFileWriter::properties() const {
 return contents_->properties();
}
Expand Down

10 changes: 10 additions & 0 deletions
cpp/src/parquet/file_writer.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -163,6 +163,9 @@ class PARQUET_EXPORT ParquetFileWriter {
 return key_value_metadata_;
 }

 virtual void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;

 // Return const-pointer to make it clear that this object is not to be copied
 const SchemaDescriptor* schema() const { return &schema_; }

Expand DownExpand Up@@ -209,6 +212,13 @@ class PARQUET_EXPORT ParquetFileWriter {
 /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
 RowGroupWriter* AppendBufferedRowGroup();

 /// \brief Add key-value metadata to the file.
 /// \param[in] key_value_metadata the metadata to add.
 /// \note This will overwrite any existing metadata with the same key.
 /// \throw ParquetException if Close() has been called.
 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);

 /// Number of columns.
 ///
 /// This number is fixed during the lifetime of the writer as it is determined via
Expand Down

24 changes: 19 additions & 5 deletions
cpp/src/parquet/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1740,7 +1740,6 @@ void RowGroupMetaDataBuilder::Finish(int64_t total_bytes_written,
}

// file metadata
// TODO(PARQUET-595) Support key_value_metadata
class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 public:
 explicit FileMetaDataBuilderImpl(
Expand DownExpand Up@@ -1797,15 +1796,21 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 }
 }

 std::unique_ptr<FileMetaData> Finish() {
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 int64_t total_rows = 0;
 for (auto row_group : row_groups_) {
 total_rows += row_group.num_rows;
 }
 metadata_->__set_num_rows(total_rows);
 metadata_->__set_row_groups(row_groups_);

 if (key_value_metadata_) {
 if (key_value_metadata_ || key_value_metadata) {
 if (!key_value_metadata_) {
 key_value_metadata_ = key_value_metadata;
 } else if (key_value_metadata) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
 }
 metadata_->key_value_metadata.clear();
 metadata_->key_value_metadata.reserve(key_value_metadata_->size());
 for (int64_t i = 0; i < key_value_metadata_->size(); ++i) {
Expand All@@ -1829,7 +1834,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 metadata_->__set_version(file_version);
 metadata_->__set_created_by(properties_->created_by());

 // Users cannot set the `ColumnOrder` since we donot not have user defined sort order
 // Users cannot set the `ColumnOrder` since we do not have user defined sort order
 // in the spec yet.
 // We always default to `TYPE_DEFINED_ORDER`. We can expose it in
 // the API once we have user defined sort orders in the Parquet format.
Expand DownExpand Up@@ -1910,6 +1915,12 @@ std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 new FileMetaDataBuilder(schema, std::move(props), std::move(key_value_metadata)));
}

std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props) {
 return std::unique_ptr<FileMetaDataBuilder>(
 new FileMetaDataBuilder(schema, std::move(props)));
}

FileMetaDataBuilder::FileMetaDataBuilder(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata)
Expand All@@ -1926,7 +1937,10 @@ void FileMetaDataBuilder::SetPageIndexLocation(const PageIndexLocation& location
 impl_->SetPageIndexLocation(location);
}

std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish() { return impl_->Finish(); }
std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 return impl_->Finish(key_value_metadata);
}

std::unique_ptr<FileCryptoMetaData> FileMetaDataBuilder::GetCryptoMetaData() {
 return impl_->BuildFileCryptoMetaData();
Expand Down

11 changes: 8 additions & 3 deletions
cpp/src/parquet/metadata.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -523,10 +523,14 @@ struct PageIndexLocation {

class PARQUET_EXPORT FileMetaDataBuilder {
 public:
// API convenience to get a MetaData reader
ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
@wgtmacwgtmacApr 12, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


Suggested change

ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
ARROW_DEPRECATED("Deprecated in 13.0.0. Use overload without KeyValueMetadata instead.")
Should I change this since it may not be included in the 12.0.0 release? @wjones127
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
 std::shared_ptr<const KeyValueMetadata> key_value_metadata);

 // API convenience to get a MetaData builder
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);

 ~FileMetaDataBuilder();

Expand All@@ -537,7 +541,8 @@ class PARQUET_EXPORT FileMetaDataBuilder {
 void SetPageIndexLocation(const PageIndexLocation& location);

 // Complete the Thrift structure
 std::unique_ptr<FileMetaData> Finish();
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);

 // crypto metadata
 std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
Expand Down

51 changes: 49 additions & 2 deletions
cpp/src/parquet/metadata_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

#include "arrow/util/key_value_metadata.h"
#include "parquet/file_reader.h"
#include "parquet/file_writer.h"
#include "parquet/schema.h"
#include "parquet/statistics.h"
#include "parquet/test_util.h"
Expand DownExpand Up@@ -284,16 +285,62 @@ TEST(Metadata, TestKeyValueMetadata) {
 auto kvmeta = std::make_shared<KeyValueMetadata>();
 kvmeta->Append("test_key", "test_value");

 auto f_builder = FileMetaDataBuilder::Make(&schema, props, kvmeta);
 auto f_builder = FileMetaDataBuilder::Make(&schema, props);

 // Read the metadata
 auto f_accessor = f_builder->Finish();
 auto f_accessor = f_builder->Finish(kvmeta);

 // Key value metadata
 ASSERT_TRUE(f_accessor->key_value_metadata());
 EXPECT_TRUE(f_accessor->key_value_metadata()->Equals(*kvmeta));
}

TEST(Metadata, TestAddKeyValueMetadata) {
 schema::NodeVector fields;
 fields.push_back(schema::Int32("int_col", Repetition::REQUIRED));
 auto schema = std::static_pointer_cast<schema::GroupNode>(
 schema::GroupNode::Make("schema", Repetition::REQUIRED, fields));

 auto kv_meta = std::make_shared<KeyValueMetadata>();
 kv_meta->Append("test_key_1", "test_value_1");
 kv_meta->Append("test_key_2", "test_value_2_");

 auto sink = CreateOutputStream();
 auto writer_props = parquet::WriterProperties::Builder().disable_dictionary()->build();
 auto file_writer =
 parquet::ParquetFileWriter::Open(sink, schema, writer_props, kv_meta);

 // Key value metadata that will be added to the file.
 auto kv_meta_added = std::make_shared<KeyValueMetadata>();
 kv_meta_added->Append("test_key_2", "test_value_2");
 kv_meta_added->Append("test_key_3", "test_value_3");

 file_writer->AddKeyValueMetadata(kv_meta_added);
 file_writer->Close();

 // Throw if appending key value metadata to closed file.
 auto kv_meta_ignored = std::make_shared<KeyValueMetadata>();
 kv_meta_ignored->Append("test_key_4", "test_value_4");
 EXPECT_THROW(file_writer->AddKeyValueMetadata(kv_meta_ignored), ParquetException);

 PARQUET_ASSIGN_OR_THROW(auto buffer, sink->Finish());
 auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
 auto file_reader = ParquetFileReader::Open(source);

 ASSERT_NE(nullptr, file_reader->metadata());
 ASSERT_NE(nullptr, file_reader->metadata()->key_value_metadata());
 auto read_kv_meta = file_reader->metadata()->key_value_metadata();

 // Verify keys that were added before file writer was closed are present.
 for (int i = 1; i <= 3; ++i) {
 auto index = std::to_string(i);
 PARQUET_ASSIGN_OR_THROW(auto value, read_kv_meta->Get("test_key_" + index));
 EXPECT_EQ("test_value_" + index, value);
 }
 // Verify keys that were added after file writer was closed are not present.
 EXPECT_FALSE(read_kv_meta->Contains("test_key_4"));
}

TEST(Metadata, TestHasBloomFilter) {
 std::string dir_string(parquet::test::get_data_dir());
 std::string path = dir_string + "/data_index_bloom_encoding_stats.parquet";
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
GH-34888: [C++][Parquet] Writer supports adding extra kv meta#34889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

wjones127
merged 5 commits into

from

Apr 13, 2023
 Merged


GH-34888: [C++][Parquet] Writer supports adding extra kv meta
#34889
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b84556c

GH-34888: [C++][Parquet] Writer supports adding extra kv meta


wgtmac Apr 4, 2023
bed0661

address comments


wgtmac Apr 9, 2023
6f6c41c

make it backward compatible


wgtmac Apr 10, 2023
9b0d022

try to fix doxygen issue


wgtmac Apr 11, 2023
5e5a39c

Update cpp/src/parquet/file_writer.cc


wgtmac Apr 12, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 24 additions & 5 deletions
cpp/src/parquet/file_writer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

#include "parquet/file_writer.h"

#include <cstddef>
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>

#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "parquet/column_writer.h"
#include "parquet/encryption/encryption_internal.h"
#include "parquet/encryption/internal_file_encryptor.h"
Expand DownExpand Up@@ -338,7 +339,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 auto file_encryption_properties = properties_->file_encryption_properties();

 if (file_encryption_properties == nullptr) { // Non encrypted file.
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 WriteFileMetaData(*file_metadata_, sink_.get());
 } else { // Encrypted file
 CloseEncryptedFile(file_encryption_properties);
Expand DownExpand Up@@ -376,6 +377,15 @@ class FileSerializer : public ParquetFileWriter::Contents {

 RowGroupWriter* AppendBufferedRowGroup() override { return AppendRowGroup(true); }

 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) override {
 if (key_value_metadata_ == nullptr) {
 key_value_metadata_ = std::move(key_value_metadata);
 } else if (key_value_metadata != nullptr) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
Comment thread
wgtmac marked this conversation as resolved.


Outdated

 }
 }

 ~FileSerializer() override {
 try {
 Close();
Expand All@@ -394,7 +404,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 properties_(std::move(properties)),
 num_row_groups_(0),
 num_rows_(0),
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_, key_value_metadata_)) {
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_)) {
 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 if (position == 0) {
 StartFile();
Expand All@@ -407,7 +417,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 // Encrypted file with encrypted footer
 if (file_encryption_properties->encrypted_footer()) {
 // encrypted footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);

 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 uint64_t metadata_start = static_cast<uint64_t>(position);
Expand All@@ -422,7 +432,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
 PARQUET_THROW_NOT_OK(sink_->Write(kParquetEMagic, 4));
 } else { // Encrypted file with plaintext footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 auto footer_signing_encryptor = file_encryptor_->GetFooterSigningEncryptor();
 WriteEncryptedFileMetadata(*file_metadata_, sink_.get(), footer_signing_encryptor,
 false);
Expand DownExpand Up@@ -613,6 +623,15 @@ RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) {
 return AppendRowGroup();
}

void ParquetFileWriter::AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 if (contents_) {
 contents_->AddKeyValueMetadata(key_value_metadata);
 } else {
 throw ParquetException("Cannot add key-value metadata to closed file");
 }
}

const std::shared_ptr<WriterProperties>& ParquetFileWriter::properties() const {
 return contents_->properties();
}
Expand Down

10 changes: 10 additions & 0 deletions
cpp/src/parquet/file_writer.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -163,6 +163,9 @@ class PARQUET_EXPORT ParquetFileWriter {
 return key_value_metadata_;
 }

 virtual void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;

 // Return const-pointer to make it clear that this object is not to be copied
 const SchemaDescriptor* schema() const { return &schema_; }

Expand DownExpand Up@@ -209,6 +212,13 @@ class PARQUET_EXPORT ParquetFileWriter {
 /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
 RowGroupWriter* AppendBufferedRowGroup();

 /// \brief Add key-value metadata to the file.
 /// \param[in] key_value_metadata the metadata to add.
 /// \note This will overwrite any existing metadata with the same key.
 /// \throw ParquetException if Close() has been called.
 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);

 /// Number of columns.
 ///
 /// This number is fixed during the lifetime of the writer as it is determined via
Expand Down

24 changes: 19 additions & 5 deletions
cpp/src/parquet/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1740,7 +1740,6 @@ void RowGroupMetaDataBuilder::Finish(int64_t total_bytes_written,
}

// file metadata
// TODO(PARQUET-595) Support key_value_metadata
class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 public:
 explicit FileMetaDataBuilderImpl(
Expand DownExpand Up@@ -1797,15 +1796,21 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 }
 }

 std::unique_ptr<FileMetaData> Finish() {
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 int64_t total_rows = 0;
 for (auto row_group : row_groups_) {
 total_rows += row_group.num_rows;
 }
 metadata_->__set_num_rows(total_rows);
 metadata_->__set_row_groups(row_groups_);

 if (key_value_metadata_) {
 if (key_value_metadata_ || key_value_metadata) {
 if (!key_value_metadata_) {
 key_value_metadata_ = key_value_metadata;
 } else if (key_value_metadata) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
 }
 metadata_->key_value_metadata.clear();
 metadata_->key_value_metadata.reserve(key_value_metadata_->size());
 for (int64_t i = 0; i < key_value_metadata_->size(); ++i) {
Expand All@@ -1829,7 +1834,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 metadata_->__set_version(file_version);
 metadata_->__set_created_by(properties_->created_by());

 // Users cannot set the `ColumnOrder` since we donot not have user defined sort order
 // Users cannot set the `ColumnOrder` since we do not have user defined sort order
 // in the spec yet.
 // We always default to `TYPE_DEFINED_ORDER`. We can expose it in
 // the API once we have user defined sort orders in the Parquet format.
Expand DownExpand Up@@ -1910,6 +1915,12 @@ std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 new FileMetaDataBuilder(schema, std::move(props), std::move(key_value_metadata)));
}

std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props) {
 return std::unique_ptr<FileMetaDataBuilder>(
 new FileMetaDataBuilder(schema, std::move(props)));
}

FileMetaDataBuilder::FileMetaDataBuilder(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata)
Expand All@@ -1926,7 +1937,10 @@ void FileMetaDataBuilder::SetPageIndexLocation(const PageIndexLocation& location
 impl_->SetPageIndexLocation(location);
}

std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish() { return impl_->Finish(); }
std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 return impl_->Finish(key_value_metadata);
}

std::unique_ptr<FileCryptoMetaData> FileMetaDataBuilder::GetCryptoMetaData() {
 return impl_->BuildFileCryptoMetaData();
Expand Down

11 changes: 8 additions & 3 deletions
cpp/src/parquet/metadata.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -523,10 +523,14 @@ struct PageIndexLocation {

class PARQUET_EXPORT FileMetaDataBuilder {
 public:
// API convenience to get a MetaData reader
ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
@wgtmacwgtmacApr 12, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


Suggested change

ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
ARROW_DEPRECATED("Deprecated in 13.0.0. Use overload without KeyValueMetadata instead.")
Should I change this since it may not be included in the 12.0.0 release? @wjones127
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
 std::shared_ptr<const KeyValueMetadata> key_value_metadata);

 // API convenience to get a MetaData builder
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);

 ~FileMetaDataBuilder();

Expand All@@ -537,7 +541,8 @@ class PARQUET_EXPORT FileMetaDataBuilder {
 void SetPageIndexLocation(const PageIndexLocation& location);

 // Complete the Thrift structure
 std::unique_ptr<FileMetaData> Finish();
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);

 // crypto metadata
 std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
Expand Down

51 changes: 49 additions & 2 deletions
cpp/src/parquet/metadata_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

#include "arrow/util/key_value_metadata.h"
#include "parquet/file_reader.h"
#include "parquet/file_writer.h"
#include "parquet/schema.h"
#include "parquet/statistics.h"
#include "parquet/test_util.h"
Expand DownExpand Up@@ -284,16 +285,62 @@ TEST(Metadata, TestKeyValueMetadata) {
 auto kvmeta = std::make_shared<KeyValueMetadata>();
 kvmeta->Append("test_key", "test_value");

 auto f_builder = FileMetaDataBuilder::Make(&schema, props, kvmeta);
 auto f_builder = FileMetaDataBuilder::Make(&schema, props);

 // Read the metadata
 auto f_accessor = f_builder->Finish();
 auto f_accessor = f_builder->Finish(kvmeta);

 // Key value metadata
 ASSERT_TRUE(f_accessor->key_value_metadata());
 EXPECT_TRUE(f_accessor->key_value_metadata()->Equals(*kvmeta));
}

TEST(Metadata, TestAddKeyValueMetadata) {
 schema::NodeVector fields;
 fields.push_back(schema::Int32("int_col", Repetition::REQUIRED));
 auto schema = std::static_pointer_cast<schema::GroupNode>(
 schema::GroupNode::Make("schema", Repetition::REQUIRED, fields));

 auto kv_meta = std::make_shared<KeyValueMetadata>();
 kv_meta->Append("test_key_1", "test_value_1");
 kv_meta->Append("test_key_2", "test_value_2_");

 auto sink = CreateOutputStream();
 auto writer_props = parquet::WriterProperties::Builder().disable_dictionary()->build();
 auto file_writer =
 parquet::ParquetFileWriter::Open(sink, schema, writer_props, kv_meta);

 // Key value metadata that will be added to the file.
 auto kv_meta_added = std::make_shared<KeyValueMetadata>();
 kv_meta_added->Append("test_key_2", "test_value_2");
 kv_meta_added->Append("test_key_3", "test_value_3");

 file_writer->AddKeyValueMetadata(kv_meta_added);
 file_writer->Close();

 // Throw if appending key value metadata to closed file.
 auto kv_meta_ignored = std::make_shared<KeyValueMetadata>();
 kv_meta_ignored->Append("test_key_4", "test_value_4");
 EXPECT_THROW(file_writer->AddKeyValueMetadata(kv_meta_ignored), ParquetException);

 PARQUET_ASSIGN_OR_THROW(auto buffer, sink->Finish());
 auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
 auto file_reader = ParquetFileReader::Open(source);

 ASSERT_NE(nullptr, file_reader->metadata());
 ASSERT_NE(nullptr, file_reader->metadata()->key_value_metadata());
 auto read_kv_meta = file_reader->metadata()->key_value_metadata();

 // Verify keys that were added before file writer was closed are present.
 for (int i = 1; i <= 3; ++i) {
 auto index = std::to_string(i);
 PARQUET_ASSIGN_OR_THROW(auto value, read_kv_meta->Get("test_key_" + index));
 EXPECT_EQ("test_value_" + index, value);
 }
 // Verify keys that were added after file writer was closed are not present.
 EXPECT_FALSE(read_kv_meta->Contains("test_key_4"));
}

TEST(Metadata, TestHasBloomFilter) {
 std::string dir_string(parquet::test::get_data_dir());
 std::string path = dir_string + "/data_index_bloom_encoding_stats.parquet";
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-34888: [C++][Parquet] Writer supports adding extra kv meta#34889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

wjones127
merged 5 commits into

from

Apr 13, 2023
 Merged


GH-34888: [C++][Parquet] Writer supports adding extra kv meta
#34889
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b84556c

GH-34888: [C++][Parquet] Writer supports adding extra kv meta


wgtmac Apr 4, 2023
bed0661

address comments


wgtmac Apr 9, 2023
6f6c41c

make it backward compatible


wgtmac Apr 10, 2023
9b0d022

try to fix doxygen issue


wgtmac Apr 11, 2023
5e5a39c

Update cpp/src/parquet/file_writer.cc


wgtmac Apr 12, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 24 additions & 5 deletions
cpp/src/parquet/file_writer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

#include "parquet/file_writer.h"

#include <cstddef>
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>

#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "parquet/column_writer.h"
#include "parquet/encryption/encryption_internal.h"
#include "parquet/encryption/internal_file_encryptor.h"
Expand DownExpand Up@@ -338,7 +339,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 auto file_encryption_properties = properties_->file_encryption_properties();

 if (file_encryption_properties == nullptr) { // Non encrypted file.
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 WriteFileMetaData(*file_metadata_, sink_.get());
 } else { // Encrypted file
 CloseEncryptedFile(file_encryption_properties);
Expand DownExpand Up@@ -376,6 +377,15 @@ class FileSerializer : public ParquetFileWriter::Contents {

 RowGroupWriter* AppendBufferedRowGroup() override { return AppendRowGroup(true); }

 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) override {
 if (key_value_metadata_ == nullptr) {
 key_value_metadata_ = std::move(key_value_metadata);
 } else if (key_value_metadata != nullptr) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
Comment thread
wgtmac marked this conversation as resolved.


Outdated

 }
 }

 ~FileSerializer() override {
 try {
 Close();
Expand All@@ -394,7 +404,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 properties_(std::move(properties)),
 num_row_groups_(0),
 num_rows_(0),
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_, key_value_metadata_)) {
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_)) {
 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 if (position == 0) {
 StartFile();
Expand All@@ -407,7 +417,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 // Encrypted file with encrypted footer
 if (file_encryption_properties->encrypted_footer()) {
 // encrypted footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);

 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 uint64_t metadata_start = static_cast<uint64_t>(position);
Expand All@@ -422,7 +432,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
 PARQUET_THROW_NOT_OK(sink_->Write(kParquetEMagic, 4));
 } else { // Encrypted file with plaintext footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 auto footer_signing_encryptor = file_encryptor_->GetFooterSigningEncryptor();
 WriteEncryptedFileMetadata(*file_metadata_, sink_.get(), footer_signing_encryptor,
 false);
Expand DownExpand Up@@ -613,6 +623,15 @@ RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) {
 return AppendRowGroup();
}

void ParquetFileWriter::AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 if (contents_) {
 contents_->AddKeyValueMetadata(key_value_metadata);
 } else {
 throw ParquetException("Cannot add key-value metadata to closed file");
 }
}

const std::shared_ptr<WriterProperties>& ParquetFileWriter::properties() const {
 return contents_->properties();
}
Expand Down

10 changes: 10 additions & 0 deletions
cpp/src/parquet/file_writer.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -163,6 +163,9 @@ class PARQUET_EXPORT ParquetFileWriter {
 return key_value_metadata_;
 }

 virtual void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;

 // Return const-pointer to make it clear that this object is not to be copied
 const SchemaDescriptor* schema() const { return &schema_; }

Expand DownExpand Up@@ -209,6 +212,13 @@ class PARQUET_EXPORT ParquetFileWriter {
 /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
 RowGroupWriter* AppendBufferedRowGroup();

 /// \brief Add key-value metadata to the file.
 /// \param[in] key_value_metadata the metadata to add.
 /// \note This will overwrite any existing metadata with the same key.
 /// \throw ParquetException if Close() has been called.
 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);

 /// Number of columns.
 ///
 /// This number is fixed during the lifetime of the writer as it is determined via
Expand Down

24 changes: 19 additions & 5 deletions
cpp/src/parquet/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1740,7 +1740,6 @@ void RowGroupMetaDataBuilder::Finish(int64_t total_bytes_written,
}

// file metadata
// TODO(PARQUET-595) Support key_value_metadata
class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 public:
 explicit FileMetaDataBuilderImpl(
Expand DownExpand Up@@ -1797,15 +1796,21 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 }
 }

 std::unique_ptr<FileMetaData> Finish() {
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 int64_t total_rows = 0;
 for (auto row_group : row_groups_) {
 total_rows += row_group.num_rows;
 }
 metadata_->__set_num_rows(total_rows);
 metadata_->__set_row_groups(row_groups_);

 if (key_value_metadata_) {
 if (key_value_metadata_ || key_value_metadata) {
 if (!key_value_metadata_) {
 key_value_metadata_ = key_value_metadata;
 } else if (key_value_metadata) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
 }
 metadata_->key_value_metadata.clear();
 metadata_->key_value_metadata.reserve(key_value_metadata_->size());
 for (int64_t i = 0; i < key_value_metadata_->size(); ++i) {
Expand All@@ -1829,7 +1834,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 metadata_->__set_version(file_version);
 metadata_->__set_created_by(properties_->created_by());

 // Users cannot set the `ColumnOrder` since we donot not have user defined sort order
 // Users cannot set the `ColumnOrder` since we do not have user defined sort order
 // in the spec yet.
 // We always default to `TYPE_DEFINED_ORDER`. We can expose it in
 // the API once we have user defined sort orders in the Parquet format.
Expand DownExpand Up@@ -1910,6 +1915,12 @@ std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 new FileMetaDataBuilder(schema, std::move(props), std::move(key_value_metadata)));
}

std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props) {
 return std::unique_ptr<FileMetaDataBuilder>(
 new FileMetaDataBuilder(schema, std::move(props)));
}

FileMetaDataBuilder::FileMetaDataBuilder(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata)
Expand All@@ -1926,7 +1937,10 @@ void FileMetaDataBuilder::SetPageIndexLocation(const PageIndexLocation& location
 impl_->SetPageIndexLocation(location);
}

std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish() { return impl_->Finish(); }
std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 return impl_->Finish(key_value_metadata);
}

std::unique_ptr<FileCryptoMetaData> FileMetaDataBuilder::GetCryptoMetaData() {
 return impl_->BuildFileCryptoMetaData();
Expand Down

11 changes: 8 additions & 3 deletions
cpp/src/parquet/metadata.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -523,10 +523,14 @@ struct PageIndexLocation {

class PARQUET_EXPORT FileMetaDataBuilder {
 public:
// API convenience to get a MetaData reader
ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
@wgtmacwgtmacApr 12, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


Suggested change

ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
ARROW_DEPRECATED("Deprecated in 13.0.0. Use overload without KeyValueMetadata instead.")
Should I change this since it may not be included in the 12.0.0 release? @wjones127
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
 std::shared_ptr<const KeyValueMetadata> key_value_metadata);

 // API convenience to get a MetaData builder
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);

 ~FileMetaDataBuilder();

Expand All@@ -537,7 +541,8 @@ class PARQUET_EXPORT FileMetaDataBuilder {
 void SetPageIndexLocation(const PageIndexLocation& location);

 // Complete the Thrift structure
 std::unique_ptr<FileMetaData> Finish();
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);

 // crypto metadata
 std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
Expand Down

51 changes: 49 additions & 2 deletions
cpp/src/parquet/metadata_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

#include "arrow/util/key_value_metadata.h"
#include "parquet/file_reader.h"
#include "parquet/file_writer.h"
#include "parquet/schema.h"
#include "parquet/statistics.h"
#include "parquet/test_util.h"
Expand DownExpand Up@@ -284,16 +285,62 @@ TEST(Metadata, TestKeyValueMetadata) {
 auto kvmeta = std::make_shared<KeyValueMetadata>();
 kvmeta->Append("test_key", "test_value");

 auto f_builder = FileMetaDataBuilder::Make(&schema, props, kvmeta);
 auto f_builder = FileMetaDataBuilder::Make(&schema, props);

 // Read the metadata
 auto f_accessor = f_builder->Finish();
 auto f_accessor = f_builder->Finish(kvmeta);

 // Key value metadata
 ASSERT_TRUE(f_accessor->key_value_metadata());
 EXPECT_TRUE(f_accessor->key_value_metadata()->Equals(*kvmeta));
}

TEST(Metadata, TestAddKeyValueMetadata) {
 schema::NodeVector fields;
 fields.push_back(schema::Int32("int_col", Repetition::REQUIRED));
 auto schema = std::static_pointer_cast<schema::GroupNode>(
 schema::GroupNode::Make("schema", Repetition::REQUIRED, fields));

 auto kv_meta = std::make_shared<KeyValueMetadata>();
 kv_meta->Append("test_key_1", "test_value_1");
 kv_meta->Append("test_key_2", "test_value_2_");

 auto sink = CreateOutputStream();
 auto writer_props = parquet::WriterProperties::Builder().disable_dictionary()->build();
 auto file_writer =
 parquet::ParquetFileWriter::Open(sink, schema, writer_props, kv_meta);

 // Key value metadata that will be added to the file.
 auto kv_meta_added = std::make_shared<KeyValueMetadata>();
 kv_meta_added->Append("test_key_2", "test_value_2");
 kv_meta_added->Append("test_key_3", "test_value_3");

 file_writer->AddKeyValueMetadata(kv_meta_added);
 file_writer->Close();

 // Throw if appending key value metadata to closed file.
 auto kv_meta_ignored = std::make_shared<KeyValueMetadata>();
 kv_meta_ignored->Append("test_key_4", "test_value_4");
 EXPECT_THROW(file_writer->AddKeyValueMetadata(kv_meta_ignored), ParquetException);

 PARQUET_ASSIGN_OR_THROW(auto buffer, sink->Finish());
 auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
 auto file_reader = ParquetFileReader::Open(source);

 ASSERT_NE(nullptr, file_reader->metadata());
 ASSERT_NE(nullptr, file_reader->metadata()->key_value_metadata());
 auto read_kv_meta = file_reader->metadata()->key_value_metadata();

 // Verify keys that were added before file writer was closed are present.
 for (int i = 1; i <= 3; ++i) {
 auto index = std::to_string(i);
 PARQUET_ASSIGN_OR_THROW(auto value, read_kv_meta->Get("test_key_" + index));
 EXPECT_EQ("test_value_" + index, value);
 }
 // Verify keys that were added after file writer was closed are not present.
 EXPECT_FALSE(read_kv_meta->Contains("test_key_4"));
}

TEST(Metadata, TestHasBloomFilter) {
 std::string dir_string(parquet::test::get_data_dir());
 std::string path = dir_string + "/data_index_bloom_encoding_stats.parquet";
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-34888: [C++][Parquet] Writer supports adding extra kv meta#34889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

wjones127
merged 5 commits into

from

Apr 13, 2023
 Merged


GH-34888: [C++][Parquet] Writer supports adding extra kv meta
#34889
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b84556c

GH-34888: [C++][Parquet] Writer supports adding extra kv meta


wgtmac Apr 4, 2023
bed0661

address comments


wgtmac Apr 9, 2023
6f6c41c

make it backward compatible


wgtmac Apr 10, 2023
9b0d022

try to fix doxygen issue


wgtmac Apr 11, 2023
5e5a39c

Update cpp/src/parquet/file_writer.cc


wgtmac Apr 12, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 24 additions & 5 deletions
cpp/src/parquet/file_writer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

#include "parquet/file_writer.h"

#include <cstddef>
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>

#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "parquet/column_writer.h"
#include "parquet/encryption/encryption_internal.h"
#include "parquet/encryption/internal_file_encryptor.h"
Expand DownExpand Up@@ -338,7 +339,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 auto file_encryption_properties = properties_->file_encryption_properties();

 if (file_encryption_properties == nullptr) { // Non encrypted file.
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 WriteFileMetaData(*file_metadata_, sink_.get());
 } else { // Encrypted file
 CloseEncryptedFile(file_encryption_properties);
Expand DownExpand Up@@ -376,6 +377,15 @@ class FileSerializer : public ParquetFileWriter::Contents {

 RowGroupWriter* AppendBufferedRowGroup() override { return AppendRowGroup(true); }

 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) override {
 if (key_value_metadata_ == nullptr) {
 key_value_metadata_ = std::move(key_value_metadata);
 } else if (key_value_metadata != nullptr) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
Comment thread
wgtmac marked this conversation as resolved.


Outdated

 }
 }

 ~FileSerializer() override {
 try {
 Close();
Expand All@@ -394,7 +404,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 properties_(std::move(properties)),
 num_row_groups_(0),
 num_rows_(0),
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_, key_value_metadata_)) {
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_)) {
 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 if (position == 0) {
 StartFile();
Expand All@@ -407,7 +417,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 // Encrypted file with encrypted footer
 if (file_encryption_properties->encrypted_footer()) {
 // encrypted footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);

 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 uint64_t metadata_start = static_cast<uint64_t>(position);
Expand All@@ -422,7 +432,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
 PARQUET_THROW_NOT_OK(sink_->Write(kParquetEMagic, 4));
 } else { // Encrypted file with plaintext footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 auto footer_signing_encryptor = file_encryptor_->GetFooterSigningEncryptor();
 WriteEncryptedFileMetadata(*file_metadata_, sink_.get(), footer_signing_encryptor,
 false);
Expand DownExpand Up@@ -613,6 +623,15 @@ RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) {
 return AppendRowGroup();
}

void ParquetFileWriter::AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 if (contents_) {
 contents_->AddKeyValueMetadata(key_value_metadata);
 } else {
 throw ParquetException("Cannot add key-value metadata to closed file");
 }
}

const std::shared_ptr<WriterProperties>& ParquetFileWriter::properties() const {
 return contents_->properties();
}
Expand Down

10 changes: 10 additions & 0 deletions
cpp/src/parquet/file_writer.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -163,6 +163,9 @@ class PARQUET_EXPORT ParquetFileWriter {
 return key_value_metadata_;
 }

 virtual void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;

 // Return const-pointer to make it clear that this object is not to be copied
 const SchemaDescriptor* schema() const { return &schema_; }

Expand DownExpand Up@@ -209,6 +212,13 @@ class PARQUET_EXPORT ParquetFileWriter {
 /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
 RowGroupWriter* AppendBufferedRowGroup();

 /// \brief Add key-value metadata to the file.
 /// \param[in] key_value_metadata the metadata to add.
 /// \note This will overwrite any existing metadata with the same key.
 /// \throw ParquetException if Close() has been called.
 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);

 /// Number of columns.
 ///
 /// This number is fixed during the lifetime of the writer as it is determined via
Expand Down

24 changes: 19 additions & 5 deletions
cpp/src/parquet/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1740,7 +1740,6 @@ void RowGroupMetaDataBuilder::Finish(int64_t total_bytes_written,
}

// file metadata
// TODO(PARQUET-595) Support key_value_metadata
class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 public:
 explicit FileMetaDataBuilderImpl(
Expand DownExpand Up@@ -1797,15 +1796,21 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 }
 }

 std::unique_ptr<FileMetaData> Finish() {
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 int64_t total_rows = 0;
 for (auto row_group : row_groups_) {
 total_rows += row_group.num_rows;
 }
 metadata_->__set_num_rows(total_rows);
 metadata_->__set_row_groups(row_groups_);

 if (key_value_metadata_) {
 if (key_value_metadata_ || key_value_metadata) {
 if (!key_value_metadata_) {
 key_value_metadata_ = key_value_metadata;
 } else if (key_value_metadata) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
 }
 metadata_->key_value_metadata.clear();
 metadata_->key_value_metadata.reserve(key_value_metadata_->size());
 for (int64_t i = 0; i < key_value_metadata_->size(); ++i) {
Expand All@@ -1829,7 +1834,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 metadata_->__set_version(file_version);
 metadata_->__set_created_by(properties_->created_by());

 // Users cannot set the `ColumnOrder` since we donot not have user defined sort order
 // Users cannot set the `ColumnOrder` since we do not have user defined sort order
 // in the spec yet.
 // We always default to `TYPE_DEFINED_ORDER`. We can expose it in
 // the API once we have user defined sort orders in the Parquet format.
Expand DownExpand Up@@ -1910,6 +1915,12 @@ std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 new FileMetaDataBuilder(schema, std::move(props), std::move(key_value_metadata)));
}

std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props) {
 return std::unique_ptr<FileMetaDataBuilder>(
 new FileMetaDataBuilder(schema, std::move(props)));
}

FileMetaDataBuilder::FileMetaDataBuilder(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata)
Expand All@@ -1926,7 +1937,10 @@ void FileMetaDataBuilder::SetPageIndexLocation(const PageIndexLocation& location
 impl_->SetPageIndexLocation(location);
}

std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish() { return impl_->Finish(); }
std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 return impl_->Finish(key_value_metadata);
}

std::unique_ptr<FileCryptoMetaData> FileMetaDataBuilder::GetCryptoMetaData() {
 return impl_->BuildFileCryptoMetaData();
Expand Down

11 changes: 8 additions & 3 deletions
cpp/src/parquet/metadata.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -523,10 +523,14 @@ struct PageIndexLocation {

class PARQUET_EXPORT FileMetaDataBuilder {
 public:
// API convenience to get a MetaData reader
ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
@wgtmacwgtmacApr 12, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


Suggested change

ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
ARROW_DEPRECATED("Deprecated in 13.0.0. Use overload without KeyValueMetadata instead.")
Should I change this since it may not be included in the 12.0.0 release? @wjones127
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
 std::shared_ptr<const KeyValueMetadata> key_value_metadata);

 // API convenience to get a MetaData builder
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);

 ~FileMetaDataBuilder();

Expand All@@ -537,7 +541,8 @@ class PARQUET_EXPORT FileMetaDataBuilder {
 void SetPageIndexLocation(const PageIndexLocation& location);

 // Complete the Thrift structure
 std::unique_ptr<FileMetaData> Finish();
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);

 // crypto metadata
 std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
Expand Down

51 changes: 49 additions & 2 deletions
cpp/src/parquet/metadata_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

#include "arrow/util/key_value_metadata.h"
#include "parquet/file_reader.h"
#include "parquet/file_writer.h"
#include "parquet/schema.h"
#include "parquet/statistics.h"
#include "parquet/test_util.h"
Expand DownExpand Up@@ -284,16 +285,62 @@ TEST(Metadata, TestKeyValueMetadata) {
 auto kvmeta = std::make_shared<KeyValueMetadata>();
 kvmeta->Append("test_key", "test_value");

 auto f_builder = FileMetaDataBuilder::Make(&schema, props, kvmeta);
 auto f_builder = FileMetaDataBuilder::Make(&schema, props);

 // Read the metadata
 auto f_accessor = f_builder->Finish();
 auto f_accessor = f_builder->Finish(kvmeta);

 // Key value metadata
 ASSERT_TRUE(f_accessor->key_value_metadata());
 EXPECT_TRUE(f_accessor->key_value_metadata()->Equals(*kvmeta));
}

TEST(Metadata, TestAddKeyValueMetadata) {
 schema::NodeVector fields;
 fields.push_back(schema::Int32("int_col", Repetition::REQUIRED));
 auto schema = std::static_pointer_cast<schema::GroupNode>(
 schema::GroupNode::Make("schema", Repetition::REQUIRED, fields));

 auto kv_meta = std::make_shared<KeyValueMetadata>();
 kv_meta->Append("test_key_1", "test_value_1");
 kv_meta->Append("test_key_2", "test_value_2_");

 auto sink = CreateOutputStream();
 auto writer_props = parquet::WriterProperties::Builder().disable_dictionary()->build();
 auto file_writer =
 parquet::ParquetFileWriter::Open(sink, schema, writer_props, kv_meta);

 // Key value metadata that will be added to the file.
 auto kv_meta_added = std::make_shared<KeyValueMetadata>();
 kv_meta_added->Append("test_key_2", "test_value_2");
 kv_meta_added->Append("test_key_3", "test_value_3");

 file_writer->AddKeyValueMetadata(kv_meta_added);
 file_writer->Close();

 // Throw if appending key value metadata to closed file.
 auto kv_meta_ignored = std::make_shared<KeyValueMetadata>();
 kv_meta_ignored->Append("test_key_4", "test_value_4");
 EXPECT_THROW(file_writer->AddKeyValueMetadata(kv_meta_ignored), ParquetException);

 PARQUET_ASSIGN_OR_THROW(auto buffer, sink->Finish());
 auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
 auto file_reader = ParquetFileReader::Open(source);

 ASSERT_NE(nullptr, file_reader->metadata());
 ASSERT_NE(nullptr, file_reader->metadata()->key_value_metadata());
 auto read_kv_meta = file_reader->metadata()->key_value_metadata();

 // Verify keys that were added before file writer was closed are present.
 for (int i = 1; i <= 3; ++i) {
 auto index = std::to_string(i);
 PARQUET_ASSIGN_OR_THROW(auto value, read_kv_meta->Get("test_key_" + index));
 EXPECT_EQ("test_value_" + index, value);
 }
 // Verify keys that were added after file writer was closed are not present.
 EXPECT_FALSE(read_kv_meta->Contains("test_key_4"));
}

TEST(Metadata, TestHasBloomFilter) {
 std::string dir_string(parquet::test::get_data_dir());
 std::string path = dir_string + "/data_index_bloom_encoding_stats.parquet";
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
GH-34888: [C++][Parquet] Writer supports adding extra kv meta#34889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

wjones127
merged 5 commits into

from

Apr 13, 2023
 Merged


GH-34888: [C++][Parquet] Writer supports adding extra kv meta
#34889
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b84556c

GH-34888: [C++][Parquet] Writer supports adding extra kv meta


wgtmac Apr 4, 2023
bed0661

address comments


wgtmac Apr 9, 2023
6f6c41c

make it backward compatible


wgtmac Apr 10, 2023
9b0d022

try to fix doxygen issue


wgtmac Apr 11, 2023
5e5a39c

Update cpp/src/parquet/file_writer.cc


wgtmac Apr 12, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


29 changes: 24 additions & 5 deletions
cpp/src/parquet/file_writer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

#include "parquet/file_writer.h"

#include <cstddef>
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>

#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "parquet/column_writer.h"
#include "parquet/encryption/encryption_internal.h"
#include "parquet/encryption/internal_file_encryptor.h"
Expand DownExpand Up@@ -338,7 +339,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 auto file_encryption_properties = properties_->file_encryption_properties();

 if (file_encryption_properties == nullptr) { // Non encrypted file.
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 WriteFileMetaData(*file_metadata_, sink_.get());
 } else { // Encrypted file
 CloseEncryptedFile(file_encryption_properties);
Expand DownExpand Up@@ -376,6 +377,15 @@ class FileSerializer : public ParquetFileWriter::Contents {

 RowGroupWriter* AppendBufferedRowGroup() override { return AppendRowGroup(true); }

 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) override {
 if (key_value_metadata_ == nullptr) {
 key_value_metadata_ = std::move(key_value_metadata);
 } else if (key_value_metadata != nullptr) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
Comment thread
wgtmac marked this conversation as resolved.


Outdated

 }
 }

 ~FileSerializer() override {
 try {
 Close();
Expand All@@ -394,7 +404,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 properties_(std::move(properties)),
 num_row_groups_(0),
 num_rows_(0),
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_, key_value_metadata_)) {
 metadata_(FileMetaDataBuilder::Make(&schema_, properties_)) {
 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 if (position == 0) {
 StartFile();
Expand All@@ -407,7 +417,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 // Encrypted file with encrypted footer
 if (file_encryption_properties->encrypted_footer()) {
 // encrypted footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);

 PARQUET_ASSIGN_OR_THROW(int64_t position, sink_->Tell());
 uint64_t metadata_start = static_cast<uint64_t>(position);
Expand All@@ -422,7 +432,7 @@ class FileSerializer : public ParquetFileWriter::Contents {
 sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
 PARQUET_THROW_NOT_OK(sink_->Write(kParquetEMagic, 4));
 } else { // Encrypted file with plaintext footer
 file_metadata_ = metadata_->Finish();
 file_metadata_ = metadata_->Finish(key_value_metadata_);
 auto footer_signing_encryptor = file_encryptor_->GetFooterSigningEncryptor();
 WriteEncryptedFileMetadata(*file_metadata_, sink_.get(), footer_signing_encryptor,
 false);
Expand DownExpand Up@@ -613,6 +623,15 @@ RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) {
 return AppendRowGroup();
}

void ParquetFileWriter::AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 if (contents_) {
 contents_->AddKeyValueMetadata(key_value_metadata);
 } else {
 throw ParquetException("Cannot add key-value metadata to closed file");
 }
}

const std::shared_ptr<WriterProperties>& ParquetFileWriter::properties() const {
 return contents_->properties();
}
Expand Down

10 changes: 10 additions & 0 deletions
cpp/src/parquet/file_writer.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -163,6 +163,9 @@ class PARQUET_EXPORT ParquetFileWriter {
 return key_value_metadata_;
 }

 virtual void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;

 // Return const-pointer to make it clear that this object is not to be copied
 const SchemaDescriptor* schema() const { return &schema_; }

Expand DownExpand Up@@ -209,6 +212,13 @@ class PARQUET_EXPORT ParquetFileWriter {
 /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
 RowGroupWriter* AppendBufferedRowGroup();

 /// \brief Add key-value metadata to the file.
 /// \param[in] key_value_metadata the metadata to add.
 /// \note This will overwrite any existing metadata with the same key.
 /// \throw ParquetException if Close() has been called.
 void AddKeyValueMetadata(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);

 /// Number of columns.
 ///
 /// This number is fixed during the lifetime of the writer as it is determined via
Expand Down

24 changes: 19 additions & 5 deletions
cpp/src/parquet/metadata.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1740,7 +1740,6 @@ void RowGroupMetaDataBuilder::Finish(int64_t total_bytes_written,
}

// file metadata
// TODO(PARQUET-595) Support key_value_metadata
class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 public:
 explicit FileMetaDataBuilderImpl(
Expand DownExpand Up@@ -1797,15 +1796,21 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 }
 }

 std::unique_ptr<FileMetaData> Finish() {
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 int64_t total_rows = 0;
 for (auto row_group : row_groups_) {
 total_rows += row_group.num_rows;
 }
 metadata_->__set_num_rows(total_rows);
 metadata_->__set_row_groups(row_groups_);

 if (key_value_metadata_) {
 if (key_value_metadata_ || key_value_metadata) {
 if (!key_value_metadata_) {
 key_value_metadata_ = key_value_metadata;
 } else if (key_value_metadata) {
 key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata);
 }
 metadata_->key_value_metadata.clear();
 metadata_->key_value_metadata.reserve(key_value_metadata_->size());
 for (int64_t i = 0; i < key_value_metadata_->size(); ++i) {
Expand All@@ -1829,7 +1834,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
 metadata_->__set_version(file_version);
 metadata_->__set_created_by(properties_->created_by());

 // Users cannot set the `ColumnOrder` since we donot not have user defined sort order
 // Users cannot set the `ColumnOrder` since we do not have user defined sort order
 // in the spec yet.
 // We always default to `TYPE_DEFINED_ORDER`. We can expose it in
 // the API once we have user defined sort orders in the Parquet format.
Expand DownExpand Up@@ -1910,6 +1915,12 @@ std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 new FileMetaDataBuilder(schema, std::move(props), std::move(key_value_metadata)));
}

std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props) {
 return std::unique_ptr<FileMetaDataBuilder>(
 new FileMetaDataBuilder(schema, std::move(props)));
}

FileMetaDataBuilder::FileMetaDataBuilder(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata)
Expand All@@ -1926,7 +1937,10 @@ void FileMetaDataBuilder::SetPageIndexLocation(const PageIndexLocation& location
 impl_->SetPageIndexLocation(location);
}

std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish() { return impl_->Finish(); }
std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
 return impl_->Finish(key_value_metadata);
}

std::unique_ptr<FileCryptoMetaData> FileMetaDataBuilder::GetCryptoMetaData() {
 return impl_->BuildFileCryptoMetaData();
Expand Down

11 changes: 8 additions & 3 deletions
cpp/src/parquet/metadata.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -523,10 +523,14 @@ struct PageIndexLocation {

class PARQUET_EXPORT FileMetaDataBuilder {
 public:
// API convenience to get a MetaData reader
ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
@wgtmacwgtmacApr 12, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


Suggested change

ARROW_DEPRECATED("Deprecated in 12.0.0. Use overload without KeyValueMetadata instead.")
ARROW_DEPRECATED("Deprecated in 13.0.0. Use overload without KeyValueMetadata instead.")
Should I change this since it may not be included in the 12.0.0 release? @wjones127
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props,
 std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
 std::shared_ptr<const KeyValueMetadata> key_value_metadata);

 // API convenience to get a MetaData builder
 static std::unique_ptr<FileMetaDataBuilder> Make(
 const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);

 ~FileMetaDataBuilder();

Expand All@@ -537,7 +541,8 @@ class PARQUET_EXPORT FileMetaDataBuilder {
 void SetPageIndexLocation(const PageIndexLocation& location);

 // Complete the Thrift structure
 std::unique_ptr<FileMetaData> Finish();
 std::unique_ptr<FileMetaData> Finish(
 const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);

 // crypto metadata
 std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
Expand Down

51 changes: 49 additions & 2 deletions
cpp/src/parquet/metadata_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

#include "arrow/util/key_value_metadata.h"
#include "parquet/file_reader.h"
#include "parquet/file_writer.h"
#include "parquet/schema.h"
#include "parquet/statistics.h"
#include "parquet/test_util.h"
Expand DownExpand Up@@ -284,16 +285,62 @@ TEST(Metadata, TestKeyValueMetadata) {
 auto kvmeta = std::make_shared<KeyValueMetadata>();
 kvmeta->Append("test_key", "test_value");

 auto f_builder = FileMetaDataBuilder::Make(&schema, props, kvmeta);
 auto f_builder = FileMetaDataBuilder::Make(&schema, props);

 // Read the metadata
 auto f_accessor = f_builder->Finish();
 auto f_accessor = f_builder->Finish(kvmeta);

 // Key value metadata
 ASSERT_TRUE(f_accessor->key_value_metadata());
 EXPECT_TRUE(f_accessor->key_value_metadata()->Equals(*kvmeta));
}

TEST(Metadata, TestAddKeyValueMetadata) {
 schema::NodeVector fields;
 fields.push_back(schema::Int32("int_col", Repetition::REQUIRED));
 auto schema = std::static_pointer_cast<schema::GroupNode>(
 schema::GroupNode::Make("schema", Repetition::REQUIRED, fields));

 auto kv_meta = std::make_shared<KeyValueMetadata>();
 kv_meta->Append("test_key_1", "test_value_1");
 kv_meta->Append("test_key_2", "test_value_2_");

 auto sink = CreateOutputStream();
 auto writer_props = parquet::WriterProperties::Builder().disable_dictionary()->build();
 auto file_writer =
 parquet::ParquetFileWriter::Open(sink, schema, writer_props, kv_meta);

 // Key value metadata that will be added to the file.
 auto kv_meta_added = std::make_shared<KeyValueMetadata>();
 kv_meta_added->Append("test_key_2", "test_value_2");
 kv_meta_added->Append("test_key_3", "test_value_3");

 file_writer->AddKeyValueMetadata(kv_meta_added);
 file_writer->Close();

 // Throw if appending key value metadata to closed file.
 auto kv_meta_ignored = std::make_shared<KeyValueMetadata>();
 kv_meta_ignored->Append("test_key_4", "test_value_4");
 EXPECT_THROW(file_writer->AddKeyValueMetadata(kv_meta_ignored), ParquetException);

 PARQUET_ASSIGN_OR_THROW(auto buffer, sink->Finish());
 auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
 auto file_reader = ParquetFileReader::Open(source);

 ASSERT_NE(nullptr, file_reader->metadata());
 ASSERT_NE(nullptr, file_reader->metadata()->key_value_metadata());
 auto read_kv_meta = file_reader->metadata()->key_value_metadata();

 // Verify keys that were added before file writer was closed are present.
 for (int i = 1; i <= 3; ++i) {
 auto index = std::to_string(i);
 PARQUET_ASSIGN_OR_THROW(auto value, read_kv_meta->Get("test_key_" + index));
 EXPECT_EQ("test_value_" + index, value);
 }
 // Verify keys that were added after file writer was closed are not present.
 EXPECT_FALSE(read_kv_meta->Contains("test_key_4"));
}

TEST(Metadata, TestHasBloomFilter) {
 std::string dir_string(parquet::test::get_data_dir());
 std::string path = dir_string + "/data_index_bloom_encoding_stats.parquet";
Expand Down