diff --git a/be/benchmark/benchmark_zone_map_index.hpp b/be/benchmark/benchmark_zone_map_index.hpp index 2fe0d41733b9a2..a93ccb010a17b4 100644 --- a/be/benchmark/benchmark_zone_map_index.hpp +++ b/be/benchmark/benchmark_zone_map_index.hpp @@ -34,7 +34,6 @@ #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" -#include "storage/field.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/tablet/tablet_schema.h" #include "util/slice.h" @@ -116,9 +115,8 @@ std::unique_ptr make_writer() { col = make_column(FieldType::OLAP_FIELD_TYPE_VARCHAR, 64, 1); dtype = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false, 0, 0, 64); } - std::unique_ptr field(StorageFieldFactory::create(*col)); std::unique_ptr w; - (void)ZoneMapIndexWriter::create(dtype, field.get(), w); + (void)ZoneMapIndexWriter::create(dtype, col.get(), w); return w; } diff --git a/be/src/core/data_type/data_type_factory.cpp b/be/src/core/data_type/data_type_factory.cpp index 7ccf7a6cd00560..256493cba04e34 100644 --- a/be/src/core/data_type/data_type_factory.cpp +++ b/be/src/core/data_type/data_type_factory.cpp @@ -65,14 +65,9 @@ #include "core/data_type/define_primitive_type.h" #include "core/types.h" #include "core/uint128.h" -#include "storage/field.h" #include "storage/olap_common.h" namespace doris { -DataTypePtr DataTypeFactory::create_data_type(const doris::StorageField& col_desc) { - return create_data_type(col_desc.get_desc(), col_desc.is_nullable()); -} - DataTypePtr DataTypeFactory::create_data_type(const TabletColumn& col_desc, bool is_nullable) { DataTypePtr nested = nullptr; if (col_desc.type() == FieldType::OLAP_FIELD_TYPE_AGG_STATE) { diff --git a/be/src/core/data_type/data_type_factory.hpp b/be/src/core/data_type/data_type_factory.hpp index 15c55a99450063..e169375e3fec8c 100644 --- a/be/src/core/data_type/data_type_factory.hpp +++ b/be/src/core/data_type/data_type_factory.hpp @@ -33,7 +33,6 @@ namespace arrow { class DataType; } // namespace arrow namespace doris { -class StorageField; class PColumnMeta; enum class FieldType; @@ -52,7 +51,6 @@ class DataTypeFactory { return instance; } - DataTypePtr create_data_type(const doris::StorageField& col_desc); DataTypePtr create_data_type(const TabletColumn& col_desc, bool is_nullable = false); DataTypePtr create_data_type(const PColumnMeta& pcolumn); diff --git a/be/src/core/value/map_value.h b/be/src/core/value/map_value.h deleted file mode 100644 index 68480cdbcb5e26..00000000000000 --- a/be/src/core/value/map_value.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -namespace doris { - -/** - * MapValue is for map type in memory - */ -class MapValue { -public: - MapValue() = default; - - explicit MapValue(int32_t length) : _key_data(nullptr), _value_data(nullptr), _length(length) {} - - MapValue(void* k_data, void* v_data, int32_t length) - : _key_data(k_data), _value_data(v_data), _length(length) {} - - int32_t size() const { return _length; } - - int32_t length() const { return _length; } - - const void* key_data() const { return _key_data; } - void* mutable_key_data() const { return _key_data; } - const void* value_data() const { return _value_data; } - void* mutable_value_data() const { return _value_data; } - - void set_length(int32_t length) { _length = length; } - void set_key(void* data) { _key_data = data; } - void set_value(void* data) { _value_data = data; } - -private: - // child column data pointer - void* _key_data = nullptr; - void* _value_data = nullptr; - // length for map size - int32_t _length; - -}; //map-value -} // namespace doris diff --git a/be/src/core/value/struct_value.h b/be/src/core/value/struct_value.h deleted file mode 100644 index fa79a3e0a9bd87..00000000000000 --- a/be/src/core/value/struct_value.h +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -namespace doris { - -class StructValue { -public: - StructValue() = default; - - explicit StructValue(uint32_t size) : _values(nullptr), _size(size), _has_null(false) {} - StructValue(void** values, uint32_t size) : _values(values), _size(size), _has_null(false) {} - StructValue(void** values, uint32_t size, bool has_null) - : _values(values), _size(size), _has_null(has_null) {} - - //void to_struct_val(StructVal* val) const; - //static StructValue from_struct_val(const StructVal& val); - - uint32_t size() const { return _size; } - void set_size(uint32_t size) { _size = size; } - bool has_null() const { return _has_null; } - void set_has_null(bool has_null) { _has_null = has_null; } - bool is_null_at(uint32_t index) const { - return this->_has_null && this->_values[index] == nullptr; - } - - const void** values() const { return const_cast(_values); } - void** mutable_values() { return _values; } - void set_values(void** values) { _values = values; } - const void* child_value(uint32_t index) const { return _values[index]; } - void* mutable_child_value(uint32_t index) { return _values[index]; } - void set_child_value(void* value, uint32_t index) { _values[index] = value; } - -private: - // pointer to the start of the vector of children pointers. These pointers are - // point to children values where a null pointer means that this child is NULL. - void** _values = nullptr; - // the number of values in this struct value. - uint32_t _size; - // child has no null value if has_null is false. - // child may has null value if has_null is true. - bool _has_null; -}; -} // namespace doris \ No newline at end of file diff --git a/be/src/io/cache/cache_block_meta_store.cpp b/be/src/io/cache/cache_block_meta_store.cpp index 369d81537969e0..fa82cee3ce99e8 100644 --- a/be/src/io/cache/cache_block_meta_store.cpp +++ b/be/src/io/cache/cache_block_meta_store.cpp @@ -33,8 +33,6 @@ #include "common/status.h" #include "exec/common/hex.h" -#include "storage/field.h" -#include "storage/field.h" // For OLAP_FIELD_TYPE_BIGINT #include "storage/key_coder.h" #include "storage/olap_common.h" #include "util/threadpool.h" diff --git a/be/src/runtime/collection_value.cpp b/be/src/runtime/collection_value.cpp deleted file mode 100644 index 1d501720695f38..00000000000000 --- a/be/src/runtime/collection_value.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "runtime/collection_value.h" - -#include - -namespace doris { - -void CollectionValue::shallow_copy(const CollectionValue* value) { - _length = value->_length; - _null_signs = value->_null_signs; - _data = value->_data; - _has_null = value->_has_null; -} - -void CollectionValue::copy_null_signs(const CollectionValue* other) { - if (other->_has_null) { - memcpy(_null_signs, other->_null_signs, other->size()); - } else { - _null_signs = nullptr; - } -} - -} // namespace doris diff --git a/be/src/runtime/collection_value.h b/be/src/runtime/collection_value.h deleted file mode 100644 index da916a9a1ae357..00000000000000 --- a/be/src/runtime/collection_value.h +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include -#include - -namespace doris { - -using MemFootprint = std::pair; -using GenMemFootprintFunc = std::function; - -/** - * The format of array-typed slot. - * A new array needs to be initialized before using it. - */ -class CollectionValue { -public: - CollectionValue() = default; - - explicit CollectionValue(uint64_t length) - : _data(nullptr), _length(length), _has_null(false), _null_signs(nullptr) {} - - CollectionValue(void* data, uint64_t length) - : _data(data), _length(length), _has_null(false), _null_signs(nullptr) {} - - CollectionValue(void* data, uint64_t length, bool* null_signs) - : _data(data), _length(length), _has_null(true), _null_signs(null_signs) {} - - CollectionValue(void* data, uint64_t length, bool has_null, bool* null_signs) - : _data(data), _length(length), _has_null(has_null), _null_signs(null_signs) {} - - bool is_null_at(uint64_t index) const { return this->_has_null && this->_null_signs[index]; } - - uint64_t size() const { return _length; } - - uint64_t length() const { return _length; } - - void shallow_copy(const CollectionValue* other); - - void copy_null_signs(const CollectionValue* other); - - const void* data() const { return _data; } - bool has_null() const { return _has_null; } - const bool* null_signs() const { return _null_signs; } - void* mutable_data() { return _data; } - bool* mutable_null_signs() { return _null_signs; } - void set_length(uint64_t length) { _length = length; } - void set_has_null(bool has_null) { _has_null = has_null; } - void set_data(void* data) { _data = data; } - void set_null_signs(bool* null_signs) { _null_signs = null_signs; } - -private: - // child column data - void* _data = nullptr; - uint64_t _length = 0; - // item has no null value if has_null is false. - // item ```may``` has null value if has_null is true. - bool _has_null = false; - // null bitmap - bool* _null_signs = nullptr; -}; -} // namespace doris diff --git a/be/src/storage/field.h b/be/src/storage/field.h deleted file mode 100644 index 3fa84c36f7b42d..00000000000000 --- a/be/src/storage/field.h +++ /dev/null @@ -1,382 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "core/arena.h" -#include "core/value/map_value.h" -#include "runtime/collection_value.h" -#include "storage/key_coder.h" -#include "storage/olap_common.h" -#include "storage/olap_define.h" -#include "storage/tablet/tablet_schema.h" -#include "storage/types.h" -#include "storage/utils.h" -#include "util/hash_util.hpp" -#include "util/json/path_in_data.h" -#include "util/slice.h" - -namespace doris { -// A Field is used to represent a column in memory format. -// User can use this class to access or deal with column data in memory. -class StorageField { -public: - StorageField(const TabletColumn& column) - : _type(column.type()), - _desc(column), - _length(column.length()), - _key_coder(get_key_coder(column.type())), - _name(column.name()), - _index_size(column.index_length()), - _is_nullable(column.is_nullable()), - _unique_id(column.unique_id()), - _parent_unique_id(column.parent_unique_id()), - _is_extracted_column(column.is_extracted_column()), - _path(column.path_info_ptr()) {} - - virtual ~StorageField() = default; - - size_t size() const { return field_type_size(_type); } - size_t length() const { return _length; } - size_t field_size() const { return size() + 1; } - size_t index_size() const { return _index_size; } - int32_t unique_id() const { return _unique_id; } - int32_t parent_unique_id() const { return _parent_unique_id; } - bool is_extracted_column() const { return _is_extracted_column; } - const std::string& name() const { return _name; } - const PathInDataPtr& path() const { return _path; } - - virtual StorageField* clone() const { - auto* local = new StorageField(_desc); - this->clone(local); - return local; - } - - FieldType type() const { return _type; } - bool is_nullable() const { return _is_nullable; } - - // similar to `full_encode_ascending`, but only encode part (the first `index_size` bytes) of the value. - // only applicable to string type - void encode_ascending(const void* value, std::string* buf) const { - _key_coder->encode_ascending(value, _index_size, buf); - } - - // encode the provided `value` into `buf`. - void full_encode_ascending(const void* value, std::string* buf) const { - _key_coder->full_encode_ascending(value, buf); - } - - const KeyCoder* key_coder() const { return _key_coder; } - void add_sub_field(std::unique_ptr sub_field) { - _sub_fields.emplace_back(std::move(sub_field)); - } - StorageField* get_sub_field(size_t i) const { return _sub_fields[i].get(); } - size_t get_sub_field_count() const { return _sub_fields.size(); } - - void set_precision(int32_t precision) { _precision = precision; } - void set_scale(int32_t scale) { _scale = scale; } - int32_t get_precision() const { return _precision; } - int32_t get_scale() const { return _scale; } - const TabletColumn& get_desc() const { return _desc; } - - int32_t get_unique_id() const { - return is_extracted_column() ? parent_unique_id() : unique_id(); - } - -protected: - FieldType _type; - TabletColumn _desc; - // unit : byte - // except for strings, other types have fixed lengths - // Note that, the struct type itself has fixed length, but due to - // its number of subfields is a variable, so the actual length of - // a struct field is not fixed. - size_t _length; - - void clone(StorageField* other) const { - other->_type = this->_type; - other->_key_coder = this->_key_coder; - other->_name = this->_name; - other->_index_size = this->_index_size; - other->_is_nullable = this->_is_nullable; - other->_sub_fields.clear(); - other->_precision = this->_precision; - other->_scale = this->_scale; - other->_unique_id = this->_unique_id; - other->_parent_unique_id = this->_parent_unique_id; - other->_is_extracted_column = this->_is_extracted_column; - for (const auto& f : _sub_fields) { - StorageField* item = f->clone(); - other->add_sub_field(std::unique_ptr(item)); - } - } - -private: - // maximum length of Field, unit : bytes - // usually equal to length, except for variable-length strings - const KeyCoder* _key_coder; - std::string _name; - size_t _index_size; - bool _is_nullable; - std::vector> _sub_fields; - int32_t _precision; - int32_t _scale; - int32_t _unique_id; - int32_t _parent_unique_id; - bool _is_extracted_column = false; - PathInDataPtr _path; -}; - -class MapField : public StorageField { -public: - MapField(const TabletColumn& column) : StorageField(column) {} -}; - -class StructField : public StorageField { -public: - StructField(const TabletColumn& column) : StorageField(column) {} -}; - -class ArrayField : public StorageField { -public: - ArrayField(const TabletColumn& column) : StorageField(column) {} -}; - -class CharField : public StorageField { -public: - CharField(const TabletColumn& column) : StorageField(column) {} - - CharField* clone() const override { - auto* local = new CharField(_desc); - StorageField::clone(local); - return local; - } -}; - -class VarcharField : public StorageField { -public: - VarcharField(const TabletColumn& column) : StorageField(column) {} - - VarcharField* clone() const override { - auto* local = new VarcharField(_desc); - StorageField::clone(local); - return local; - } -}; -class StringField : public StorageField { -public: - StringField(const TabletColumn& column) : StorageField(column) {} - - StringField* clone() const override { - auto* local = new StringField(_desc); - StorageField::clone(local); - return local; - } -}; - -class BitmapAggField : public StorageField { -public: - BitmapAggField(const TabletColumn& column) : StorageField(column) {} - - BitmapAggField* clone() const override { - auto* local = new BitmapAggField(_desc); - StorageField::clone(local); - return local; - } -}; - -class QuantileStateAggField : public StorageField { -public: - QuantileStateAggField(const TabletColumn& column) : StorageField(column) {} - - QuantileStateAggField* clone() const override { - auto* local = new QuantileStateAggField(_desc); - StorageField::clone(local); - return local; - } -}; - -class AggStateField : public StorageField { -public: - AggStateField(const TabletColumn& column) : StorageField(column) {} - - AggStateField* clone() const override { - auto* local = new AggStateField(_desc); - StorageField::clone(local); - return local; - } -}; - -class HllAggField : public StorageField { -public: - HllAggField(const TabletColumn& column) : StorageField(column) {} - - HllAggField* clone() const override { - auto* local = new HllAggField(_desc); - StorageField::clone(local); - return local; - } -}; - -class StorageFieldFactory { -public: - static StorageField* create(const TabletColumn& column) { - // for key column - if (column.is_key()) { - switch (column.type()) { - case FieldType::OLAP_FIELD_TYPE_CHAR: - return new CharField(column); - case FieldType::OLAP_FIELD_TYPE_VARCHAR: - case FieldType::OLAP_FIELD_TYPE_STRING: - return new StringField(column); - case FieldType::OLAP_FIELD_TYPE_STRUCT: { - auto* local = new StructField(column); - for (uint32_t i = 0; i < column.get_subtype_count(); i++) { - std::unique_ptr sub_field( - StorageFieldFactory::create(column.get_sub_column(i))); - local->add_sub_field(std::move(sub_field)); - } - return local; - } - case FieldType::OLAP_FIELD_TYPE_ARRAY: { - std::unique_ptr item_field( - StorageFieldFactory::create(column.get_sub_column(0))); - auto* local = new ArrayField(column); - local->add_sub_field(std::move(item_field)); - return local; - } - case FieldType::OLAP_FIELD_TYPE_MAP: { - std::unique_ptr key_field( - StorageFieldFactory::create(column.get_sub_column(0))); - std::unique_ptr val_field( - StorageFieldFactory::create(column.get_sub_column(1))); - auto* local = new MapField(column); - local->add_sub_field(std::move(key_field)); - local->add_sub_field(std::move(val_field)); - return local; - } - case FieldType::OLAP_FIELD_TYPE_DECIMAL: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL32: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL64: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL256: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: { - StorageField* field = new StorageField(column); - field->set_precision(column.precision()); - field->set_scale(column.frac()); - return field; - } - default: - return new StorageField(column); - } - } - - // for value column - switch (column.aggregation()) { - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_SUM: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_MIN: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_MAX: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL: - switch (column.type()) { - case FieldType::OLAP_FIELD_TYPE_CHAR: - return new CharField(column); - case FieldType::OLAP_FIELD_TYPE_VARCHAR: - return new VarcharField(column); - case FieldType::OLAP_FIELD_TYPE_STRING: - return new StringField(column); - case FieldType::OLAP_FIELD_TYPE_STRUCT: { - auto* local = new StructField(column); - for (uint32_t i = 0; i < column.get_subtype_count(); i++) { - std::unique_ptr sub_field( - StorageFieldFactory::create(column.get_sub_column(i))); - local->add_sub_field(std::move(sub_field)); - } - return local; - } - case FieldType::OLAP_FIELD_TYPE_ARRAY: { - std::unique_ptr item_field( - StorageFieldFactory::create(column.get_sub_column(0))); - auto* local = new ArrayField(column); - local->add_sub_field(std::move(item_field)); - return local; - } - case FieldType::OLAP_FIELD_TYPE_MAP: { - DCHECK(column.get_subtype_count() == 2); - auto* local = new MapField(column); - std::unique_ptr key_field( - StorageFieldFactory::create(column.get_sub_column(0))); - std::unique_ptr value_field( - StorageFieldFactory::create(column.get_sub_column(1))); - local->add_sub_field(std::move(key_field)); - local->add_sub_field(std::move(value_field)); - return local; - } - case FieldType::OLAP_FIELD_TYPE_DECIMAL: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL32: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL64: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL256: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: { - StorageField* field = new StorageField(column); - field->set_precision(column.precision()); - field->set_scale(column.frac()); - return field; - } - default: - return new StorageField(column); - } - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_HLL_UNION: - return new HllAggField(column); - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_BITMAP_UNION: - return new BitmapAggField(column); - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_QUANTILE_UNION: - return new QuantileStateAggField(column); - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_GENERIC: - return new AggStateField(column); - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_UNKNOWN: - CHECK(false) << ", value column no agg type"; - return nullptr; - } - return nullptr; - } - - static StorageField* create_by_type(const FieldType& type) { - TabletColumn column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, type); - return create(column); - } -}; -} // namespace doris diff --git a/be/src/storage/index/ann/ann_index_writer.cpp b/be/src/storage/index/ann/ann_index_writer.cpp index e441f1a2c4dd27..28d348cc319a48 100644 --- a/be/src/storage/index/ann/ann_index_writer.cpp +++ b/be/src/storage/index/ann/ann_index_writer.cpp @@ -133,11 +133,6 @@ Status AnnIndexColumnWriter::add_array_values(size_t field_size, const void* val return Status::OK(); } -Status AnnIndexColumnWriter::add_array_values(size_t field_size, const CollectionValue* values, - size_t count) { - return Status::InternalError("Ann index should not be used on nullable column"); -} - Status AnnIndexColumnWriter::add_nulls(uint32_t count) { return Status::InternalError("Ann index should not be used on nullable column"); } diff --git a/be/src/storage/index/ann/ann_index_writer.h b/be/src/storage/index/ann/ann_index_writer.h index 1625fa65c6559b..7b7e63f8574439 100644 --- a/be/src/storage/index/ann/ann_index_writer.h +++ b/be/src/storage/index/ann/ann_index_writer.h @@ -29,7 +29,6 @@ #include "common/config.h" #include "core/pod_array.h" -#include "runtime/collection_value.h" #include "storage/index/ann/ann_index.h" #include "storage/index/index_file_writer.h" #include "storage/index/index_writer.h" @@ -68,8 +67,6 @@ class AnnIndexColumnWriter : public IndexColumnWriter { Status add_values(const std::string fn, const void* values, size_t count) override; Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* null_map, const uint8_t* offsets_ptr, size_t count) override; - Status add_array_values(size_t field_size, const CollectionValue* values, - size_t count) override; int64_t size() const override; Status finish() override; diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp index d64d6b56f56aed..2325d280471337 100644 --- a/be/src/storage/index/index_writer.cpp +++ b/be/src/storage/index/index_writer.cpp @@ -16,9 +16,10 @@ // under the License. #include "common/exception.h" -#include "storage/field.h" #include "storage/index/ann/ann_index_writer.h" #include "storage/index/inverted/inverted_index_writer.h" +#include "storage/tablet/tablet_schema.h" +#include "storage/types.h" namespace doris::segment_v2 { @@ -45,33 +46,33 @@ bool IndexColumnWriter::check_support_ann_index(const TabletColumn& column) { } // create index writer -Status IndexColumnWriter::create(const StorageField* field, std::unique_ptr* res, +Status IndexColumnWriter::create(const TabletColumn* column, + std::unique_ptr* res, IndexFileWriter* index_file_writer, const TabletIndex* index_meta) { - FieldType type = field->type(); + FieldType type = column->type(); std::string field_name; auto storage_format = index_file_writer->get_storage_format(); if (storage_format == InvertedIndexStorageFormatPB::V1) { - field_name = field->name(); + field_name = column->name(); } else { - if (field->is_extracted_column()) { + if (column->is_extracted_column()) { // variant sub col // field_name format: parent_unique_id.sub_col_name - field_name = std::to_string(field->parent_unique_id()) + "." + field->name(); + field_name = std::to_string(column->parent_unique_id()) + "." + column->name(); } else { - field_name = std::to_string(field->unique_id()); + field_name = std::to_string(column->unique_id()); } } if (index_meta->is_inverted_index()) { bool single_field = true; if (type == FieldType::OLAP_FIELD_TYPE_ARRAY) { - const auto& column = field->get_desc(); - bool has_item_subcolumn = column.get_subtype_count() > 0; + bool has_item_subcolumn = column->get_subtype_count() > 0; DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_array_typeinfo_is_nullptr", { has_item_subcolumn = false; }) if (has_item_subcolumn) { - type = column.get_sub_column(0).type(); + type = column->get_sub_column(0).type(); single_field = false; } else { return Status::NotSupported("unsupported array type for inverted index: " + diff --git a/be/src/storage/index/index_writer.h b/be/src/storage/index/index_writer.h index 38fd20122415c2..a0760f99000fcb 100644 --- a/be/src/storage/index/index_writer.h +++ b/be/src/storage/index/index_writer.h @@ -35,10 +35,6 @@ namespace doris { -class CollectionValue; - -class StorageField; - class TabletIndex; class TabletColumn; @@ -47,7 +43,7 @@ class IndexFileWriter; class IndexColumnWriter { public: - static Status create(const StorageField* field, std::unique_ptr* res, + static Status create(const TabletColumn* column, std::unique_ptr* res, IndexFileWriter* index_file_writer, const TabletIndex* inverted_index); virtual Status init() = 0; @@ -55,8 +51,6 @@ class IndexColumnWriter { virtual ~IndexColumnWriter() = default; virtual Status add_values(const std::string name, const void* values, size_t count) = 0; - virtual Status add_array_values(size_t field_size, const CollectionValue* values, - size_t count) = 0; virtual Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* null_map, const uint8_t* offsets_ptr, diff --git a/be/src/storage/index/inverted/inverted_index_reader.cpp b/be/src/storage/index/inverted/inverted_index_reader.cpp index 8eae5aaab662b1..0e83238064297e 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.cpp +++ b/be/src/storage/index/inverted/inverted_index_reader.cpp @@ -44,7 +44,6 @@ #include "core/type_limit.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_reader_helper.h" #include "storage/index/inverted/analyzer/analyzer.h" diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index 89524a9126b984..0f82b5225e666e 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -532,60 +532,6 @@ Status InvertedIndexColumnWriter::add_array_values(size_t field_size return Status::OK(); } -template -Status InvertedIndexColumnWriter::add_array_values(size_t field_size, - const CollectionValue* values, - size_t count) { - if constexpr (field_is_slice_type(field_type)) { - DBUG_EXECUTE_IF("InvertedIndexColumnWriter::add_array_values_field_is_nullptr", - { _field = nullptr; }) - DBUG_EXECUTE_IF( - "InvertedIndexColumnWriter::add_array_values_index_writer_is_" - "nullptr", - { _index_writer = nullptr; }) - if (_field == nullptr || _index_writer == nullptr) { - LOG(ERROR) << "field or index writer is null in inverted index writer."; - return Status::InternalError("field or index writer is null in inverted index writer"); - } - for (int i = 0; i < count; ++i) { - const auto* item_data_ptr = values->data(); - std::vector strings; - - for (size_t j = 0; j < values->length(); ++j) { - auto* v = (Slice*)item_data_ptr; - - if (!values->is_null_at(j)) { - strings.emplace_back(v->get_data(), v->get_size()); - } - item_data_ptr = (uint8_t*)item_data_ptr + field_size; - } - auto value = join(strings, " "); - RETURN_IF_ERROR(new_inverted_index_field(value.c_str(), value.length())); - _rid++; - RETURN_IF_ERROR(add_document()); - values++; - } - } else if constexpr (field_is_numeric_type(field_type)) { - for (int i = 0; i < count; ++i) { - const auto* item_data_ptr = values->data(); - - for (size_t j = 0; j < values->length(); ++j) { - const auto* p = reinterpret_cast(item_data_ptr); - if (values->is_null_at(j)) { - // bkd do not index null values, so we do nothing here. - } else { - RETURN_IF_ERROR(add_value(*p)); - } - item_data_ptr = (uint8_t*)item_data_ptr + field_size; - } - _row_ids_seen_for_bkd++; - _rid++; - values++; - } - } - return Status::OK(); -} - template Status InvertedIndexColumnWriter::add_numeric_values(const void* values, size_t count) { auto p = reinterpret_cast(values); diff --git a/be/src/storage/index/inverted/inverted_index_writer.h b/be/src/storage/index/inverted/inverted_index_writer.h index de0b370acbe268..3ef7b4d9319e39 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.h +++ b/be/src/storage/index/inverted/inverted_index_writer.h @@ -72,8 +72,6 @@ class InvertedIndexColumnWriter : public IndexColumnWriter { Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* nested_null_map, const uint8_t* offsets_ptr, size_t count) override; - Status add_array_values(size_t field_size, const CollectionValue* values, - size_t count) override; Status add_numeric_values(const void* values, size_t count); Status add_value(const CppType& value); int64_t size() const override; diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp index b1735be6543867..632936a20210c5 100644 --- a/be/src/storage/index/zone_map/zone_map_index.cpp +++ b/be/src/storage/index/zone_map/zone_map_index.cpp @@ -32,11 +32,11 @@ #include "core/string_ref.h" #include "core/value/decimalv2_value.h" #include "core/value/vdatetime_value.h" -#include "storage/field.h" #include "storage/index/indexed_column_reader.h" #include "storage/index/indexed_column_writer.h" #include "storage/olap_common.h" #include "storage/segment/encoding_info.h" +#include "storage/tablet/tablet_schema.h" #include "storage/types.h" #include "util/slice.h" #include "util/unaligned.h" @@ -365,9 +365,9 @@ ZoneMapIndexReader::~ZoneMapIndexReader() = default; M(TYPE_DECIMAL128I) \ M(TYPE_DECIMAL256) -Status ZoneMapIndexWriter::create(DataTypePtr data_type, StorageField* field, +Status ZoneMapIndexWriter::create(DataTypePtr data_type, const TabletColumn* column, std::unique_ptr& res) { - switch (field->type()) { + switch (column->type()) { #define M(NAME) \ case FieldType::OLAP_FIELD_##NAME: { \ res.reset(new TypedZoneMapIndexWriter(std::move(data_type))); \ diff --git a/be/src/storage/index/zone_map/zone_map_index.h b/be/src/storage/index/zone_map/zone_map_index.h index 930248de9a0b6b..96d9a7a26e5919 100644 --- a/be/src/storage/index/zone_map/zone_map_index.h +++ b/be/src/storage/index/zone_map/zone_map_index.h @@ -31,8 +31,8 @@ #include "core/data_type/define_primitive_type.h" #include "core/string_ref.h" #include "io/fs/file_reader_writer_fwd.h" -#include "storage/field.h" #include "storage/metadata_adder.h" +#include "storage/tablet/tablet_schema.h" #include "util/once.h" namespace doris { @@ -87,7 +87,7 @@ struct ZoneMap { class ZoneMapIndexWriter { public: - static Status create(DataTypePtr data_type, StorageField* field, + static Status create(DataTypePtr data_type, const TabletColumn* column, std::unique_ptr& res); ZoneMapIndexWriter() = default; diff --git a/be/src/storage/iterator/olap_data_convertor.cpp b/be/src/storage/iterator/olap_data_convertor.cpp index 3f56e91e6f3bee..42b792ab1c7f18 100644 --- a/be/src/storage/iterator/olap_data_convertor.cpp +++ b/be/src/storage/iterator/olap_data_convertor.cpp @@ -994,7 +994,6 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap( _value_convertor->set_source_column(value_typed_column, start_offset, elem_size); RETURN_IF_ERROR(_value_convertor->convert_to_olap()); - // todo (Amory). put this value into MapValue _results[0] = (void*)elem_size; _results[1] = _offsets.data(); _results[2] = _key_convertor->get_data(); diff --git a/be/src/storage/iterator/olap_data_convertor.h b/be/src/storage/iterator/olap_data_convertor.h index 409f73618e5820..82376ac890c32a 100644 --- a/be/src/storage/iterator/olap_data_convertor.h +++ b/be/src/storage/iterator/olap_data_convertor.h @@ -46,7 +46,6 @@ #include "core/string_ref.h" #include "core/types.h" #include "core/uint24.h" -#include "runtime/collection_value.h" #include "util/slice.h" namespace doris { diff --git a/be/src/storage/iterator/vertical_merge_iterator.cpp b/be/src/storage/iterator/vertical_merge_iterator.cpp index 9aedae0b1be082..07b3c957653ead 100644 --- a/be/src/storage/iterator/vertical_merge_iterator.cpp +++ b/be/src/storage/iterator/vertical_merge_iterator.cpp @@ -34,7 +34,6 @@ #include "core/string_ref.h" #include "core/types.h" #include "io/cache/block_file_cache_factory.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/olap_common.h" diff --git a/be/src/storage/iterator/vgeneric_iterators.cpp b/be/src/storage/iterator/vgeneric_iterators.cpp index b18305dae99431..8d3b1bb50ee2f3 100644 --- a/be/src/storage/iterator/vgeneric_iterators.cpp +++ b/be/src/storage/iterator/vgeneric_iterators.cpp @@ -26,7 +26,6 @@ #include "core/block/column_with_type_and_name.h" #include "core/column/column.h" #include "core/data_type/data_type.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/olap_common.h" #include "storage/schema.h" diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index ae960dbc456f91..1fb680ee747cc7 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -115,9 +115,9 @@ struct TabletSize { size_t tablet_size; }; -// Define all data types supported by StorageField. -// If new filed_type is defined, not only new TypeInfo may need be defined, -// but also some functions like get_type_info in types.cpp need to be changed. +// Storage-engine cell types, used by TabletColumn / KeyCoder and the +// data_type traits chain. When adding a new value, also extend CppTypeTraits, +// FieldTypeTraits and the field_type_size() switch in storage/types.h. enum class FieldType { OLAP_FIELD_TYPE_TINYINT = 1, // MYSQL_TYPE_TINY OLAP_FIELD_TYPE_UNSIGNED_TINYINT = 2, @@ -162,10 +162,10 @@ enum class FieldType { OLAP_FIELD_TYPE_TIMESTAMPTZ = 40, }; -// Define all aggregation methods supported by StorageField +// Define all aggregation methods supported by TabletColumn // Note that in practice, not all types can use all the following aggregation methods // For example, it is meaningless to use SUM for the string type (but it will not cause the program to crash) -// The implementation of the StorageField class does not perform such checks, and should be constrained when creating the table +// The implementation of the TabletColumn class does not perform such checks, and should be constrained when creating the table enum class FieldAggregationMethod { OLAP_FIELD_AGGREGATION_NONE = 0, OLAP_FIELD_AGGREGATION_SUM = 1, @@ -290,8 +290,6 @@ struct Vertex { Vertex(int64_t v) : value(v) {} }; -class StorageField; - // ReaderStatistics used to collect statistics when scan data from storage struct OlapReaderStatistics { int64_t io_ns = 0; diff --git a/be/src/storage/row_cursor.cpp b/be/src/storage/row_cursor.cpp index ef649a6a092979..e98112beddbab6 100644 --- a/be/src/storage/row_cursor.cpp +++ b/be/src/storage/row_cursor.cpp @@ -27,7 +27,7 @@ #include "common/consts.h" #include "core/data_type/primitive_type.h" #include "core/field.h" -#include "storage/field.h" +#include "storage/key_coder.h" #include "storage/olap_common.h" #include "storage/olap_define.h" #include "storage/tablet/tablet_schema.h" @@ -125,7 +125,7 @@ RowCursor RowCursor::clone() const { void RowCursor::pad_char_fields() { for (size_t i = 0; i < _fields.size(); ++i) { - const StorageField* col = _schema->column(cast_set(i)); + const TabletColumn* col = _schema->column(cast_set(i)); if (col->type() == FieldType::OLAP_FIELD_TYPE_CHAR && !_fields[i].is_null()) { String padded = _fields[i].get(); padded.resize(col->length(), '\0'); @@ -144,40 +144,41 @@ std::string RowCursor::to_string() const { result.append("1&NULL"); } else { result.append("0&"); - result.append(_fields[i].to_debug_string( - _schema->column(cast_set(i))->get_scale())); + result.append( + _fields[i].to_debug_string(_schema->column(cast_set(i))->frac())); } } return result; } -void RowCursor::_encode_field(const StorageField* storage_field, const Field& f, bool full_encode, - std::string* buf) const { - FieldType ft = storage_field->type(); +void RowCursor::_encode_column_value(const TabletColumn* column, const Field& value, + bool full_encode, std::string* buf) const { + FieldType ft = column->type(); + const KeyCoder* coder = get_key_coder(ft); if (field_is_slice_type(ft)) { // String types: CHAR, VARCHAR, STRING — all stored as String in Field. - const String& str = f.get(); + const String& str = value.get(); if (ft == FieldType::OLAP_FIELD_TYPE_CHAR) { // CHAR type: must pad with \0 to the declared column length - size_t col_len = storage_field->length(); + size_t col_len = column->length(); String padded(col_len, '\0'); memcpy(padded.data(), str.data(), std::min(str.size(), col_len)); Slice slice(padded.data(), col_len); if (full_encode) { - storage_field->full_encode_ascending(&slice, buf); + coder->full_encode_ascending(&slice, buf); } else { - storage_field->encode_ascending(&slice, buf); + coder->encode_ascending(&slice, column->index_length(), buf); } } else { // VARCHAR / STRING: use actual length Slice slice(str.data(), str.size()); if (full_encode) { - storage_field->full_encode_ascending(&slice, buf); + coder->full_encode_ascending(&slice, buf); } else { - storage_field->encode_ascending(&slice, buf); + coder->encode_ascending(&slice, column->index_length(), buf); } } return; @@ -186,11 +187,10 @@ void RowCursor::_encode_field(const StorageField* storage_field, const Field& f, // Non-string scalar keys are fixed-width; their KeyCoder::encode_ascending // ignores `index_size` and delegates to full_encode_ascending, so the // `full_encode` flag here is a no-op and we always call the full helper. - const KeyCoder* coder = storage_field->key_coder(); switch (ft) { -#define CASE(FT, PT) \ - case FieldType::FT: \ - full_encode_field_as_key(f, coder, buf); \ +#define CASE(FT, PT) \ + case FieldType::FT: \ + full_encode_field_as_key(value, coder, buf); \ break; DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(CASE) #undef CASE @@ -204,8 +204,8 @@ template void RowCursor::encode_key_with_padding(std::string* buf, size_t num_keys, bool padding_minimal) const { for (uint32_t cid = 0; cid < num_keys; cid++) { - auto* storage_field = _schema->column(cid); - if (storage_field == nullptr) { + auto* column = _schema->column(cid); + if (column == nullptr) { if (padding_minimal) { buf->push_back(KeyConsts::KEY_MINIMAL_MARKER); } else { @@ -224,7 +224,7 @@ void RowCursor::encode_key_with_padding(std::string* buf, size_t num_keys, } buf->push_back(KeyConsts::KEY_NORMAL_MARKER); - _encode_field(storage_field, _fields[cid], is_mow, buf); + _encode_column_value(column, _fields[cid], is_mow, buf); } } @@ -240,7 +240,7 @@ void RowCursor::encode_key(std::string* buf, size_t num_keys) const { continue; } buf->push_back(KeyConsts::KEY_NORMAL_MARKER); - _encode_field(_schema->column(cid), _fields[cid], full_encode, buf); + _encode_column_value(_schema->column(cid), _fields[cid], full_encode, buf); } } diff --git a/be/src/storage/row_cursor.h b/be/src/storage/row_cursor.h index 58a07d9ce8c874..bc40439b19aa7d 100644 --- a/be/src/storage/row_cursor.h +++ b/be/src/storage/row_cursor.h @@ -34,7 +34,6 @@ #include "storage/tablet/tablet_schema.h" namespace doris { -class StorageField; // Delegate the operation of a row of data. // Stores values as core::Field objects instead of raw byte buffers. @@ -65,7 +64,7 @@ class RowCursor { size_t field_count() const { return _fields.size(); } - const StorageField* column_schema(uint32_t cid) const { return _schema->column(cid); } + const TabletColumn* column(uint32_t cid) const { return _schema->column(cid); } const Schema* schema() const { return _schema.get(); } // Returns a deep copy of this RowCursor with the same schema and field values. @@ -95,7 +94,7 @@ class RowCursor { void encode_single_field(uint32_t cid, std::string* buf, bool full_encode) const { const auto& f = _fields[cid]; DCHECK(!f.is_null()); - _encode_field(_schema->column(cid), f, full_encode, buf); + _encode_column_value(_schema->column(cid), f, full_encode, buf); } private: @@ -107,8 +106,8 @@ class RowCursor { // Helper: encode a single non-null field for the given column. // Converts the core::Field to storage format and calls KeyCoder. - void _encode_field(const StorageField* storage_field, const Field& f, bool full_encode, - std::string* buf) const; + void _encode_column_value(const TabletColumn* column, const Field& value, bool full_encode, + std::string* buf) const; std::unique_ptr _schema; std::vector _fields; diff --git a/be/src/storage/schema.cpp b/be/src/storage/schema.cpp index 63718a20d4b5ce..12904338fde59d 100644 --- a/be/src/storage/schema.cpp +++ b/be/src/storage/schema.cpp @@ -63,11 +63,9 @@ void Schema::_copy_from(const Schema& other) { _tso_col_idx = other._tso_col_idx; _mem_size = other._mem_size; - // Deep copy _cols - // TODO(lingbin): really need clone? - _cols.resize(other._cols.size(), nullptr); + _cols.resize(other._cols.size()); for (auto cid : _col_ids) { - _cols[cid] = other._cols[cid]->clone(); + _cols[cid] = other._cols[cid]; } } @@ -76,29 +74,21 @@ void Schema::_init(const std::vector& cols, const std::vector col_id_set(col_ids.begin(), col_ids.end()); for (int cid = 0; cid < cols.size(); ++cid) { if (col_id_set.find(cid) == col_id_set.end()) { continue; } - _cols[cid] = StorageFieldFactory::create(*cols[cid]); + _cols[cid] = cols[cid]; } } -Schema::~Schema() { - for (auto col : _cols) { - delete col; - } -} - -DataTypePtr Schema::get_data_type_ptr(const StorageField& field) { - return DataTypeFactory::instance().create_data_type(field); -} +Schema::~Schema() = default; -IColumn::MutablePtr Schema::get_column_by_field(const StorageField& field) { - return get_data_type_ptr(field)->create_column(); +DataTypePtr Schema::get_data_type_ptr(const TabletColumn& column) { + return DataTypeFactory::instance().create_data_type(column); } IColumn::MutablePtr Schema::get_predicate_column_ptr(const FieldType& type, bool is_nullable, diff --git a/be/src/storage/schema.h b/be/src/storage/schema.h index 2eabb12853c8b4..8850440067432c 100644 --- a/be/src/storage/schema.h +++ b/be/src/storage/schema.h @@ -32,7 +32,6 @@ #include "io/io_common.h" #include "runtime/thread_context.h" #include "storage/binlog.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/tablet/tablet_schema.h" #include "storage/utils.h" @@ -137,16 +136,14 @@ class Schema { ~Schema(); - static DataTypePtr get_data_type_ptr(const doris::StorageField& field); - - static IColumn::MutablePtr get_column_by_field(const doris::StorageField& field); + static DataTypePtr get_data_type_ptr(const TabletColumn& column); static IColumn::MutablePtr get_predicate_column_ptr(const FieldType& type, bool is_nullable, const ReaderType reader_type); - const std::vector& columns() const { return _cols; } + const std::vector& columns() const { return _cols; } - const doris::StorageField* column(ColumnId cid) const { return _cols[cid]; } + const TabletColumn* column(ColumnId cid) const { return _cols[cid].get(); } size_t num_key_columns() const { return _num_key_columns; } @@ -178,7 +175,7 @@ class Schema { std::vector _unique_ids; // NOTE: _cols[cid] can only be accessed when the cid is // contained in _col_ids - std::vector _cols; + std::vector _cols; size_t _num_key_columns; int32_t _delete_sign_idx = -1; diff --git a/be/src/storage/schema_change/schema_change.cpp b/be/src/storage/schema_change/schema_change.cpp index aaf89d3697f598..009231f2da7b01 100644 --- a/be/src/storage/schema_change/schema_change.cpp +++ b/be/src/storage/schema_change/schema_change.cpp @@ -56,7 +56,6 @@ #include "runtime/runtime_state.h" #include "storage/data_dir.h" #include "storage/delete/delete_handler.h" -#include "storage/field.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_writer.h" #include "storage/iterator/olap_data_convertor.h" @@ -88,8 +87,6 @@ namespace doris { -class CollectionValue; - using namespace ErrorCode; constexpr int ALTER_TABLE_BATCH_SIZE = 4064; @@ -1533,12 +1530,6 @@ Status SchemaChangeJob::parse_request(const SchemaChangeParams& sc_params, Status SchemaChangeJob::_init_column_mapping(ColumnMapping* column_mapping, const TabletColumn& column_schema, const std::string& value) { - auto t = StorageFieldFactory::create(column_schema); - Defer defer([t]() { delete t; }); - if (t == nullptr) { - return Status::Uninitialized("Unsupport field creation of {}", column_schema.name()); - } - if (!column_schema.is_nullable() || value.length() != 0) { RETURN_IF_ERROR(column_schema.get_vec_type()->get_serde()->from_fe_string( value, column_mapping->default_value)); diff --git a/be/src/storage/schema_change/schema_change.h b/be/src/storage/schema_change/schema_change.h index 3facaf914a57bc..1220e05d818aac 100644 --- a/be/src/storage/schema_change/schema_change.h +++ b/be/src/storage/schema_change/schema_change.h @@ -54,7 +54,6 @@ namespace doris { class DeleteHandler; -class StorageField; class TAlterInvertedIndexReq; class TAlterTabletReqV2; class TExpr; diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index 1ba371233c4671..15d4f558f9b054 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -30,8 +30,6 @@ #include "core/data_type/data_type_factory.hpp" #include "core/types.h" #include "io/fs/file_writer.h" -#include "runtime/collection_value.h" -#include "storage/field.h" #include "storage/index/bloom_filter/bloom_filter_index_writer.h" #include "storage/index/inverted/inverted_index_writer.h" #include "storage/index/ordinal_page_index.h" @@ -139,18 +137,16 @@ inline ScalarColumnWriter* get_null_writer(const ColumnWriterOptions& opts, null_options.need_bloom_filter = false; null_options.encoding_preference = opts.encoding_preference; - TabletColumn null_column = - TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, null_type, false, - null_options.meta->unique_id(), null_options.meta->length()); - null_column.set_name("nullable"); - null_column.set_index_length(-1); // no short key index - std::unique_ptr null_field(StorageFieldFactory::create(null_column)); - return new ScalarColumnWriter(null_options, std::move(null_field), file_writer); + auto null_column_ptr = std::make_shared( + FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, null_type, false, + null_options.meta->unique_id(), null_options.meta->length()); + null_column_ptr->set_name("nullable"); + null_column_ptr->set_index_length(-1); // no short key index + return new ScalarColumnWriter(null_options, std::move(null_column_ptr), file_writer); } -ColumnWriter::ColumnWriter(std::unique_ptr field, bool is_nullable, - ColumnMetaPB* meta) - : _field(std::move(field)), _is_nullable(is_nullable), _column_meta(meta) { +ColumnWriter::ColumnWriter(TabletColumnPtr column, bool is_nullable, ColumnMetaPB* meta) + : _column(std::move(column)), _is_nullable(is_nullable), _column_meta(meta) { _data_type = DataTypeFactory::instance().create_data_type(*_column_meta); } Status ColumnWriter::create_struct_writer(const ColumnWriterOptions& opts, @@ -180,8 +176,7 @@ Status ColumnWriter::create_struct_writer(const ColumnWriterOptions& opts, get_null_writer(opts, file_writer, column->get_subtype_count() + 1); *writer = std::unique_ptr(new StructColumnWriter( - opts, std::unique_ptr(StorageFieldFactory::create(*column)), null_writer, - sub_column_writers)); + opts, std::make_shared(*column), null_writer, sub_column_writers)); return Status::OK(); } @@ -219,21 +214,20 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts, length_options.need_bloom_filter = false; length_options.encoding_preference = opts.encoding_preference; - TabletColumn length_column = - TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, - length_options.meta->is_nullable(), length_options.meta->unique_id(), - length_options.meta->length()); - length_column.set_name("length"); - length_column.set_index_length(-1); // no short key index - std::unique_ptr bigint_field(StorageFieldFactory::create(length_column)); + auto length_column_ptr = std::make_shared( + FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, + length_options.meta->is_nullable(), length_options.meta->unique_id(), + length_options.meta->length()); + length_column_ptr->set_name("length"); + length_column_ptr->set_index_length(-1); // no short key index auto* length_writer = - new OffsetColumnWriter(length_options, std::move(bigint_field), file_writer); + new OffsetColumnWriter(length_options, std::move(length_column_ptr), file_writer); ScalarColumnWriter* null_writer = get_null_writer(opts, file_writer, 3); - *writer = std::unique_ptr(new ArrayColumnWriter( - opts, std::unique_ptr(StorageFieldFactory::create(*column)), - length_writer, null_writer, std::move(item_writer))); + *writer = std::unique_ptr( + new ArrayColumnWriter(opts, std::make_shared(*column), length_writer, + null_writer, std::move(item_writer))); return Status::OK(); } @@ -283,22 +277,21 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta length_options.need_bloom_filter = false; length_options.encoding_preference = opts.encoding_preference; - TabletColumn length_column = - TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, - length_options.meta->is_nullable(), length_options.meta->unique_id(), - length_options.meta->length()); - length_column.set_name("length"); - length_column.set_index_length(-1); // no short key index - std::unique_ptr bigint_field(StorageFieldFactory::create(length_column)); + auto length_column_ptr = std::make_shared( + FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, + length_options.meta->is_nullable(), length_options.meta->unique_id(), + length_options.meta->length()); + length_column_ptr->set_name("length"); + length_column_ptr->set_index_length(-1); // no short key index auto* length_writer = - new OffsetColumnWriter(length_options, std::move(bigint_field), file_writer); + new OffsetColumnWriter(length_options, std::move(length_column_ptr), file_writer); ScalarColumnWriter* null_writer = get_null_writer(opts, file_writer, column->get_subtype_count() + 2); - *writer = std::unique_ptr(new MapColumnWriter( - opts, std::unique_ptr(StorageFieldFactory::create(*column)), null_writer, - length_writer, inner_writer_list)); + *writer = std::unique_ptr( + new MapColumnWriter(opts, std::make_shared(*column), null_writer, + length_writer, inner_writer_list)); return Status::OK(); } @@ -312,9 +305,8 @@ Status ColumnWriter::create_agg_state_writer(const ColumnWriterOptions& opts, auto type = agg_state_type->get_serialized_type()->get_primitive_type(); if (type == PrimitiveType::TYPE_STRING || type == PrimitiveType::INVALID_TYPE || type == PrimitiveType::TYPE_FIXED_LENGTH_OBJECT || type == PrimitiveType::TYPE_BITMAP) { - *writer = std::unique_ptr(new ScalarColumnWriter( - opts, std::unique_ptr(StorageFieldFactory::create(*column)), - file_writer)); + *writer = std::unique_ptr( + new ScalarColumnWriter(opts, std::make_shared(*column), file_writer)); } else if (type == PrimitiveType::TYPE_ARRAY) { RETURN_IF_ERROR(create_array_writer(opts, column, file_writer, writer)); } else if (type == PrimitiveType::TYPE_MAP) { @@ -338,28 +330,25 @@ Status ColumnWriter::create_variant_writer(const ColumnWriterOptions& opts, if (column->is_extracted_column()) { if (column->name().find(DOC_VALUE_COLUMN_PATH) != std::string::npos) { *writer = std::make_unique( - opts, column, - std::unique_ptr(StorageFieldFactory::create(*column))); + opts, std::make_shared(*column)); return Status::OK(); } VLOG_DEBUG << "gen subwriter for " << column->path_info_ptr()->get_path(); - *writer = std::make_unique( - opts, column, std::unique_ptr(StorageFieldFactory::create(*column))); + *writer = std::make_unique(opts, + std::make_shared(*column)); return Status::OK(); } - *writer = std::make_unique( - opts, column, std::unique_ptr(StorageFieldFactory::create(*column))); + *writer = std::make_unique(opts, std::make_shared(*column)); return Status::OK(); } //Todo(Amory): here should according nullable and offset and need sub to simply this function Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn* column, io::FileWriter* file_writer, std::unique_ptr* writer) { - std::unique_ptr field(StorageFieldFactory::create(*column)); - DCHECK(field.get() != nullptr); + auto column_ptr = std::make_shared(*column); if (is_scalar_type(column->type())) { *writer = std::unique_ptr( - new ScalarColumnWriter(opts, std::move(field), file_writer)); + new ScalarColumnWriter(opts, std::move(column_ptr), file_writer)); return Status::OK(); } else { switch (column->type()) { @@ -386,7 +375,7 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn* } default: return Status::NotSupported("unsupported type for ColumnWriter: {}", - std::to_string(int(field->type()))); + std::to_string(int(column_ptr->type()))); } } } @@ -417,7 +406,7 @@ Status ColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** pt if (non_null_count == 0) { // All NULL: skip run-length iteration, directly append all nulls RETURN_IF_ERROR(append_nulls(num_rows)); - *ptr += get_field()->size() * num_rows; + *ptr += cell_size() * num_rows; return Status::OK(); } @@ -445,10 +434,10 @@ Status ColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** pt auto step = next_run_step(); if (null_map[offset]) { RETURN_IF_ERROR(append_nulls(step)); - *ptr += get_field()->size() * step; + *ptr += cell_size() * step; } else { // TODO: - // 1. `*ptr += get_field()->size() * step;` should do in this function, not append_data; + // 1. `*ptr += cell_size() * step;` should do in this function, not append_data; // 2. support array vectorized load and ptr offset add RETURN_IF_ERROR(append_data(ptr, step)); } @@ -470,10 +459,9 @@ Status ColumnWriter::append(const uint8_t* nullmap, const void* data, size_t num /////////////////////////////////////////////////////////////////////////////////// -ScalarColumnWriter::ScalarColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, +ScalarColumnWriter::ScalarColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, io::FileWriter* file_writer) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta), + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta), _opts(opts), _file_writer(file_writer), _data_size(0) { @@ -499,7 +487,7 @@ Status ScalarColumnWriter::init() { PageBuilder* page_builder = nullptr; - RETURN_IF_ERROR(EncodingInfo::get(get_field()->type(), _opts.meta->encoding(), + RETURN_IF_ERROR(EncodingInfo::get(get_column()->type(), _opts.meta->encoding(), _opts.encoding_preference, &_encoding_info)); _opts.meta->set_encoding(_encoding_info->encoding()); // create page builder @@ -510,7 +498,7 @@ Status ScalarColumnWriter::init() { RETURN_IF_ERROR(_encoding_info->create_page_builder(opts, &page_builder)); if (page_builder == nullptr) { return Status::NotSupported("Failed to create page builder for type {} and encoding {}", - get_field()->type(), _opts.meta->encoding()); + get_column()->type(), _opts.meta->encoding()); } // should store more concrete encoding type instead of DEFAULT_ENCODING // because the default encoding of a data type can be changed in the future @@ -518,7 +506,7 @@ Status ScalarColumnWriter::init() { VLOG_DEBUG << fmt::format( "[verbose] scalar column writer init, column_id={}, type={}, encoding={}, " "is_nullable={}", - _opts.meta->column_id(), get_field()->type(), + _opts.meta->column_id(), get_column()->type(), EncodingTypePB_Name(_opts.meta->encoding()), _opts.meta->is_nullable()); _page_builder.reset(page_builder); // create ordinal builder @@ -529,7 +517,7 @@ Status ScalarColumnWriter::init() { } if (_opts.need_zone_map) { RETURN_IF_ERROR( - ZoneMapIndexWriter::create(_data_type, get_field(), _zone_map_index_builder)); + ZoneMapIndexWriter::create(_data_type, get_column(), _zone_map_index_builder)); } if (_opts.need_inverted_index) { @@ -543,10 +531,6 @@ Status ScalarColumnWriter::init() { size_t count) override { return Status::OK(); } - Status add_array_values(size_t field_size, const CollectionValue* values, - size_t count) override { - return Status::OK(); - } Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* null_map, const uint8_t* offsets_ptr, size_t count) override { @@ -567,19 +551,19 @@ Status ScalarColumnWriter::init() { break; }); - RETURN_IF_ERROR(IndexColumnWriter::create(get_field(), &_inverted_index_builders[i], - _opts.index_file_writer, - _opts.inverted_indexes[i])); + RETURN_IF_ERROR(IndexColumnWriter::create( + get_column(), &_inverted_index_builders[i], _opts.index_file_writer, + _opts.inverted_indexes[i])); } } while (false); } if (_opts.need_bloom_filter) { if (_opts.is_ngram_bf_index) { RETURN_IF_ERROR(NGramBloomFilterIndexWriterImpl::create( - BloomFilterOptions(), get_field()->type(), _opts.gram_size, _opts.gram_bf_size, + BloomFilterOptions(), get_column()->type(), _opts.gram_size, _opts.gram_bf_size, &_bloom_filter_index_builder)); } else { - RETURN_IF_ERROR(BloomFilterIndexWriter::create(_opts.bf_options, get_field()->type(), + RETURN_IF_ERROR(BloomFilterIndexWriter::create(_opts.bf_options, get_column()->type(), &_bloom_filter_index_builder)); } } @@ -629,7 +613,7 @@ Status ScalarColumnWriter::_internal_append_data_in_current_page(const uint8_t* } if (_opts.need_inverted_index) { for (const auto& builder : _inverted_index_builders) { - RETURN_IF_ERROR(builder->add_values(get_field()->name(), data, *num_written)); + RETURN_IF_ERROR(builder->add_values(get_column()->name(), data, *num_written)); } } if (_opts.need_bloom_filter) { @@ -648,7 +632,7 @@ Status ScalarColumnWriter::_internal_append_data_in_current_page(const uint8_t* Status ScalarColumnWriter::append_data_in_current_page(const uint8_t** data, size_t* num_written) { RETURN_IF_ERROR(append_data_in_current_page(*data, num_written)); - *data += get_field()->size() * (*num_written); + *data += cell_size() * (*num_written); return Status::OK(); } @@ -698,7 +682,7 @@ Status ScalarColumnWriter::append_nullable(const uint8_t* null_map, const uint8_ if (non_null_count == 0) { // All NULL: skip data writing, only update null bitmap and indexes RETURN_IF_ERROR(append_nulls(num_rows)); - *ptr += get_field()->size() * num_rows; + *ptr += cell_size() * num_rows; return Status::OK(); } @@ -712,10 +696,10 @@ Status ScalarColumnWriter::append_nullable(const uint8_t* null_map, const uint8_ size_t run_length = run.len; if (run.is_null) { RETURN_IF_ERROR(append_nulls(run_length)); - *ptr += get_field()->size() * run_length; + *ptr += cell_size() * run_length; } else { // TODO: - // 1. `*ptr += get_field()->size() * step;` should do in this function, not append_data; + // 1. `*ptr += cell_size() * step;` should do in this function, not append_data; // 2. support array vectorized load and ptr offset add RETURN_IF_ERROR(append_data(ptr, run_length)); } @@ -890,12 +874,11 @@ Status ScalarColumnWriter::finish_current_page() { // offset column writer //////////////////////////////////////////////////////////////////////////////// -OffsetColumnWriter::OffsetColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, +OffsetColumnWriter::OffsetColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, io::FileWriter* file_writer) - : ScalarColumnWriter(opts, std::move(field), file_writer) { + : ScalarColumnWriter(opts, std::move(column), file_writer) { // now we only explain data in offset column as uint64 - DCHECK(get_field()->type() == FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT); + DCHECK(get_column()->type() == FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT); } OffsetColumnWriter::~OffsetColumnWriter() = default; @@ -930,10 +913,9 @@ void OffsetColumnWriter::put_extra_info_in_page(DataPageFooterPB* footer) { } StructColumnWriter::StructColumnWriter( - const ColumnWriterOptions& opts, std::unique_ptr field, - ScalarColumnWriter* null_writer, + const ColumnWriterOptions& opts, TabletColumnPtr column, ScalarColumnWriter* null_writer, std::vector>& sub_column_writers) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta), _opts(opts) { + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta), _opts(opts) { for (auto& sub_column_writer : sub_column_writers) { _sub_column_writers.push_back(std::move(sub_column_writer)); } @@ -1038,12 +1020,11 @@ Status StructColumnWriter::finish_current_page() { return Status::NotSupported("struct writer has no data, can not finish_current_page"); } -ArrayColumnWriter::ArrayColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, +ArrayColumnWriter::ArrayColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, OffsetColumnWriter* offset_writer, ScalarColumnWriter* null_writer, std::unique_ptr item_writer) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta), + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta), _item_writer(std::move(item_writer)), _opts(opts) { _offset_writer.reset(offset_writer); @@ -1061,7 +1042,7 @@ Status ArrayColumnWriter::init() { if (_opts.need_inverted_index) { auto* writer = dynamic_cast(_item_writer.get()); if (writer != nullptr) { - RETURN_IF_ERROR(IndexColumnWriter::create(get_field(), &_inverted_index_writer, + RETURN_IF_ERROR(IndexColumnWriter::create(get_column(), &_inverted_index_writer, _opts.index_file_writer, _opts.inverted_indexes[0])); } @@ -1112,7 +1093,8 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { if (writer != nullptr) { //NOTE: use array field name as index field, but item_writer size should be used when moving item_data_ptr RETURN_IF_ERROR(_inverted_index_writer->add_array_values( - _item_writer->get_field()->size(), reinterpret_cast(data), + field_type_size(_item_writer->get_column()->type()), + reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); } } @@ -1123,13 +1105,14 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { if (writer != nullptr) { //NOTE: use array field name as index field, but item_writer size should be used when moving item_data_ptr RETURN_IF_ERROR(_ann_index_writer->add_array_values( - _item_writer->get_field()->size(), reinterpret_cast(data), + field_type_size(_item_writer->get_column()->type()), + reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); } else { return Status::NotSupported( "Ann index can only be build on array with scalar type. but got {} as " "nested", - _item_writer->get_field()->type()); + _item_writer->get_column()->type()); } } @@ -1209,11 +1192,10 @@ Status ArrayColumnWriter::finish_current_page() { } /// ============================= MapColumnWriter =====================//// -MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, +MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, ScalarColumnWriter* null_writer, OffsetColumnWriter* offset_writer, std::vector>& kv_writers) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta), _opts(opts) { + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta), _opts(opts) { CHECK_EQ(kv_writers.size(), 2); _offsets_writer.reset(offset_writer); if (is_nullable()) { @@ -1348,11 +1330,9 @@ Status MapColumnWriter::write_inverted_index() { return Status::OK(); } -VariantColumnWriter::VariantColumnWriter(const ColumnWriterOptions& opts, - const TabletColumn* column, - std::unique_ptr field) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta) { - _impl = std::make_unique(opts, column); +VariantColumnWriter::VariantColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column) + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) { + _impl = std::make_unique(opts, get_column()); } Status VariantColumnWriter::init() { diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index 8a87be44ebe0a1..44567c2d8d3f6c 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -31,7 +31,6 @@ #include "common/status.h" // for Status #include "core/column/column_variant.h" -#include "storage/field.h" // for StorageField #include "storage/index/ann/ann_index_writer.h" #include "storage/index/bloom_filter/bloom_filter.h" #include "storage/index/inverted/inverted_index_writer.h" @@ -39,8 +38,10 @@ #include "storage/segment/options.h" #include "storage/segment/variant/nested_group_provider.h" #include "storage/segment/variant/variant_statistics.h" -#include "util/bitmap.h" // for BitmapChange -#include "util/slice.h" // for OwnedSlice +#include "storage/tablet/tablet_schema.h" // for TabletColumnPtr +#include "storage/types.h" // for field_type_size +#include "util/bitmap.h" // for BitmapChange +#include "util/slice.h" // for OwnedSlice namespace doris { @@ -127,8 +128,7 @@ class ColumnWriter { const TabletColumn* column, io::FileWriter* file_writer, std::unique_ptr* writer); - explicit ColumnWriter(std::unique_ptr field, bool is_nullable, - ColumnMetaPB* meta); + explicit ColumnWriter(TabletColumnPtr column, bool is_nullable, ColumnMetaPB* meta); virtual ~ColumnWriter() = default; @@ -194,7 +194,11 @@ class ColumnWriter { bool is_nullable() const { return _is_nullable; } - StorageField* get_field() const { return _field.get(); } + const TabletColumn* get_column() const { return _column.get(); } + + // Per-row in-memory cell footprint of this writer's column, used to step + // the input pointer across rows in append_*/null-run loops. + size_t cell_size() const { return field_type_size(_column->type()); } ColumnMetaPB* get_column_meta() const { return _column_meta; } @@ -202,7 +206,7 @@ class ColumnWriter { DataTypePtr _data_type; private: - std::unique_ptr _field; + TabletColumnPtr _column; bool _is_nullable; ColumnMetaPB* _column_meta; std::vector _null_bitmap; @@ -220,7 +224,7 @@ class FlushPageCallback { // to file class ScalarColumnWriter : public ColumnWriter { public: - ScalarColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr field, + ScalarColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, io::FileWriter* file_writer); ~ScalarColumnWriter() override; @@ -341,7 +345,7 @@ class ScalarColumnWriter : public ColumnWriter { // in footer.next_array_item_ordinal which in finish_cur_page() callback put_extra_info_in_page() class OffsetColumnWriter final : public ScalarColumnWriter, FlushPageCallback { public: - OffsetColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr field, + OffsetColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, io::FileWriter* file_writer); ~OffsetColumnWriter() override; @@ -358,8 +362,7 @@ class OffsetColumnWriter final : public ScalarColumnWriter, FlushPageCallback { class StructColumnWriter final : public ColumnWriter { public: - explicit StructColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, + explicit StructColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, ScalarColumnWriter* null_writer, std::vector>& sub_column_writers); ~StructColumnWriter() override = default; @@ -426,7 +429,7 @@ class StructColumnWriter final : public ColumnWriter { class ArrayColumnWriter final : public ColumnWriter { public: - explicit ArrayColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr field, + explicit ArrayColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, OffsetColumnWriter* offset_writer, ScalarColumnWriter* null_writer, std::unique_ptr item_writer); ~ArrayColumnWriter() override = default; @@ -500,7 +503,7 @@ class ArrayColumnWriter final : public ColumnWriter { class MapColumnWriter final : public ColumnWriter { public: - explicit MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr field, + explicit MapColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, ScalarColumnWriter* null_writer, OffsetColumnWriter* offsets_writer, std::vector>& _kv_writers); @@ -574,8 +577,7 @@ class MapColumnWriter final : public ColumnWriter { // used for compaction to write sub variant column class VariantSubcolumnWriter : public ColumnWriter { public: - explicit VariantSubcolumnWriter(const ColumnWriterOptions& opts, const TabletColumn* column, - std::unique_ptr field); + explicit VariantSubcolumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column); ~VariantSubcolumnWriter() override = default; @@ -626,7 +628,6 @@ class VariantSubcolumnWriter : public ColumnWriter { ordinal_t _next_rowid = 0; size_t none_null_size = 0; ColumnVariant::MutablePtr _column; - const TabletColumn* _tablet_column = nullptr; ColumnWriterOptions _opts; std::unique_ptr _writer; TabletIndexes _indexes; @@ -637,8 +638,7 @@ class VariantSubcolumnWriter : public ColumnWriter { class VariantColumnWriter : public ColumnWriter { public: - explicit VariantColumnWriter(const ColumnWriterOptions& opts, const TabletColumn* column, - std::unique_ptr field); + explicit VariantColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column); ~VariantColumnWriter() override = default; diff --git a/be/src/storage/segment/row_binlog_segment_writer.cpp b/be/src/storage/segment/row_binlog_segment_writer.cpp index 98c123f3aec279..11dafa3a78a4b2 100644 --- a/be/src/storage/segment/row_binlog_segment_writer.cpp +++ b/be/src/storage/segment/row_binlog_segment_writer.cpp @@ -498,7 +498,7 @@ Status RowBinlogSourceDataWriter::fill_normal_columns( const auto& including_cids = partial_source_cids.empty() ? _normal_column_ids : partial_source_cids; for (size_t cid : including_cids) { - DCHECK(column_writers[start + cid]->get_field()->type() == + DCHECK(column_writers[start + cid]->get_column()->type() == _opt.source.tablet_schema->columns()[cid]->type()) << cid; RETURN_IF_ERROR(column_writers[start + cid]->append(_converted_columns[cid]->get_nullmap(), diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index abacf4d68690d9..1c5578c1388146 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -57,6 +57,7 @@ #include "storage/index/short_key_index.h" #include "storage/iterator/vgeneric_iterators.h" #include "storage/iterators.h" +#include "storage/key_coder.h" #include "storage/olap_common.h" #include "storage/predicate/block_column_predicate.h" #include "storage/predicate/column_predicate.h" diff --git a/be/src/storage/segment/segment.h b/be/src/storage/segment/segment.h index 8806505d1e14bd..f18be6c093ade8 100644 --- a/be/src/storage/segment/segment.h +++ b/be/src/storage/segment/segment.h @@ -38,7 +38,6 @@ #include "io/fs/file_system.h" #include "runtime/descriptors.h" #include "storage/cache/page_cache.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/schema.h" #include "storage/segment/page_handle.h" @@ -187,9 +186,9 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd int cid, const Schema& schema, const std::map& target_cast_type_for_variants, const StorageReadOptions& read_options) { - const doris::StorageField* col = schema.column(cid); + const TabletColumn* col = schema.column(cid); DCHECK(col != nullptr) << "Column not found in schema for cid=" << cid; - DataTypePtr storage_column_type = get_data_type_of(col->get_desc(), read_options); + DataTypePtr storage_column_type = get_data_type_of(*col, read_options); if (storage_column_type == nullptr || col->type() != FieldType::OLAP_FIELD_TYPE_VARIANT || !target_cast_type_for_variants.contains(col->name())) { // Default column iterator or not variant column diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index f3931c72f6dfe7..8ac492769ea53a 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -76,7 +76,6 @@ #include "runtime/thread_context.h" #include "storage/binlog.h" #include "storage/compaction/collection_similarity.h" -#include "storage/field.h" #include "storage/id_manager.h" #include "storage/index/ann/ann_index.h" #include "storage/index/ann/ann_index_iterator.h" @@ -541,12 +540,12 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { _storage_name_and_type.resize(_schema->columns().size()); auto storage_format = _opts.tablet_schema->get_inverted_index_storage_format(); for (int i = 0; i < _schema->columns().size(); ++i) { - const StorageField* col = _schema->column(i); + const TabletColumn* col = _schema->column(i); if (col) { - auto storage_type = _segment->get_data_type_of(col->get_desc(), _opts); + auto storage_type = _segment->get_data_type_of(*col, _opts); if (storage_type == nullptr) { - storage_type = DataTypeFactory::instance().create_data_type(col->get_desc(), - col->is_nullable()); + storage_type = + DataTypeFactory::instance().create_data_type(*col, col->is_nullable()); } // Currently, when writing a lucene index, the field of the document is column_name, and the column name is // bound to the index field. Since version 1.2, the data file storage has been changed from column_name to @@ -568,7 +567,9 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { } } _storage_name_and_type[i] = std::make_pair(field_name, storage_type); - if (int32_t uid = col->get_unique_id(); !_variant_sparse_column_cache.contains(uid)) { + if (int32_t uid = + col->is_extracted_column() ? col->parent_unique_id() : col->unique_id(); + !_variant_sparse_column_cache.contains(uid)) { DCHECK(uid >= 0); _variant_sparse_column_cache.emplace(uid, std::make_unique()); @@ -804,10 +805,11 @@ Status SegmentIterator::_get_row_ranges_by_keys() { } // Read & seek key columns is a waste of time when no key column in _schema - if (std::none_of( - _schema->columns().begin(), _schema->columns().end(), [&](const StorageField* col) { - return col && _opts.tablet_schema->column_by_uid(col->unique_id()).is_key(); - })) { + if (std::none_of(_schema->columns().begin(), _schema->columns().end(), + [&](const TabletColumnPtr& col) { + return col && + _opts.tablet_schema->column_by_uid(col->unique_id()).is_key(); + })) { return Status::OK(); } @@ -839,29 +841,27 @@ Status SegmentIterator::_get_row_ranges_by_keys() { // Set up environment for the following seek. Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_range) { - std::vector key_fields; + std::vector key_columns; std::set column_set; if (key_range.lower_key != nullptr) { for (auto cid : key_range.lower_key->schema()->column_ids()) { column_set.emplace(cid); - key_fields.emplace_back(key_range.lower_key->column_schema(cid)); + key_columns.emplace_back(key_range.lower_key->column(cid)); } } if (key_range.upper_key != nullptr) { for (auto cid : key_range.upper_key->schema()->column_ids()) { if (column_set.count(cid) == 0) { - key_fields.emplace_back(key_range.upper_key->column_schema(cid)); + key_columns.emplace_back(key_range.upper_key->column(cid)); column_set.emplace(cid); } } } if (!_seek_schema) { - // Schema constructors accept a vector of TabletColumnPtr. Convert - // StorageField pointers to TabletColumnPtr by copying their descriptors. std::vector cols; - cols.reserve(key_fields.size()); - for (const StorageField* f : key_fields) { - cols.emplace_back(std::make_shared(f->get_desc())); + cols.reserve(key_columns.size()); + for (const TabletColumn* col : key_columns) { + cols.emplace_back(std::make_shared(*col)); } _seek_schema = std::make_unique(cols, cols.size()); } @@ -871,7 +871,7 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra int i = 0; for (auto cid : _seek_schema->column_ids()) { auto column_desc = _seek_schema->column(cid); - _seek_block[i] = Schema::get_column_by_field(*column_desc); + _seek_block[i] = Schema::get_data_type_ptr(*column_desc)->create_column(); i++; } } @@ -2193,18 +2193,18 @@ bool SegmentIterator::_can_evaluated_by_vectorized(std::shared_ptrcolumns().size(), false); for (size_t i = 0; i < _schema->num_column_ids(); i++) { auto cid = _schema->column_id(i); - const StorageField* column_desc = _schema->column(cid); + const TabletColumn* column_desc = _schema->column(cid); // The additional deleted filter condition will be in the materialized column at the end of the block. // After _output_column_by_sel_idx, it will be erased, so we do not need to shrink it. @@ -2292,7 +2292,9 @@ Status SegmentIterator::_init_current_block(Block* block, "col_path {}", block->get_by_position(i).type->get_name(), file_column_type->get_name(), column_desc->name(), - column_desc->path() == nullptr ? "" : column_desc->path()->get_path()); + column_desc->path_info_ptr() == nullptr + ? "" + : column_desc->path_info_ptr()->get_path()); // TODO reuse current_columns[cid] = file_column_type->create_column(); current_columns[cid]->reserve(nrows_read_limit); @@ -2924,8 +2926,8 @@ Status SegmentIterator::_convert_to_expected_type(const std::vector& c if (!_current_return_columns[i] || _converted_column_ids[i] || _is_pred_column[i]) { continue; } - const StorageField* field_type = _schema->column(i); - DataTypePtr expected_type = Schema::get_data_type_ptr(*field_type); + const TabletColumn* column_desc = _schema->column(i); + DataTypePtr expected_type = Schema::get_data_type_ptr(*column_desc); DataTypePtr file_column_type = _storage_name_and_type[i].second; if (!file_column_type->equals(*expected_type)) { ColumnPtr expected; @@ -2934,11 +2936,12 @@ Status SegmentIterator::_convert_to_expected_type(const std::vector& c expected_type, &expected)); _current_return_columns[i] = expected->assume_mutable(); _converted_column_ids[i] = true; - VLOG_DEBUG << fmt::format( - "Convert {} fom file column type {} to {}, num_rows {}", - field_type->path() == nullptr ? "" : field_type->path()->get_path(), - file_column_type->get_name(), expected_type->get_name(), - _current_return_columns[i]->size()); + VLOG_DEBUG << fmt::format("Convert {} fom file column type {} to {}, num_rows {}", + column_desc->path_info_ptr() == nullptr + ? "" + : column_desc->path_info_ptr()->get_path(), + file_column_type->get_name(), expected_type->get_name(), + _current_return_columns[i]->size()); } } return Status::OK(); diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 67236e15c19e51..5105d050da1872 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -44,7 +44,6 @@ #include "exprs/vexpr_fwd.h" #include "io/fs/file_reader_writer_fwd.h" #include "runtime/runtime_profile.h" -#include "storage/field.h" #include "storage/index/ann/ann_topn_runtime.h" #include "storage/index/index_iterator.h" #include "storage/iterators.h" @@ -209,7 +208,7 @@ class SegmentIterator : public RowwiseIterator { // CHAR type in storage layer padding the 0 in length. But query engine need ignore the padding 0. // so segment iterator need to shrink char column before output it. only use in vec query engine. void _vec_init_char_column_id(Block* block); - bool _has_char_type(const StorageField& column_desc); + bool _has_char_type(const TabletColumn& column_desc); uint32_t segment_id() const { return _segment->id(); } uint32_t num_rows() const { return _segment->num_rows(); } @@ -256,8 +255,7 @@ class SegmentIterator : public RowwiseIterator { if (block_cid >= block->columns()) { continue; } - DataTypePtr storage_type = - _segment->get_data_type_of(_schema->column(cid)->get_desc(), _opts); + DataTypePtr storage_type = _segment->get_data_type_of(*_schema->column(cid), _opts); if (storage_type && !storage_type->equals(*block->get_by_position(block_cid).type)) { // Do additional cast MutableColumnPtr tmp = storage_type->create_column(); diff --git a/be/src/storage/segment/variant/binary_column_extract_iterator.h b/be/src/storage/segment/variant/binary_column_extract_iterator.h index 0e5632b9853400..a58cf10a6b0e5a 100644 --- a/be/src/storage/segment/variant/binary_column_extract_iterator.h +++ b/be/src/storage/segment/variant/binary_column_extract_iterator.h @@ -41,7 +41,6 @@ #include "core/types.h" #include "exprs/function/function_helpers.h" #include "io/io_common.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" diff --git a/be/src/storage/segment/variant/hierarchical_data_iterator.h b/be/src/storage/segment/variant/hierarchical_data_iterator.h index 3e3816736a4851..ae7f96526a633f 100644 --- a/be/src/storage/segment/variant/hierarchical_data_iterator.h +++ b/be/src/storage/segment/variant/hierarchical_data_iterator.h @@ -41,7 +41,6 @@ #include "core/types.h" #include "exprs/function/function_helpers.h" #include "io/io_common.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" diff --git a/be/src/storage/segment/variant/sparse_column_merge_iterator.h b/be/src/storage/segment/variant/sparse_column_merge_iterator.h index 9a37fd1f73a52b..77fcd8fe95cbc8 100644 --- a/be/src/storage/segment/variant/sparse_column_merge_iterator.h +++ b/be/src/storage/segment/variant/sparse_column_merge_iterator.h @@ -41,7 +41,6 @@ #include "core/types.h" #include "exprs/function/function_helpers.h" #include "io/io_common.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index 8ad08640ba19bd..95f266e15c44cb 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -452,7 +452,6 @@ Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnW const DataTypePtr& type, const ColumnPtr& values_column, const std::vector& rowids, size_t total_rows) { DCHECK_EQ(values_column->size(), rowids.size()); - const size_t cell_size = writer->get_field()->size(); auto base_type = type; if (base_type->is_nullable()) { @@ -512,6 +511,9 @@ Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnW return writer->append_nulls(total_rows); } + // Non-ARRAY scalar path: writer cell is strided by sizeof(CppType). + const size_t cell_size = field_type_size(writer->get_column()->type()); + converter->add_column_data_convertor(tablet_column); RETURN_IF_ERROR(converter->set_source_content_with_specifid_column({values_column, type, ""}, 0, rowids.size(), cid)); @@ -1209,8 +1211,7 @@ Status VariantColumnWriterImpl::_process_root_column(ColumnVariant* ptr, size_t num_rows, int& column_id) { // root column _root_writer = std::make_unique( - _opts, std::unique_ptr(StorageFieldFactory::create(*_tablet_column)), - _opts.file_writer); + _opts, std::make_shared(*_tablet_column), _opts.file_writer); RETURN_IF_ERROR(_root_writer->init()); // make sure the root type @@ -1608,10 +1609,8 @@ Status VariantColumnWriterImpl::append_nullable(const uint8_t* null_map, const u } VariantSubcolumnWriter::VariantSubcolumnWriter(const ColumnWriterOptions& opts, - const TabletColumn* column, - std::unique_ptr field) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta) { - _tablet_column = column; + TabletColumnPtr column) + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) { _opts = opts; _column = ColumnVariant::create(0, false); } @@ -1645,7 +1644,7 @@ Status VariantSubcolumnWriter::finalize() { DCHECK(ptr->is_finalized()); const auto& parent_column = - _opts.rowset_ctx->tablet_schema->column_by_uid(_tablet_column->parent_unique_id()); + _opts.rowset_ctx->tablet_schema->column_by_uid(get_column()->parent_unique_id()); TabletColumn flush_column; if (ptr->get_subcolumns().get_root()->data.get_least_common_base_type_id() == @@ -1655,10 +1654,10 @@ Status VariantSubcolumnWriter::finalize() { ptr->ensure_root_node_type(flush_type); } flush_column = variant_util::get_column_by_type( - ptr->get_root_type(), _tablet_column->name(), + ptr->get_root_type(), get_column()->name(), variant_util::ExtraInfo {.unique_id = -1, - .parent_unique_id = _tablet_column->parent_unique_id(), - .path_info = *_tablet_column->path_info_ptr()}); + .parent_unique_id = get_column()->parent_unique_id(), + .path_info = *get_column()->path_info_ptr()}); int64_t none_null_value_size = ptr->get_subcolumns().get_root()->data.get_non_null_value_size(); bool need_record_none_null_value_size = (!flush_column.path_info_ptr()->get_is_typed()) && @@ -1734,11 +1733,9 @@ Status VariantSubcolumnWriter::append_nullable(const uint8_t* null_map, const ui } VariantDocCompactWriter::VariantDocCompactWriter(const ColumnWriterOptions& opts, - const TabletColumn* column, - std::unique_ptr field) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta) { + TabletColumnPtr column) + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) { _opts = opts; - _tablet_column = column; _column = ColumnVariant::create(0, false); } @@ -1841,7 +1838,7 @@ Status VariantDocCompactWriter::_write_doc_value_column(const TabletColumn& pare ColumnVariant* variant_column, OlapBlockDataConvertor* converter, int column_id, size_t num_rows) { - std::string doc_value_column_path = _tablet_column->path_info_ptr()->get_path(); + std::string doc_value_column_path = get_column()->path_info_ptr()->get_path(); size_t pos = doc_value_column_path.rfind("b"); int bucket_value = std::stoi(doc_value_column_path.substr(pos + 1)); TabletColumn doc_value_column = @@ -1867,7 +1864,7 @@ Status VariantDocCompactWriter::finalize() { auto* variant_column = assert_cast(_column.get()); const auto& parent_column = - _opts.rowset_ctx->tablet_schema->column_by_uid(_tablet_column->parent_unique_id()); + _opts.rowset_ctx->tablet_schema->column_by_uid(get_column()->parent_unique_id()); size_t num_rows = variant_column->size(); auto converter = std::make_unique(); diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.h b/be/src/storage/segment/variant/variant_column_writer_impl.h index 4cb2724c046e64..639e58bb65f8e7 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.h +++ b/be/src/storage/segment/variant/variant_column_writer_impl.h @@ -223,8 +223,7 @@ class VariantColumnWriterImpl { class VariantDocCompactWriter : public ColumnWriter { public: - explicit VariantDocCompactWriter(const ColumnWriterOptions& opts, const TabletColumn* column, - std::unique_ptr field); + explicit VariantDocCompactWriter(const ColumnWriterOptions& opts, TabletColumnPtr column); ~VariantDocCompactWriter() override = default; @@ -279,7 +278,6 @@ class VariantDocCompactWriter : public ColumnWriter { ordinal_t _next_rowid = 0; MutableColumnPtr _column; - const TabletColumn* _tablet_column = nullptr; ColumnWriterOptions _opts; bool _is_finalized = false; bool _data_written = false; diff --git a/be/src/storage/segment/variant/variant_streaming_compaction_writer.cpp b/be/src/storage/segment/variant/variant_streaming_compaction_writer.cpp index 0dcf05e095e302..48ea040c7bafe8 100644 --- a/be/src/storage/segment/variant/variant_streaming_compaction_writer.cpp +++ b/be/src/storage/segment/variant/variant_streaming_compaction_writer.cpp @@ -55,8 +55,7 @@ Status VariantStreamingCompactionWriter::init() { Status VariantStreamingCompactionWriter::_init_root_writer() { _root_writer = std::make_unique( - _opts, std::unique_ptr(StorageFieldFactory::create(*_tablet_column)), - _opts.file_writer); + _opts, std::make_shared(*_tablet_column), _opts.file_writer); RETURN_IF_ERROR(_root_writer->init()); _opts.meta->set_num_rows(0); return Status::OK(); diff --git a/be/src/storage/task/index_builder.cpp b/be/src/storage/task/index_builder.cpp index 71176c99ce834d..b7626d553ede15 100644 --- a/be/src/storage/task/index_builder.cpp +++ b/be/src/storage/task/index_builder.cpp @@ -21,7 +21,6 @@ #include "common/logging.h" #include "common/status.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" @@ -488,7 +487,6 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta DCHECK(output_rowset_schema->has_inverted_index_with_index_id(index_id)); _olap_data_convertor->add_column_data_convertor(column); return_columns.emplace_back(column_idx); - std::unique_ptr field(StorageFieldFactory::create(column)); if (inverted_index.index_type == TIndexType::INVERTED) { // inverted index @@ -500,7 +498,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta std::unique_ptr inverted_index_builder; try { RETURN_IF_ERROR(segment_v2::IndexColumnWriter::create( - field.get(), &inverted_index_builder, index_file_writer.get(), + &column, &inverted_index_builder, index_file_writer.get(), index_meta)); DBUG_EXECUTE_IF( "IndexBuilder::handle_single_rowset_index_column_writer_create_" @@ -530,8 +528,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta std::unique_ptr index_writer; try { RETURN_IF_ERROR(segment_v2::IndexColumnWriter::create( - field.get(), &index_writer, index_file_writer.get(), - index_meta)); + &column, &index_writer, index_file_writer.get(), index_meta)); DBUG_EXECUTE_IF( "IndexBuilder::handle_single_rowset_index_column_writer_create_" "error", @@ -688,9 +685,8 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, continue; } } - auto column = tablet_schema->column(column_idx); + const auto& column = tablet_schema->column(column_idx); auto writer_sign = std::make_pair(segment_idx, index_id); - std::unique_ptr field(StorageFieldFactory::create(column)); auto converted_result = _olap_data_convertor->convert_column_data(i); DBUG_EXECUTE_IF("IndexBuilder::_write_inverted_index_data_convert_column_data_error", { converted_result.first = Status::Error( @@ -703,10 +699,10 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, const auto* ptr = (const uint8_t*)converted_result.second->get_data(); const auto* null_map = converted_result.second->get_nullmap(); if (null_map) { - RETURN_IF_ERROR(_add_nullable(column_name, writer_sign, field.get(), null_map, &ptr, + RETURN_IF_ERROR(_add_nullable(column_name, writer_sign, &column, null_map, &ptr, block->rows())); } else { - RETURN_IF_ERROR(_add_data(column_name, writer_sign, field.get(), &ptr, block->rows())); + RETURN_IF_ERROR(_add_data(column_name, writer_sign, &column, &ptr, block->rows())); } } _olap_data_convertor->clear_source_content(); @@ -716,11 +712,11 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, Status IndexBuilder::_add_nullable(const std::string& column_name, const std::pair& index_writer_sign, - StorageField* field, const uint8_t* null_map, + const TabletColumn* column, const uint8_t* null_map, const uint8_t** ptr, size_t num_rows) { // TODO: need to process null data for inverted index - if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - DCHECK(field->get_sub_field_count() == 1); + if (column->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + DCHECK(column->get_subtype_count() == 1); // [size, offset_ptr, item_data_ptr, item_nullmap_ptr] const auto* data_ptr = reinterpret_cast(*ptr); // total number length @@ -730,7 +726,8 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, auto data = *(data_ptr + 2); auto nested_null_map = *(data_ptr + 3); RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_array_values( - field->get_sub_field(0)->size(), reinterpret_cast(data), + field_type_size(column->get_sub_column(0).type()), + reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); DBUG_EXECUTE_IF("IndexBuilder::_add_nullable_add_array_values_error", { _CLTHROWA(CL_ERR_IO, "debug point: _add_nullable_add_array_values_error"); @@ -766,7 +763,7 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_values(column_name, *ptr, step)); } - *ptr += field->size() * step; + *ptr += field_type_size(column->type()) * step; offset += step; DBUG_EXECUTE_IF("IndexBuilder::_add_nullable_throw_exception", { _CLTHROWA(CL_ERR_IO, "debug point: _add_nullable_throw_exception"); }) @@ -781,10 +778,10 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, Status IndexBuilder::_add_data(const std::string& column_name, const std::pair& index_writer_sign, - StorageField* field, const uint8_t** ptr, size_t num_rows) { + const TabletColumn* column, const uint8_t** ptr, size_t num_rows) { try { - if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - DCHECK(field->get_sub_field_count() == 1); + if (column->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + DCHECK(column->get_subtype_count() == 1); // [size, offset_ptr, item_data_ptr, item_nullmap_ptr] const auto* data_ptr = reinterpret_cast(*ptr); // total number length @@ -795,7 +792,8 @@ Status IndexBuilder::_add_data(const std::string& column_name, auto data = *(data_ptr + 2); auto nested_null_map = *(data_ptr + 3); RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_array_values( - field->get_sub_field(0)->size(), reinterpret_cast(data), + field_type_size(column->get_sub_column(0).type()), + reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); } } else { diff --git a/be/src/storage/task/index_builder.h b/be/src/storage/task/index_builder.h index e3b536f54614a7..bf417182b7ff3c 100644 --- a/be/src/storage/task/index_builder.h +++ b/be/src/storage/task/index_builder.h @@ -17,7 +17,6 @@ #pragma once -#include "storage/field.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/iterator/olap_data_convertor.h" @@ -36,8 +35,6 @@ class IndexFileWriter; } // namespace segment_v2 class OlapBlockDataConvertor; -class StorageField; - class StorageEngine; class RowsetWriter; @@ -63,11 +60,12 @@ class IndexBuilder { Status _write_inverted_index_data(TabletSchemaSPtr tablet_schema, int64_t segment_idx, Block* block); Status _add_data(const std::string& column_name, - const std::pair& index_writer_sign, StorageField* field, - const uint8_t** ptr, size_t num_rows); + const std::pair& index_writer_sign, + const TabletColumn* column, const uint8_t** ptr, size_t num_rows); Status _add_nullable(const std::string& column_name, - const std::pair& index_writer_sign, StorageField* field, - const uint8_t* null_map, const uint8_t** ptr, size_t num_rows); + const std::pair& index_writer_sign, + const TabletColumn* column, const uint8_t* null_map, const uint8_t** ptr, + size_t num_rows); private: StorageEngine& _engine; diff --git a/be/src/storage/types.h b/be/src/storage/types.h index 07b882d6fdb397..7d66631ef16180 100644 --- a/be/src/storage/types.h +++ b/be/src/storage/types.h @@ -40,11 +40,8 @@ #include "core/uint24.h" #include "core/value/ipv4_value.h" #include "core/value/ipv6_value.h" -#include "core/value/map_value.h" -#include "core/value/struct_value.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_timestamptz.h" -#include "runtime/collection_value.h" #include "storage/olap_common.h" #include "storage/olap_define.h" #include "util/slice.h" @@ -58,8 +55,27 @@ static const std::vector DATE_FORMATS { "%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d", "%y/%m/%d", }; +// Maps a storage FieldType to its in-memory cell representation. +// +// ARRAY / MAP / STRUCT are intentionally NOT specialized here: they are +// containers of other types, so only their element types have a storage-layer +// cell representation. The primary template below uses a deferred +// static_assert so that instantiating `CppTypeTraits` for any +// unspecialized FieldType — most importantly ARRAY/MAP/STRUCT — fails at +// build time with a clear message, rather than at runtime. +namespace detail { +template +inline constexpr bool cpp_type_traits_unspecialized = false; +} // namespace detail + template -struct CppTypeTraits {}; +struct CppTypeTraits { + static_assert(detail::cpp_type_traits_unspecialized, + "CppTypeTraits not specialized for this FieldType. " + "ARRAY / MAP / STRUCT and similar container types have no " + "storage-layer cell representation — operate on the element " + "type instead."); +}; template <> struct CppTypeTraits { @@ -214,18 +230,6 @@ struct CppTypeTraits { using CppType = Slice; }; -template <> -struct CppTypeTraits { - using CppType = StructValue; -}; -template <> -struct CppTypeTraits { - using CppType = CollectionValue; -}; -template <> -struct CppTypeTraits { - using CppType = MapValue; -}; template struct BaseFieldTypeTraits : public CppTypeTraits { using CppType = typename CppTypeTraits::CppType; @@ -372,6 +376,27 @@ struct TypeTraits : public FieldTypeTraits { static const int32_t size = sizeof(CppType); }; +// In-memory storage cell footprint for one value of `field_type`, +// i.e. sizeof(CppTypeTraits::CppType). +// +// This is NOT the schema-declared length: +// - CHAR(N) / VARCHAR(N) / STRING / JSONB / VARIANT / HLL / BITMAP / +// QUANTILE_STATE / AGG_STATE all return sizeof(Slice) == 16 (the ptr+len +// descriptor in a row buffer); for the user-declared N see +// TabletColumn::get_field_length_by_type. +// +// ARRAY / MAP / STRUCT are containers of other types — only their element +// types have a storage-layer cell size. The container itself has no such size +// at this layer, so it is not handled here. Passing one in is a programming +// error and trips the default LOG(FATAL) below. +// +// VARIANT root data is still routed through ColumnReader/EncodingInfo at read +// time, so VARIANT keeps its full traits chain even though the column-writer +// step path doesn't reach it. +// +// Used for cell-level pointer arithmetic on row buffers, BKD bytes_per_dim +// fallback when the index file has no header, and per-row footprint estimation +// during compaction. inline size_t field_type_size(FieldType field_type) { switch (field_type) { #define DORIS_FIELD_TYPE_SIZE_CASE(ft) \ @@ -409,9 +434,6 @@ inline size_t field_type_size(FieldType field_type) { DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_BITMAP) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_QUANTILE_STATE) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_AGG_STATE) - DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_STRUCT) - DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_ARRAY) - DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_MAP) #undef DORIS_FIELD_TYPE_SIZE_CASE default: LOG(FATAL) << "field_type_size: unsupported FieldType " << int(field_type); diff --git a/be/test/exec/scan/vgeneric_iterators_test.cpp b/be/test/exec/scan/vgeneric_iterators_test.cpp index d02cc8dd34dfab..34349b8bd36038 100644 --- a/be/test/exec/scan/vgeneric_iterators_test.cpp +++ b/be/test/exec/scan/vgeneric_iterators_test.cpp @@ -28,7 +28,6 @@ #include "core/data_type/data_type.h" #include "core/field.h" #include "gtest/gtest_pred_impl.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" diff --git a/be/test/exprs/function/cast/function_variant_cast_test.cpp b/be/test/exprs/function/cast/function_variant_cast_test.cpp index 2ee76058bc69f8..8f710188b40290 100644 --- a/be/test/exprs/function/cast/function_variant_cast_test.cpp +++ b/be/test/exprs/function/cast/function_variant_cast_test.cpp @@ -33,7 +33,6 @@ #include "exprs/function/simple_function_factory.h" #include "gtest/gtest_pred_impl.h" #include "runtime/runtime_state.h" -#include "storage/field.h" namespace doris { static doris::Field construct_variant_map( diff --git a/be/test/load/memtable/memtable_flush_executor_test.cpp b/be/test/load/memtable/memtable_flush_executor_test.cpp index eacf2cadcb9a9c..7916d94db409c1 100644 --- a/be/test/load/memtable/memtable_flush_executor_test.cpp +++ b/be/test/load/memtable/memtable_flush_executor_test.cpp @@ -38,7 +38,6 @@ #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" -#include "storage/field.h" #include "storage/options.h" #include "storage/rowset/group_rowset_writer.h" #include "storage/rowset/rowset_writer.h" diff --git a/be/test/storage/compaction/ordered_data_compaction_test.cpp b/be/test/storage/compaction/ordered_data_compaction_test.cpp index 006d48358c467e..fa050f6a68b40e 100644 --- a/be/test/storage/compaction/ordered_data_compaction_test.cpp +++ b/be/test/storage/compaction/ordered_data_compaction_test.cpp @@ -49,7 +49,6 @@ #include "storage/compaction/cumulative_compaction.h" #include "storage/data_dir.h" #include "storage/delete/delete_handler.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/options.h" #include "storage/rowset/beta_rowset.h" diff --git a/be/test/storage/compaction/vertical_compaction_test.cpp b/be/test/storage/compaction/vertical_compaction_test.cpp index 9623f230dbbaa4..58e0cec0762753 100644 --- a/be/test/storage/compaction/vertical_compaction_test.cpp +++ b/be/test/storage/compaction/vertical_compaction_test.cpp @@ -48,7 +48,6 @@ #include "json2pb/json_to_pb.h" #include "runtime/exec_env.h" #include "storage/delete/delete_handler.h" -#include "storage/field.h" #include "storage/iterator/vertical_merge_iterator.h" #include "storage/merger.h" #include "storage/olap_common.h" diff --git a/be/test/storage/index/ann/ann_index_smoke_test.cpp b/be/test/storage/index/ann/ann_index_smoke_test.cpp index 32de8d0626bd61..3b5886cdffc456 100644 --- a/be/test/storage/index/ann/ann_index_smoke_test.cpp +++ b/be/test/storage/index/ann/ann_index_smoke_test.cpp @@ -22,7 +22,6 @@ #include #include -#include "storage/field.h" #include "storage/index/ann/ann_index.h" #include "storage/index/ann/ann_index_writer.h" #include "storage/index/ann/ann_search_params.h" @@ -55,11 +54,6 @@ class AnnIndexTest : public testing::Test { _tablet_column_array = std::make_unique(); _tablet_column_float = std::make_unique(); - EXPECT_CALL(*_tablet_column_array, type()) - .WillRepeatedly(testing::Return(FieldType::OLAP_FIELD_TYPE_ARRAY)); - - StorageField field(*_tablet_column_array); - EXPECT_CALL(*_index_file_writer, open(_index_meta.get())) .WillOnce(testing::Return(_ram_dir)); diff --git a/be/test/storage/index/ann/ann_index_writer_test.cpp b/be/test/storage/index/ann/ann_index_writer_test.cpp index 9035c79320cd5e..bb30f9e19794af 100644 --- a/be/test/storage/index/ann/ann_index_writer_test.cpp +++ b/be/test/storage/index/ann/ann_index_writer_test.cpp @@ -26,8 +26,6 @@ #include #include -#include "runtime/collection_value.h" -#include "storage/field.h" #include "storage/index/ann/vector_search_utils.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_fs_directory.h" @@ -235,22 +233,6 @@ TEST_F(AnnIndexWriterTest, TestAddArrayValuesWrongDimension) { EXPECT_TRUE(status.is()); } -TEST_F(AnnIndexWriterTest, TestAddArrayValuesWithCollectionValue) { - auto writer = - std::make_unique(_index_file_writer.get(), _tablet_index.get()); - - auto fs_dir = std::make_shared(); - fs_dir->init(doris::io::global_local_filesystem(), "./ut_dir/tmp_vector_search", nullptr); - EXPECT_CALL(*_index_file_writer, open(testing::_)).WillOnce(testing::Return(fs_dir)); - - ASSERT_TRUE(writer->init().ok()); - - // This should return an error as ANN index doesn't support nullable columns - Status status = writer->add_array_values(sizeof(float), nullptr, 1); - EXPECT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); -} - TEST_F(AnnIndexWriterTest, TestAddValues) { auto writer = std::make_unique(_index_file_writer.get(), _tablet_index.get()); @@ -517,8 +499,8 @@ TEST_F(AnnIndexWriterTest, TestCreateFromIndexColumnWriter) { tablet_schema->append_column(array_column); // Get field for array column - std::unique_ptr field(StorageFieldFactory::create(array_column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(array_column); + ASSERT_NE(field, nullptr); auto fs_dir = std::make_shared(); fs_dir->init(doris::io::global_local_filesystem(), "./ut_dir/tmp_vector_search", nullptr); @@ -526,7 +508,7 @@ TEST_F(AnnIndexWriterTest, TestCreateFromIndexColumnWriter) { // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, _index_file_writer.get(), + auto status = IndexColumnWriter::create(field, &column_writer, _index_file_writer.get(), _tablet_index.get()); EXPECT_TRUE(status.ok()); diff --git a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp index 162543c3033dc8..2a59fb86acc5e8 100644 --- a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp +++ b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp @@ -33,6 +33,7 @@ #include "storage/compaction/base_compaction.h" #include "storage/index/index_file_reader.h" #include "storage/index/inverted/query/query_factory.h" +#include "storage/key_coder.h" #include "storage/rowset/beta_rowset.h" #include "storage/rowset/beta_rowset_writer.h" #include "storage/rowset/rowset_factory.h" diff --git a/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp b/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp index 3fc08841df774b..2da72758470a27 100644 --- a/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp +++ b/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp @@ -24,7 +24,6 @@ #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_cache.h" @@ -131,16 +130,16 @@ class PhraseEdgeQueryTest : public testing::Test { std::make_unique(fs, *index_path_prefix, std::string {rowset_id}, seg_id, format, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Write string values diff --git a/be/test/storage/index/inverted/query/phrase_prefix_query_test.cpp b/be/test/storage/index/inverted/query/phrase_prefix_query_test.cpp index cd9c8eb214d9e3..739720d61bd9d2 100644 --- a/be/test/storage/index/inverted/query/phrase_prefix_query_test.cpp +++ b/be/test/storage/index/inverted/query/phrase_prefix_query_test.cpp @@ -23,7 +23,6 @@ #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_cache.h" @@ -130,16 +129,16 @@ class PhrasePrefixQueryTest : public testing::Test { std::make_unique(fs, *index_path_prefix, std::string {rowset_id}, seg_id, format, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Write string values diff --git a/be/test/storage/index/inverted/query/phrase_query_test.cpp b/be/test/storage/index/inverted/query/phrase_query_test.cpp index 877a91d1f20571..40f7f59349dbb5 100644 --- a/be/test/storage/index/inverted/query/phrase_query_test.cpp +++ b/be/test/storage/index/inverted/query/phrase_query_test.cpp @@ -23,7 +23,6 @@ #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_cache.h" @@ -131,16 +130,16 @@ class PhraseQueryTest : public testing::Test { std::make_unique(fs, *index_path_prefix, std::string {rowset_id}, seg_id, format, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Write string values diff --git a/be/test/storage/metadata_adder_test.cpp b/be/test/storage/metadata_adder_test.cpp index 8a2d8ec13e5277..b436a6c670cb2d 100644 --- a/be/test/storage/metadata_adder_test.cpp +++ b/be/test/storage/metadata_adder_test.cpp @@ -21,7 +21,6 @@ #include "core/data_type/data_type_factory.hpp" #include "core/data_type/define_primitive_type.h" -#include "storage/field.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" @@ -123,7 +122,7 @@ TEST_F(MetadataAdderTest, meta_load_with_pb_test) { { auto fs = io::global_local_filesystem(); TabletColumnPtr int_column = create_int_key(0); - StorageField* int_field = StorageFieldFactory::create(*int_column); + const TabletColumn* int_field = int_column.get(); auto int_data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_INT, false); // 1 load first column @@ -161,7 +160,7 @@ TEST_F(MetadataAdderTest, meta_load_with_pb_test) { // load second column segment_v2::ColumnIndexMetaPB index_meta2; TabletColumnPtr varchar_column = create_varchar_key(0); - StorageField* str_field = StorageFieldFactory::create(*varchar_column); + const TabletColumn* str_field = varchar_column.get(); auto str_data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false); std::string file2 = kTestDir + "/copy_obj2"; @@ -193,9 +192,6 @@ TEST_F(MetadataAdderTest, meta_load_with_pb_test) { ASSERT_TRUE(MetadataAdder::get_all_segments_size() == mem_size2 + mem_size); - - delete int_field; - delete str_field; } ASSERT_TRUE(MetadataAdder::get_all_segments_size() == 0); diff --git a/be/test/storage/segment/column_reader_writer_test.cpp b/be/test/storage/segment/column_reader_writer_test.cpp index 191db3e116ffe6..98e6b74b266da3 100644 --- a/be/test/storage/segment/column_reader_writer_test.cpp +++ b/be/test/storage/segment/column_reader_writer_test.cpp @@ -233,209 +233,6 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, } } -template -void test_array_nullable_data(CollectionValue* src_data, uint8_t* src_is_null, int num_rows, - std::string test_name) { - CollectionValue* src = src_data; - ColumnMetaPB meta; - TabletColumn list_column(OLAP_FIELD_AGGREGATION_NONE, FieldType::OLAP_FIELD_TYPE_ARRAY); - int32 item_length = 0; - if (item_type == FieldType::OLAP_FIELD_TYPE_CHAR || - item_type == FieldType::OLAP_FIELD_TYPE_VARCHAR) { - item_length = 10; - } - TabletColumn item_column(OLAP_FIELD_AGGREGATION_NONE, item_type, true, 0, item_length); - list_column.add_sub_column(item_column); - Field* field = StorageFieldFactory::create(list_column); - - // write data - std::string fname = TEST_DIR + "/" + test_name; - auto fs = io::global_local_filesystem(); - { - io::FileWriterPtr file_writer; - Status st = fs->create_file(fname, &file_writer); - EXPECT_TRUE(st.ok()) << st; - - ColumnWriterOptions writer_opts; - writer_opts.meta = &meta; - writer_opts.meta->set_column_id(0); - writer_opts.meta->set_unique_id(0); - writer_opts.meta->set_type(FieldType::OLAP_FIELD_TYPE_ARRAY); - writer_opts.meta->set_length(0); - writer_opts.meta->set_encoding(array_encoding); - writer_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F); - writer_opts.meta->set_is_nullable(true); - writer_opts.data_page_size = 5 * 8; - - ColumnMetaPB* child_meta = meta.add_children_columns(); - - child_meta->set_column_id(1); - child_meta->set_unique_id(1); - child_meta->set_type(item_type); - child_meta->set_length(item_length); - child_meta->set_encoding(item_encoding); - child_meta->set_compression(segment_v2::CompressionTypePB::LZ4F); - child_meta->set_is_nullable(true); - - std::unique_ptr writer; - ColumnWriter::create(writer_opts, &list_column, file_writer.get(), &writer); - st = writer->init(); - EXPECT_TRUE(st.ok()) << st.to_string(); - - for (int i = 0; i < num_rows; ++i) { - st = writer->append(BitmapTest(src_is_null, i), src + i); - EXPECT_TRUE(st.ok()); - } - - st = writer->finish(); - EXPECT_TRUE(st.ok()); - - st = writer->write_data(); - EXPECT_TRUE(st.ok()); - st = writer->write_ordinal_index(); - EXPECT_TRUE(st.ok()); - - // close the file - EXPECT_TRUE(file_writer->close().ok()); - } - auto type_info = get_type_info(&meta); - io::FileReaderSPtr file_reader; - ASSERT_EQ(fs->open_file(fname, &file_reader), Status::OK()); - // read and check - { - ColumnReaderOptions reader_opts; - std::shared_ptr reader; - auto st = ColumnReader::create(reader_opts, meta, num_rows, file_reader, &reader); - EXPECT_TRUE(st.ok()); - - ColumnIteratorUPtr iter; - st = reader->new_iterator(&iter); - EXPECT_TRUE(st.ok()); - - ColumnIteratorOptions iter_opts; - OlapReaderStatistics stats; - iter_opts.stats = &stats; - iter_opts.file_reader = file_reader.get(); - st = iter->init(iter_opts); - EXPECT_TRUE(st.ok()); - // sequence read - { - Arena pool; - std::unique_ptr cvb; - ColumnVectorBatch::create(0, true, type_info.get(), field, &cvb); - cvb->resize(1024); - ColumnBlock col(cvb.get(), &pool); - - int idx = 0; - while (true) { - size_t rows_read = 1024; - ColumnBlockView dst(&col); - st = iter->next_batch(&rows_read, &dst); - EXPECT_TRUE(st.ok()); - for (int j = 0; j < rows_read; ++j) { - EXPECT_EQ(BitmapTest(src_is_null, idx), col.is_null(j)); - if (!col.is_null(j)) { - EXPECT_TRUE(type_info->equal(&src[idx], col.cell_ptr(j))); - } - ++idx; - } - if (rows_read < 1024) { - break; - } - } - } - // seek read - { - Arena pool; - std::unique_ptr cvb; - ColumnVectorBatch::create(0, true, type_info.get(), field, &cvb); - cvb->resize(1024); - ColumnBlock col(cvb.get(), &pool); - - for (int rowid = 0; rowid < num_rows; rowid += 4025) { - st = iter->seek_to_ordinal(rowid); - EXPECT_TRUE(st.ok()); - - int idx = rowid; - size_t rows_read = 1024; - ColumnBlockView dst(&col); - - st = iter->next_batch(&rows_read, &dst); - EXPECT_TRUE(st.ok()); - for (int j = 0; j < rows_read; ++j) { - EXPECT_EQ(BitmapTest(src_is_null, idx), col.is_null(j)); - if (!col.is_null(j)) { - EXPECT_TRUE(type_info->equal(&src[idx], col.cell_ptr(j))); - } - ++idx; - } - } - } - delete iter; - } - delete field; -} - -TEST_F(ColumnReaderWriterTest, test_array_type) { - size_t num_array = LOOP_LESS_OR_MORE(1024, 24 * 1024); - size_t num_item = num_array * 3; - - uint8_t* array_is_null = new uint8_t[BitmapSize(num_array)]; - CollectionValue* array_val = new CollectionValue[num_array]; - bool* item_is_null = new bool[num_item]; - uint8_t* item_val = new uint8_t[num_item]; - for (int i = 0; i < num_item; ++i) { - item_val[i] = i; - item_is_null[i] = (i % 4) == 0; - if (i % 3 == 0) { - size_t array_index = i / 3; - bool is_null = (array_index % 4) == 1; - BitmapChange(array_is_null, array_index, is_null); - if (is_null) { - continue; - } - array_val[array_index].set_data(&item_val[i]); - array_val[array_index].set_null_signs(&item_is_null[i]); - array_val[array_index].set_length(3); - } - } - test_array_nullable_data( - array_val, array_is_null, num_array, "null_array_bs"); - - delete[] array_val; - delete[] item_val; - delete[] item_is_null; - - array_val = new CollectionValue[num_array]; - Slice* varchar_vals = new Slice[3]; - item_is_null = new bool[3]; - for (int i = 0; i < 3; ++i) { - item_is_null[i] = i == 1; - if (i != 1) { - set_column_value_by_type(FieldType::OLAP_FIELD_TYPE_VARCHAR, i, (char*)&varchar_vals[i], - &_pool); - } - } - for (int i = 0; i < num_array; ++i) { - bool is_null = (i % 4) == 1; - BitmapChange(array_is_null, i, is_null); - if (is_null) { - continue; - } - array_val[i].set_data(varchar_vals); - array_val[i].set_null_signs(item_is_null); - array_val[i].set_length(3); - } - test_array_nullable_data( - array_val, array_is_null, num_array, "null_array_chars"); - - delete[] array_val; - delete[] varchar_vals; - delete[] item_is_null; - - delete[] array_is_null; -} - TEST_F(ColumnReaderWriterTest, test_array_append_nulls) { ColumnMetaPB meta; TabletColumn list_column(OLAP_FIELD_AGGREGATION_NONE, FieldType::OLAP_FIELD_TYPE_ARRAY); @@ -837,29 +634,5 @@ TEST_F(ColumnReaderWriterTest, test_v_default_value) { test_v_read_default_value(v_decimal, &decimal); } -TEST_F(ColumnReaderWriterTest, test_single_empty_array) { - size_t num_array = 1; - std::unique_ptr array_is_null(new uint8_t[BitmapSize(num_array)]()); - CollectionValue array(0); - test_array_nullable_data( - &array, array_is_null.get(), num_array, "test_single_empty_array"); -} - -TEST_F(ColumnReaderWriterTest, test_mixed_empty_arrays) { - size_t num_array = 3; - std::unique_ptr array_is_null(new uint8_t[BitmapSize(num_array)]()); - std::unique_ptr collection_values(new CollectionValue[num_array]); - int data[] = {1, 2, 3}; - for (int i = 0; i < num_array; ++i) { - if (i % 2 == 1) { - new (&collection_values[i]) CollectionValue(0); - } else { - new (&collection_values[i]) CollectionValue(&data, 3, false, nullptr); - } - } - test_array_nullable_data( - collection_values.get(), array_is_null.get(), num_array, "test_mixed_empty_arrays"); -} - } // namespace segment_v2 } // namespace doris diff --git a/be/test/storage/segment/inverted_index_array_test.cpp b/be/test/storage/segment/inverted_index_array_test.cpp index 5eb93a75bb578a..7fc63e6fcd0b78 100644 --- a/be/test/storage/segment/inverted_index_array_test.cpp +++ b/be/test/storage/segment/inverted_index_array_test.cpp @@ -42,7 +42,6 @@ #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_compound_reader.h" @@ -53,6 +52,7 @@ #include "storage/iterator/olap_data_convertor.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" +#include "storage/types.h" #include "util/faststring.h" #include "util/slice.h" @@ -203,7 +203,7 @@ class InvertedIndexArrayTest : public testing::Test { return tablet_schema; } - void test_non_null_string(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_non_null_string(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -270,7 +270,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Get the length of the subfield, used for inverted index writing - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); // Call the inverted index writing interface, passing in item_data, item_nullmap, offsets_ptr, and the number of rows (the number of array rows in the Block) st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); @@ -289,7 +289,7 @@ class InvertedIndexArrayTest : public testing::Test { &idx_meta); } - void test_string(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_string(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -357,7 +357,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Get the length of the subfield, used for inverted index writing - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); // Call the inverted index writing interface, passing in item_data, item_nullmap, offsets_ptr, and the number of rows (the number of array rows in the Block) st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); @@ -375,7 +375,7 @@ class InvertedIndexArrayTest : public testing::Test { &idx_meta); } - void test_null_write_v2(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_null_write_v2(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -469,7 +469,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Call the inverted index writing interface, passing in the converted nested data, nullmap, and offsets - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); EXPECT_EQ(st, Status::OK()); @@ -491,7 +491,7 @@ class InvertedIndexArrayTest : public testing::Test { InvertedIndexStorageFormatPB::V2, &idx_meta); } - void test_null_write(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_null_write(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -582,7 +582,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Call the inverted index writing interface, passing in the converted nested data, nullmap, and offsets - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); EXPECT_EQ(st, Status::OK()); @@ -604,7 +604,7 @@ class InvertedIndexArrayTest : public testing::Test { InvertedIndexStorageFormatPB::V1, &idx_meta); } - void test_multi_block_write(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_multi_block_write(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -678,7 +678,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* offsets_ptr = reinterpret_cast(data_ptr[1]); const void* item_data = reinterpret_cast(data_ptr[2]); const auto* item_nullmap = reinterpret_cast(data_ptr[3]); - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, row_num); EXPECT_EQ(st, Status::OK()); @@ -727,7 +727,7 @@ class InvertedIndexArrayTest : public testing::Test { const void* item_data = reinterpret_cast(data_ptr[2]); const auto* item_nullmap = reinterpret_cast(data_ptr[3]); - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, row_num); EXPECT_EQ(st, Status::OK()); @@ -774,7 +774,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* offsets_ptr = reinterpret_cast(data_ptr[1]); const void* item_data = reinterpret_cast(data_ptr[2]); const auto* item_nullmap = reinterpret_cast(data_ptr[3]); - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, row_num); EXPECT_EQ(st, Status::OK()); @@ -796,7 +796,7 @@ class InvertedIndexArrayTest : public testing::Test { InvertedIndexStorageFormatPB::V1, &idx_meta); } - void test_array_numeric(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_array_numeric(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -882,7 +882,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // get the size of the sub field (4 bytes for INT type) - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); EXPECT_EQ(st, Status::OK()); @@ -938,7 +938,7 @@ class InvertedIndexArrayTest : public testing::Test { } } - void test_array_all_null(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_array_all_null(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -993,7 +993,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); const auto* null_map = accessor->get_nullmap(); - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); EXPECT_EQ(st, Status::OK()); @@ -1055,10 +1055,9 @@ TEST_F(InvertedIndexArrayTest, ArrayString) { arraySubColumn.set_name("arr_sub_string"); arraySubColumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING); arrayTabletColumn.add_sub_column(arraySubColumn); - StorageField* field = StorageFieldFactory::create(arrayTabletColumn); + const TabletColumn* field = &(arrayTabletColumn); test_string("rowset_id", 0, field); test_non_null_string("rowset_id_non_null", 0, field); - delete field; } TEST_F(InvertedIndexArrayTest, ComplexNullCases) { @@ -1071,11 +1070,10 @@ TEST_F(InvertedIndexArrayTest, ComplexNullCases) { arraySubColumn.set_name("arr_sub_string"); arraySubColumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING); arrayTabletColumn.add_sub_column(arraySubColumn); - StorageField* field = StorageFieldFactory::create(arrayTabletColumn); + const TabletColumn* field = &(arrayTabletColumn); test_null_write("complex_null", 0, field); test_null_write_v2("complex_null_v2", 0, field); test_array_all_null("complex_null_all_null", 0, field); - delete field; } TEST_F(InvertedIndexArrayTest, MultiBlockWrite) { @@ -1088,9 +1086,8 @@ TEST_F(InvertedIndexArrayTest, MultiBlockWrite) { arraySubColumn.set_name("arr_sub_string"); arraySubColumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING); arrayTabletColumn.add_sub_column(arraySubColumn); - StorageField* field = StorageFieldFactory::create(arrayTabletColumn); + const TabletColumn* field = &(arrayTabletColumn); test_multi_block_write("multi_block", 0, field); - delete field; } TEST_F(InvertedIndexArrayTest, ArrayInt) { @@ -1103,8 +1100,7 @@ TEST_F(InvertedIndexArrayTest, ArrayInt) { arraySubColumn.set_name("arr_sub_int"); arraySubColumn.set_type(FieldType::OLAP_FIELD_TYPE_INT); arrayTabletColumn.add_sub_column(arraySubColumn); - StorageField* field = StorageFieldFactory::create(arrayTabletColumn); + const TabletColumn* field = &(arrayTabletColumn); test_array_numeric("int_test", 0, field); - delete field; } } // namespace doris::segment_v2 diff --git a/be/test/storage/segment/inverted_index_reader_test.cpp b/be/test/storage/segment/inverted_index_reader_test.cpp index 2305833a402778..982d2c3f14155f 100644 --- a/be/test/storage/segment/inverted_index_reader_test.cpp +++ b/be/test/storage/segment/inverted_index_reader_test.cpp @@ -32,12 +32,12 @@ #include "core/field.h" #include "core/value/vdatetime_value.h" #include "runtime/runtime_state.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_iterator.h" #include "storage/index/inverted/inverted_index_writer.h" +#include "storage/key_coder.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" #include "util/slice.h" @@ -136,16 +136,16 @@ class InvertedIndexReaderTest : public testing::Test { std::make_unique(fs, *index_path_prefix, std::string {rowset_id}, seg_id, format, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Write string values @@ -190,16 +190,16 @@ class InvertedIndexReaderTest : public testing::Test { fs, *index_path_prefix, std::string {rowset_id}, seg_id, InvertedIndexStorageFormatPB::V2, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Add NULL values @@ -258,16 +258,16 @@ class InvertedIndexReaderTest : public testing::Test { fs, *index_path_prefix, std::string {rowset_id}, seg_id, InvertedIndexStorageFormatPB::V2, std::move(file_writer)); - // Get c1 column StorageField + // Get c1 column descriptor const TabletColumn& column = tablet_schema->column(0); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Add integer values @@ -2712,12 +2712,12 @@ class InvertedIndexReaderTest : public testing::Test { double_schema->append_column(double_column); const TabletColumn& column = double_schema->column(0); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; for (const auto& value : values) { @@ -3491,12 +3491,12 @@ class InvertedIndexReaderTest : public testing::Test { seg_id, format, std::move(file_writer)); const TabletColumn& column = tablet_schema->column(col_id); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; for (const auto& value : values) { @@ -3996,12 +3996,12 @@ class InvertedIndexReaderTest : public testing::Test { InvertedIndexStorageFormatPB::V2, std::move(file_writer)); const TabletColumn& test_column = tablet_schema->column(0); - std::unique_ptr field(StorageFieldFactory::create(test_column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(test_column); + ASSERT_NE(field, nullptr); std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); // This should fail for unsupported types, demonstrating the default case // If it succeeds, we can still test with invalid query parameters diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index c91c44b0911ea3..8b7f56221cf8cf 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -31,12 +31,12 @@ #include #include +#include "core/block/block.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_number.h" #include "core/field.h" #include "io/fs/local_file_system.h" #include "runtime/runtime_state.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" @@ -44,6 +44,7 @@ #include "storage/index/inverted/inverted_index_reader.h" #include "storage/iterator/olap_data_convertor.h" #include "storage/tablet/tablet_schema.h" +#include "storage/types.h" #include "util/faststring.h" #include "util/slice.h" @@ -402,13 +403,13 @@ class InvertedIndexWriterTest : public testing::Test { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add some string values @@ -458,13 +459,13 @@ class InvertedIndexWriterTest : public testing::Test { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add string values @@ -522,13 +523,13 @@ class InvertedIndexWriterTest : public testing::Test { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add null values @@ -598,13 +599,13 @@ class InvertedIndexWriterTest : public testing::Test { // Get field for column c1 const TabletColumn& column = tablet_schema->column(0); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add integer values @@ -661,8 +662,8 @@ class InvertedIndexWriterTest : public testing::Test { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Save original config value bool original_config_value = config::enable_inverted_index_correct_term_write; @@ -672,8 +673,8 @@ class InvertedIndexWriterTest : public testing::Test { // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add string values with Unicode characters above 0xFFFF @@ -846,8 +847,8 @@ TEST_F(InvertedIndexWriterTest, CompareUnicodeStringWriteResults) { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Save original config value bool original_config_value = config::enable_inverted_index_correct_term_write; @@ -857,13 +858,13 @@ TEST_F(InvertedIndexWriterTest, CompareUnicodeStringWriteResults) { // Set config to enabled for first writer config::enable_inverted_index_correct_term_write = true; - auto status = IndexColumnWriter::create(field.get(), &column_writer_enabled, + auto status = IndexColumnWriter::create(field, &column_writer_enabled, index_file_writer_enabled.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Set config to disabled for second writer config::enable_inverted_index_correct_term_write = false; - status = IndexColumnWriter::create(field.get(), &column_writer_disabled, + status = IndexColumnWriter::create(field, &column_writer_disabled, index_file_writer_disabled.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; @@ -1009,13 +1010,13 @@ TEST_F(InvertedIndexWriterTest, ErrorHandlingInFileWriter) { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Test with empty values array to trigger certain error paths @@ -1088,13 +1089,13 @@ TEST_F(InvertedIndexWriterTest, ArrayValuesWithNulls) { std::move(file_writer)); // Get field for array column - std::unique_ptr field(StorageFieldFactory::create(array_column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(array_column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Construct arrays with mixed null and non-null elements (reference inverted_index_array_test.cpp) @@ -1143,7 +1144,7 @@ TEST_F(InvertedIndexWriterTest, ArrayValuesWithNulls) { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Get the length of the subfield - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); // Call the inverted index writing interface status = column_writer->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, @@ -1217,13 +1218,13 @@ TEST_F(InvertedIndexWriterTest, NumericArrayWithErrorConditions) { std::move(file_writer)); // Get field for array column - std::unique_ptr field(StorageFieldFactory::create(array_column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(array_column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Construct numeric arrays (reference inverted_index_array_test.cpp) @@ -1274,7 +1275,7 @@ TEST_F(InvertedIndexWriterTest, NumericArrayWithErrorConditions) { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Get the length of the subfield - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); // Call the inverted index writing interface status = column_writer->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, @@ -1332,13 +1333,13 @@ TEST_F(InvertedIndexWriterTest, CopyFileErrorHandling) { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add some values to create index files @@ -1356,77 +1357,6 @@ TEST_F(InvertedIndexWriterTest, CopyFileErrorHandling) { EXPECT_TRUE(status.ok()) << status; } -// Test case for Collection value processing -TEST_F(InvertedIndexWriterTest, CollectionValueProcessing) { - auto tablet_schema = create_schema(); - - // Create index meta - auto index_meta_pb = std::make_unique(); - index_meta_pb->set_index_type(IndexType::INVERTED); - index_meta_pb->set_index_id(1); - index_meta_pb->set_index_name("test"); - index_meta_pb->clear_col_unique_id(); - index_meta_pb->add_col_unique_id(1); // c2 column id - - TabletIndex idx_meta; - idx_meta.init_from_pb(*index_meta_pb.get()); - - std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( - local_segment_path(kTestDir, "test_collection", 0))}; - std::string index_path = InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix); - - io::FileWriterPtr file_writer; - io::FileWriterOptions opts; - auto fs = io::global_local_filesystem(); - Status sts = fs->create_file(index_path, &file_writer, &opts); - ASSERT_TRUE(sts.ok()) << sts; - - auto index_file_writer = std::make_unique( - fs, index_path_prefix, "test_collection", 0, InvertedIndexStorageFormatPB::V2, - std::move(file_writer)); - - // Get field for column c2 - const TabletColumn& column = tablet_schema->column(1); // c2 is the second column - ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); - - // Create column writer - std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); - EXPECT_TRUE(status.ok()) << status; - - // Create collection values for testing - std::vector test_strings = {"apple", "banana", "cherry"}; - std::vector slices; - for (const auto& s : test_strings) { - slices.emplace_back(s); - } - - // Create CollectionValue instances - std::vector collections; - CollectionValue collection1; - collection1.set_data(reinterpret_cast(slices.data())); - collection1.set_length(3); - bool null_signs[] = {false, false, false}; - collection1.set_null_signs(null_signs); - collections.push_back(collection1); - - // Test add_array_values with CollectionValue - status = column_writer->add_array_values(sizeof(Slice), collections.data(), 1); - EXPECT_TRUE(status.ok()) << status; - - // Finish and write - status = column_writer->finish(); - EXPECT_TRUE(status.ok()) << status; - - status = index_file_writer->begin_close(); - EXPECT_TRUE(status.ok()) << status; - status = index_file_writer->finish_close(); - EXPECT_TRUE(status.ok()) << status; -} - // Test case for BKD writer error conditions TEST_F(InvertedIndexWriterTest, BKDWriterErrorConditions) { auto tablet_schema = create_schema(); @@ -1463,13 +1393,13 @@ TEST_F(InvertedIndexWriterTest, BKDWriterErrorConditions) { // Get field for column c1 const TabletColumn& column = tablet_schema->column(0); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add some numeric values with edge cases @@ -1525,13 +1455,13 @@ TEST_F(InvertedIndexWriterTest, FileCreationAndOutputErrorHandling) { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add some values to ensure files are created diff --git a/be/test/storage/segment/segment_corruption_test.cpp b/be/test/storage/segment/segment_corruption_test.cpp index affeac8fc3770a..0212fd90d6252c 100644 --- a/be/test/storage/segment/segment_corruption_test.cpp +++ b/be/test/storage/segment/segment_corruption_test.cpp @@ -30,7 +30,6 @@ #include "io/cache/block_file_cache_factory.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_cache.h" diff --git a/be/test/storage/segment/zone_map_index_test.cpp b/be/test/storage/segment/zone_map_index_test.cpp index 2a8c90594670b3..4ec894feac6827 100644 --- a/be/test/storage/segment/zone_map_index_test.cpp +++ b/be/test/storage/segment/zone_map_index_test.cpp @@ -37,7 +37,6 @@ #include "exprs/function/cast/cast_to_string.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/predicate/comparison_predicate.h" #include "storage/tablet/tablet_schema.h" @@ -60,7 +59,7 @@ class ColumnZoneMapTest : public testing::Test { } void TearDown() override { EXPECT_TRUE(_fs->delete_directory(kTestDir).ok()); } - void test_string(std::string testname, StorageField* field, DataTypePtr data_type_ptr) { + void test_string(std::string testname, const TabletColumn* field, DataTypePtr data_type_ptr) { std::string filename = kTestDir + "/" + testname; auto fs = io::global_local_filesystem(); @@ -129,10 +128,10 @@ class ColumnZoneMapTest : public testing::Test { } else { tab_col = create_string_key(0); } - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); // Create a string longer than MAX_ZONE_MAP_INDEX_SIZE (512) std::string short_string = "mmmm"; @@ -249,7 +248,7 @@ class ColumnZoneMapTest : public testing::Test { auto data_type = DataTypeFactory::instance().create_data_type(TYPE_CHAR, true, 0, 0, length); auto tab_col = create_char_key(0, true, length); - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::string s_less_than_schema_length1(length - 1, 'a'); std::string s_less_than_schema_length1_expect(length, 'a'); s_less_than_schema_length1_expect[length - 1] = '\0'; @@ -258,7 +257,7 @@ class ColumnZoneMapTest : public testing::Test { s_less_than_schema_length2_expect[length - 1] = '\0'; s_less_than_schema_length2_expect[length - 2] = '\0'; std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); Slice slices[] = {Slice(s_less_than_schema_length1), Slice(s_less_than_schema_length2)}; writer->add_values(&slices, 2); if (pass_all) { @@ -343,10 +342,10 @@ class ColumnZoneMapTest : public testing::Test { precision, scale); TabletColumnPtr tab_col; tab_col = create_decimalv2_key(0, true); - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); decimal12_t decimal1 {.integer = 123, .fraction = 456}; decimal12_t decimal2 {.integer = 223, .fraction = 4567}; @@ -431,10 +430,10 @@ class ColumnZoneMapTest : public testing::Test { auto data_type = DataTypeFactory::instance().create_data_type(TYPE_DATE, true); TabletColumnPtr tab_col; tab_col = create_datev1_key(0, true); - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); VecDateTimeValue value1(false, TIME_DATE, 0, 0, 0, 2026, 2, 1); VecDateTimeValue value2(false, TIME_DATE, 0, 0, 0, 2026, 2, 2); @@ -512,10 +511,10 @@ class ColumnZoneMapTest : public testing::Test { auto data_type = DataTypeFactory::instance().create_data_type(TYPE_DATETIME, true); TabletColumnPtr tab_col; tab_col = create_datetimev1_key(0, true); - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); VecDateTimeValue value1(false, TIME_DATETIME, 18, 12, 10, 2026, 2, 1); VecDateTimeValue value2(false, TIME_DATETIME, 18, 13, 0, 2026, 2, 2); @@ -586,7 +585,7 @@ TEST_F(ColumnZoneMapTest, NormalTestIntPage) { auto fs = io::global_local_filesystem(); TabletColumnPtr int_column = create_int_key(0); - StorageField* field = StorageFieldFactory::create(*int_column); + const TabletColumn* field = &(*int_column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_INT, false); std::unique_ptr builder(nullptr); @@ -635,35 +634,31 @@ TEST_F(ColumnZoneMapTest, NormalTestIntPage) { EXPECT_EQ(true, zone_maps[2].has_null()); EXPECT_EQ(false, zone_maps[2].has_not_null()); - delete field; } // Test for string TEST_F(ColumnZoneMapTest, NormalTestVarcharPage) { TabletColumnPtr varchar_column = create_varchar_key(0); - StorageField* field = StorageFieldFactory::create(*varchar_column); + const TabletColumn* field = &(*varchar_column); auto str_data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false); test_string("NormalTestVarcharPage", field, str_data_type_ptr); - delete field; } // Test for string TEST_F(ColumnZoneMapTest, NormalTestCharPage) { TabletColumnPtr char_column = create_char_key(0); - StorageField* field = StorageFieldFactory::create(*char_column); + const TabletColumn* field = &(*char_column); auto char_data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_CHAR, false); test_string("NormalTestCharPage", field, char_data_type_ptr); - delete field; } // Test for zone map limit TEST_F(ColumnZoneMapTest, ZoneMapCut) { TabletColumnPtr varchar_column = create_varchar_key(0); varchar_column->set_index_length(1024); - StorageField* field = StorageFieldFactory::create(*varchar_column); + const TabletColumn* field = &(*varchar_column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false); test_string("ZoneMapCut", field, data_type_ptr); - delete field; } TEST_F(ColumnZoneMapTest, StringColumnTruncation) { @@ -716,7 +711,7 @@ TEST_F(ColumnZoneMapTest, NormalTestFloatPage) { auto fs = io::global_local_filesystem(); auto column = create_float_column(0, true); - StorageField* field = StorageFieldFactory::create(*column); + const TabletColumn* field = &(*column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, false); std::unique_ptr builder(nullptr); @@ -798,7 +793,6 @@ TEST_F(ColumnZoneMapTest, NormalTestFloatPage) { EXPECT_EQ(true, zone_maps[2].has_null()); EXPECT_EQ(false, zone_maps[2].has_not_null()); - delete field; } TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { @@ -806,7 +800,7 @@ TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { auto fs = io::global_local_filesystem(); auto column = create_float_column(0, true); - StorageField* field = StorageFieldFactory::create(*column); + const TabletColumn* field = &(*column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, false); std::unique_ptr builder(nullptr); @@ -889,7 +883,6 @@ TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { EXPECT_EQ(true, zone_maps[2].has_null()); EXPECT_EQ(false, zone_maps[2].has_not_null()); - delete field; } TabletColumnPtr create_timestamptz_column(int32_t id, bool is_nullable) { @@ -910,7 +903,7 @@ TEST_F(ColumnZoneMapTest, TimestamptzPage) { auto fs = io::global_local_filesystem(); auto column = create_timestamptz_column(0, true); - StorageField* field = StorageFieldFactory::create(*column); + const TabletColumn* field = &(*column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMPTZ, false); std::unique_ptr builder(nullptr); @@ -1066,7 +1059,6 @@ TEST_F(ColumnZoneMapTest, TimestamptzPage) { // page 5 EXPECT_EQ(true, zone_maps[4].has_null()); EXPECT_EQ(false, zone_maps[4].has_not_null()); - delete field; } // Regression test for "all-null page after a value page" — int variant. @@ -1078,11 +1070,11 @@ TEST_F(ColumnZoneMapTest, TimestamptzPage) { // a no-op. This test pins that behavior. TEST_F(ColumnZoneMapTest, AllNullPageAfterIntValues_SegmentMinMaxPreserved) { TabletColumnPtr int_column = create_int_key(0); - std::unique_ptr field(StorageFieldFactory::create(*int_column)); + const TabletColumn* field = &(*int_column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_INT, false); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type_ptr, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type_ptr, field, writer).ok()); // Page 1: integers spanning [100, 200]. std::vector values = {100, 150, 200}; @@ -1133,10 +1125,10 @@ TEST_F(ColumnZoneMapTest, AllNullPageAfterIntValues_SegmentMinMaxPreserved) { TEST_F(ColumnZoneMapTest, AllNullPageAfterMaxLenStringPage_NoSegmentMaxDoubleIncrement) { auto data_type = DataTypeFactory::instance().create_data_type(TYPE_STRING, true); auto tab_col = create_string_key(0); - std::unique_ptr field(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = &(*tab_col); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); // Page 1: one string of exactly MAX_ZONE_MAP_INDEX_SIZE bytes, all 'x'. std::string long_x(MAX_ZONE_MAP_INDEX_SIZE, 'x'); diff --git a/be/test/storage/storage_types_test.cpp b/be/test/storage/storage_types_test.cpp index f0d285b5861dc2..45a89abe87e013 100644 --- a/be/test/storage/storage_types_test.cpp +++ b/be/test/storage/storage_types_test.cpp @@ -24,8 +24,6 @@ #include "core/decimal12.h" #include "core/uint24.h" #include "gtest/gtest_pred_impl.h" -#include "runtime/collection_value.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/tablet/tablet_schema.h" #include "storage/types.h" @@ -46,12 +44,10 @@ void common_test(typename TypeTraits::CppType src_val) { template void test_char(Slice src_val) { - StorageField* field = StorageFieldFactory::create_by_type(fieldType); - field->_length = src_val.size; - + auto field = std::make_unique(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, + fieldType, false, 0, src_val.size); EXPECT_EQ(field->type(), fieldType); - EXPECT_EQ(sizeof(src_val), field->size()); - delete field; + EXPECT_EQ(sizeof(src_val), field_type_size(field->type())); } template <> @@ -64,7 +60,7 @@ void common_test(Slice src_val) { test_char(src_val); } -TEST(TypesTest, cmp_and_minmax) { +TEST(TypesTest, field_type_size_matches_cpp_type) { common_test(true); common_test(112); common_test(static_cast(54321)); @@ -88,81 +84,6 @@ TEST(TypesTest, cmp_and_minmax) { common_test(slice); } -template -void common_test_array(CollectionValue src_val) { - TabletColumn list_column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, - FieldType::OLAP_FIELD_TYPE_ARRAY); - int32_t item_length = 0; - if (item_type == FieldType::OLAP_FIELD_TYPE_CHAR || - item_type == FieldType::OLAP_FIELD_TYPE_VARCHAR) { - item_length = 10; - } - TabletColumn item_column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, item_type, true, - 0, item_length); - list_column.add_sub_column(item_column); - - ASSERT_EQ(item_type, list_column.get_sub_column(0).type()); -} - -TEST(ArrayTypeTest, copy_and_equal) { - bool bool_array[3] = {true, false, true}; - bool null_signs[3] = {true, true, true}; - common_test_array(CollectionValue(bool_array, 3, null_signs)); - - uint8_t tiny_int_array[3] = {3, 4, 5}; - common_test_array( - CollectionValue(tiny_int_array, 3, null_signs)); - - int16_t small_int_array[3] = {123, 234, 345}; - common_test_array( - CollectionValue(small_int_array, 3, null_signs)); - - int32_t int_array[3] = {-123454321, 123454321, 323412343}; - common_test_array(CollectionValue(int_array, 3, null_signs)); - - uint32_t uint_array[3] = {123454321, 2342341, 52435234}; - common_test_array( - CollectionValue(uint_array, 3, null_signs)); - - int64_t bigint_array[3] = {123454321123456789L, 23534543234L, -123454321123456789L}; - common_test_array( - CollectionValue(bigint_array, 3, null_signs)); - - __int128 large_int_array[3] = {1234567899L, 1234567899L, -12345631899L}; - common_test_array( - CollectionValue(large_int_array, 3, null_signs)); - - float float_array[3] = {1.11, 2.22, -3.33}; - common_test_array( - CollectionValue(float_array, 3, null_signs)); - - double double_array[3] = {12221.11, 12221.11, -12221.11}; - common_test_array( - CollectionValue(double_array, 3, null_signs)); - - decimal12_t decimal_array[3] = {{123, 234}, {345, 453}, {4524, 2123}}; - common_test_array( - CollectionValue(decimal_array, 3, null_signs)); - - uint24_t date_array[3] = {(1988 << 9) | (2 << 5) | 1, (1998 << 9) | (2 << 5) | 1, - (2008 << 9) | (2 << 5) | 1}; - common_test_array(CollectionValue(date_array, 3, null_signs)); - - uint32_t date_v2_array[3] = {(1988 << 9) | (2 << 5) | 1, (1998 << 9) | (2 << 5) | 1, - (2008 << 9) | (2 << 5) | 1}; - common_test_array( - CollectionValue(date_v2_array, 3, null_signs)); - - int64_t datetime_array[3] = {19880201010203L, 19980201010203L, 20080204010203L}; - common_test_array( - CollectionValue(datetime_array, 3, null_signs)); - - Slice char_array[3] = {"12345abcde", "12345abcde", "asdf322"}; - common_test_array(CollectionValue(char_array, 3, null_signs)); - common_test_array( - CollectionValue(char_array, 3, null_signs)); -} - TEST(TypesTest, has_char_type) { // Test basic types TabletColumn char_column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE,