Skip to content

[fix](variant) Handle truncated sparse path stats when reading variant#64205

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eldenmoon merged 8 commits into from
Jun 17, 2026
Merged

[fix](variant) Handle truncated sparse path stats when reading variant #64205

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
cf31ab2
[fix](variant) Handle truncated sparse path stats when reading variant
eldenmoon Jun 8, 2026
c481e4f
Merge branch 'master' into branch-fix-tc-962905-variant-sparse-limit
eldenmoon Jun 9, 2026
370e493
[fix](variant) Reject zero sparse statistics limit
eldenmoon Jun 15, 2026
3919d40
Merge remote-tracking branch 'upstream/master' into branch-fix-tc-962…
eldenmoon Jun 15, 2026
1fdb31a
[fix](variant) Update nested schema change output
eldenmoon Jun 15, 2026
ae0105e
Merge branch 'master' into branch-fix-tc-962905-variant-sparse-limit
eldenmoon Jun 15, 2026
f6a9302
Merge branch 'master' into branch-fix-tc-962905-variant-sparse-limit
eldenmoon Jun 16, 2026
3a3c2e2
Merge branch 'master' into branch-fix-tc-962905-variant-sparse-limit
eldenmoon Jun 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions be/src/storage/segment/variant/variant_column_reader.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -192,9 +192,9 @@ bool VariantColumnReader::is_exceeded_sparse_column_limit() const {
}

bool VariantColumnReader::_is_exceeded_sparse_column_limit_unlocked() const {
bool exceeded_sparse_column_limit = !_statistics->sparse_column_non_null_size.empty() &&
_statistics->sparse_column_non_null_size.size() >=
_variant_sparse_column_statistics_size;
const bool exceeded_sparse_column_limit = !_statistics->sparse_column_non_null_size.empty() &&
_statistics->sparse_column_non_null_size.size() >=
_variant_sparse_column_statistics_size;
DBUG_EXECUTE_IF("exceeded_sparse_column_limit_must_be_false", {
if (exceeded_sparse_column_limit) {
throw doris::Exception(
Expand DownExpand Up@@ -882,8 +882,12 @@ Status VariantColumnReader::_build_read_plan(ReadPlan* plan, const TabletColumn&
}

// Check if path is prefix, example sparse columns path: a.b.c, a.b.e, access prefix: a.b.
// Or access root path
if (_has_prefix_path_unlocked(relative_path)) {
// Or access root path. If sparse stats reached the configured limit, an exact sparse path can
// still have unrecorded sparse children such as a.b.c.
const bool has_prefix_path = _has_prefix_path_unlocked(relative_path);
const bool sparse_stats_may_have_unrecorded_children =
exceeded_sparse_column_limit && existed_in_sparse_column;
if (has_prefix_path || sparse_stats_may_have_unrecorded_children) {

@github-actionsgithub-actionsBotJun 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This only treats truncated stats as ambiguous when the exact target path is recorded in sparse stats, but the same ambiguity exists when the target path is a materialized leaf and an unrecorded sparse child exists. A concrete shape is variant_max_subcolumns_count = 1, sparse stats limit = 1, with b having the highest non-null count so it is kept as an extracted leaf, while overflow sparse paths include a and b.c; serialize_sparse_columns() iterates the remaining std::map, so the single recorded sparse stat can be a and b.c is physically present but absent from stats. For v['b'], node is the leaf for b, _has_prefix_path_unlocked("b") is false because there is no child node and no recorded b. sparse prefix, and existed_in_sparse_column is false, so this branch is skipped and _try_build_leaf_plan() returns only the b leaf. Rows whose b value only comes from sparse b.c are not merged. When sparse stats are truncated, a materialized exact node also needs a hierarchical read unless there is a persisted proof that no sparse child with this prefix exists.

// Example {"b" : {"c":456,"e":7.111}}
// b.c is sparse column, b.e is subcolumn, so b is both the prefix of sparse column and
// subcolumn
Expand DownExpand Up@@ -951,7 +955,8 @@ Status VariantColumnReader::_build_read_plan(ReadPlan* plan, const TabletColumn&
}

if (exceeded_sparse_column_limit) {
// maybe exist prefix path in sparse column
// Sparse stats are truncated, so a missing exact sparse path does not prove that the
// path is absent. It may still be nested under a recorded sparse object.
plan->kind = ReadKind::HIERARCHICAL;
plan->type = create_variant_type(target_col);
plan->relative_path = relative_path;
Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2118,12 +2118,12 @@ public static int analyzeVariantMaxSparseColumnStatisticsSize(Map<String, String
properties.get(PROPERTIES_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE);
try {
maxSparseColumnStatisticsSize = Integer.parseInt(maxSparseColumnStatisticsSizeStr);
if (maxSparseColumnStatisticsSize < 0 || maxSparseColumnStatisticsSize > 50000) {
throw new AnalysisException("variant_max_sparse_column_statistics_size must between 0 and 50000 ");
}
} catch (Exception e) {
} catch (NumberFormatException e) {
throw new AnalysisException("variant_max_sparse_column_statistics_size format error:" + e.getMessage());
}
if (maxSparseColumnStatisticsSize < 1 || maxSparseColumnStatisticsSize > 50000) {
throw new AnalysisException("variant_max_sparse_column_statistics_size must between 1 and 50000 ");
}

properties.remove(PROPERTIES_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE);
}
Expand Down
9 changes: 9 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3597,6 +3597,7 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) {
@VarAttrDef.VarAttr(
name = DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE,
needForward = true,
checker = "checkDefaultVariantMaxSparseColumnStatisticsSize",
fuzzy = true

@github-actionsgithub-actionsBotJun 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This checker rejects new SET statements, but it does not cover upgraded metadata that was valid before this PR. SessionVariable.readFromJson() loads persisted int fields directly, and VariableMgr.replayGlobalVariableV2() catches and ignores checker failures, so an image/edit log containing default_variant_max_sparse_column_statistics_size = 0 can still leave new sessions with value 0 after upgrade. Existing table metadata can also already contain 0, as shown by this PR changing the nested schema-change expected output from 0 to 1. Those zero-valued tables/sessions still reach BE, where a writer with limit 0 records no sparse stats and the reader no longer treats empty stats plus a sparse reader as truncated. Please add a compatibility normalization/migration for persisted zero values, or keep a BE compatibility branch for limit == 0 while still rejecting new user input.

)
public int defaultVariantMaxSparseColumnStatisticsSize = 10000;
Expand DownExpand Up@@ -6446,6 +6447,14 @@ public void checkDefaultVariantMaxSubcolumnsCount(String variantMaxSubcolumnsCou
}
}

public void checkDefaultVariantMaxSparseColumnStatisticsSize(String variantMaxSparseColumnStatisticsSize) {
int value = Integer.valueOf(variantMaxSparseColumnStatisticsSize);
if (value < 1 || value > 50000) {
throw new UnsupportedOperationException("variant max sparse column statistics size is: "
+ variantMaxSparseColumnStatisticsSize + " it must between 1 and 50000");
}
}

public void checkHnswEfSearch(String efSearch) {
int value = Integer.valueOf(efSearch);
if (value < 1) {
Expand Down
22 changes: 22 additions & 0 deletions fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.doris.catalog.Type;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.resource.Tag;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TStorageFormat;
Expand DownExpand Up@@ -345,6 +346,19 @@ public void testAnalyzeInvertedIndexFileStorageFormat() throws AnalysisException
@Test
public void testAnalyzeVariantMaxSparseColumnStatisticsSize() throws AnalysisException {
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.PROPERTIES_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE, "0");
try {
PropertyAnalyzer.analyzeVariantMaxSparseColumnStatisticsSize(properties, 0);
Assertions.fail("Expected AnalysisException was not thrown");
} catch (AnalysisException e) {
Assertions.assertNotNull(e.getMessage());
}
properties.clear();
properties.put(PropertyAnalyzer.PROPERTIES_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE, "1");
Assertions.assertEquals(1, PropertyAnalyzer.analyzeVariantMaxSparseColumnStatisticsSize(properties, 0));
Assertions.assertFalse(properties.containsKey(
PropertyAnalyzer.PROPERTIES_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE));
properties.clear();
properties.put(PropertyAnalyzer.PROPERTIES_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE, "-1");
try {
PropertyAnalyzer.analyzeVariantMaxSparseColumnStatisticsSize(properties, 0);
Expand All@@ -370,6 +384,14 @@ public void testAnalyzeVariantMaxSparseColumnStatisticsSize() throws AnalysisExc
}
}

@Test
public void testCheckDefaultVariantMaxSparseColumnStatisticsSize() {
SessionVariable sessionVariable = new SessionVariable();
Assertions.assertThrows(UnsupportedOperationException.class,
() -> sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("0"));
sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("1");
}

@Test
public void testAnalyzeSequenceMap() throws AnalysisException {
List<Column> columns = Lists.newArrayList();
Expand Down
12 changes: 6 additions & 6 deletions regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,7 @@ col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "0","variant_sparse_hash_shard_count" = "1")> Yes false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "1","variant_sparse_hash_shard_count" = "1")> Yes false \N NONE

-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1, 2, 3]}
Expand All@@ -67,7 +67,7 @@ col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> No false \N NONE
col5 struct<f1:int> No false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "0","variant_sparse_hash_shard_count" = "1")> No false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "1","variant_sparse_hash_shard_count" = "1")> No false \N NONE

-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1, 2, 3]}
Expand All@@ -81,7 +81,7 @@ col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "0","variant_sparse_hash_shard_count" = "1")> Yes false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "1","variant_sparse_hash_shard_count" = "1")> Yes false \N NONE

-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1, 2, 3]}
Expand All@@ -95,7 +95,7 @@ col2 int No false \N NONE
col3 array<int> No false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> No false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "0","variant_sparse_hash_shard_count" = "1")> No false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "1","variant_sparse_hash_shard_count" = "1")> No false \N NONE

-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1, 2, 3]}
Expand All@@ -109,7 +109,7 @@ col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "0","variant_sparse_hash_shard_count" = "1")> Yes false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "1","variant_sparse_hash_shard_count" = "1")> Yes false \N NONE

-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1, 2, 3]}
Expand All@@ -123,7 +123,7 @@ col2 int No false \N NONE
col3 array<int> No false \N NONE
col4 map<int,int> No false \N NONE
col5 struct<f1:int> Yes false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "0","variant_sparse_hash_shard_count" = "1")> No false \N NONE
col6 variant<PROPERTIES ("variant_max_subcolumns_count" = "0","variant_enable_typed_paths_to_sparse" = "false","variant_max_sparse_column_statistics_size" = "1","variant_sparse_hash_shard_count" = "1")> No false \N NONE

-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1, 2, 3]}
Expand Down
6 changes: 3 additions & 3 deletions regression-test/data/variant_p0/predefine/test_variant_compaction_with_sparse_limit.out
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,8 +34,8 @@
16 {"a":"1223"}
17 {"a":[1]}
17 {"a":[1]}
18 {"a":["1",2,1.1]}
18 {"a":["1",2,1.1]}
18 {"a":["1", 2, 1.1]}
18 {"a":["1", 2, 1.1]}
19 {"a":1,"b":{"c":1}}
19 {"a":1,"b":{"c":1}}
20 {"a":1,"b":{"c":[{"a":1}]}}
Expand DownExpand Up@@ -193,7 +193,7 @@
15 {"a":1}
16 {"a":"1223"}
17 {"a":[1]}
18 {"a":["1",2,1.1]}
18 {"a":["1", 2, 1.1]}
19 {"a":1,"b":{"c":1}}
20 {"a":1,"b":{"c":[{"a":1}]}}
21 {"a":1,"b":{"c":[{"a":1}]}}
Expand Down
2 changes: 1 addition & 1 deletion ...sion-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@ suite("create_nestedtypes_with_schemachange", "p0") {
// create basic type
sql "set default_variant_max_subcolumns_count = 0"
sql "set default_variant_enable_typed_paths_to_sparse = false"
sql "set default_variant_max_sparse_column_statistics_size = 0"
sql "set default_variant_max_sparse_column_statistics_size = 1"
sql "set default_variant_sparse_hash_shard_count = 0"
sql "set default_variant_enable_doc_mode = false"
sql "DROP TABLE IF EXISTS $testTablex"
Expand Down
24 changes: 22 additions & 2 deletions regression-test/suites/variant_p0/predefine/test_variant_compaction_with_sparse_limit.groovy
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,24 @@ suite("test_compaction_variant_predefine_with_sparse_limit", "nonConcurrent") {
}

int max_sparse_column_statistics_size = 2
def create_table = { tableName, buckets="auto", key_type="DUPLICATE" ->
test {
sql """ set default_variant_max_sparse_column_statistics_size = 0 """
exception "variant max sparse column statistics size"
}
sql "DROP TABLE IF EXISTS variant_sparse_stats_zero"
test {
sql """
CREATE TABLE variant_sparse_stats_zero (
k bigint,
v variant <properties("variant_max_sparse_column_statistics_size" = "0")>
)
DUPLICATE KEY(`k`)
DISTRIBUTED BY HASH(k) BUCKETS 1
properties("replication_num" = "1");
"""
exception "variant_max_sparse_column_statistics_size must between 1 and 50000"
}
def create_table = { tableName, buckets="auto", key_type="DUPLICATE", max_subcolumns_count=2048 ->
sql "DROP TABLE IF EXISTS ${tableName}"
def var_def = "variant <MATCH_NAME 'sala' : int, MATCH_NAME 'ddd' : double, MATCH_NAME 'z' : double, properties(\"variant_max_sparse_column_statistics_size\" = \"${max_sparse_column_statistics_size}\")>"
if (key_type == "AGGREGATE") {
Expand All@@ -60,13 +77,16 @@ suite("test_compaction_variant_predefine_with_sparse_limit", "nonConcurrent") {
def create_tbl_res = sql """ show create table ${tableName} """
logger.info("${create_tbl_res}")
assertTrue(create_tbl_res.toString().contains("variant_max_sparse_column_statistics_size"))
assertTrue(create_tbl_res.toString().contains("\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\""))
}
def key_types = ["DUPLICATE", "UNIQUE", "AGGREGATE"]
// def key_types = ["AGGREGATE"]
for (int i = 0; i < key_types.size(); i++) {
def max_subcolumns_count = key_types[i] == "AGGREGATE" ? 2048 : 1
sql """ set default_variant_max_subcolumns_count = ${max_subcolumns_count} """
def tableName = "simple_variant_${key_types[i]}"
// 1. simple cases
create_table.call(tableName, "1", key_types[i])
create_table.call(tableName, "1", key_types[i], max_subcolumns_count)
def insert1 = {
sql """insert into ${tableName} values (1, '{"x" : [1]}'),(13, '{"a" : 1}');"""
sql """insert into ${tableName} values (2, '{"a" : "1"}'),(14, '{"a" : [[[1]]]}');"""
Expand Down
Loading