From 276cfc47bc47047868624efa70628ce6dc6158ca Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 20 Apr 2026 22:02:47 +0800 Subject: [PATCH] [improvement](be) Use plain encoding for V3 floating-point columns ### What problem does this PR solve? Issue Number: None Related PR: #63622 Problem Summary: Segment V3 continued to select BIT_SHUFFLE for FLOAT and DOUBLE columns. The original implementation required a tablet-schema flag propagated through proto, FE, Cloud, and BE writers. After #63622 split encoding selection into explicit V2 and V3 maps and persisted storage_format in TabletSchema, that propagation chain is obsolete. Express the behavior directly in the V3 default map, preserve the V2 BIT_SHUFFLE default for compatibility, and refresh all affected scalar and materialized Variant Segment goldens. ### Release note Segment V3 now uses PLAIN encoding by default for FLOAT and DOUBLE columns. Segment V2 behavior is unchanged. ### Check List (For Author) - Test: Unit Test - `./run-be-ut.sh --run --filter='SegmentFlusherFormatTest.*KeepTheirSegmentBytes:SegmentFlusherTransformFormatTest.PartialUpdateAndRowBinlogPathsKeepTheirSegmentBytes' -j100` (6 tests passed in two generation runs; all 164 generated Segment files were byte-identical between runs) - `./run-be-ut.sh --run --filter='EncodingInfoTest.*:SegmentFlusherFormatTest.*:SegmentFlusherTransformFormatTest.*' -j100` (17 tests passed against the checked-in goldens) - `build-support/clang-format.sh` - `build-support/check-format.sh` - `git diff --check` - Behavior changed: Yes. Newly written V3 FLOAT and DOUBLE pages use PLAIN instead of BIT_SHUFFLE; V2 remains BIT_SHUFFLE. - Does this need documentation: No --- be/src/storage/segment/encoding_info.cpp | 4 ++-- .../storage/segment/encoding_info_test.cpp | 20 ++++++++++++------ .../segment_0.dat | Bin 6800 -> 6719 bytes .../segment_1.dat | Bin 6795 -> 6714 bytes .../segment_0.dat | Bin 8451 -> 8359 bytes .../segment_1.dat | Bin 8239 -> 8149 bytes .../segment_0.dat | Bin 5680 -> 5656 bytes .../segment_1.dat | Bin 5680 -> 5656 bytes .../segment_0.dat | Bin 5496 -> 5472 bytes .../segment_1.dat | Bin 5496 -> 5472 bytes .../segment_0.dat | Bin 5478 -> 5454 bytes .../segment_0.dat | Bin 5294 -> 5270 bytes 12 files changed, 15 insertions(+), 9 deletions(-) diff --git a/be/src/storage/segment/encoding_info.cpp b/be/src/storage/segment/encoding_info.cpp index d2b9d99f114e85..85d01bf1e0ce09 100644 --- a/be/src/storage/segment/encoding_info.cpp +++ b/be/src/storage/segment/encoding_info.cpp @@ -386,8 +386,8 @@ EncodingInfoResolver::EncodingInfoResolver() { _set_v3_default(); _set_v3_default(); _set_v3_default(); - _set_v3_default(); - _set_v3_default(); + _set_v3_default(); + _set_v3_default(); _set_v3_default(); _set_v3_default(); _set_v3_default(); diff --git a/be/test/storage/segment/encoding_info_test.cpp b/be/test/storage/segment/encoding_info_test.cpp index 6a3b6e76e3258f..d32c72e4200836 100644 --- a/be/test/storage/segment/encoding_info_test.cpp +++ b/be/test/storage/segment/encoding_info_test.cpp @@ -91,10 +91,15 @@ TEST_F(EncodingInfoTest, v2_vs_v3_defaults) { check_split(FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, "QUANTILE_STATE"); check_split(FieldType::OLAP_FIELD_TYPE_AGG_STATE, "AGG_STATE"); - // Signed integers: V2=BIT_SHUFFLE, V3=PLAIN. + // Signed integers and floating-point types: V2=BIT_SHUFFLE, V3=PLAIN. + auto check_plain_v3 = [](FieldType type, const std::string& name) { + EXPECT_EQ(BIT_SHUFFLE, get_v2_default_encoding(type)) << name << " v2 default"; + EXPECT_EQ(PLAIN_ENCODING, get_v3_default_encoding(type)) << name << " v3 default"; + }; constexpr FieldType bigint_type = FieldType::OLAP_FIELD_TYPE_BIGINT; - EXPECT_EQ(BIT_SHUFFLE, get_v2_default_encoding(bigint_type)); - EXPECT_EQ(PLAIN_ENCODING, get_v3_default_encoding(bigint_type)); + check_plain_v3(bigint_type, "BIGINT"); + check_plain_v3(FieldType::OLAP_FIELD_TYPE_FLOAT, "FLOAT"); + check_plain_v3(FieldType::OLAP_FIELD_TYPE_DOUBLE, "DOUBLE"); // Value-seek default is only registered for VARCHAR (the only production caller). EXPECT_EQ(PREFIX_ENCODING, @@ -345,9 +350,10 @@ struct EncodingMapEntry { const char* name; }; -// Expected V3 default per type. Differs from V2 default in two type families: -// - binary blobs (HLL/BITMAP/QUANTILE_STATE/AGG_STATE): PLAIN_ENCODING_V2 (vs V2's PLAIN) +// Expected V3 default per type. Differs from V2 default in three type families: +// - binary blobs (HLL/BITMAP/QUANTILE_STATE/AGG_STATE): PLAIN_ENCODING_V3 (vs V2's PLAIN) // - signed integers (TINYINT..LARGEINT): PLAIN_ENCODING (vs V2's BIT_SHUFFLE) +// - floating-point types (FLOAT/DOUBLE): PLAIN_ENCODING (vs V2's BIT_SHUFFLE) const std::vector kV3DefaultExpect = { {FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING, "TINYINT"}, {FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING, "SMALLINT"}, @@ -356,8 +362,8 @@ const std::vector kV3DefaultExpect = { {FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING, "LARGEINT"}, {FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE, "UNSIGNED_BIGINT"}, {FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE, "UNSIGNED_INT"}, - {FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE, "FLOAT"}, - {FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE, "DOUBLE"}, + {FieldType::OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING, "FLOAT"}, + {FieldType::OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING, "DOUBLE"}, {FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING, "CHAR"}, {FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING, "VARCHAR"}, {FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING, "STRING"}, diff --git a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_0.dat index 3b5529606419bfa03b9ce15e51f6789a7030d5ef..493a0bad583d5d2231e92e555fcf3197f6667928 100644 GIT binary patch delta 437 zcmbPWy5D4j0b2kA!-D+`3=H-R3^I-!i~<5yY#a;%%n}SdK*9N@@v_W7TKZnI)2RcW z_gnQCJnE}-wd>mO@Z~j`9}YnEGRW$qdJ?KPUu0Xw%Cv-k@Am@Yt=lDxi5 ze;~|$-ay6`lOOWdGG3Ydkw;^4J6{B2!{i5iGZ@zZNsY-1`QsUHfLWRXg^Vqe83i;Z z9~Ovb+yG{I3)VB;5t`f}q(Aw$U^8RKGg(ojj%kO; ze4gsgKKZWXYL2DCm)^P<1Q`MV{ZE0f delta 538 zcmZXNPe@cz6vpp4_s_eHZ{}TR=Dm5d5N1Rgg@`8HNz`%~cp< z+Dcg2%Jnc{pM(&=z~ONT&|oS;2la57{LJk~yQAlaIzpp0#eYHLR?cmkCT9xpFp1S??yP)ZsJBrJO zmff4U6SoQtr|a;OmeRLyr8@~?Y&cWH-IH-6Glj<|w&qRZvn;K9Pw+!FVW-U`F60vX z%gn(GI-8AXwfi>gS=Yond3u|j#q~T@b1nEto!l(m^Qo4NVF?X{8Y1!$Q2;S8YSJ z1FjBC5O$UHfBb=K8q^zJ!mWfCjW#r>8J1v|*1|FGd%>JB5aDI17c-_#byKEt@pO7W kV=fx7Y0ER$lXRwNoJXN4UwsqO{o-@|{;)YP8eJLu16U$^^#A|> diff --git a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_1.dat index 88ab18cdf7fd3cf1d6bfc0ca7eb298cfa7a8238d..edcd401c259eead2329cde183ca36aae4618c859 100644 GIT binary patch delta 472 zcmeA+-DR>tlP!ROfx(`ELB^4RVZnY5MgajUHVy^>W(fu!pv07YF2&41+N#IkQD3F2 zUDt+(FR#h`aFD*&>~!kDXOMasWcANDnAT6WXSUqT%wEjOG?#y}yrBBz`8+;MCm~D` zUSFoK5M~!|Ak#ttumTpoP^PmG=6oKG$z6N_Og|x9X8r=ErGj8ZbNJ(#E<%_p0)rJe-HdA|vkBKT-4veOE7neK{ARu!Gj*g1Kx=w7C+qLXXHx|kk{f^E?kFJ} zF*w0!F!{QK0@F{P$v-7r_$P`B=m;oX6w&{7#3_dE*q5KW_OmFj-@k&I)g?W4u(R$$~e5G^IE1ZjI zU%UV<`cSOWMtli2{*uzEiENZ)Xe!Q&CC@hKPBF!WV*8hS8Y|Fv51xr_oMih3_;hHnKJ=t>3hvirw)AH z-?ic4%WE<}9Kb5YkX1f@AKgD$klkwYb+%)iOk4OSGs>$^J|^J9^Z>%N7W8H6;RlOu z6bxkA!9RJSkSx;`vB?L7)F*ohg)%(>i!e@{ykDq+al+&ULUK&!#3v^RPiNc%=5$Md zmD-EMGrj-|oPY>C6DeexA_z9ERT8XZzGytt0f@*Uh=`k5J<}VA$bZSnJH_=Ui;6ch z%@6|X+9L%N>0℘r*1FoFY-j^a0G{-zF_!E5K;L&wgL1uE8R6Elb5KoR#X7WF&=}ZemCOZntG0INfCcTI0j0jlQ zbJ@w0WxAMtKzK{!zz$E9EoE9F3YLEe7Ev)f!Dx1YQGr8LKvaTDfklJKfYpN0iJybf zikpj*iRsxK`zb%7 delta 682 zcmZ9IPiPZC6vk)XZsH_ql1YMHK<=e4PcaSDh3(hh7BFkHBRon0J#sz_QQV5Dz_*#h(QSf zB9n%UMoBwF0OV0JxeoV4tXHaKKzmxf<;X}YjUP@7(Quo%T}HxhvC-bVVF*+ z(ot@AS zd(BR#4t$2F6hl_&G;QVP7`7jrOs{z-I|`~#t`+cMn$8Dg>QDY9;L3Cu#4KZa&o{YI zLYAppY;vKH`sDwDfsC^!2MWnCod5|`F&+nT7+WV72x&|<7LI594CWsK^KS{qGtCzO zX=nTo7HAYHWI7ENVA>-AHqS~lp6NS8UG~;iuzy_(w|HT`amJ5NM_f~510*OgXmm$1W(vyQFE1CX7c+aILzm=?G zTs^r#LXL6C*QP+?@un2k>kB8CtxeUXu#yf%CVPc@_RXV o?oT`dRsxJb{u*F3$s4jyk{3{n*epM}QhpuBgZ|_^7lR-}0M%!mMgRZ+ delta 678 zcmZWlJ!n%=6u#%Yy!19P&CSnydHH#HDM?8dNjivXGP79AW!(2CY- zp`bme9TZWBprCD$qEZ|>h!reUv_pnsGq^dl1;Gmb-22{A^uXbs@8|q*m;0yoOwZ|; zdKhC|rcb7fh{J%vG&0oOkS}r&#j2k(p;@?1dOlxYKZWu=CR4Sll(BcsK#ge{$Xa%* z5h!f{44Ec5kctMH%>*Ranh!LoL!P0sZQXTz>#hq8vP|899XS!_rk7y=ZqTSI!cu1?UEPRo86c$yD_bg=| zj@4n)#Vv^^u^DkE-AO8OHD1S$9`PwjCymGW%R@I)^wTWUka-24MBRo(#f;&yhsM&> zObp}Yn5(@@(?M$jpLytNhBmG9*z(YPhA4RvN8-u}zo|DhiZd?2Ih$IkNnG)yA9O1{ zf;S9z;%ru4Q>N3I0i5-i4aIEF=C~mYE0FV*{1tr{>VmDNHMvSU0`O7D69FqAEbL*SbW%%LbgTj&F|WJ~5sn$9c{NZ?}i` ISC01n1#1PTZvX%Q diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat index b607d5d4309fa060738a5b9def85ae214b7c98c7..151025699d90bcb3b5aeb743ac813a297ac83212 100644 GIT binary patch delta 251 zcmdm>Gec)X1dAL481OhSGjlKs@L92OFbFV9F!6vyb3@i`?qgA6VVt#jBS#hs555Zay zCIeOrMkjs_Mk{VEPA*oCD;xq!lWz*@GM<{uF4V+$dvcdh4&$WB--VhP+b35G+cTb@ zyjwVw_Y=2(g#e=$E5|Ax0h!4HoC=d|MWk7d@d&6*=HpbFTqh#MxNGuE5qG9HJd+=Z pG|EgA5O5M;)B@Q6VH&UkHGkk2P>i@dxn6W7hpdKZzl%YTApl*BKpX%7 delta 299 zcmbQCvq5J=1dEvp0|NsG6ay&%c?L!XIb}vhV}<}Amyv;)nS)V4&5DhKL4a9;i3ce5 z?90@y&1Eb~EF7D71#|=$4VWe$f)QchEJC9^kf6US`jt_RtrWaehx+} zZZ1wPR*owi0#=g`3hFXGn*3g{iShH~BB30{Rg>=vHFM187BCiIjNqIs&m}v#U09m& z`Q*jIq5NGu0u};{UaTC4cm!l3*ub(XA~Ha$9YyRJFHLR~ac64boqS59QD&upfRg~D g7RVk5(|{GIuR}mUG2->)bkUU@`8ltGTnvH?0qwp=&j0`b diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat index f8b08ffaed0127c1b2dc65dfb120e5c8c6f58fb5..db4be9934a784ed6a539062efc2fa9a19c6ff069 100644 GIT binary patch delta 251 zcmdm>Gec)X7>gVO81OhSvvM#B@L92OFbFV9F!2CIjW0dQ-rU2Y!ooOX^G1#=7RHs6 zWq9o-OY4#vPMqinueq t;+gzNq*10{K)^|WQ43@PglWJE)clTLKr!Otk(-0|NsG6ay&%c?L!XIb}vhV}<}Amyv;)m4i`0&5DhKL4a9;i3ccl z)mzzra|w$I3&#dt0UZHG1E$Fbx#TzR=Wt<}?8K`uS(#U!^(Lo)M8sr9eyPd7IHj3- zI5*ev>M%0inY@}$j;V)#^J%^wMy3wF$&muv8MjOp6qI8;IoUw4R)o!f)q>H9pM%kg zn~Rf+mE#hJfYs!Kg1U?kCchVKV*EI{NGOMK#pL@!%^Y*N1&jq4BRD6^bIDF_7nWvx zI(e~hD1Qf!fQ0~~7c0jB9s!vMHn6OUhz!tbM-h9*3zHi~+?kqqC!Z2&lvyqy;3UAP g1+oXiG++hlYZDMqjCeUYU34YK&K-fzTnvH?0hVz_%K!iX diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat index 3a201138c91372b6f154952dc30a620c4be52f9b..f180609ba788ad500d28f54e0d457d5e9d106958 100644 GIT binary patch delta 253 zcmeyN^+0Pw1dAL481OhSGjlKs@L92OFbFV9F!6vyb3@i`?qgZO%rT2sKu3VlfN8QI zkK*Qd4ks4I)sr9c$WLzKv7f{%#q^D1vkb2eBjd5no_xKGjPoX6=ikBfiFfiuff^Ae z111YbCw>k_D{d}ME>?~!90E#{6$Nz}Pfhk0Y+}4U`I2A`D;1mgmS~nS6prVe(2IdA8e}0um7nlMnF8On${9!PLjOnT1z}k@4PU zL%v={#%+`L^Y38l;+tG4P$R--z+}Pb#LvNK#m&XZ#maGoL%?b>r=Tw5qsiuiO^lx> z?-I;mTs2u;sF`v8(77FF>;t{YAVDw_;IK(3$6Tvo_kxOoJuCO%6Jstry z0Y<36I$gVO81OhSvvM#B@L92OFbFV9F!2CIjW0dQ-rU2oikV{uuYisKqXE-o zK_11;@f=Ppj4LNUk(-0|NsG6ay&%c?L!XIb}vhV}<}Amyv;)m4i`0&5DhKL4a9;i3ccl z)mzzra|z2TX2uPh5fJ6ktS4>tHYG#}}d9{!|it+g7Q@py2Ow)NM7YN8Q9-l1EKacV7dJ!ALe`@-&wdnc=lG|4>T7qAgv)B>3a dVH&Ukl^^66P>h&2`KZWD4xj4tW-bOnh5)&_Ko|f3 delta 302 zcmXXPwo>&1WcW+FCyagN2l$mf$2p!AS%mtE)KM$-#CI#O>lBNEdOk zQZXP|1;I%-2kB5<3T_>obkO)24(G?=JbEMVYEE37VT|#AB!ndftT+u&XX}XtCI}p` z5~MjpA)8W@Z1jG77&S##NQDjBp=Nto?Fsl&Jg4u>Uip?GP zq*<%O#|qXFLYZnxfgY?p#Pnuu8iNcr5Y{A4D!O6%x81f~v9Dr>QlYz1@T- kGMu*iVc;@?_w)8AeJNQ#8(0o+=+Zgi{c&q))!(Rn1Ah8T$N&HU diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_uncompressed/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_uncompressed/segment_0.dat index db20be8790f7ab0880b854b1daa0dc0592622d61..d7c8fcb0a5ef07335e219cbead664fd20995df38 100644 GIT binary patch delta 264 zcmZ3dIZbneCW{;c81OhSGjlKs@L92OFbFV9F!6vyb3@i`j%QiK%<-B>Ku3VlfN63e zm;7c8j&&@oe>emrA|^i+ked9JM`7|cAm$O4Vm!V%fLE81ar)#szIlv?C(H8hV0zCx z`8t1%2$KPm1)~!`2cs1?7bh1h$5IXfrOD|6x{R|YPZellTr*ihFo*H+?st=dzf3mLV(eWmE$+JfXw7X4u#35g``=g^9ZO-j^|JU zTi-odUD%y*@8o>pCYcv}0yYASS|Gb1OamsM-s5}%iV;gD^NP&mxZBy_?P3sQ2moY0 BK{WsX delta 250 zcmbQHxlVI~CX1O00|NsG6ay&%c?L!XIb}vhV}<}Amyv;)nS)V4&5DhKL4a9;i3ce5 z?90@y&F(CVn30olLey_KL?`~Hy0-tE5}j}0jtUW0=kTwC)WxzF&>#Dn8Wyc za;{)A<-W}=U?ITh#mX_6XYxcLTcE+`h3pv@Pv#ePXS_N&T)0W5 nfnUHzfKdx%B!p?e1XO;XPe3tZ&*ZPdGdXN7Dok=Q2r>i!e5gKe