Support bitmap_intersect

be/src/exprs/bitmap_function.cpp

@@ -302,6 +302,31 @@ void BitmapFunctions::bitmap_union(FunctionContext* ctx, const StringVal& src, S
  }
 }
 
+// the dst value could be null
+void BitmapFunctions::nullable_bitmap_init(FunctionContext* ctx, StringVal* dst) {
+ dst->is_null = true;
+}
+
+void BitmapFunctions::bitmap_intersect(FunctionContext* ctx, const StringVal& src, StringVal* dst) {
+ if (src.is_null) {
+ return;
+ }
+ // if dst is null, the src input is the first value
+ if (dst->is_null) {
+ dst->is_null = false;
+ dst->len = sizeof(BitmapValue);
+ dst->ptr = (uint8_t*)new BitmapValue((char*) src.ptr);
+ return;
+ }
+ auto dst_bitmap = reinterpret_cast<BitmapValue*>(dst->ptr);
+ // zero size means the src input is a agg object
+ if (src.len == 0) {
+ (*dst_bitmap) &= *reinterpret_cast<BitmapValue*>(src.ptr);
+ } else {
+ (*dst_bitmap) &= BitmapValue((char*) src.ptr);
+ }
+}
+
 BigIntVal BitmapFunctions::bitmap_count(FunctionContext* ctx, const StringVal& src) {
  if (src.is_null) {
  return 0;
@@ -343,12 +368,17 @@ StringVal BitmapFunctions::bitmap_hash(doris_udf::FunctionContext* ctx, const do
 }
 
 StringVal BitmapFunctions::bitmap_serialize(FunctionContext* ctx, const StringVal& src) {
+ if (src.is_null) {
+ return src;
+ }
+
  auto src_bitmap = reinterpret_cast<BitmapValue*>(src.ptr);
  StringVal result = serialize(ctx, src_bitmap);
  delete src_bitmap;
  return result;
 }
 
+// This is a init function for intersect_count not for bitmap_intersect.
 template<typename T, typename ValType>
 void BitmapFunctions::bitmap_intersect_init(FunctionContext* ctx, StringVal* dst) {
  dst->is_null = false;
@@ -510,6 +540,7 @@ template void BitmapFunctions::bitmap_update_int<IntVal>(
 template void BitmapFunctions::bitmap_update_int<BigIntVal>(
  FunctionContext* ctx, const BigIntVal& src, StringVal* dst);
 
+// this is init function for intersect_count not for bitmap_intersect
 template void BitmapFunctions::bitmap_intersect_init<int8_t, TinyIntVal>(
  FunctionContext* ctx, StringVal* dst);
 template void BitmapFunctions::bitmap_intersect_init<int16_t, SmallIntVal>(

be/src/exprs/bitmap_function.h

@@ -51,6 +51,9 @@ class BitmapFunctions {
  static BigIntVal bitmap_get_value(FunctionContext* ctx, const StringVal& src);
 
  static void bitmap_union(FunctionContext* ctx, const StringVal& src, StringVal* dst);
+ // the dst value could be null
+ static void nullable_bitmap_init(FunctionContext* ctx, StringVal* dst);
+ static void bitmap_intersect(FunctionContext* ctx, const StringVal& src, StringVal* dst);
  static BigIntVal bitmap_count(FunctionContext* ctx, const StringVal& src);
 
  static StringVal bitmap_serialize(FunctionContext* ctx, const StringVal& src);
@@ -68,8 +71,9 @@ class BitmapFunctions {
  static BooleanVal bitmap_contains(FunctionContext* ctx, const StringVal& src, const BigIntVal& input);
  static BooleanVal bitmap_has_any(FunctionContext* ctx, const StringVal& lhs, const StringVal& rhs);
 
- // bitmap_intersect
+ // intersect count
  template<typename T, typename ValType>
+ // this is init function for intersect_count not for bitmap_intersect
  static void bitmap_intersect_init(FunctionContext* ctx, StringVal* dst);
  template<typename T, typename ValType>
  static void bitmap_intersect_update(FunctionContext* ctx, const StringVal& src, const ValType& key,

be/test/exprs/bitmap_function_test.cpp

@@ -172,6 +172,39 @@ TEST_F(BitmapFunctionsTest, bitmap_union) {
  ASSERT_EQ(expected, result);
 }
 
+// test bitmap_intersect
+TEST_F(BitmapFunctionsTest, bitmap_intersect) {
+ StringVal dst;
+ BitmapFunctions::bitmap_intersect_init_real(ctx, &dst);
+
+ BitmapValue bitmap1(1);
+ bitmap1.add(2);
+ bitmap1.add(3);
+ StringVal src1 = convert_bitmap_to_string(ctx, bitmap1);
+ BitmapFunctions::bitmap_intersect(ctx, src1, &dst);
+
+ BitmapValue bitmap2(1);
+ bitmap2.add(2);
+ StringVal src2 = convert_bitmap_to_string(ctx, bitmap2);
+ BitmapFunctions::bitmap_intersect(ctx, src2, &dst);
+
+ StringVal serialized = BitmapFunctions::bitmap_serialize(ctx, dst);
+ BigIntVal result = BitmapFunctions::bitmap_count(ctx, serialized);
+ BigIntVal expected(2);
+ ASSERT_EQ(expected, result); 
+}
+
+// test bitmap_intersect with null dst
+TEST_F(BitmapFunctionsTest, bitmap_intersect_empty) {
+ StringVal dst;
+ BitmapFunctions::bitmap_intersect_init_real(ctx, &dst);
+
+ StringVal serialized = BitmapFunctions::bitmap_serialize(ctx, dst);
+ BigIntVal result = BitmapFunctions::bitmap_count(ctx, serialized);
+ BigIntVal expected(0);
+ ASSERT_EQ(expected, result); 
+}
+
 TEST_F(BitmapFunctionsTest, bitmap_count) {
  BitmapValue bitmap(1024);
  bitmap.add(1);
@@ -186,6 +219,7 @@ TEST_F(BitmapFunctionsTest, bitmap_count) {
  ASSERT_EQ(BigIntVal(0), null_bitmap);
 }
 
+// test intersect_count
 template<typename ValType, typename ValueType>
 void test_bitmap_intersect(FunctionContext* ctx, ValType key1, ValType key2) {
  StringVal bitmap_column("placeholder");

docs/.vuepress/sidebar/en.js

@@ -257,6 +257,8 @@ module.exports = [
  "bitmap_or",
  "bitmap_to_string",
  "to_bitmap",
+ "bitmap_intersect",
+ "bitmap_union",
  ],
  },
  {

docs/.vuepress/sidebar/zh-CN.js

@@ -269,6 +269,8 @@ module.exports = [
  "bitmap_or",
  "bitmap_to_string",
  "to_bitmap",
+ "bitmap_intersect",
+ "bitmap_union",
  ],
  },
  {

docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_intersect.md

@@ -0,0 +1,61 @@
+---
+{
+ "title": "bitmap_intersect",
+ "language": "en"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# bitmap_intersect
+## description
+
+Aggregation function, used to calculate the bitmap intersection after grouping. Common usage scenarios such as: calculating user retention rate.
+
+### Syntax
+
+`BITMAP BITMAP_INTERSECT(BITMAP value)`
+
+Enter a set of bitmap values, find the intersection of the set of bitmap values, and return.
+
+## example
+
+Table schema
+
+```
+KeysType: AGG_KEY
+Columns: tag varchar, date datetime, user_id bitmap bitmap_union
+```
+
+```
+Find the retention of users between 2020-05-18 and 2020-05-19 under different tags.
+mysql> select tag, bitmap_intersect(user_id) from (select tag, date, bitmap_union(user_id) user_id from table where date in ('2020-05-18', '2020-05-19') group by tag, date) a group by tag;
+```
+
+Used in combination with the bitmap_to_string function to obtain the specific data of the intersection
+
+```
+Who are the users retained under different tags between 2020-05-18 and 2020-05-19?
+mysql> select tag, bitmap_to_string(bitmap_intersect(user_id)) from (select tag, date, bitmap_union(user_id) user_id from table where date in ('2020-05-18', '2020-05-19') group by tag, date) a group by tag;
+```
+
+## keyword
+
+ BITMAP_INTERSECT, BITMAP

docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_union.md

@@ -0,0 +1,58 @@
+---
+{
+ "title": "bitmap_union",
+ "language": "en"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# bitmap_union
+## description
+
+Aggregate function, used to calculate the grouped bitmap union. Common usage scenarios such as: calculating PV, UV.
+
+### Syntax
+
+`BITMAP BITMAP_UNION(BITMAP value)`
+
+Enter a set of bitmap values, find the union of this set of bitmap values, and return.
+
+## example
+
+```
+mysql> select page_id, bitmap_union(user_id) from table group by page_id;
+```
+
+Combined with the bitmap_count function, the PV data of the web page can be obtained
+
+```
+mysql> select page_id, bitmap_count(bitmap_union(user_id)) from table group by page_id;
+```
+
+When the user_id field is int, the above query semantics is equivalent to
+
+```
+mysql> select page_id, count(distinct user_id) from table group by page_id;
+```
+
+## keyword
+
+ BITMAP_UNION, BITMAP

docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_intersect.md

@@ -0,0 +1,62 @@
+---
+{
+ "title": "bitmap_intersect",
+ "language": "zh-CN"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# bitmap_intersect
+## description
+
+聚合函数，用于计算分组后的 bitmap 交集。常见使用场景如：计算用户留存率。
+
+### Syntax
+
+`BITMAP BITMAP_INTERSECT(BITMAP value)`
+
+输入一组 bitmap 值，求这一组 bitmap 值的交集，并返回。
+
+## example
+
+表结构
+
+```
+KeysType: AGG_KEY
+Columns: tag varchar, date datetime, user_id bitmap bitmap_union
+
+```
+
+```
+求今天和昨天不同 tag 下的用户留存
+mysql> select tag, bitmap_intersect(user_id) from (select tag, date, bitmap_union(user_id) user_id from table where date in ('2020-05-18', '2020-05-19') group by tag, date) a group by tag;
+```
+
+和 bitmap_to_string 函数组合使用可以获取交集的具体数据
+
+```
+求今天和昨天不同 tag 下留存的用户都是哪些
+mysql> select tag, bitmap_to_string(bitmap_intersect(user_id)) from (select tag, date, bitmap_union(user_id) user_id from table where date in ('2020-05-18', '2020-05-19') group by tag, date) a group by tag;
+```
+
+## keyword
+
+ BITMAP_INTERSECT, BITMAP

docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_union.md

@@ -0,0 +1,58 @@
+---
+{
+ "title": "bitmap_union",
+ "language": "zh-CN"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# bitmap_union
+## description
+
+聚合函数，用于计算分组后的 bitmap 并集。常见使用场景如：计算PV，UV。
+
+### Syntax
+
+`BITMAP BITMAP_UNION(BITMAP value)`
+
+输入一组 bitmap 值，求这一组 bitmap 值的并集，并返回。
+
+## example
+
+```
+mysql> select page_id, bitmap_union(user_id) from table group by page_id;
+```
+
+和 bitmap_count 函数组合使用可以求得网页的 PV 数据
+
+```
+mysql> select page_id, bitmap_count(bitmap_union(user_id)) from table group by page_id;
+```
+
+当 user_id 字段为 int 时，上面查询语义等同于
+
+```
+mysql> select page_id, count(distinct user_id) from table group by page_id;
+```
+
+## keyword
+
+ BITMAP_UNION, BITMAP

fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java

@@ -407,7 +407,8 @@ private void analyzeBuiltinAggFunction(Analyzer analyzer) throws AnalysisExcepti
 
  if (fnName.getFunction().equalsIgnoreCase(FunctionSet.BITMAP_COUNT)
  || fnName.getFunction().equalsIgnoreCase(FunctionSet.BITMAP_UNION)
- || fnName.getFunction().equalsIgnoreCase(FunctionSet.BITMAP_UNION_COUNT)) {
+ || fnName.getFunction().equalsIgnoreCase(FunctionSet.BITMAP_UNION_COUNT)
+ || fnName.getFunction().equalsIgnoreCase(FunctionSet.BITMAP_INTERSECT)) {
  if (children.size() != 1) {
  throw new AnalysisException(fnName + " function could only have one child");
  }