[Feature](datatype) Add IPv4/v6 data type for doris

be/src/exprs/create_predicate_function.h

@@ -104,7 +104,9 @@ class PredicateFunctionCreator {
  M(TYPE_DECIMAL32) \
  M(TYPE_DECIMAL64) \
  M(TYPE_DECIMAL128I) \
- M(TYPE_DECIMAL256)
+ M(TYPE_DECIMAL256) \
+ M(TYPE_IPV4) \
+ M(TYPE_IPV6)
 
 template <class Traits, size_t N = 0>
 typename Traits::BasePtr create_predicate_function(PrimitiveType type) {

be/src/olap/delete_handler.cpp

@@ -213,6 +213,10 @@ bool DeleteHandler::is_condition_value_valid(const TabletColumn& column,
  return valid_datetime(value_str, column.frac());
  case FieldType::OLAP_FIELD_TYPE_BOOL:
  return valid_bool(value_str);
+ case FieldType::OLAP_FIELD_TYPE_IPV4:
+ return valid_ipv4(value_str);
+ case FieldType::OLAP_FIELD_TYPE_IPV6:
+ return valid_ipv6(value_str);
  default:
  LOG(WARNING) << "unknown field type. [type=" << int(field_type) << "]";
  }

be/src/olap/key_coder.cpp

@@ -81,6 +81,8 @@ class KeyCoderResolver {
  add_mapping<FieldType::OLAP_FIELD_TYPE_DECIMAL64>();
  add_mapping<FieldType::OLAP_FIELD_TYPE_DECIMAL128I>();
  add_mapping<FieldType::OLAP_FIELD_TYPE_DECIMAL256>();
+ add_mapping<FieldType::OLAP_FIELD_TYPE_IPV4>();
+ add_mapping<FieldType::OLAP_FIELD_TYPE_IPV6>();
  }
 
  template <FieldType field_type>

be/src/olap/olap_common.h

@@ -145,6 +145,8 @@ enum class FieldType {
  OLAP_FIELD_TYPE_VARIANT = 35,
  OLAP_FIELD_TYPE_AGG_STATE = 36,
  OLAP_FIELD_TYPE_DECIMAL256 = 37,
+ OLAP_FIELD_TYPE_IPV4 = 38,
+ OLAP_FIELD_TYPE_IPV6 = 39,
 };
 
 // Define all aggregation methods supported by Field

be/src/olap/predicate_creator.h

@@ -237,6 +237,24 @@ std::unique_ptr<PredicateCreator<ConditionType>> get_creator(const FieldType& ty
  return value;
  });
  }
+ case FieldType::OLAP_FIELD_TYPE_IPV4: {
+ return std::make_unique<CustomPredicateCreator<TYPE_IPV4, PT, ConditionType>>(
+ [](const std::string& condition) {
+ vectorized::IPv4 value;
+ bool res = IPv4Value::from_string(value, condition);
+ DCHECK(res);
+ return value;
+ });
+ }
+ case FieldType::OLAP_FIELD_TYPE_IPV6: {
+ return std::make_unique<CustomPredicateCreator<TYPE_IPV6, PT, ConditionType>>(
+ [](const std::string& condition) {
+ vectorized::IPv6 value;
+ bool res = IPv6Value::from_string(value, condition);
+ DCHECK(res);
+ return value;
+ });
+ }
  default:
  return nullptr;
  }

be/src/olap/rowset/segment_v2/encoding_info.cpp

@@ -325,6 +325,12 @@ EncodingInfoResolver::EncodingInfoResolver() {
  _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, PLAIN_ENCODING>();
  _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, true>();
 
+ _add_map<FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE>();
+
+ _add_map<FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE>();
+ _add_map<FieldType::OLAP_FIELD_TYPE_IPV6, PLAIN_ENCODING>();
+ _add_map<FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE, true>();
+
  _add_map<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING>();
 
  _add_map<FieldType::OLAP_FIELD_TYPE_OBJECT, PLAIN_ENCODING>();

be/src/olap/rowset/segment_v2/zone_map_index.cpp

@@ -196,6 +196,8 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory,
  M(TYPE_DATETIME) \
  M(TYPE_DATEV2) \
  M(TYPE_DATETIMEV2) \
+ M(TYPE_IPV4) \
+ M(TYPE_IPV6) \
  M(TYPE_VARCHAR) \
  M(TYPE_STRING) \
  M(TYPE_DECIMAL32) \

be/src/olap/schema.cpp

@@ -202,6 +202,12 @@ vectorized::IColumn::MutablePtr Schema::get_predicate_column_ptr(const Field& fi
  case FieldType::OLAP_FIELD_TYPE_DECIMAL256:
  ptr = doris::vectorized::PredicateColumnType<TYPE_DECIMAL256>::create();
  break;
+ case FieldType::OLAP_FIELD_TYPE_IPV4:
+ ptr = doris::vectorized::PredicateColumnType<TYPE_IPV4>::create();
+ break;
+ case FieldType::OLAP_FIELD_TYPE_IPV6:
+ ptr = doris::vectorized::PredicateColumnType<TYPE_IPV6>::create();
+ break;
  default:
  LOG(FATAL) << "Unexpected type when choosing predicate column, type=" << int(field.type());
  }

be/src/olap/tablet_schema.cpp

@@ -70,6 +70,10 @@ FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) {
  type = FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT;
  } else if (0 == upper_type_str.compare("UNSIGNED_BIGINT")) {
  type = FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT;
+ } else if (0 == upper_type_str.compare("IPV4")) {
+ type = FieldType::OLAP_FIELD_TYPE_IPV4;
+ } else if (0 == upper_type_str.compare("IPV6")) {
+ type = FieldType::OLAP_FIELD_TYPE_IPV6;
  } else if (0 == upper_type_str.compare("FLOAT")) {
  type = FieldType::OLAP_FIELD_TYPE_FLOAT;
  } else if (0 == upper_type_str.compare("DISCRETE_DOUBLE")) {
@@ -192,6 +196,12 @@ std::string TabletColumn::get_string_by_field_type(FieldType type) {
  case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT:
  return "UNSIGNED_BIGINT";
 
+ case FieldType::OLAP_FIELD_TYPE_IPV4:
+ return "IPV4";
+
+ case FieldType::OLAP_FIELD_TYPE_IPV6:
+ return "IPV6";
+
  case FieldType::OLAP_FIELD_TYPE_FLOAT:
  return "FLOAT";
 
@@ -316,6 +326,10 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3
  return 8;
  case TPrimitiveType::LARGEINT:
  return 16;
+ case TPrimitiveType::IPV4:
+ return 4;
+ case TPrimitiveType::IPV6:
+ return 16;
  case TPrimitiveType::DATE:
  return 3;
  case TPrimitiveType::DATEV2:

be/src/olap/types.cpp

@@ -99,6 +99,8 @@ const TypeInfo* get_scalar_type_info(FieldType field_type) {
  get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARIANT>(),
  get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_AGG_STATE>(),
  get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_DECIMAL256>(),
+ get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_IPV4>(),
+ get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_IPV6>(),
  nullptr};
  return field_type_array[int(field_type)];
 }

be/src/olap/types.h

@@ -83,7 +83,7 @@ class TypeInfo {
 
  virtual void direct_copy(void* dest, const void* src) const = 0;
 
- // Use only in zone map to cut data.
+ // Use only in zone map to cut data.StringParser::string_to_unsigned_int<uint32_t>
  virtual void direct_copy_may_cut(void* dest, const void* src) const = 0;
 
  virtual Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
@@ -722,6 +722,16 @@ struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIME> {
  using UnsignedCppType = uint64_t;
 };
 template <>
+struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_IPV4> {
+ using CppType = uint32_t;
+ using UnsignedCppType = uint32_t;
+};
+template <>
+struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_IPV6> {
+ using CppType = uint128_t;
+ using UnsignedCppType = uint128_t;
+};
+template <>
 struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_CHAR> {
  using CppType = Slice;
 };
@@ -778,6 +788,8 @@ struct BaseFieldtypeTraits : public CppTypeTraits<field_type> {
  static inline CppType get_cpp_type_value(const void* address) {
  if constexpr (field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT) {
  return get_int128_from_unalign(address);
+ } else if constexpr (field_type == FieldType::OLAP_FIELD_TYPE_IPV6) {
+ return get_uint128_from_unalign(address);
  }
  return *reinterpret_cast<const CppType*>(address);
  }
@@ -964,6 +976,101 @@ struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_LARGEINT>
  }
 };
 
+template <>
+struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_IPV4>
+ : public BaseFieldtypeTraits<FieldType::OLAP_FIELD_TYPE_IPV4> {
+ static Status from_string(void* buf, const std::string& scan_key, const int precision,
+ const int scale) {
+ StringParser::ParseResult result = StringParser::PARSE_SUCCESS;
+ uint32_t value = StringParser::string_to_unsigned_int<uint32_t>(scan_key.c_str(),
+ scan_key.size(), &result);
+
+ if (result == StringParser::PARSE_FAILURE) {
+ return Status::Error<ErrorCode::INVALID_ARGUMENT>(
+ "FieldTypeTraits<OLAP_FIELD_TYPE_IPV4>::from_string meet PARSE_FAILURE");
+ }
+ *reinterpret_cast<uint32_t*>(buf) = value;
+ return Status::OK();
+ }
+
+ static std::string to_string(const void* src) {
+ uint32_t value = *reinterpret_cast<const uint32_t*>(src);
+ std::stringstream ss;
+ ss << ((value >> 24) & 0xFF) << '.' << ((value >> 16) & 0xFF) << '.'
+ << ((value >> 8) & 0xFF) << '.' << (value & 0xFF);
+ return ss.str();
+ }
+};
+
+template <>
+struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_IPV6>
+ : public BaseFieldtypeTraits<FieldType::OLAP_FIELD_TYPE_IPV6> {
+ static Status from_string(void* buf, const std::string& scan_key, const int precision,
+ const int scale) {
+ std::istringstream iss(scan_key);
+ std::string token;
+ uint128_t result = 0;
+ int count = 0;
+
+ while (std::getline(iss, token, ':')) {
+ if (token.empty()) {
+ count += 8 - count;
+ break;
+ }
+
+ if (count > 8) {
+ return Status::Error<ErrorCode::INVALID_ARGUMENT>(
+ "FieldTypeTraits<OLAP_FIELD_TYPE_IPV6>::from_string meet PARSE_FAILURE");
+ }
+
+ uint16_t value = 0;
+ std::istringstream ss(token);
+ if (!(ss >> std::hex >> value)) {
+ return Status::Error<ErrorCode::INVALID_ARGUMENT>(
+ "FieldTypeTraits<OLAP_FIELD_TYPE_IPV6>::from_string meet PARSE_FAILURE");
+ }
+
+ result = (result << 16) | value;
+ count++;
+ }
+
+ if (count < 8) {
+ return Status::Error<ErrorCode::INVALID_ARGUMENT>(
+ "FieldTypeTraits<OLAP_FIELD_TYPE_IPV6>::from_string meet PARSE_FAILURE");
+ }
+
+ *reinterpret_cast<uint128_t*>(buf) = result;
+ return Status::OK();
+ }
+
+ static std::string to_string(const void* src) {
+ std::stringstream result;
+ uint128_t ipv6 = *reinterpret_cast<const uint128_t*>(src);
+
+ for (int i = 0; i < 8; i++) {
+ uint16_t part = static_cast<uint16_t>((ipv6 >> (112 - i * 16)) & 0xFFFF);
+ result << std::to_string(part);
+ if (i != 7) {
+ result << ":";
+ }
+ }
+
+ return result.str();
+ }
+
+ static void set_to_max(void* buf) {
+ *reinterpret_cast<PackedInt128*>(buf) =
+ static_cast<int128_t>(999999999999999999ll) * 100000000000000000ll * 1000ll +
+ static_cast<int128_t>(99999999999999999ll) * 1000ll + 999ll;
+ }
+
+ static void set_to_min(void* buf) {
+ *reinterpret_cast<PackedInt128*>(buf) =
+ -(static_cast<int128_t>(999999999999999999ll) * 100000000000000000ll * 1000ll +
+ static_cast<int128_t>(99999999999999999ll) * 1000ll + 999ll);
+ }
+};
+
 template <>
 struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_FLOAT>
  : public NumericFieldtypeTraits<FieldType::OLAP_FIELD_TYPE_FLOAT, true> {

be/src/olap/utils.cpp

@@ -49,6 +49,8 @@
 #include "io/fs/local_file_system.h"
 #include "olap/olap_common.h"
 #include "util/string_parser.hpp"
+#include "vec/runtime/ipv4_value.h"
+#include "vec/runtime/ipv6_value.h"
 
 namespace doris {
 using namespace ErrorCode;
@@ -648,6 +650,24 @@ bool valid_bool(const std::string& value_str) {
  return result == StringParser::PARSE_SUCCESS;
 }
 
+bool valid_ipv4(const std::string& value_str) {
+ if (value_str.size() == 0) {
+ return false;
+ }
+
+ vectorized::IPv4 value = 0;
+ return IPv4Value::from_string(value, value_str);
+}
+
+bool valid_ipv6(const std::string& value_str) {
+ if (value_str.size() == 0) {
+ return false;
+ }
+
+ vectorized::IPv6 value;
+ return IPv6Value::from_string(value, value_str);
+}
+
 void write_log_info(char* buf, size_t buf_len, const char* fmt, ...) {
  va_list args;
  va_start(args, fmt);

be/src/olap/utils.h

@@ -232,6 +232,10 @@ bool valid_datetime(const std::string& value_str, const uint32_t scale);
 
 bool valid_bool(const std::string& value_str);
 
+bool valid_ipv4(const std::string& value_str);
+
+bool valid_ipv6(const std::string& value_str);
+
 constexpr bool is_string_type(const FieldType& field_type) {
  return field_type == FieldType::OLAP_FIELD_TYPE_VARCHAR ||
  field_type == FieldType::OLAP_FIELD_TYPE_CHAR ||

be/src/runtime/define_primitive_type.h

@@ -64,6 +64,8 @@ enum PrimitiveType : PrimitiveNative {
  TYPE_LAMBDA_FUNCTION, /* 33 */
  TYPE_AGG_STATE, /* 34 */
  TYPE_DECIMAL256, /* 35 */
+ TYPE_IPV4, /* 36 */
+ TYPE_IPV6 /* 37 */
 };
 
 constexpr PrimitiveNative BEGIN_OF_PRIMITIVE_TYPE = INVALID_TYPE;

be/src/runtime/primitive_type.cpp

@@ -113,6 +113,12 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) {
  case TPrimitiveType::BINARY:
  return TYPE_BINARY;
 
+ case TPrimitiveType::IPV4:
+ return TYPE_IPV4;
+
+ case TPrimitiveType::IPV6:
+ return TYPE_IPV6;
+
  case TPrimitiveType::DECIMALV2:
  return TYPE_DECIMALV2;
 
@@ -229,6 +235,12 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) {
  case TYPE_BINARY:
  return TPrimitiveType::BINARY;
 
+ case TYPE_IPV4:
+ return TPrimitiveType::IPV4;
+
+ case TYPE_IPV6:
+ return TPrimitiveType::IPV6;
+
  case TYPE_DECIMALV2:
  return TPrimitiveType::DECIMALV2;
 
@@ -336,6 +348,12 @@ std::string type_to_string(PrimitiveType t) {
  case TYPE_BINARY:
  return "BINARY";
 
+ case TYPE_IPV4:
+ return "IPV4";
+
+ case TYPE_IPV6:
+ return "IPV6";
+
  case TYPE_DECIMALV2:
  return "DECIMALV2";
 
@@ -448,6 +466,12 @@ std::string type_to_odbc_string(PrimitiveType t) {
  case TYPE_BINARY:
  return "binary";
 
+ case TYPE_IPV4:
+ return "ipv4";
+
+ case TYPE_IPV6:
+ return "ipv6";
+
  case TYPE_DECIMALV2:
  return "decimalv2";