[Feat](nereids) support generated column

fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4

@@ -115,6 +115,7 @@ AT: 'AT';
 AUTHORS: 'AUTHORS';
 AUTO: 'AUTO';
 AUTO_INCREMENT: 'AUTO_INCREMENT';
+ALWAYS: 'ALWAYS';
 BACKEND: 'BACKEND';
 BACKENDS: 'BACKENDS';
 BACKUP: 'BACKUP';
@@ -276,6 +277,7 @@ FRONTENDS: 'FRONTENDS';
 FULL: 'FULL';
 FUNCTION: 'FUNCTION';
 FUNCTIONS: 'FUNCTIONS';
+GENERATED: 'GENERATED';
 GENERIC: 'GENERIC';
 GLOBAL: 'GLOBAL';
 GRANT: 'GRANT';

fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4

@@ -585,14 +585,15 @@ columnDef
  : colName=identifier type=dataType
  KEY?
  (aggType=aggTypeDef)?
+ ((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)?
  ((NOT)? nullable=NULL)?
  (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)?
  (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | stringValue=STRING_LITERAL
  | CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))?
  (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)?
  (COMMENT comment=STRING_LITERAL)?
  ;
-
+
 indexDefs
  : indexes+=indexDef (COMMA indexes+=indexDef)*
  ;
@@ -1028,6 +1029,7 @@ nonReserved
  | AGG_STATE
  | AGGREGATE
  | ALIAS
+ | ALWAYS
  | ANALYZED
  | ARRAY
  | ARRAY_RANGE
@@ -1142,6 +1144,7 @@ nonReserved
  | FREE
  | FRONTENDS
  | FUNCTION
+ | GENERATED
  | GENERIC
  | GLOBAL
  | GRAPH

fe/fe-core/src/main/cup/sql_parser.cup

@@ -43,6 +43,7 @@ import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.GeneratedColumnInfo;
 import org.apache.doris.catalog.MapType;
 import org.apache.doris.catalog.StructField;
 import org.apache.doris.catalog.StructType;
@@ -62,6 +63,7 @@ import org.apache.doris.resource.workloadschedpolicy.WorkloadActionMeta;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import java_cup.runtime.Symbol;
@@ -261,6 +263,7 @@ terminal String
  KW_ALIAS,
  KW_ALL,
  KW_ALTER,
+ KW_ALWAYS,
  KW_ANALYZE,
  KW_AND,
  KW_ANTI,
@@ -408,6 +411,7 @@ terminal String
  KW_FULL,
  KW_FUNCTION,
  KW_FUNCTIONS,
+ KW_GENERATED,
  KW_GENERIC,
  KW_GLOBAL,
  KW_GRANT,
@@ -1004,6 +1008,9 @@ nonterminal String stage_name;
 nonterminal StageAndPattern stage_and_pattern;
 nonterminal List<Expr> copy_select_expr_list;
 
+//genearted column
+nonterminal Boolean opt_generated_always;
+
 precedence nonassoc COMMA;
 precedence nonassoc STRING_LITERAL;
 precedence nonassoc KW_COLUMNS;
@@ -3968,6 +3975,11 @@ column_definition ::=
  ColumnDef columnDef = new ColumnDef(columnName, typeDef, isKey, aggType, nullable_type, autoIncInitValue, defaultValue, comment);
  RESULT = columnDef;
  :}
+ | ident:columnName type_def:typeDef opt_is_key:isKey opt_generated_always KW_AS LPAREN expr:expr RPAREN opt_nullable_type:nullable_type opt_comment:comment
+ {:
+ ColumnDef columnDef = new ColumnDef(columnName, typeDef, isKey, nullable_type, comment, Optional.of(new GeneratedColumnInfo(null, expr)));
+ RESULT = columnDef;
+ :}
  ;
 
 index_definition ::=
@@ -7942,7 +7954,16 @@ opt_work ::=
  RESULT = null;
  :}
  ;
-
+opt_generated_always ::=
+ /* empty */
+ {:
+ RESULT = null;
+ :}
+ | KW_GENERATED KW_ALWAYS
+ {:
+ RESULT = null;
+ :}
+ ;
 opt_chain ::=
  {:
  RESULT = null;
@@ -7982,6 +8003,8 @@ type_func_name_keyword ::=
 keyword ::=
  KW_AFTER:id
  {: RESULT = id; :}
+ | KW_ALWAYS:id
+ {: RESULT = id; :}
  | KW_AGGREGATE:id
  {: RESULT = id; :}
  | KW_ALIAS:id
@@ -8142,6 +8165,8 @@ keyword ::=
  {: RESULT = id; :}
  | KW_GLOBAL:id
  {: RESULT = id; :}
+ | KW_GENERATED:id
+ {: RESULT = id; :}
  | KW_GENERIC:id
  {: RESULT = id; :}
  | KW_GRAPH:id

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java

@@ -28,6 +28,7 @@
 import org.apache.doris.analysis.CreateMaterializedViewStmt;
 import org.apache.doris.analysis.DropColumnClause;
 import org.apache.doris.analysis.DropIndexClause;
+import org.apache.doris.analysis.Expr;
 import org.apache.doris.analysis.IndexDef;
 import org.apache.doris.analysis.ModifyColumnClause;
 import org.apache.doris.analysis.ModifyTablePropertiesClause;
@@ -42,6 +43,7 @@
 import org.apache.doris.catalog.DistributionInfo.DistributionInfoType;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.EnvFactory;
+import org.apache.doris.catalog.GeneratedColumnInfo;
 import org.apache.doris.catalog.HashDistributionInfo;
 import org.apache.doris.catalog.Index;
 import org.apache.doris.catalog.KeysType;
@@ -377,6 +379,23 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
  }
  }
 
+ // generated column check
+ Map<String, Column> nameToColumn = new HashMap<>();
+ for (Column c : indexSchemaMap.get(baseIndexId)) {
+ nameToColumn.put(c.getName(), c);
+ }
+ if (null == targetIndexName) {
+ if (nameToColumn.containsKey(dropColName)) {
+ Column column = nameToColumn.get(dropColName);
+ Set<String> generatedColumnsThatReferToThis = column.getGeneratedColumnsThatReferToThis();
+ if (!generatedColumnsThatReferToThis.isEmpty()) {
+ throw new DdlException(
+ "Column '" + dropColName + "' has a generated column dependency on :"
+ + generatedColumnsThatReferToThis);
+ }
+ }
+ }
+
  Iterator<Index> it = indexes.iterator();
  while (it.hasNext()) {
  Index index = it.next();
@@ -403,6 +422,8 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
  if (column.getName().equalsIgnoreCase(dropColName)) {
  baseIter.remove();
  found = true;
+ // find generated column referred column
+ removeColumnWhenDropGeneratedColumn(column, nameToColumn);
  break;
  }
  }
@@ -528,6 +549,7 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol
  Map<Long, LinkedList<Column>> indexSchemaMap) throws DdlException {
  Column modColumn = alterClause.getColumn();
  boolean lightSchemaChange = false;
+
  if (KeysType.AGG_KEYS == olapTable.getKeysType()) {
  if (modColumn.isKey() && null != modColumn.getAggregationType()) {
  throw new DdlException("Can not assign aggregation method on key column: " + modColumn.getName());
@@ -784,12 +806,15 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol
  if (targetIndexName == null) {
  targetIndexName = baseIndexName;
  }
-
  long targetIndexId = olapTable.getIndexIdByName(targetIndexName);
 
  LinkedList<Column> newSchema = new LinkedList<Column>();
  List<Column> targetIndexSchema = indexSchemaMap.get(targetIndexId);
-
+ // When rollup is specified, there is no need to check the order of generated columns.
+ // When rollup is not specified and the order of baseIndex needs to be modified, the order needs to be checked.
+ if (alterClause.getRollupName() == null) {
+ checkOrder(targetIndexSchema, orderedColNames);
+ }
  // check and create new ordered column list
  Set<String> colNameSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
  for (String colName : orderedColNames) {
@@ -1004,6 +1029,10 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
  }
  }
 
+ if (newColumn.getGeneratedColumnInfo() != null) {
+ throw new DdlException("Not supporting alter table add generated columns.");
+ }
+
  /*
  * add new column to indexes.
  * UNIQUE:
@@ -3194,4 +3223,61 @@ public boolean updateBinlogConfig(Database db, OlapTable olapTable, List<AlterCl
 
  return false;
  }
+
+ private void removeColumnWhenDropGeneratedColumn(Column dropColumn, Map<String, Column> nameToColumn) {
+ GeneratedColumnInfo generatedColumnInfo = dropColumn.getGeneratedColumnInfo();
+ if (generatedColumnInfo == null) {
+ return;
+ }
+ String dropColName = dropColumn.getName();
+ Expr expr = generatedColumnInfo.getExpr();
+ Set<Expr> slotRefsInGeneratedExpr = new HashSet<>();
+ expr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr);
+ for (Expr slotRef : slotRefsInGeneratedExpr) {
+ String name = ((SlotRef) slotRef).getColumnName();
+ if (!nameToColumn.containsKey(name)) {
+ continue;
+ }
+ Column c = nameToColumn.get(name);
+ Set<String> sets = c.getGeneratedColumnsThatReferToThis();
+ sets.remove(dropColName);
+ }
+ }
+
+ private void checkOrder(List<Column> targetIndexSchema, List<String> orderedColNames) throws DdlException {
+ Set<String> nameSet = new HashSet<>();
+ for (Column column : targetIndexSchema) {
+ if (column.isVisible() && null == column.getGeneratedColumnInfo()) {
+ nameSet.add(column.getName());
+ }
+ }
+ for (String colName : orderedColNames) {
+ Column oneCol = null;
+ for (Column column : targetIndexSchema) {
+ if (column.getName().equalsIgnoreCase(colName) && column.isVisible()) {
+ oneCol = column;
+ break;
+ }
+ }
+ if (oneCol == null) {
+ throw new DdlException("Column[" + colName + "] not exists");
+ }
+ if (null == oneCol.getGeneratedColumnInfo()) {
+ continue;
+ }
+ Expr expr = oneCol.getGeneratedColumnInfo().getExpr();
+ Set<Expr> slotRefsInGeneratedExpr = new HashSet<>();
+ expr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr);
+ for (Expr slotRef : slotRefsInGeneratedExpr) {
+ String slotName = ((SlotRef) slotRef).getColumnName();
+ if (!nameSet.contains(slotName)) {
+ throw new DdlException("The specified column order is incorrect, `" + colName
+ + "` should come after `" + slotName
+ + "`, because both of them are generated columns, and `"
+ + colName + "` refers to `" + slotName + "`.");
+ }
+ }
+ nameSet.add(colName);
+ }
+ }
 }