[opt](schema) Optimize tables schema scan status

be/src/exec/operator/schema_scan_operator.cpp

@@ -25,6 +25,7 @@
 #include "core/data_type/data_type_factory.hpp"
 #include "exec/operator/operator.h"
 #include "runtime/runtime_profile.h"
+#include "util/string_util.h"
 
 namespace doris {
 class RuntimeState;
@@ -186,6 +187,8 @@ Status SchemaScanOperatorX::prepare(RuntimeState* state) {
  for (; j < columns_desc.size(); ++j) {
  if (boost::iequals(_dest_tuple_desc->slots()[i]->col_name(), columns_desc[j].name)) {
  _slot_offsets[i] = j;
+ _common_scanner_param->required_columns.insert(
+ to_upper(_dest_tuple_desc->slots()[i]->col_name()));
  break;
  }
  }

be/src/information_schema/schema_scanner.h

@@ -24,6 +24,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <set>
 #include <string>
 #include <vector>
 
@@ -69,6 +70,7 @@ struct SchemaScannerCommonParam {
  int64_t thread_id;
  const std::string* catalog = nullptr;
  std::set<TNetworkAddress> fe_addr_list;
+ std::set<std::string> required_columns;
 };
 
 // scanner parameter from frontend

be/src/information_schema/schema_tables_scanner.cpp

@@ -114,6 +114,7 @@ Status SchemaTablesScanner::_get_new_table() {
  if (nullptr != _param->common_param->table) {
  table_params.__set_table(*(_param->common_param->table));
  }
+ table_params.__set_required_columns(_param->common_param->required_columns);
  if (nullptr != _param->common_param->current_user_ident) {
  table_params.__set_current_user_ident(*(_param->common_param->current_user_ident));
  } else {

fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTablet.java

@@ -34,7 +34,6 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
-import java.util.stream.LongStream;
 
 public class LocalTablet extends Tablet {
  private static final Logger LOG = LogManager.getLogger(LocalTablet.class);
@@ -133,17 +132,34 @@ public Replica getReplicaById(long replicaId) {
  // due to dataSize not write to image
  @Override
  public long getDataSize(boolean singleReplica, boolean filterSizeZero) {
- LongStream s = getReplicas().stream().filter(r -> r.getState() == ReplicaState.NORMAL)
- .filter(r -> !filterSizeZero || r.getDataSize() > 0)
- .mapToLong(Replica::getDataSize);
- return singleReplica ? Double.valueOf(s.average().orElse(0)).longValue() : s.sum();
+ long dataSize = 0;
+ int replicaNum = 0;
+ for (Replica replica : getReplicas()) {
+ if (replica.getState() != ReplicaState.NORMAL) {
+ continue;
+ }
+ long replicaDataSize = replica.getDataSize();
+ if (filterSizeZero && replicaDataSize <= 0) {
+ continue;
+ }
+ dataSize += replicaDataSize;
+ replicaNum++;
+ }
+ return singleReplica && replicaNum > 0 ? dataSize / replicaNum : dataSize;
  }
 
  @Override
  public long getRowCount(boolean singleReplica) {
- LongStream s = getReplicas().stream().filter(r -> r.getState() == ReplicaState.NORMAL)
- .mapToLong(Replica::getRowCount);
- return singleReplica ? Double.valueOf(s.average().orElse(0)).longValue() : s.sum();
+ long rowCount = 0;
+ int replicaNum = 0;
+ for (Replica replica : getReplicas()) {
+ if (replica.getState() != ReplicaState.NORMAL) {
+ continue;
+ }
+ rowCount += replica.getRowCount();
+ replicaNum++;
+ }
+ return singleReplica && replicaNum > 0 ? rowCount / replicaNum : rowCount;
  }
 
  // Get the least row count among all valid replicas.

fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

@@ -1911,37 +1911,43 @@ public long getRowCountForPartitionIndex(long partitionId, long indexId, boolean
 
  @Override
  public long getAvgRowLength() {
- long rowCount = 0;
- long dataSize = 0;
- for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
- rowCount += entry.getValue().getBaseIndex().getRowCount();
- dataSize += entry.getValue().getBaseIndex().getDataSize(false, false);
- }
- if (rowCount > 0) {
- return dataSize / rowCount;
- } else {
- return 0;
- }
+ return getTableStatusStats().getAvgRowLength();
  }
 
  @Override
  public long getDataLength() {
- long dataSize = 0;
- for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
- dataSize += entry.getValue().getBaseIndex().getLocalSegmentSize();
- dataSize += entry.getValue().getBaseIndex().getRemoteSegmentSize();
- }
- return dataSize;
+ return getTableStatusStats().getDataLength();
  }
 
  @Override
  public long getIndexLength() {
- long indexSize = 0;
+ return getTableStatusStats().getIndexLength();
+ }
+
+ @Override
+ public TableStatusStats getTableStatusStats() {
+ long rowCount = 0;
+ long rowCountForAvg = 0;
+ long dataSizeForAvg = 0;
+ long dataLength = 0;
+ long indexLength = 0;
  for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
- indexSize += entry.getValue().getBaseIndex().getLocalIndexSize();
- indexSize += entry.getValue().getBaseIndex().getRemoteIndexSize();
+ MaterializedIndex baseIndex = entry.getValue().getBaseIndex();
+ long baseIndexRowCount = baseIndex.getRowCount();
+ rowCount += baseIndexRowCount == UNKNOWN_ROW_COUNT ? 0 : baseIndexRowCount;
+ rowCountForAvg += baseIndexRowCount;
+ for (Tablet tablet : baseIndex.getTablets()) {
+ for (Replica replica : tablet.getReplicas()) {
+ if (replica.getState() == Replica.ReplicaState.NORMAL) {
+ dataSizeForAvg += replica.getDataSize();
+ }
+ dataLength += replica.getLocalSegmentSize() + replica.getRemoteSegmentSize();
+ indexLength += replica.getLocalInvertedIndexSize() + replica.getRemoteInvertedIndexSize();
+ }
+ }
  }
- return indexSize;
+ long avgRowLength = rowCountForAvg > 0 ? dataSizeForAvg / rowCountForAvg : 0;
+ return new TableStatusStats(rowCount, dataLength, avgRowLength, indexLength);
  }
 
  // Get the signature string of this table with specified partitions.

fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java

@@ -46,6 +46,36 @@
 public interface TableIf {
  Logger LOG = LogManager.getLogger(TableIf.class);
 
+ class TableStatusStats {
+ private final long rows;
+ private final long dataLength;
+ private final long avgRowLength;
+ private final long indexLength;
+
+ public TableStatusStats(long rows, long dataLength, long avgRowLength, long indexLength) {
+ this.rows = rows;
+ this.dataLength = dataLength;
+ this.avgRowLength = avgRowLength;
+ this.indexLength = indexLength;
+ }
+
+ public long getRows() {
+ return rows;
+ }
+
+ public long getDataLength() {
+ return dataLength;
+ }
+
+ public long getAvgRowLength() {
+ return avgRowLength;
+ }
+
+ public long getIndexLength() {
+ return indexLength;
+ }
+ }
+
  long UNKNOWN_ROW_COUNT = -1;
 
  default void readLock() {
@@ -176,6 +206,10 @@ default int getBaseColumnIdxByName(String colName) {
 
  long getIndexLength();
 
+ default TableStatusStats getTableStatusStats() {
+ return new TableStatusStats(getCachedRowCount(), getDataLength(), getAvgRowLength(), getIndexLength());
+ }
+
  long getLastCheckTime();
 
  String getComment(boolean escapeQuota);

fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java

@@ -350,6 +350,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Random;
@@ -660,6 +661,11 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE
  TListTableStatusResult result = new TListTableStatusResult();
  List<TTableStatus> tablesResult = Lists.newArrayList();
  result.setTables(tablesResult);
+ Set<String> requiredColumns = params.isSetRequiredColumns()
+ ? params.getRequiredColumns().stream()
+ .map(column -> column.toUpperCase(Locale.ROOT))
+ .collect(Collectors.toSet())
+ : null;
  PatternMatcher matcher = null;
  String specifiedTable = null;
  if (params.isSetPattern()) {
@@ -739,17 +745,40 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE
  TTableStatus status = new TTableStatus();
  status.setName(table.getName());
  status.setType(table.getMysqlType());
- status.setEngine(table.getEngine());
  status.setComment(table.getComment());
- status.setCreateTime(table.getCreateTime());
- status.setLastCheckTime(lastCheckTime / 1000);
- status.setUpdateTime(table.getUpdateTime() / 1000);
- status.setCheckTime(lastCheckTime / 1000);
- status.setCollation("utf-8");
- status.setRows(table.getCachedRowCount());
- status.setDataLength(table.getDataLength());
- status.setAvgRowLength(table.getAvgRowLength());
- status.setIndexLength(table.getIndexLength());
+ if (needTableStatusColumn(requiredColumns, "ENGINE")) {
+ status.setEngine(table.getEngine());
+ }
+ if (needTableStatusColumn(requiredColumns, "CREATE_TIME")) {
+ status.setCreateTime(table.getCreateTime());
+ }
+ if (needTableStatusColumn(requiredColumns, "LAST_CHECK_TIME")
+ || needTableStatusColumn(requiredColumns, "CHECK_TIME")) {
+ status.setLastCheckTime(lastCheckTime / 1000);
+ }
+ if (needTableStatusColumn(requiredColumns, "UPDATE_TIME")) {
+ status.setUpdateTime(table.getUpdateTime() / 1000);
+ }
+ if (needTableStatusColumn(requiredColumns, "CHECK_TIME")) {
+ status.setCheckTime(lastCheckTime / 1000);
+ }
+ if (needTableStatusColumn(requiredColumns, "TABLE_COLLATION")) {
+ status.setCollation("utf-8");
+ }
+ TableIf.TableStatusStats tableStatusStats =
+ needTableStatusStats(requiredColumns) ? table.getTableStatusStats() : null;
+ if (needTableStatusColumn(requiredColumns, "TABLE_ROWS")) {
+ status.setRows(tableStatusStats.getRows());
+ }
+ if (needTableStatusColumn(requiredColumns, "DATA_LENGTH")) {
+ status.setDataLength(tableStatusStats.getDataLength());
+ }
+ if (needTableStatusColumn(requiredColumns, "AVG_ROW_LENGTH")) {
+ status.setAvgRowLength(tableStatusStats.getAvgRowLength());
+ }
+ if (needTableStatusColumn(requiredColumns, "INDEX_LENGTH")) {
+ status.setIndexLength(tableStatusStats.getIndexLength());
+ }
  if (table instanceof View) {
  status.setDdlSql(((View) table).getInlineViewDef());
  }
@@ -766,6 +795,17 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE
  return result;
  }
 
+ private static boolean needTableStatusColumn(Set<String> requiredColumns, String columnName) {
+ return requiredColumns == null || requiredColumns.contains(columnName);
+ }
+
+ private static boolean needTableStatusStats(Set<String> requiredColumns) {
+ return needTableStatusColumn(requiredColumns, "TABLE_ROWS")
+ || needTableStatusColumn(requiredColumns, "DATA_LENGTH")
+ || needTableStatusColumn(requiredColumns, "AVG_ROW_LENGTH")
+ || needTableStatusColumn(requiredColumns, "INDEX_LENGTH");
+ }
+
  public TListTableMetadataNameIdsResult listTableMetadataNameIds(TGetTablesParams params) throws TException {
  if (LOG.isDebugEnabled()) {
  LOG.debug("get list simple table request: {}", params);

fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java

@@ -52,6 +52,42 @@
 
 public class OlapTableTest {
 
+ @Test
+ public void testGetTableStatusStatsUsesSinglePassSemantics() {
+ OlapTable olapTable = new OlapTable();
+ MaterializedIndex index = new MaterializedIndex(10, MaterializedIndex.IndexState.NORMAL);
+ index.setRowCount(20);
+
+ List<Replica> replicas = Lists.newArrayList(
+ mockReplica(Replica.ReplicaState.NORMAL, 10, 100, 1, 20, 2),
+ mockReplica(Replica.ReplicaState.DECOMMISSION, 30, 300, 3, 40, 4),
+ mockReplica(Replica.ReplicaState.NORMAL, 50, 500, 5, 60, 6));
+ Tablet tablet = Mockito.mock(Tablet.class);
+ Mockito.when(tablet.getReplicas()).thenReturn(replicas);
+ index.appendTablets(Lists.newArrayList(tablet));
+
+ Partition partition = new Partition(11, "p1", index, null);
+ olapTable.addPartition(partition);
+
+ TableIf.TableStatusStats stats = olapTable.getTableStatusStats();
+ Assert.assertEquals(20L, stats.getRows());
+ Assert.assertEquals(909L, stats.getDataLength());
+ Assert.assertEquals(3L, stats.getAvgRowLength());
+ Assert.assertEquals(132L, stats.getIndexLength());
+ }
+
+ private Replica mockReplica(Replica.ReplicaState state, long dataSize, long localSegmentSize,
+ long remoteSegmentSize, long localIndexSize, long remoteIndexSize) {
+ Replica replica = Mockito.mock(Replica.class);
+ Mockito.when(replica.getState()).thenReturn(state);
+ Mockito.when(replica.getDataSize()).thenReturn(dataSize);
+ Mockito.when(replica.getLocalSegmentSize()).thenReturn(localSegmentSize);
+ Mockito.when(replica.getRemoteSegmentSize()).thenReturn(remoteSegmentSize);
+ Mockito.when(replica.getLocalInvertedIndexSize()).thenReturn(localIndexSize);
+ Mockito.when(replica.getRemoteInvertedIndexSize()).thenReturn(remoteIndexSize);
+ return replica;
+ }
+
  @Test
  public void test() throws IOException {
 

fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java

@@ -105,6 +105,23 @@ public void getMethodTest() {
  Assert.assertEquals("tabletId=" + newTabletId, tablet.toString());
  }
 
+ @Test
+ public void testGetReplicaStatsOnlyUsesNormalReplicas() {
+ Tablet statsTablet = new LocalTablet(2);
+ invertedIndex.addTablet(2, new TabletMeta(10, 20, 30, 40, 1, TStorageMedium.HDD));
+ statsTablet.addReplica(new LocalReplica(11L, 1L, 100L, 0, 10L, 0L, 100L, ReplicaState.NORMAL, 0L, 100L));
+ statsTablet.addReplica(new LocalReplica(12L, 2L, 100L, 0, 20L, 0L, 200L, ReplicaState.NORMAL, 0L, 100L));
+ statsTablet.addReplica(new LocalReplica(13L, 3L, 100L, 0, 0L, 0L, 300L, ReplicaState.NORMAL, 0L, 100L));
+ statsTablet.addReplica(new LocalReplica(14L, 4L, 100L, 0, 40L, 0L, 400L, ReplicaState.DECOMMISSION,
+ 0L, 100L));
+
+ Assert.assertEquals(30L, statsTablet.getDataSize(false, false));
+ Assert.assertEquals(10L, statsTablet.getDataSize(true, false));
+ Assert.assertEquals(30L, statsTablet.getDataSize(false, true));
+ Assert.assertEquals(600L, statsTablet.getRowCount(false));
+ Assert.assertEquals(200L, statsTablet.getRowCount(true));
+ }
+
  @Test
  public void deleteReplicaTest() {
  // delete replica1