Fixed multiple bugs related to schema

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java

@@ -101,6 +101,10 @@ public String getName() {
  @Override
  public void setName(final String name) {
  tableNodeInfo.setName(name);
+
+ // Full path ends with name, so it needs to be updated when name changes
+ // Clear cache and compute again when fullPath is next accessed
+ this.fullPath = null;
  }
 
  @Override

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DeleteDevicesProcedure.java

@@ -204,7 +204,6 @@ protected void onAllReplicasetFailure(
  };
  deleteDevicesExecutor.execute();
 
- setNextState(CONSTRUCT_BLACK_LIST);
  deletedDevicesNum =
  !isFailed()
  ? deleteDevicesExecutor.getSuccessResult().stream()

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java

@@ -88,7 +88,7 @@
  private ClusterSchemaFetcher() {}
 
  @Override
  public ClusterSchemaTree fetchSchema(
  PathPatternTree patternTree,
  boolean withTemplate,
  MPPQueryContext context,
@@ -329,9 +329,11 @@
  if (!config.isAutoCreateSchemaEnabled()) {
  // disable auto-create for non-system series
  indexOfDevicesWithMissingMeasurements.removeIf(
- i -> !devicePathList.get(i).startsWith("root." + SystemConstant.SYSTEM_PREFIX_KEY));
- indexOfDevicesWithMissingMeasurements.removeIf(
- i -> !devicePathList.get(i).startsWith("root." + SystemConstant.AUDIT_PREFIX_KEY));
+ i ->
+ !devicePathList.get(i).startsWith("root." + SystemConstant.SYSTEM_PREFIX_KEY)
+ && !devicePathList
+ .get(i)
+ .startsWith("root." + SystemConstant.AUDIT_PREFIX_KEY));
  if (indexOfDevicesWithMissingMeasurements.isEmpty()) {
  return schemaTree;
  }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/NormalSchemaFetcher.java

@@ -196,7 +196,7 @@
  return indexOfMissingMeasurements;
  }
 
  void processNormalTimeSeries(
  List<? extends ISchemaComputationWithAutoCreation> schemaComputationWithAutoCreationList,
  MPPQueryContext context) {
  // [Step 0] Record the input value.
@@ -330,15 +330,13 @@
  indexOfDevicesNeedAutoCreateSchema.removeIf(
  i ->
  !schemaComputationWithAutoCreationList
- .get(i)
- .getDevicePath()
- .startsWith("root." + SystemConstant.SYSTEM_PREFIX_KEY));
- indexOfDevicesNeedAutoCreateSchema.removeIf(
- i ->
- !schemaComputationWithAutoCreationList
- .get(i)
- .getDevicePath()
- .startsWith("root." + SystemConstant.AUDIT_PREFIX_KEY));
+ .get(i)
+ .getDevicePath()
+ .startsWith("root." + SystemConstant.SYSTEM_PREFIX_KEY)
+ && !schemaComputationWithAutoCreationList
+ .get(i)
+ .getDevicePath()
+ .startsWith("root." + SystemConstant.AUDIT_PREFIX_KEY));
  }
 
  // [Step 5] Auto Create and process the missing schema

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java

@@ -38,7 +38,7 @@
 import org.apache.iotdb.db.queryengine.plan.relational.metadata.AlignedDeviceEntry;
 import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry;
 import org.apache.iotdb.db.queryengine.plan.relational.metadata.NonAlignedDeviceEntry;
-import org.apache.iotdb.db.queryengine.plan.relational.metadata.cache.DeviceSchemaRequestCache;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.DeviceSchemaRequestCache;
 import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.IDeviceSchema;
 import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache;
 import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceNormalSchema;
@@ -169,7 +169,7 @@
  for (int i = 0; i < tsBlock.get().getPositionCount(); i++) {
  final String[] nodes = new String[tagLength + 1];
  final Map<String, Binary> attributeMap = new HashMap<>();
-constructNodsArrayAndAttributeMap(
+constructNodesArrayAndAttributeMap(
  attributeMap, nodes, table, columnHeaderList, columns, tableInstance, i);
 
  fetchedDeviceSchema.put(IDeviceID.Factory.DEFAULT_FACTORY.create(nodes), attributeMap);
@@ -234,7 +234,7 @@
 
  // Used by show/count device and update device.
  // Update / Delete device will not access cache
  public boolean parseFilter4TraverseDevice(
  final TsTable tableInstance,
  final List<Expression> expressionList,
  final AbstractTraverseDevice statement,
@@ -566,7 +566,7 @@
  for (int i = 0; i < tsBlock.getPositionCount(); i++) {
  final String[] nodes = new String[tableInstance.getTagNum() + 1];
  final Map<String, Binary> attributeMap = new HashMap<>();
-constructNodsArrayAndAttributeMap(
+constructNodesArrayAndAttributeMap(
  attributeMap,
  nodes,
  tableInstance.getTableName(),
@@ -597,7 +597,7 @@
  final Column[] columns = tsBlock.getValueColumns();
  for (int i = 0; i < tsBlock.getPositionCount(); i++) {
  final String[] nodes = new String[tableInstance.getTagNum()];
-constructNodsArrayAndAttributeMap(
+constructNodesArrayAndAttributeMap(
  Collections.emptyMap(), nodes, null, columnHeaderList, columns, tableInstance, i);
  final IDeviceID deviceID =
  DataNodeTreeViewSchemaUtils.convertToIDeviceID(tableInstance, nodes);
@@ -608,13 +608,13 @@
  mppQueryContext.reserveMemoryForFrontEnd(deviceEntry.ramBytesUsed());
  deviceEntryMap
  .computeIfAbsent(
- columns[columns.length - 1].getBinary(0).getStringValue(TSFileConfig.STRING_CHARSET),
+ columns[columns.length - 1].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET),
  k -> new ArrayList<>())
  .add(deviceEntry);
  }
  }
 
- private void constructNodsArrayAndAttributeMap(
+ private void constructNodesArrayAndAttributeMap(
  final Map<String, Binary> attributeMap,
  final String[] nodes,
  final String tableName,

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/cache/DeviceSchemaRequestCache.java → iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/DeviceSchemaRequestCache.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iotdb.db.queryengine.plan.relational.metadata.cache;
+package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache;
 
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice;
@@ -29,7 +29,7 @@
 
 import java.util.Map;
 
 public class DeviceSchemaRequestCache {
  private static final DeviceSchemaRequestCache INSTANCE = new DeviceSchemaRequestCache();
 
  private final Cache<FetchDevice, FetchMissingDeviceSchema> pendingRequests =
@@ -53,7 +53,7 @@
  }
 
  public static class FetchMissingDeviceSchema {
  private volatile Map<IDeviceID, Map<String, Binary>> result;
  private volatile boolean done = false;
  private volatile long queryId = -1;
 
@@ -69,7 +69,7 @@
  if (this.queryId != -1) {
  if (!done) {
  try {
  this.wait(
  IoTDBDescriptor.getInstance().getConfig().getDeviceSchemaRequestCacheWaitTimeMs());
  } catch (InterruptedException e) {
  Thread.currentThread().interrupt();

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java

@@ -37,6 +37,7 @@
 import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName;
 import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegion;
 import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
+import org.apache.iotdb.rpc.subscription.annotation.TableModel;
 
 import org.apache.tsfile.enums.TSDataType;
 import org.apache.tsfile.file.metadata.IDeviceID;
@@ -667,6 +668,7 @@ public void invalidate(
  }
  }
 
+ @TableModel
  public void invalidate(
  final String database,
  final String tableName,
@@ -680,7 +682,7 @@ public void invalidate(
  isAttributeColumn
  ? entry -> -entry.invalidateAttributeColumn(columnName)
  : entry -> -entry.invalidateLastCache(columnName);
- dualKeyCache.update(new TableId(null, tableName), deviceID -> true, updateFunction);
+ dualKeyCache.update(new TableId(database, tableName), deviceID -> true, updateFunction);
  } finally {
  readWriteLock.writeLock().unlock();
  }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java

@@ -158,6 +158,8 @@ public static Expression deserialize(ByteBuffer byteBuffer) {
  case 28:
  expression = new WhenClause(byteBuffer);
  break;
+ // 29: CurrentDatabase, 30: CurrentUser
+ // Do not deserialize here and throw exception because it's unexpected
  case 31:
  expression = new Row(byteBuffer);
  break;

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RelationalAuthorStatement.java

@@ -380,7 +380,7 @@ public TSStatus checkStatementIsValid(String currentUser) {
  }
  break;
  case DROP_ROLE:
- if (AuthorityChecker.SUPER_USER.equals(userName)) {
+ if (AuthorityChecker.SUPER_USER.equals(roleName)) {
  return AuthorityChecker.getTSStatus(false, "Cannot drop role with admin name");
  }
  break;

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java

@@ -214,10 +214,12 @@ public void clear() {
  memoryUsage.getAndSet(0);
  measurementNumber.getAndSet(0);
  devicesNumber.getAndSet(0);
- viewNumber.getAndAdd(0);
+ viewNumber.getAndSet(0);
  templateUsage.forEach(
  (templateId, cnt) -> schemaEngineStatistics.deactivateTemplate(templateId, cnt));
  templateUsage.clear();
+ tableDeviceNumber.clear();
+ tableAttributeMemory.clear();
  }
 
  @Override

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java

@@ -39,7 +39,7 @@ public enum SchemaRegionPlanType {
  ACTIVATE_TEMPLATE_IN_CLUSTER((byte) 63),
  PRE_DELETE_TIMESERIES_IN_CLUSTER((byte) 64),
  ROLLBACK_PRE_DELETE_TIMESERIES((byte) 65),
- ALTER_ENCODING_COMPRESSOR((byte) 66),
+ ALTER_ENCODING_COMPRESSOR((byte) 80),
 
  // endregion
 

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java

@@ -753,7 +753,7 @@
  * @param plan CreateAlignedTimeSeriesPlan
  */
  @Override
  public void createAlignedTimeSeries(final ICreateAlignedTimeSeriesPlan plan)
  throws MetadataException {
  if (!regionStatistics.isAllowToCreateNewSeries()) {
  throw new SeriesOverflowException(
@@ -1002,7 +1002,7 @@
  writeToMLog(plan);
  }
  // update statistics
- regionStatistics.addView(1L);
+ regionStatistics.addView(pathList.size());
  }
 
  @Override

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java

@@ -107,12 +107,15 @@ public boolean hasNext() {
  if (skipPreDeletedSchema
  && nextMatchedNode.isMeasurement()
  && nextMatchedNode.getAsMeasurementMNode().isPreDeleted()) {
+ releaseSkippedNode(nextMatchedNode);
  nextMatchedNode = null;
  }
  }
  return true;
  }
 
+ protected abstract void releaseSkippedNode(final N node);
+
  @Override
  public N next() {
  if (!hasNext()) {

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MemoryTraverserIterator.java

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator;
 
 import org.apache.iotdb.commons.exception.MetadataException;
@@ -30,11 +31,16 @@
 // only use for IConfigMNode and IMemMNode
 public class MemoryTraverserIterator<N extends IMNode<N>> extends AbstractTraverserIterator<N> {
  public MemoryTraverserIterator(
- IMTreeStore<N> store,
- IDeviceMNode<N> parent,
- Map<Integer, Template> templateMap,
- IMNodeFactory<N> nodeFactory)
+final IMTreeStore<N> store,
+final IDeviceMNode<N> parent,
+final Map<Integer, Template> templateMap,
+final IMNodeFactory<N> nodeFactory)
  throws MetadataException {
  super(store, parent, templateMap, nodeFactory);
  }
+
+ @Override
+ protected void releaseSkippedNode(final N node) {
+ // Do nothing
+ }
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/iterator/CachedTraverserIterator.java

@@ -48,4 +48,11 @@ public void close() {
  }
  super.close();
  }
+
+ @Override
+ protected void releaseSkippedNode(final ICachedMNode node) {
+ if (usingDirectChildrenIterator) {
+ store.unPin(node);
+ }
+ }
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/WrappedSegment.java

@@ -105,17 +105,16 @@ public WrappedSegment(ByteBuffer buffer, boolean override) {
  }
  }
 
- public WrappedSegment(ByteBuffer buffer) throws RecordDuplicatedException {
+ public WrappedSegment(ByteBuffer buffer) {
  this(buffer, true);
  }
 
  @TestOnly
- public WrappedSegment(int size) throws RecordDuplicatedException {
+ public WrappedSegment(int size) {
  this(ByteBuffer.allocate(size));
  }
 
- public static ISegment<ByteBuffer, ICachedMNode> initAsSegment(ByteBuffer buffer)
- throws RecordDuplicatedException {
+ public static ISegment<ByteBuffer, ICachedMNode> initAsSegment(ByteBuffer buffer) {
  if (buffer == null) {
  return null;
  }
@@ -559,7 +558,6 @@ public Queue<ICachedMNode> getAllRecords() throws MetadataException {
 
  @Override
  public int updateRecord(String key, ByteBuffer uBuffer) throws MetadataException {
-
  int idx = binarySearchOnKeys(key);
  if (idx < 0) {
  throw new MetadataException(String.format("Record[key:%s] Not Existed.", key));
@@ -614,8 +612,8 @@ public int removeRecord(String key) {
  }
 
  // shift offsets forward
- if (idx != recordNum) {
- int shift = recordNum - idx;
+ if (idx != recordNum - 1) {
+ int shift = recordNum - idx - 1;
  this.buffer.position(SchemaFileConfig.SEG_HEADER_SIZE + idx * 2);
  ShortBuffer lb = this.buffer.asReadOnlyBuffer().asShortBuffer();
  lb.get();

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java

@@ -29,6 +29,7 @@
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion;
+import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType;
 import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ISchemaInfo;
 import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo;
 import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory;
@@ -41,6 +42,7 @@
 import org.junit.Test;
 
 import java.io.File;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -230,4 +232,14 @@ public void testSnapshotPerformance() throws Exception {
  config.setSchemaRegionConsensusProtocolClass(schemaRegionConsensusProtocolClass);
  }
  }
+
+ @Test
+ public void testSchemaRegionPlanType() throws Exception {
+ Assert.assertEquals(
+ SchemaRegionPlanType.values().length,
+ Arrays.stream(SchemaRegionPlanType.values())
+ .map(SchemaRegionPlanType::getPlanType)
+ .distinct()
+ .count());
+ }
 }

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java

@@ -119,6 +119,9 @@ private static void internalExtract(
  final List<SchemaFilter> result,
  final SchemaFilter schemaFilter,
  final SchemaFilterType filterType) {
+ if (schemaFilter == null) {
+ return;
+ }
  if (schemaFilter.getSchemaFilterType().equals(filterType)) {
  result.add(schemaFilter);
  }

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java

@@ -339,6 +339,9 @@ public void deserialize(InputStream bufferedInputStream)
  public void clear() {
  ttlCacheTree.removeAllChildren();
  ttlCacheTree.addChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD, Long.MAX_VALUE);
+
+ // root.**
+ ttlCount = 1;
  }
 
  static class CacheNode {