From 79390c5a9d529fb2c7b11e3f7cc6bb13fc41c268 Mon Sep 17 00:00:00 2001 From: Gokcen Iskender Date: Wed, 11 Nov 2020 16:37:39 -0800 Subject: [PATCH] PHOENIX-6220 CREATE INDEX shouldn't ignore IMMUTABLE_STORAGE_SCHEME and COLUMN_ENDCODED_BYTES --- .../apache/phoenix/end2end/CreateTableIT.java | 90 +++++++++++++++ .../apache/phoenix/schema/MetaDataClient.java | 104 +++++++++++++----- 2 files changed, 166 insertions(+), 28 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java index 4f6ccab976d..00996681b91 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java @@ -622,6 +622,96 @@ public void testMultiTenantImmutableTableMetadata() throws Exception { } } + @Test + public void testCreateIndexWithDifferentStorageAndEncoding() throws Exception { + verifyIndexSchemeChange(false, false); + verifyIndexSchemeChange(false, true); + verifyIndexSchemeChange(true, false); + verifyIndexSchemeChange(true, true); + + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + String createTableDDL = "create IMMUTABLE TABLE " + tableName + "(id char(1) NOT NULL, col1 char(1), col2 char(1) " + + "CONSTRAINT NAME_PK PRIMARY KEY (id)) IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS"; + String createIndexDDL = "create INDEX " + indexName + " ON " + tableName + " (col1) INCLUDE (col2) IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN"; + + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute(createTableDDL); + assertColumnEncodingMetadata(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS, + ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, tableName, conn); + + boolean failed = false; + try { + conn.createStatement().execute(createIndexDDL); + } catch (SQLException e) { + assertEquals(e.getErrorCode(), SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE.getErrorCode()); + failed = true; + } + assertEquals(true, failed); + } + } + + private void verifyIndexSchemeChange(boolean immutable, boolean multiTenant) throws Exception{ + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + String nonEncodedOneCellPerColumnTable = generateUniqueName(); + String createTableDDL; + String createIndexDDL; + + String createTableBaseDDL = "create " + (immutable? " IMMUTABLE ":"") + " TABLE %s (" + + " id char(1) NOT NULL," + " col1 integer NOT NULL," + + " col2 bigint NOT NULL," + + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)) MULTI_TENANT=" + (multiTenant? "true,":"false,"); + + String createIndexBaseDDL = "create index %s ON %s (col1) INCLUDE (col2) "; + + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + createTableDDL = String.format(createTableBaseDDL, nonEncodedOneCellPerColumnTable); + createTableDDL += "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0"; + conn.createStatement().execute(createTableDDL); + assertColumnEncodingMetadata(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, + ImmutableStorageScheme.ONE_CELL_PER_COLUMN, + nonEncodedOneCellPerColumnTable, conn); + + String idxName = "IDX_" + generateUniqueName(); + // Don't specify anything to see if it inherits from parent + createIndexDDL = String.format(createIndexBaseDDL, idxName, nonEncodedOneCellPerColumnTable); + conn.createStatement().execute(createIndexDDL); + assertColumnEncodingMetadata(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, + ImmutableStorageScheme.ONE_CELL_PER_COLUMN, + idxName, conn); + + idxName = "IDX_" + generateUniqueName(); + createIndexDDL = String.format(createIndexBaseDDL, idxName, nonEncodedOneCellPerColumnTable); + createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS"; + conn.createStatement().execute(createIndexDDL); + // Check if it sets the encoding to 2 + assertColumnEncodingMetadata(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS, + ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, + idxName, conn); + + idxName = "IDX_" + generateUniqueName(); + createIndexDDL = String.format(createIndexBaseDDL, idxName, nonEncodedOneCellPerColumnTable); + createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=3"; + conn.createStatement().execute(createIndexDDL); + assertColumnEncodingMetadata(QualifierEncodingScheme.THREE_BYTE_QUALIFIERS, + ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, + idxName, conn); + + createIndexDDL = String.format(createIndexBaseDDL, idxName, nonEncodedOneCellPerColumnTable); + createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=0"; + // should fail + boolean failed = false; + try { + conn.createStatement().execute(createIndexDDL); + } catch (SQLException e) { + failed = true; + assertEquals(SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES.getErrorCode(),e.getErrorCode()); + } + assertEquals(true, failed); + } + } + private void verifyUCFValueInSysCat(String tableName, String createTableString, Properties props, long expectedUCFInSysCat) throws SQLException { String readSysCatQuery = "SELECT TABLE_NAME, UPDATE_CACHE_FREQUENCY FROM SYSTEM.CATALOG " diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 0a45981240e..4b50f288a6c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -2534,28 +2534,46 @@ else if (!SchemaUtil.isSystemTable(Bytes.toBytes(SchemaUtil.getTableName(schemaN * */ if (parent != null) { - encodingScheme = parent.getEncodingScheme(); - immutableStorageScheme = parent.getImmutableStorageScheme(); - } else { - Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps); - if (encodingSchemeSerializedByte == null) { - // Ignore default if transactional and column encoding is not supported (as with OMID) - if (transactionProvider == null || !transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { - encodingSchemeSerializedByte = (byte)connection.getQueryServices().getProps().getInt(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB, - QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES); - encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte); - } + Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps); + // Table has encoding scheme defined + if (encodingSchemeSerializedByte != null) { + encodingScheme = getEncodingScheme(tableProps, schemaName, tableName, transactionProvider); } else { - encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte); - if (encodingScheme != NON_ENCODED_QUALIFIERS && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { - throw new SQLExceptionInfo.Builder( - SQLExceptionCode.UNSUPPORTED_COLUMN_ENCODING_FOR_TXN_PROVIDER) - .setSchemaName(schemaName).setTableName(tableName) - .setMessage(transactionProvider.name()) - .build() - .buildException(); + encodingScheme = parent.getEncodingScheme(); + } + + ImmutableStorageScheme immutableStorageSchemeProp = (ImmutableStorageScheme) TableProperty.IMMUTABLE_STORAGE_SCHEME.getValue(tableProps); + if (immutableStorageSchemeProp == null) { + immutableStorageScheme = parent.getImmutableStorageScheme(); + } else { + immutableStorageScheme = getImmutableStorageSchemeForIndex(immutableStorageSchemeProp, schemaName, tableName, transactionProvider); + } + + if (immutableStorageScheme == SINGLE_CELL_ARRAY_WITH_OFFSETS) { + if (encodingScheme == NON_ENCODED_QUALIFIERS) { + if (encodingSchemeSerializedByte != null) { + // encoding scheme is set as non-encoded on purpose, so we should fail + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES) + .setSchemaName(schemaName).setTableName(tableName).build().buildException(); + } else { + // encoding scheme is inherited from parent but it is not compatible with Single Cell. + encodingScheme = + QualifierEncodingScheme.fromSerializedValue( + (byte) QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES); + } } } + + if (parent.getImmutableStorageScheme() == SINGLE_CELL_ARRAY_WITH_OFFSETS && immutableStorageScheme == ONE_CELL_PER_COLUMN) { + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE) + .setSchemaName(schemaName).setTableName(tableName).build() + .buildException(); + } + LOGGER.info(String.format("STORAGE--ENCODING: %s--%s", immutableStorageScheme, encodingScheme)); + } else { + encodingScheme = getEncodingScheme(tableProps, schemaName, tableName, transactionProvider); + if (isImmutableRows) { ImmutableStorageScheme immutableStorageSchemeProp = (ImmutableStorageScheme) TableProperty.IMMUTABLE_STORAGE_SCHEME @@ -2586,15 +2604,7 @@ else if (!SchemaUtil.isSystemTable(Bytes.toBytes(SchemaUtil.getTableName(schemaN } } } else { - immutableStorageScheme = immutableStorageSchemeProp; - if (immutableStorageScheme != ONE_CELL_PER_COLUMN && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { - throw new SQLExceptionInfo.Builder( - SQLExceptionCode.UNSUPPORTED_STORAGE_FORMAT_FOR_TXN_PROVIDER) - .setSchemaName(schemaName).setTableName(tableName) - .setMessage(transactionProvider.name()) - .build() - .buildException(); - } + immutableStorageScheme = getImmutableStorageSchemeForIndex(immutableStorageSchemeProp, schemaName, tableName, transactionProvider); } if (immutableStorageScheme != ONE_CELL_PER_COLUMN && encodingScheme == NON_ENCODED_QUALIFIERS) { @@ -3133,6 +3143,44 @@ public boolean isViewReferenced() { } } + private QualifierEncodingScheme getEncodingScheme(Map tableProps, String schemaName, String tableName, TransactionFactory.Provider transactionProvider) + throws SQLException { + QualifierEncodingScheme encodingScheme = null; + Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps); + if (encodingSchemeSerializedByte == null) { + // Ignore default if transactional and column encoding is not supported (as with OMID) + if (transactionProvider == null || !transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { + encodingSchemeSerializedByte = (byte)connection.getQueryServices().getProps().getInt(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB, + QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES); + encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte); + } else { + encodingScheme = NON_ENCODED_QUALIFIERS; + } + } else { + encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte); + if (encodingScheme != NON_ENCODED_QUALIFIERS && transactionProvider != null && transactionProvider.getTransactionProvider() + .isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING)) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNSUPPORTED_COLUMN_ENCODING_FOR_TXN_PROVIDER) + .setSchemaName(schemaName).setTableName(tableName).setMessage(transactionProvider.name()).build().buildException(); + } + } + + return encodingScheme; + } + + private ImmutableStorageScheme getImmutableStorageSchemeForIndex(ImmutableStorageScheme immutableStorageSchemeProp, String schemaName, String tableName, TransactionFactory.Provider transactionProvider) + throws SQLException { + if (immutableStorageSchemeProp != ONE_CELL_PER_COLUMN && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.UNSUPPORTED_STORAGE_FORMAT_FOR_TXN_PROVIDER) + .setSchemaName(schemaName).setTableName(tableName) + .setMessage(transactionProvider.name()) + .build() + .buildException(); + } + return immutableStorageSchemeProp; + } + /* This method handles mutation codes sent by phoenix server, except for TABLE_NOT_FOUND which * is considered to be a success code. If TABLE_ALREADY_EXISTS in hbase, we don't need to add * it in ConnectionQueryServices and we return result as true. However if code is