Skip to content

PHOENIX-6220 CREATE INDEX shouldn't ignore IMMUTABLE_STORAGE_SCHEME a…#963

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gokceni wants to merge 1 commit into from
Closed

PHOENIX-6220 CREATE INDEX shouldn't ignore IMMUTABLE_STORAGE_SCHEME a… #963

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -622,6 +622,96 @@ public void testMultiTenantImmutableTableMetadata() throws Exception {
}
}

@Test
public void testCreateIndexWithDifferentStorageAndEncoding() throws Exception {
verifyIndexSchemeChange(false, false);
verifyIndexSchemeChange(false, true);
verifyIndexSchemeChange(true, false);
verifyIndexSchemeChange(true, true);

String tableName = generateUniqueName();
String indexName = generateUniqueName();
String createTableDDL = "create IMMUTABLE TABLE " + tableName + "(id char(1) NOT NULL, col1 char(1), col2 char(1) "
+ "CONSTRAINT NAME_PK PRIMARY KEY (id)) IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS";
String createIndexDDL = "create INDEX " + indexName + " ON " + tableName + " (col1) INCLUDE (col2) IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN";

Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute(createTableDDL);
assertColumnEncodingMetadata(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS,
ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, tableName, conn);

boolean failed = false;
try {
conn.createStatement().execute(createIndexDDL);
} catch (SQLException e) {
assertEquals(e.getErrorCode(), SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE.getErrorCode());
failed = true;
}
assertEquals(true, failed);
}
}

private void verifyIndexSchemeChange(boolean immutable, boolean multiTenant) throws Exception{
Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
String nonEncodedOneCellPerColumnTable = generateUniqueName();
String createTableDDL;
String createIndexDDL;

String createTableBaseDDL = "create " + (immutable? " IMMUTABLE ":"") + " TABLE %s ("
+ " id char(1) NOT NULL," + " col1 integer NOT NULL,"
+ " col2 bigint NOT NULL,"
+ " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)) MULTI_TENANT=" + (multiTenant? "true,":"false,");

String createIndexBaseDDL = "create index %s ON %s (col1) INCLUDE (col2) ";

try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
createTableDDL = String.format(createTableBaseDDL, nonEncodedOneCellPerColumnTable);
createTableDDL += "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0";
conn.createStatement().execute(createTableDDL);
assertColumnEncodingMetadata(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS,
ImmutableStorageScheme.ONE_CELL_PER_COLUMN,
nonEncodedOneCellPerColumnTable, conn);

String idxName = "IDX_" + generateUniqueName();
// Don't specify anything to see if it inherits from parent
createIndexDDL = String.format(createIndexBaseDDL, idxName, nonEncodedOneCellPerColumnTable);
conn.createStatement().execute(createIndexDDL);
assertColumnEncodingMetadata(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS,
ImmutableStorageScheme.ONE_CELL_PER_COLUMN,
idxName, conn);

idxName = "IDX_" + generateUniqueName();
createIndexDDL = String.format(createIndexBaseDDL, idxName, nonEncodedOneCellPerColumnTable);
createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS";
conn.createStatement().execute(createIndexDDL);
// Check if it sets the encoding to 2
assertColumnEncodingMetadata(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS,
ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS,
idxName, conn);

idxName = "IDX_" + generateUniqueName();
createIndexDDL = String.format(createIndexBaseDDL, idxName, nonEncodedOneCellPerColumnTable);
createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=3";
conn.createStatement().execute(createIndexDDL);
assertColumnEncodingMetadata(QualifierEncodingScheme.THREE_BYTE_QUALIFIERS,
ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS,
idxName, conn);

createIndexDDL = String.format(createIndexBaseDDL, idxName, nonEncodedOneCellPerColumnTable);
createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=0";
// should fail
boolean failed = false;
try {
conn.createStatement().execute(createIndexDDL);
} catch (SQLException e) {
failed = true;
assertEquals(SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES.getErrorCode(),e.getErrorCode());
}
assertEquals(true, failed);
}
}

private void verifyUCFValueInSysCat(String tableName, String createTableString,
Properties props, long expectedUCFInSysCat) throws SQLException {
String readSysCatQuery = "SELECT TABLE_NAME, UPDATE_CACHE_FREQUENCY FROM SYSTEM.CATALOG "
Expand Down
104 changes: 76 additions & 28 deletions phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2534,28 +2534,46 @@ else if (!SchemaUtil.isSystemTable(Bytes.toBytes(SchemaUtil.getTableName(schemaN
*
*/
if (parent != null) {
encodingScheme = parent.getEncodingScheme();
immutableStorageScheme = parent.getImmutableStorageScheme();
} else {
Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps);
if (encodingSchemeSerializedByte == null) {
// Ignore default if transactional and column encoding is not supported (as with OMID)
if (transactionProvider == null || !transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
encodingSchemeSerializedByte = (byte)connection.getQueryServices().getProps().getInt(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB,
QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES);
encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte);
}
Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps);
// Table has encoding scheme defined
if (encodingSchemeSerializedByte != null) {
encodingScheme = getEncodingScheme(tableProps, schemaName, tableName, transactionProvider);
} else {
encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte);
if (encodingScheme != NON_ENCODED_QUALIFIERS && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.UNSUPPORTED_COLUMN_ENCODING_FOR_TXN_PROVIDER)
.setSchemaName(schemaName).setTableName(tableName)
.setMessage(transactionProvider.name())
.build()
.buildException();
encodingScheme = parent.getEncodingScheme();
}

ImmutableStorageScheme immutableStorageSchemeProp = (ImmutableStorageScheme) TableProperty.IMMUTABLE_STORAGE_SCHEME.getValue(tableProps);
if (immutableStorageSchemeProp == null) {
immutableStorageScheme = parent.getImmutableStorageScheme();
} else {
immutableStorageScheme = getImmutableStorageSchemeForIndex(immutableStorageSchemeProp, schemaName, tableName, transactionProvider);
}

if (immutableStorageScheme == SINGLE_CELL_ARRAY_WITH_OFFSETS) {
if (encodingScheme == NON_ENCODED_QUALIFIERS) {
if (encodingSchemeSerializedByte != null) {
// encoding scheme is set as non-encoded on purpose, so we should fail
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES)
.setSchemaName(schemaName).setTableName(tableName).build().buildException();
} else {
// encoding scheme is inherited from parent but it is not compatible with Single Cell.
encodingScheme =
QualifierEncodingScheme.fromSerializedValue(
(byte) QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES);
}
}
}

if (parent.getImmutableStorageScheme() == SINGLE_CELL_ARRAY_WITH_OFFSETS && immutableStorageScheme == ONE_CELL_PER_COLUMN) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE)
.setSchemaName(schemaName).setTableName(tableName).build()
.buildException();
}
LOGGER.info(String.format("STORAGE--ENCODING: %s--%s", immutableStorageScheme, encodingScheme));
} else {
encodingScheme = getEncodingScheme(tableProps, schemaName, tableName, transactionProvider);

if (isImmutableRows) {
ImmutableStorageScheme immutableStorageSchemeProp =
(ImmutableStorageScheme) TableProperty.IMMUTABLE_STORAGE_SCHEME
Expand DownExpand Up@@ -2586,15 +2604,7 @@ else if (!SchemaUtil.isSystemTable(Bytes.toBytes(SchemaUtil.getTableName(schemaN
}
}
} else {
immutableStorageScheme = immutableStorageSchemeProp;
if (immutableStorageScheme != ONE_CELL_PER_COLUMN && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.UNSUPPORTED_STORAGE_FORMAT_FOR_TXN_PROVIDER)
.setSchemaName(schemaName).setTableName(tableName)
.setMessage(transactionProvider.name())
.build()
.buildException();
}
immutableStorageScheme = getImmutableStorageSchemeForIndex(immutableStorageSchemeProp, schemaName, tableName, transactionProvider);
}
if (immutableStorageScheme != ONE_CELL_PER_COLUMN
&& encodingScheme == NON_ENCODED_QUALIFIERS) {
Expand DownExpand Up@@ -3133,6 +3143,44 @@ public boolean isViewReferenced() {
}
}

private QualifierEncodingScheme getEncodingScheme(Map<String, Object> tableProps, String schemaName, String tableName, TransactionFactory.Provider transactionProvider)
throws SQLException {
QualifierEncodingScheme encodingScheme = null;
Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps);
if (encodingSchemeSerializedByte == null) {
// Ignore default if transactional and column encoding is not supported (as with OMID)
if (transactionProvider == null || !transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
encodingSchemeSerializedByte = (byte)connection.getQueryServices().getProps().getInt(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB,
QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES);
encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte);
} else {
encodingScheme = NON_ENCODED_QUALIFIERS;
}
} else {
encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte);
if (encodingScheme != NON_ENCODED_QUALIFIERS && transactionProvider != null && transactionProvider.getTransactionProvider()
.isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING)) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNSUPPORTED_COLUMN_ENCODING_FOR_TXN_PROVIDER)
.setSchemaName(schemaName).setTableName(tableName).setMessage(transactionProvider.name()).build().buildException();
}
}

return encodingScheme;
}

private ImmutableStorageScheme getImmutableStorageSchemeForIndex(ImmutableStorageScheme immutableStorageSchemeProp, String schemaName, String tableName, TransactionFactory.Provider transactionProvider)
throws SQLException {
if (immutableStorageSchemeProp != ONE_CELL_PER_COLUMN && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.UNSUPPORTED_STORAGE_FORMAT_FOR_TXN_PROVIDER)
.setSchemaName(schemaName).setTableName(tableName)
.setMessage(transactionProvider.name())
.build()
.buildException();
}
return immutableStorageSchemeProp;
}

/* This method handles mutation codes sent by phoenix server, except for TABLE_NOT_FOUND which
* is considered to be a success code. If TABLE_ALREADY_EXISTS in hbase, we don't need to add
* it in ConnectionQueryServices and we return result as true. However if code is
Expand Down