Skip to content

PHOENIX-6148: [SchemaExtractionTool] DDL parsing exception in Phoenix…#900

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
swaroopak merged 1 commit into from
Sep 30, 2020
Merged

PHOENIX-6148: [SchemaExtractionTool] DDL parsing exception in Phoenix… #900

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1211,4 +1211,13 @@ public static boolean isLogTable(String schemaName, String tableName) {
public static String getNormalizedColumnName(ColumnParseNode columnParseNode) {
return columnParseNode.getName();
}


public static String getPTableFullNameWithQuotes(String pSchemaName, String pTableName) {
String pTableFullName = getQualifiedTableName(pSchemaName, pTableName);
if(!(Character.isAlphabetic(pTableName.charAt(0)))) {
pTableFullName = pSchemaName+".\""+pTableName+"\"";
}
return pTableFullName;
}
}
32 changes: 20 additions & 12 deletions phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,6 +19,8 @@

import org.apache.phoenix.end2end.ParallelStatsEnabledIT;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.parse.ParseException;
import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
Expand All@@ -39,6 +41,7 @@
import java.util.Arrays;
import java.util.Properties;

import static junit.framework.TestCase.fail;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;

public class SchemaExtractionToolIT extends ParallelStatsEnabledIT {
Expand DownExpand Up@@ -202,7 +205,7 @@ public void testCreateTableWithArrayColumn() throws Exception {
"b_char CHAR(10) NOT NULL, " +
"c_var_array VARCHAR ARRAY, " +
"d_char_array CHAR(15) ARRAY[3] CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
"TTL=2592000, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1";
"TTL=2592000, IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN', REPLICATION_SCOPE=1";
List<String> queries = new ArrayList<String>(){};
queries.add(query);
String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
Expand All@@ -213,7 +216,8 @@ public void testCreateTableWithArrayColumn() throws Exception {
public void testCreateTableWithDefaultCFProperties() throws Exception {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
String properties = "KEEP_DELETED_CELLS=TRUE, TTL=1209600, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1, DEFAULT_COLUMN_FAMILY=cv, SALT_BUCKETS=16, MULTI_TENANT=true";
String properties = "KEEP_DELETED_CELLS=TRUE, TTL=1209600, IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN', "
+ "REPLICATION_SCOPE=1, DEFAULT_COLUMN_FAMILY='cv', SALT_BUCKETS=16, MULTI_TENANT=true, TIME_TEST='72HOURS'";
String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
String query = "create table " + pTableFullName +
"(a_char CHAR(15) NOT NULL, " +
Expand All@@ -233,8 +237,8 @@ public void testCreateTableWithCFProperties() throws Exception {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
String properties = "\"av\".VERSIONS=2, \"bv\".VERSIONS=2, " +
"DATA_BLOCK_ENCODING=DIFF, " +
"IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, MULTI_TENANT=true";
"DATA_BLOCK_ENCODING='DIFF', " +
"IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN', SALT_BUCKETS=16, MULTI_TENANT=true";
String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
String query = "create table " + pTableFullName +
"(a_char CHAR(15) NOT NULL, " +
Expand All@@ -253,8 +257,8 @@ public void testCreateTableWithMultipleCF() throws Exception {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
String properties = "\"av\".VERSIONS=2, \"bv\".VERSIONS=3, " +
"\"cv\".VERSIONS=4, DATA_BLOCK_ENCODING=DIFF, " +
"IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, MULTI_TENANT=true";
"\"cv\".VERSIONS=4, DATA_BLOCK_ENCODING='DIFF', " +
"IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN', SALT_BUCKETS=16, MULTI_TENANT=true";
String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
final String query = "create table " + pTableFullName +
"(a_char CHAR(15) NOT NULL, " +
Expand All@@ -271,13 +275,12 @@ public void testCreateTableWithMultipleCF() throws Exception {

@Test
public void testCreateTableWithMultipleCFProperties() throws Exception {
String tableName = generateUniqueName();
String tableName = "07"+generateUniqueName();
String schemaName = generateUniqueName();
String properties = "\"av\".DATA_BLOCK_ENCODING=DIFF, \"bv\".DATA_BLOCK_ENCODING=DIFF, \"cv\".DATA_BLOCK_ENCODING=DIFF, " +
"IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, MULTI_TENANT=true";
String simplifiedProperties = "DATA_BLOCK_ENCODING=DIFF, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, MULTI_TENANT=true";
String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
String query = "create table " + pTableFullName +
String properties = "\"av\".DATA_BLOCK_ENCODING='DIFF', \"bv\".DATA_BLOCK_ENCODING='DIFF', \"cv\".DATA_BLOCK_ENCODING='DIFF', " +
"IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN', SALT_BUCKETS=16, MULTI_TENANT=true, BLOOMFITER='ROW'";
String simplifiedProperties = "DATA_BLOCK_ENCODING='DIFF', IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN', SALT_BUCKETS=16, MULTI_TENANT=true, BLOOMFITER='ROW'";
String query = "create table " + schemaName+".\""+tableName+"\"" +
"(a_char CHAR(15) NOT NULL, " +
"b_char CHAR(10) NOT NULL, " +
"\"av\".\"_\" CHAR(1), " +
Expand All@@ -286,6 +289,11 @@ public void testCreateTableWithMultipleCFProperties() throws Exception {
List<String> queries = new ArrayList<String>(){};
queries.add(query);
String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
try {
new SQLParser(result).parseStatement();
} catch (ParseException pe) {
fail("This should not happen!");
}
Assert.assertTrue(compareProperties(simplifiedProperties, getProperties(result)));
}

Expand Down
17 changes: 13 additions & 4 deletions phoenix-tools/src/main/java/org/apache/phoenix/schema/SchemaExtractionProcessor.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.schema;

import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
Expand DownExpand Up@@ -201,7 +202,7 @@ protected String extractCreateViewDDL(PTable table) throws SQLException {

private String generateCreateViewDDL(String columnInfoString, String baseTableFullName,
String whereClause, String pSchemaName, String pTableName) {
String viewFullName = SchemaUtil.getQualifiedTableName(pSchemaName, pTableName);
String viewFullName = SchemaUtil.getPTableFullNameWithQuotes(pSchemaName, pTableName);
StringBuilder outputBuilder = new StringBuilder(String.format(CREATE_VIEW, viewFullName,
columnInfoString, baseTableFullName, whereClause));
return outputBuilder.toString();
Expand All@@ -224,9 +225,10 @@ public String extractCreateTableDDL(PTable table) throws IOException, SQLExcepti

return generateTableDDLString(columnInfoString, propertiesString, pSchemaName, pTableName);
}

private String generateTableDDLString(String columnInfoString, String propertiesString,
String pSchemaName, String pTableName) {
String pTableFullName = SchemaUtil.getQualifiedTableName(pSchemaName, pTableName);
String pTableFullName = SchemaUtil.getPTableFullNameWithQuotes(pSchemaName, pTableName);
StringBuilder outputBuilder = new StringBuilder(String.format(CREATE_TABLE, pTableFullName));
outputBuilder.append(columnInfoString).append(" ").append(propertiesString);
return outputBuilder.toString();
Expand DownExpand Up@@ -325,8 +327,15 @@ private String convertPropertiesToString() {
if (optionBuilder.length() != 0) {
optionBuilder.append(", ");
}
key = columnFamilyName.equals(QueryConstants.DEFAULT_COLUMN_FAMILY)? key : String.format("\"%s\".%s", columnFamilyName, key);
optionBuilder.append(key+"="+value);
key = columnFamilyName.equals(QueryConstants.DEFAULT_COLUMN_FAMILY)?
key : String.format("\"%s\".%s", columnFamilyName, key);
// properties value that corresponds to a number will not need single quotes around it
// properties value that corresponds to a boolean value will not need single quotes around it
if(!(NumberUtils.isNumber(value)) &&
!(value.equalsIgnoreCase(Boolean.TRUE.toString()) ||value.equalsIgnoreCase(Boolean.FALSE.toString()))) {
value= "'" + value + "'";
}
optionBuilder.append(key + "=" + value);
}
}
return optionBuilder.toString();
Expand Down