diff --git a/bin/performance.py b/bin/performance.py index 2f7fad1002f..c16dd5a31ae 100755 --- a/bin/performance.py +++ b/bin/performance.py @@ -66,10 +66,40 @@ def createFileWithContent(filename, content): # HBase/Phoenix client side property override hbase_config_path = os.getenv('HBASE_CONF_DIR', phoenix_utils.current_dir) -execute = ('java -cp "%s%s%s" -Dlog4j.configuration=file:' + +java_home = os.getenv('JAVA_HOME') + +# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR +hbase_env_path = None +hbase_env_cmd = None +if os.name == 'posix': + hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh') + hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path] +elif os.name == 'nt': + hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd') + hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path] +if not hbase_env_path or not hbase_env_cmd: + print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name + sys.exit(-1) + +hbase_env = {} +if os.path.isfile(hbase_env_path): + p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE) + for x in p.stdout: + (k, _, v) = x.partition('=') + hbase_env[k.strip()] = v.strip() + +if hbase_env.has_key('JAVA_HOME'): + java_home = hbase_env['JAVA_HOME'] + +if java_home: + java_cmd = os.path.join(java_home, 'bin', 'java') +else: + java_cmd = 'java' + +execute = ('%s $PHOENIX_OPTS -cp "%s%s%s" -Dlog4j.configuration=file:' + os.path.join(phoenix_utils.current_dir, "log4j.properties") + ' org.apache.phoenix.util.PhoenixRuntime -t %s %s ') % \ - (hbase_config_path, os.pathsep, phoenix_utils.phoenix_client_jar, table, zookeeper) + (java_cmd, hbase_config_path, os.pathsep, phoenix_utils.phoenix_client_jar, table, zookeeper) # Create Table DDL createtable = "CREATE TABLE IF NOT EXISTS %s (HOST CHAR(2) NOT NULL,\ @@ -98,7 +128,7 @@ def createFileWithContent(filename, content): queryex("5 - Filter + Count", "SELECT COUNT(1) FROM %s WHERE CORE<10;" % (table)) print "\nGenerating and upserting data..." -exitcode = subprocess.call('java -jar %s %s %s' % (phoenix_utils.testjar, data, rowcount), +exitcode = subprocess.call('%s -jar %s %s %s' % (java_cmd, phoenix_utils.testjar, data, rowcount), shell=True) if exitcode != 0: sys.exit(exitcode) diff --git a/bin/phoenix_sandbox.py b/bin/phoenix_sandbox.py index 433bc988b08..4279dd60b11 100755 --- a/bin/phoenix_sandbox.py +++ b/bin/phoenix_sandbox.py @@ -43,7 +43,7 @@ with open(cp_file_path, 'rb') as cp_file: cp_components.append(cp_file.read()) -java_cmd = ("java -Dlog4j.configuration=file:%s " + +java_cmd = ("java $PHOENIX_OPTS -Dlog4j.configuration=file:%s " + "-cp %s org.apache.phoenix.Sandbox") % ( logging_config, ":".join(cp_components)) diff --git a/bin/psql.py b/bin/psql.py index d4269d11ad0..973d3de4b06 100755 --- a/bin/psql.py +++ b/bin/psql.py @@ -62,7 +62,8 @@ else: java = 'java' -java_cmd = java +' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + \ +java_cmd = java + ' $PHOENIX_OPTS ' + \ + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + \ os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \ os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ " org.apache.phoenix.util.PhoenixRuntime " + args diff --git a/bin/queryserver.py b/bin/queryserver.py index 78d340c8607..d4228b39cf2 100755 --- a/bin/queryserver.py +++ b/bin/queryserver.py @@ -119,7 +119,7 @@ # " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=n " + \ # " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true" + \ -java_cmd = '%(java)s -cp ' + hbase_config_path + os.pathsep + hadoop_config_path + os.pathsep + \ +java_cmd = '%(java)s $PHOENIX_OPTS -cp ' + hbase_config_path + os.pathsep + hadoop_config_path + os.pathsep + \ phoenix_utils.phoenix_queryserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \ " -Dproc_phoenixserver" + \ " -Dlog4j.configuration=file:" + os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ diff --git a/bin/sqlline-thin.py b/bin/sqlline-thin.py index d17d756dd9d..b37cd90009d 100755 --- a/bin/sqlline-thin.py +++ b/bin/sqlline-thin.py @@ -145,7 +145,8 @@ def get_serialization(): else: java = 'java' -java_cmd = java + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_thin_client_jar + \ +java_cmd = java + ' $PHOENIX_OPTS ' + \ + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_thin_client_jar + \ os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \ os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ " sqlline.SqlLine -d org.apache.phoenix.queryserver.client.Driver " + \ diff --git a/bin/sqlline.py b/bin/sqlline.py index d1fdf741645..f14cfc7ed50 100755 --- a/bin/sqlline.py +++ b/bin/sqlline.py @@ -87,7 +87,8 @@ def kill_child(): if os.name == 'nt': colorSetting = "false" -java_cmd = java + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + os.pathsep + phoenix_utils.hadoop_common_jar + os.pathsep + phoenix_utils.hadoop_hdfs_jar + \ +java_cmd = java + ' $PHOENIX_OPTS ' + \ + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + os.pathsep + phoenix_utils.hadoop_common_jar + os.pathsep + phoenix_utils.hadoop_hdfs_jar + \ os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \ os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ " sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver \ diff --git a/bin/traceserver.py b/bin/traceserver.py index 082b4173358..665099eed2f 100755 --- a/bin/traceserver.py +++ b/bin/traceserver.py @@ -116,7 +116,8 @@ # " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=n " + \ # " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true" + \ -java_cmd = '%(java)s -cp ' + hbase_config_path + os.pathsep + phoenix_utils.phoenix_traceserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \ +java_cmd = '%(java)s $PHOENIX_OPTS ' + \ + '-cp ' + hbase_config_path + os.pathsep + phoenix_utils.phoenix_traceserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \ " -Dproc_phoenixtraceserver" + \ " -Dlog4j.configuration=file:" + os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ " -Dpsql.root.logger=%(root_logger)s" + \ diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java index c713df47246..7e7175fc0b1 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java @@ -27,11 +27,20 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; +import java.sql.Statement; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.QueryUtil; @@ -81,20 +90,40 @@ public static Collection data() { } protected void testUpdatableViewWithIndex(Integer saltBuckets, boolean localIndex) throws Exception { - testUpdatableView(saltBuckets); - testUpdatableViewIndex(saltBuckets, localIndex); + String viewName = testUpdatableView(saltBuckets); + Pair pair = testUpdatableViewIndex(saltBuckets, localIndex); + Scan scan = pair.getSecond(); + String tableName = pair.getFirst(); + // Confirm that dropping the view also deletes the rows in the index + if (saltBuckets == null) { + try (Connection conn = DriverManager.getConnection(getUrl())) { + HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(tableName)); + ResultScanner scanner = htable.getScanner(scan); + Result result = scanner.next(); + // Confirm index has rows + assertTrue(result != null && !result.isEmpty()); + + conn.createStatement().execute("DROP VIEW " + viewName); + + // Confirm index has no rows after view is dropped + scanner = htable.getScanner(scan); + result = scanner.next(); + assertTrue(result == null || result.isEmpty()); + } + } } - protected void testUpdatableView(Integer saltBuckets) throws Exception { + protected String testUpdatableView(Integer saltBuckets) throws Exception { Connection conn = DriverManager.getConnection(getUrl()); if (saltBuckets!=null) { if (tableDDLOptions.length()!=0) tableDDLOptions+=","; tableDDLOptions+=(" SALT_BUCKETS="+saltBuckets); } + String viewName = "V"; String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))" + tableDDLOptions; conn.createStatement().execute(ddl); - ddl = "CREATE VIEW v AS SELECT * FROM " + fullTableName + " WHERE k1 = 1"; + ddl = "CREATE VIEW " + viewName + " AS SELECT * FROM " + fullTableName + " WHERE k1 = 1"; conn.createStatement().execute(ddl); for (int i = 0; i < 10; i++) { conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + (i % 4) + "," + (i+100) + "," + (i > 5 ? 2 : 1) + ")"); @@ -135,13 +164,14 @@ protected void testUpdatableView(Integer saltBuckets) throws Exception { assertEquals(121, rs.getInt(2)); assertFalse(rs.next()); conn.close(); + return viewName; } - protected void testUpdatableViewIndex(Integer saltBuckets) throws Exception { - testUpdatableViewIndex(saltBuckets, false); + protected Pair testUpdatableViewIndex(Integer saltBuckets) throws Exception { + return testUpdatableViewIndex(saltBuckets, false); } - protected void testUpdatableViewIndex(Integer saltBuckets, boolean localIndex) throws Exception { + protected Pair testUpdatableViewIndex(Integer saltBuckets, boolean localIndex) throws Exception { ResultSet rs; Connection conn = DriverManager.getConnection(getUrl()); if (localIndex) { @@ -196,28 +226,32 @@ protected void testUpdatableViewIndex(Integer saltBuckets, boolean localIndex) t query = "SELECT k1, k2, s FROM v WHERE s = 'foo'"; - rs = conn.createStatement().executeQuery(query); + Statement statement = conn.createStatement(); + rs = statement.executeQuery(query); + Scan scan = statement.unwrap(PhoenixStatement.class).getQueryPlan().getContext().getScan(); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); assertEquals(120, rs.getInt(2)); assertEquals("foo", rs.getString(3)); assertFalse(rs.next()); + String htableName; rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER _LOCAL_IDX_" + tableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n" + htableName = "_LOCAL_IDX_" + tableName; + assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER " + htableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",QueryUtil.getExplainPlan(rs)); } else { + htableName = "_IDX_" + tableName; assertEquals(saltBuckets == null - ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_" + tableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n" + ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + htableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n" + " SERVER FILTER BY FIRST KEY ONLY" - : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER _IDX_T" + (transactional ? "_TXN" : "") + " [0," + (Short.MIN_VALUE+1) + ",'foo']\n" + : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER " + htableName + " [0," + (Short.MIN_VALUE+1) + ",'foo']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); } conn.close(); + return new Pair<>(htableName,scan); } - - } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java index 0fce403af7d..6c53f1d541f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java @@ -52,6 +52,7 @@ import java.util.GregorianCalendar; import org.apache.phoenix.util.DateUtil; +import org.apache.phoenix.util.TestUtil; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -664,4 +665,13 @@ public void testNullDate() throws Exception { assertNull(rs.getDate(1, GregorianCalendar.getInstance())); assertFalse(rs.next()); } + + @Test + public void testCurrentDateWithNoTable() throws Exception { + long expectedTime = System.currentTimeMillis(); + ResultSet rs = conn.createStatement().executeQuery("SELECT CURRENT_DATE()"); + assertTrue(rs.next()); + long actualTime = rs.getDate(1).getTime(); + assertTrue(Math.abs(actualTime - expectedTime) < TestUtil.MILLIS_IN_DAY); + } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java index 364b4232c88..30de4de4816 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java @@ -1376,7 +1376,9 @@ public void testUpsertSelectWithFixedWidthNullByteSizeArray() throws Exception { @Test public void testParallelUpsertSelect() throws Exception { + long ts = nextTimestamp(); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); props.setProperty(QueryServices.MUTATE_BATCH_SIZE_ATTRIB, Integer.toString(3)); props.setProperty(QueryServices.SCAN_CACHE_SIZE_ATTRIB, Integer.toString(3)); props.setProperty(QueryServices.SCAN_RESULT_CHUNK_SIZE, Integer.toString(3)); @@ -1385,11 +1387,17 @@ public void testParallelUpsertSelect() throws Exception { conn.createStatement().execute("CREATE SEQUENCE S1"); conn.createStatement().execute("CREATE TABLE SALTEDT1 (pk INTEGER PRIMARY KEY, val INTEGER) SALT_BUCKETS=4"); conn.createStatement().execute("CREATE TABLE T2 (pk INTEGER PRIMARY KEY, val INTEGER)"); - + conn.close(); + + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10)); + conn = DriverManager.getConnection(getUrl(), props); for (int i = 0; i < 100; i++) { conn.createStatement().execute("UPSERT INTO SALTEDT1 VALUES (NEXT VALUE FOR S1, " + (i%10) + ")"); } conn.commit(); + conn.close(); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20)); + conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(true); int upsertCount = conn.createStatement().executeUpdate("UPSERT INTO T2 SELECT pk, val FROM SALTEDT1"); assertEquals(100,upsertCount); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java index 55f26cd97c4..146fb1f9357 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java @@ -450,13 +450,26 @@ private void validateViewDoesNotExist(Connection conn, String viewName) throws S } } - @Test - public void testViewUsesTableIndex() throws Exception { + @Test + public void testViewUsesTableGlobalIndex() throws Exception { + testViewUsesTableIndex(false); + } + + @Test + public void testViewUsesTableLocalIndex() throws Exception { + testViewUsesTableIndex(true); + } + + + private void testViewUsesTableIndex(boolean localIndex) throws Exception { ResultSet rs; + // Use unique name for table with local index as otherwise we run into issues + // when we attempt to drop the table (with the drop metadata option set to false + String fullTableName = this.fullTableName + (localIndex ? "_WITH_LI" : "_WITHOUT_LI"); Connection conn = DriverManager.getConnection(getUrl()); String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s1 VARCHAR, s2 VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))" + tableDDLOptions; conn.createStatement().execute(ddl); - conn.createStatement().execute("CREATE INDEX i1 ON " + fullTableName + "(k3, k2) INCLUDE(s1, s2)"); + conn.createStatement().execute("CREATE " + (localIndex ? "LOCAL " : "") + " INDEX i1 ON " + fullTableName + "(k3, k2) INCLUDE(s1, s2)"); conn.createStatement().execute("CREATE INDEX i2 ON " + fullTableName + "(k3, k2, s2)"); ddl = "CREATE VIEW v AS SELECT * FROM " + fullTableName + " WHERE s1 = 'foo'"; @@ -481,11 +494,18 @@ public void testViewUsesTableIndex() throws Exception { assertFalse(rs.next()); rs = conn.createStatement().executeQuery("EXPLAIN " + query); String queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals( - "CLIENT PARALLEL 1-WAY SKIP SCAN ON 4 KEYS OVER I1 [1,100] - [2,109]\n" + - " SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')", queryPlan); - } - + // Assert that in either case (local & global) that index from physical table used for query on view. + if (localIndex) { + assertEquals("CLIENT PARALLEL 1-WAY SKIP SCAN ON 4 KEYS OVER _LOCAL_IDX_" + fullTableName + " [-32768,1,100] - [-32768,2,109]\n" + + " SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')\n" + + "CLIENT MERGE SORT", queryPlan); + } else { + assertEquals( + "CLIENT PARALLEL 1-WAY SKIP SCAN ON 4 KEYS OVER I1 [1,100] - [2,109]\n" + + " SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')", queryPlan); + } + } + @Test public void testCreateViewDefinesPKColumn() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java index b97176f0719..10634b8b86e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java @@ -157,6 +157,29 @@ private void testOptimization(Integer saltBuckets) throws Exception { assertEquals("a", rs.getString("v1")); assertFalse(rs.next()); + query = "SELECT /*+ INDEX(" + TestUtil.DEFAULT_DATA_TABLE_NAME + " " + TestUtil.DEFAULT_INDEX_TABLE_NAME + ")*/ * FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME +" where v1='a' limit 1"; + rs = conn1.createStatement().executeQuery("EXPLAIN "+ query); + + expected = + "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + TestUtil.DEFAULT_DATA_TABLE_NAME + "\n" + + "CLIENT 1 ROW LIMIT\n" + + " SKIP-SCAN-JOIN TABLE 0\n" + + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + TestUtil.DEFAULT_INDEX_TABLE_NAME + " \\['a'\\]\n" + + " SERVER FILTER BY FIRST KEY ONLY\n" + + " DYNAMIC SERVER FILTER BY \\(\"T.T_ID\", \"T.K1\", \"T.K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)\n" + + " JOIN-SCANNER 1 ROW LIMIT"; + actual = QueryUtil.getExplainPlan(rs); + assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, Pattern.matches(expected, actual)); + + rs = conn1.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("f", rs.getString("t_id")); + assertEquals(1, rs.getInt("k1")); + assertEquals(2, rs.getInt("k2")); + assertEquals(3, rs.getInt("k3")); + assertEquals("a", rs.getString("v1")); + assertFalse(rs.next()); + query = "SELECT /*+ INDEX(" + TestUtil.DEFAULT_DATA_TABLE_NAME + " " + TestUtil.DEFAULT_INDEX_TABLE_NAME + ")*/ t_id, k1, k2, k3, V1 from " + TestUtil.DEFAULT_DATA_TABLE_FULL_NAME + " where v1<='z' and k3 > 1 order by V1,t_id"; rs = conn1.createStatement().executeQuery("EXPLAIN " + query); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java index 0d329fefdd8..71713826d96 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java @@ -20,7 +20,6 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import java.sql.Connection; import java.sql.DriverManager; @@ -29,12 +28,26 @@ import java.sql.Statement; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; import org.apache.phoenix.end2end.Shadower; -import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; @@ -47,80 +60,194 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; @RunWith(Parameterized.class) public class ImmutableIndexIT extends BaseHBaseManagedTimeIT { - - private final boolean localIndex; - private final String tableDDLOptions; - private final String tableName; + + private final boolean localIndex; + private final String tableDDLOptions; + private final String tableName; private final String indexName; private final String fullTableName; private final String fullIndexName; - - public ImmutableIndexIT(boolean localIndex, boolean transactional) { - this.localIndex = localIndex; - StringBuilder optionBuilder = new StringBuilder("IMMUTABLE_ROWS=true"); - if (transactional) { - optionBuilder.append(", TRANSACTIONAL=true"); - } - this.tableDDLOptions = optionBuilder.toString(); - this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : ""); + + private static String TABLE_NAME; + private static String INDEX_DDL; + public static final AtomicInteger NUM_ROWS = new AtomicInteger(1); + + public ImmutableIndexIT(boolean localIndex, boolean transactional) { + this.localIndex = localIndex; + StringBuilder optionBuilder = new StringBuilder("IMMUTABLE_ROWS=true"); + if (transactional) { + optionBuilder.append(", TRANSACTIONAL=true"); + } + this.tableDDLOptions = optionBuilder.toString(); + this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : ""); this.indexName = "IDX" + ( transactional ? "_TXN" : ""); this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName); this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName); - } - - @BeforeClass + } + + @BeforeClass @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class) public static void doSetup() throws Exception { - Map props = Maps.newHashMapWithExpectedSize(1); - props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true)); - setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + Map serverProps = Maps.newHashMapWithExpectedSize(1); + serverProps.put("hbase.coprocessor.region.classes", CreateIndexRegionObserver.class.getName()); + Map clientProps = Maps.newHashMapWithExpectedSize(2); + clientProps.put(QueryServices.TRANSACTIONS_ENABLED, "true"); + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); } - - @Parameters(name="localIndex = {0} , transactional = {1}") + + @Parameters(name="localIndex = {0} , transactional = {1}") public static Collection data() { return Arrays.asList(new Boolean[][] { - { false, false }, { false, true }, { true, false }, { true, true } - }); + { false, true }, { true, true } + }); } - + + @Test - public void testDropIfImmutableKeyValueColumn() throws Exception { - Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + public void testCreateIndexDuringUpsertSelect() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + props.setProperty(QueryServices.MUTATE_BATCH_SIZE_ATTRIB, Integer.toString(100)); + TABLE_NAME = fullTableName + "_testCreateIndexDuringUpsertSelect"; + String ddl ="CREATE TABLE " + TABLE_NAME + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions; + INDEX_DDL = "CREATE " + (localIndex ? "LOCAL" : "") + " INDEX IF NOT EXISTS " + indexName + " ON " + TABLE_NAME + + " (long_pk, varchar_pk)" + + " INCLUDE (long_col1, long_col2)"; + + Connection conn = DriverManager.getConnection(getUrl(), props); + try { + conn.setAutoCommit(false); + Statement stmt = conn.createStatement(); + stmt.execute(ddl); + + upsertRows(conn, TABLE_NAME, 220); + conn.commit(); + + // run the upsert select and also create an index + conn.setAutoCommit(true); + String upsertSelect = "UPSERT INTO " + TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) " + + "SELECT varchar_pk||'_upsert_select', char_pk, int_pk, long_pk, decimal_pk, date_pk FROM "+ TABLE_NAME; + conn.createStatement().execute(upsertSelect); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + TABLE_NAME); + assertTrue(rs.next()); + assertEquals(440,rs.getInt(1)); + rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + TABLE_NAME); + assertTrue(rs.next()); + assertEquals(440,rs.getInt(1)); + } + finally { + conn.close(); + } + } + + // used to create an index while a batch of rows are being written + public static class CreateIndexRegionObserver extends SimpleRegionObserver { + @Override + public void postPut(ObserverContext c, + Put put, WALEdit edit, final Durability durability) + throws HBaseIOException { + String tableName = c.getEnvironment().getRegion().getRegionInfo() + .getTable().getNameAsString(); + if (tableName.equalsIgnoreCase(TABLE_NAME) + // create the index after the second batch of 1000 rows + && Bytes.startsWith(put.getRow(), Bytes.toBytes("varchar200_upsert_select"))) { + try { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute(INDEX_DDL); + } + } catch (SQLException e) { + throw new DoNotRetryIOException(e); + } + } + } + } + + private static class UpsertRunnable implements Runnable { + private static final int NUM_ROWS_IN_BATCH = 10000; + private final String fullTableName; + + public UpsertRunnable(String fullTableName) { + this.fullTableName = fullTableName; + } + + public void run() { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + while (true) { + // write a large batch of rows + boolean fistRowInBatch = true; + for (int i=0; i> futureList = Lists.newArrayListWithExpectedSize(numThreads); + for (int i =0; i future : futureList) { + future.cancel(true); + } + threadPool.shutdownNow(); + threadPool.awaitTermination(30, TimeUnit.SECONDS); + Thread.sleep(100); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + fullTableName); + assertTrue(rs.next()); + int dataTableRowCount = rs.getInt(1); + rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexName); + assertTrue(rs.next()); + int indexTableRowCount = rs.getInt(1); + assertEquals("Data and Index table should have the same number of rows ", dataTableRowCount, indexTableRowCount); } } - } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java index 7da0d85808d..7be8d41ecc5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java @@ -95,7 +95,7 @@ private void insertRow(PreparedStatement stmt, int i) throws SQLException { private void verifyResult(ResultSet rs, int i) throws SQLException { assertTrue(rs.next()); - assertEquals("VARCHAR" + String.valueOf(i) + "_" + StringUtils.rightPad("CHAR" + String.valueOf(i), 6, ' ') + assertEquals("VARCHAR" + String.valueOf(i) + "_" + StringUtils.rightPad("CHAR" + String.valueOf(i), 10, ' ') + "_A.VARCHAR" + String.valueOf(i) + "_" + StringUtils.rightPad("B.CHAR" + String.valueOf(i), 10, ' '), rs.getString(1)); assertEquals(i * 3, rs.getInt(2)); @@ -141,7 +141,7 @@ protected void helpTestCreateAndUpdate(boolean mutable, boolean localIndex) thro // DECIMAL in the index (which is not fixed width) + " AND date_pk+1=? AND date1+1=? AND date2+1=?"; stmt = conn.prepareStatement(whereSql); - stmt.setString(1, "VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 "); + stmt.setString(1, "VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 "); stmt.setInt(2, 3); Date date = DateUtil.parseDate("2015-01-02 00:00:00"); stmt.setDate(3, date); @@ -153,8 +153,8 @@ protected void helpTestCreateAndUpdate(boolean mutable, boolean localIndex) thro assertEquals( localIndex ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_INDEX_TEST." + dataTableName - + " [-32768,'VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]\nCLIENT MERGE SORT" - : "CLIENT PARALLEL 1-WAY RANGE SCAN OVER INDEX_TEST.IDX ['VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]", + + " [-32768,'VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]\nCLIENT MERGE SORT" + : "CLIENT PARALLEL 1-WAY RANGE SCAN OVER INDEX_TEST.IDX ['VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]", QueryUtil.getExplainPlan(rs)); // verify that the correct results are returned @@ -254,20 +254,20 @@ protected void helpTestUpdate(boolean localIndex) throws Exception { + fullDataTableName; ResultSet rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ " + selectSql); assertTrue(rs.next()); - assertEquals("VARCHAR1_CHAR1 _A.VARCHAR_UPDATED_B.CHAR1 ", rs.getString(1)); + assertEquals("VARCHAR1_CHAR1 _A.VARCHAR_UPDATED_B.CHAR1 ", rs.getString(1)); assertEquals(101, rs.getLong(2)); assertTrue(rs.next()); - assertEquals("VARCHAR2_CHAR2 _A.VARCHAR2_B.CHAR2 ", rs.getString(1)); + assertEquals("VARCHAR2_CHAR2 _A.VARCHAR2_B.CHAR2 ", rs.getString(1)); assertEquals(2, rs.getLong(2)); assertFalse(rs.next()); // verify that the rows in the index table are also updated rs = conn.createStatement().executeQuery("SELECT " + selectSql); assertTrue(rs.next()); - assertEquals("VARCHAR1_CHAR1 _A.VARCHAR_UPDATED_B.CHAR1 ", rs.getString(1)); + assertEquals("VARCHAR1_CHAR1 _A.VARCHAR_UPDATED_B.CHAR1 ", rs.getString(1)); assertEquals(101, rs.getLong(2)); assertTrue(rs.next()); - assertEquals("VARCHAR2_CHAR2 _A.VARCHAR2_B.CHAR2 ", rs.getString(1)); + assertEquals("VARCHAR2_CHAR2 _A.VARCHAR2_B.CHAR2 ", rs.getString(1)); assertEquals(2, rs.getLong(2)); assertFalse(rs.next()); conn.createStatement().execute("DROP INDEX IDX ON " + fullDataTableName); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java index 13ed8aade34..20a3c48c9ee 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java @@ -49,8 +49,8 @@ import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; @@ -73,48 +73,86 @@ public static void doSetup() throws Exception { setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } - @Before - public void setUp() throws SQLException { - ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE); - ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE); - } - - private static void setupSystemTable(Long scn) throws SQLException { + private static void setupSystemTable(String fullTableName) throws SQLException { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - if (scn != null) { - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn)); - } try (Connection conn = DriverManager.getConnection(getUrl(), props)) { conn.createStatement().execute( - "create table " + QueryConstants.SYSTEM_SCHEMA_NAME + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE + TEST_TABLE_SCHEMA); + "create table " + fullTableName + TEST_TABLE_SCHEMA); } } @Test public void testUpdateCacheForTxnTable() throws Exception { - helpTestUpdateCache(true, false, null); + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + TRANSACTIONAL_DATA_TABLE; + ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE); + helpTestUpdateCache(fullTableName, null, new int[] {1, 1}); } @Test public void testUpdateCacheForNonTxnTable() throws Exception { - helpTestUpdateCache(false, false, null); + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE); + helpTestUpdateCache(fullTableName, null, new int[] {1, 3}); } @Test public void testUpdateCacheForNonTxnSystemTable() throws Exception { - helpTestUpdateCache(false, true, null); + String fullTableName = QueryConstants.SYSTEM_SCHEMA_NAME + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + setupSystemTable(fullTableName); + helpTestUpdateCache(fullTableName, null, new int[] {0, 0}); + } + + @Test + public void testUpdateCacheForNeverUpdatedTable() throws Exception { + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute( + "alter table " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=NEVER"); + } + helpTestUpdateCache(fullTableName, null, new int[] {0, 0}); + } + + @Test + public void testUpdateCacheForAlwaysUpdatedTable() throws Exception { + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("CREATE TABLE " + fullTableName + TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=always"); + } + helpTestUpdateCache(fullTableName, null, new int[] {1, 3}); + } + + @Test + public void testUpdateCacheForTimeLimitedUpdateTable() throws Exception { + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("CREATE TABLE " + fullTableName + TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=" + 10000); + } + helpTestUpdateCache(fullTableName, null, new int[] {0, 0}); + Thread.sleep(10000); + helpTestUpdateCache(fullTableName, null, new int[] {1, 0}); + } + + @Test + public void testUpdateCacheForChangingUpdateTable() throws Exception { + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("CREATE TABLE " + fullTableName + TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never"); + } + helpTestUpdateCache(fullTableName, null, new int[] {0, 0}); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS"); + } + helpTestUpdateCache(fullTableName, null, new int[] {1, 3}); } - public static void helpTestUpdateCache(boolean isTransactional, boolean isSystem, Long scn) throws Exception { - String tableName = isTransactional ? TRANSACTIONAL_DATA_TABLE : MUTABLE_INDEX_DATA_TABLE; - String schemaName; - if (isSystem) { - setupSystemTable(scn); - schemaName = QueryConstants.SYSTEM_SCHEMA_NAME; - } else { - schemaName = INDEX_DATA_SCHEMA; - } - String fullTableName = schemaName + QueryConstants.NAME_SEPARATOR + tableName; + public static void helpTestUpdateCache(String fullTableName, Long scn, int[] expectedRPCs) throws Exception { + String tableName = SchemaUtil.getTableNameFromFullName(fullTableName); + String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName); String selectSql = "SELECT * FROM "+fullTableName; // use a spyed ConnectionQueryServices so we can verify calls to getTable ConnectionQueryServices connectionQueryServices = Mockito.spy(driver.getConnectionQueryServices(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))); @@ -136,7 +174,7 @@ public static void helpTestUpdateCache(boolean isTransactional, boolean isSystem TestUtil.setRowKeyColumns(stmt, 3); stmt.execute(); conn.commit(); - int numUpsertRpcs = isSystem ? 0 : 1; + int numUpsertRpcs = expectedRPCs[0]; // verify only 0 or 1 rpc to fetch table metadata, verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong()); reset(connectionQueryServices); @@ -169,7 +207,7 @@ public static void helpTestUpdateCache(boolean isTransactional, boolean isSystem // for non-transactional tables with a scn : verify *only* one rpc occurs // for transactional tables : verify *only* one rpc occurs // for non-transactional, system tables : verify no rpc occurs - int numRpcs = isSystem ? 0 : (isTransactional || scn!=null ? 1 : 3); + int numRpcs = expectedRPCs[1]; verify(connectionQueryServices, times(numRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong()); } finally { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java index 5ff2fb0d9b6..04f751b2407 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java @@ -17,25 +17,22 @@ */ package org.apache.phoenix.rpc; +import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA; import static org.apache.phoenix.util.TestUtil.MUTABLE_INDEX_DATA_TABLE; import org.apache.phoenix.end2end.BaseClientManagedTimeIT; -import org.junit.Before; +import org.apache.phoenix.query.QueryConstants; import org.junit.Test; public class UpdateCacheWithScnIT extends BaseClientManagedTimeIT { - protected long ts; - - @Before - public void initTable() throws Exception { - ts = nextTimestamp(); - ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE, ts); - } - @Test public void testUpdateCacheWithScn() throws Exception { - UpdateCacheIT.helpTestUpdateCache(false, false, ts+2); + long ts = nextTimestamp(); + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE, ts); + // FIXME: given that the scn is advancing in the test, why aren't there more RPCs? + UpdateCacheIT.helpTestUpdateCache(fullTableName, ts+2, new int[] {1, 1}); } } diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g index 18a153ed0d0..23f7e8f8fc2 100644 --- a/phoenix-core/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g @@ -509,7 +509,8 @@ fam_prop_name returns [PropertyName ret] ; prop_value returns [Object ret] - : l=literal { $ret = l.getValue(); } + : v=identifier { $ret = v; } + | l=literal { $ret = l.getValue(); } ; column_name returns [ColumnName ret] diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java index 0828b94c1af..9b2c460d2b2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java @@ -652,7 +652,7 @@ public Void visit(DerivedTableNode subselectNode) throws SQLException { PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.emptyList(), false, Collections.emptyList(), null, null, false, false, false, null, - null, null, false, false); + null, null, false, false, 0); String alias = subselectNode.getAlias(); TableRef tableRef = new TableRef(alias, t, MetaDataProtocol.MIN_TABLE_TIMESTAMP, false); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java index b55e4aad720..b64b9b7e7a2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java @@ -1302,7 +1302,7 @@ public static PTable joinProjectedTables(PTable left, PTable right, JoinType typ left.getBucketNum(), merged,left.getParentSchemaName(), left.getParentTableName(), left.getIndexes(), left.isImmutableRows(), Collections.emptyList(), null, null, PTable.DEFAULT_DISABLE_WAL, left.isMultiTenant(), left.getStoreNulls(), left.getViewType(), left.getViewIndexId(), left.getIndexType(), - left.rowKeyOrderOptimizable(), left.isTransactional()); + left.rowKeyOrderOptimizable(), left.isTransactional(), left.getUpdateCacheFrequency()); } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java index 192a031a21b..2659b3f8d8a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java @@ -127,7 +127,7 @@ public boolean hasUDFs() { @Override public MutationState execute() throws SQLException { if (tableRefs.isEmpty()) { - return null; + return new MutationState(0, connection); } boolean wasAutoCommit = connection.getAutoCommit(); try { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java index bb0b59516e1..1a667ae1f20 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java @@ -120,7 +120,7 @@ public MutationPlan compile(final PTable indexTable) throws SQLException { DelegateMutationPlan delegate = new DelegateMutationPlan(statement.compileMutation(updateStmtStr.toString())) { @Override public MutationState execute() throws SQLException { - connection.getMutationState().commitWriteFence(dataTable); + connection.getMutationState().commitDDLFence(dataTable); return super.execute(); } }; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java index 551b05cb2ad..0fc6d74902f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java @@ -152,7 +152,7 @@ public static PTable createProjectedTable(SelectStatement select, StatementConte table.getBucketNum(), projectedColumns, table.getParentSchemaName(), table.getParentName(), table.getIndexes(), table.isImmutableRows(), Collections.emptyList(), null, null, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), - table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTable createProjectedTable(TableRef tableRef, List sourceColumnRefs, boolean retainPKColumns) throws SQLException { @@ -179,7 +179,7 @@ public static PTable createProjectedTable(TableRef tableRef, List sou retainPKColumns ? table.getBucketNum() : null, projectedColumns, null, null, Collections.emptyList(), table.isImmutableRows(), Collections.emptyList(), null, null, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), - null, table.rowKeyOrderOptimizable(), table.isTransactional()); + null, table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } // For extracting column references from single select statement diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java index 298303dbdd4..3bc1e374a9a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java @@ -82,7 +82,7 @@ public static TableRef contructSchemaTable(PhoenixStatement statement, QueryPlan PTable tempTable = PTableImpl.makePTable(statement.getConnection().getTenantId(), UNION_SCHEMA_NAME, UNION_TABLE_NAME, PTableType.SUBQUERY, null, HConstants.LATEST_TIMESTAMP, scn == null ? HConstants.LATEST_TIMESTAMP : scn, null, null, projectedColumns, null, null, null, - true, null, null, null, true, true, true, null, null, null, false, false); + true, null, null, null, true, true, true, null, null, null, false, false, 0); TableRef tableRef = new TableRef(null, tempTable, 0, false); return tableRef; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java index 236c2dca717..c95cd5df194 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java @@ -93,7 +93,8 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver { public static final String GUIDEPOST_PER_REGION = "_GUIDEPOST_PER_REGION"; public static final String UPGRADE_DESC_ROW_KEY = "_UPGRADE_DESC_ROW_KEY"; public static final String SCAN_REGION_SERVER = "_SCAN_REGION_SERVER"; - public static final String RUN_UPDATE_STATS_ASYNC = "_RunUpdateStatsAsync"; + public static final String RUN_UPDATE_STATS_ASYNC_ATTRIB = "_RunUpdateStatsAsync"; + public static final String SKIP_REGION_BOUNDARY_CHECK = "_SKIP_REGION_BOUNDARY_CHECK"; /** * Attribute name used to pass custom annotations in Scans and Mutations (later). Custom annotations @@ -145,11 +146,18 @@ private static void throwIfScanOutOfRegion(Scan scan, Region region) throws DoNo abstract protected boolean isRegionObserverFor(Scan scan); abstract protected RegionScanner doPostScannerOpen(ObserverContext c, final Scan scan, final RegionScanner s) throws Throwable; + protected boolean skipRegionBoundaryCheck(Scan scan) { + byte[] skipCheckBytes = scan.getAttribute(SKIP_REGION_BOUNDARY_CHECK); + return skipCheckBytes != null && Bytes.toBoolean(skipCheckBytes); + } + @Override public RegionScanner preScannerOpen(final ObserverContext c, final Scan scan, final RegionScanner s) throws IOException { if (isRegionObserverFor(scan)) { - throwIfScanOutOfRegion(scan, c.getEnvironment().getRegion()); + if (! skipRegionBoundaryCheck(scan)) { + throwIfScanOutOfRegion(scan, c.getEnvironment().getRegion()); + } // Muck with the start/stop row of the scan and set as reversed at the // last possible moment. You need to swap the start/stop and make the // start exclusive and the stop inclusive. diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 8c905bad250..9887e7bec13 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -62,6 +62,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID_INDEX; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT_BYTES; @@ -242,6 +243,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso private static final KeyValue BASE_COLUMN_COUNT_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.BASE_COLUMN_COUNT_BYTES); private static final KeyValue ROW_KEY_ORDER_OPTIMIZABLE_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, ROW_KEY_ORDER_OPTIMIZABLE_BYTES); private static final KeyValue TRANSACTIONAL_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TRANSACTIONAL_BYTES); + private static final KeyValue UPDATE_CACHE_FREQUENCY_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, UPDATE_CACHE_FREQUENCY_BYTES); private static final List TABLE_KV_COLUMNS = Arrays.asList( EMPTY_KEYVALUE_KV, @@ -264,7 +266,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso STORE_NULLS_KV, BASE_COLUMN_COUNT_KV, ROW_KEY_ORDER_OPTIMIZABLE_KV, - TRANSACTIONAL_KV + TRANSACTIONAL_KV, + UPDATE_CACHE_FREQUENCY_KV ); static { Collections.sort(TABLE_KV_COLUMNS, KeyValue.COMPARATOR); @@ -289,6 +292,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso private static final int BASE_COLUMN_COUNT_INDEX = TABLE_KV_COLUMNS.indexOf(BASE_COLUMN_COUNT_KV); private static final int ROW_KEY_ORDER_OPTIMIZABLE_INDEX = TABLE_KV_COLUMNS.indexOf(ROW_KEY_ORDER_OPTIMIZABLE_KV); private static final int TRANSACTIONAL_INDEX = TABLE_KV_COLUMNS.indexOf(TRANSACTIONAL_KV); + private static final int UPDATE_CACHE_FREQUENCY_INDEX = TABLE_KV_COLUMNS.indexOf(UPDATE_CACHE_FREQUENCY_KV); // KeyValues for Column private static final KeyValue DECIMAL_DIGITS_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, DECIMAL_DIGITS_BYTES); @@ -806,6 +810,10 @@ private PTable getTable(RegionScanner scanner, long clientTimeStamp, long tableT baseColumnCountKv.getValueOffset(), SortOrder.getDefault()); Cell rowKeyOrderOptimizableKv = tableKeyValues[ROW_KEY_ORDER_OPTIMIZABLE_INDEX]; boolean rowKeyOrderOptimizable = rowKeyOrderOptimizableKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(rowKeyOrderOptimizableKv.getValueArray(), rowKeyOrderOptimizableKv.getValueOffset(), rowKeyOrderOptimizableKv.getValueLength())); + Cell updateCacheFrequencyKv = tableKeyValues[UPDATE_CACHE_FREQUENCY_INDEX]; + long updateCacheFrequency = updateCacheFrequencyKv == null ? 0 : + PLong.INSTANCE.getCodec().decodeLong(updateCacheFrequencyKv.getValueArray(), + updateCacheFrequencyKv.getValueOffset(), SortOrder.getDefault()); List columns = Lists.newArrayListWithExpectedSize(columnCount); List indexes = new ArrayList(); @@ -850,7 +858,8 @@ private PTable getTable(RegionScanner scanner, long clientTimeStamp, long tableT return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp, tableSeqNum, pkName, saltBucketNum, columns, tableType == INDEX ? schemaName : null, tableType == INDEX ? dataTableName : null, indexes, isImmutableRows, physicalTables, defaultFamilyName, viewStatement, - disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable, transactional); + disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, rowKeyOrderOptimizable, transactional, updateCacheFrequency, + stats, baseColumnCount); } private PFunction getFunction(RegionScanner scanner, final boolean isReplace, long clientTimeStamp, List deleteMutationsForReplace) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java index b7b936e969a..a704e22d0ea 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java @@ -73,7 +73,7 @@ public abstract class MetaDataProtocol extends MetaDataService { public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0 = MIN_TABLE_TIMESTAMP + 7; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0 = MIN_TABLE_TIMESTAMP + 8; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0 = MIN_TABLE_TIMESTAMP + 9; - public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 11; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 13; // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0; // TODO: pare this down to minimum, as we don't need duplicates for both table and column errors, nor should we need diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java index c40e3cd435c..7950ac8f183 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java @@ -18,10 +18,6 @@ package org.apache.phoenix.coprocessor; import java.io.IOException; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.phoenix.jdbc.PhoenixDriver; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; @@ -36,28 +32,34 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; import org.apache.phoenix.cache.GlobalCache; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.jdbc.PhoenixDriver; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PIndexState; -import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.UpgradeUtil; /** @@ -96,10 +98,52 @@ public void start(CoprocessorEnvironment env) throws IOException { rebuildIndexTimeInterval = env.getConfiguration().getLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL); } - + + private static String getJdbcUrl(RegionCoprocessorEnvironment env) { + String zkQuorum = env.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM); + String zkClientPort = env.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, + Integer.toString(HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT)); + String zkParentNode = env.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkClientPort + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkParentNode; + } @Override public void postOpen(ObserverContext e) { + final RegionCoprocessorEnvironment env = e.getEnvironment(); + + Runnable r = new Runnable() { + @Override + public void run() { + HTableInterface metaTable = null; + HTableInterface statsTable = null; + try { + Thread.sleep(1000); + metaTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME)); + statsTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)); + if (UpgradeUtil.truncateStats(metaTable, statsTable)) { + LOG.info("Stats are successfully truncated for upgrade 4.7!!"); + } + } catch (Exception exception) { + LOG.warn("Exception while truncate stats..," + + " please check and delete stats manually inorder to get proper result with old client!!"); + LOG.warn(exception.getStackTrace()); + } finally { + try { + if (metaTable != null) { + metaTable.close(); + } + if (statsTable != null) { + statsTable.close(); + } + } catch (IOException e) {} + } + } + }; + (new Thread(r)).start(); + if (!enableRebuildIndex) { LOG.info("Failure Index Rebuild is skipped by configuration."); return; @@ -134,17 +178,6 @@ public BuildIndexScheduleTask(RegionCoprocessorEnvironment env) { this.env = env; } - private String getJdbcUrl() { - String zkQuorum = this.env.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM); - String zkClientPort = this.env.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, - Integer.toString(HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT)); - String zkParentNode = this.env.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT, - HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); - return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum - + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkClientPort - + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkParentNode; - } - @Override public void run() { RegionScanner scanner = null; @@ -219,7 +252,7 @@ public void run() { } if (conn == null) { - conn = DriverManager.getConnection(getJdbcUrl()).unwrap(PhoenixConnection.class); + conn = DriverManager.getConnection(getJdbcUrl(env)).unwrap(PhoenixConnection.class); } String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTable); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 1d67994747e..f332e60b960 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -666,7 +666,7 @@ private RegionScanner collectStats(final RegionScanner innerScanner, StatisticsC final Region region, final Scan scan, Configuration config) throws IOException { StatsCollectionCallable callable = new StatsCollectionCallable(stats, region, innerScanner, config); - byte[] asyncBytes = scan.getAttribute(BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC); + byte[] asyncBytes = scan.getAttribute(BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB); boolean async = false; if (asyncBytes != null) { async = Bytes.toBoolean(asyncBytes); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java index 9f965309b0e..d9dcb56c4ab 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java @@ -44,6 +44,36 @@ public interface PGuidePostsOrBuilder * optional int64 rowCount = 3; */ long getRowCount(); + + // optional int32 maxLength = 4; + /** + * optional int32 maxLength = 4; + */ + boolean hasMaxLength(); + /** + * optional int32 maxLength = 4; + */ + int getMaxLength(); + + // optional int32 encodedGuidePostsCount = 5; + /** + * optional int32 encodedGuidePostsCount = 5; + */ + boolean hasEncodedGuidePostsCount(); + /** + * optional int32 encodedGuidePostsCount = 5; + */ + int getEncodedGuidePostsCount(); + + // optional bytes encodedGuidePosts = 6; + /** + * optional bytes encodedGuidePosts = 6; + */ + boolean hasEncodedGuidePosts(); + /** + * optional bytes encodedGuidePosts = 6; + */ + com.google.protobuf.ByteString getEncodedGuidePosts(); } /** * Protobuf type {@code PGuidePosts} @@ -114,6 +144,21 @@ private PGuidePosts( rowCount_ = input.readInt64(); break; } + case 32: { + bitField0_ |= 0x00000004; + maxLength_ = input.readInt32(); + break; + } + case 40: { + bitField0_ |= 0x00000008; + encodedGuidePostsCount_ = input.readInt32(); + break; + } + case 50: { + bitField0_ |= 0x00000010; + encodedGuidePosts_ = input.readBytes(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -212,10 +257,61 @@ public long getRowCount() { return rowCount_; } + // optional int32 maxLength = 4; + public static final int MAXLENGTH_FIELD_NUMBER = 4; + private int maxLength_; + /** + * optional int32 maxLength = 4; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional int32 maxLength = 4; + */ + public int getMaxLength() { + return maxLength_; + } + + // optional int32 encodedGuidePostsCount = 5; + public static final int ENCODEDGUIDEPOSTSCOUNT_FIELD_NUMBER = 5; + private int encodedGuidePostsCount_; + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public boolean hasEncodedGuidePostsCount() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public int getEncodedGuidePostsCount() { + return encodedGuidePostsCount_; + } + + // optional bytes encodedGuidePosts = 6; + public static final int ENCODEDGUIDEPOSTS_FIELD_NUMBER = 6; + private com.google.protobuf.ByteString encodedGuidePosts_; + /** + * optional bytes encodedGuidePosts = 6; + */ + public boolean hasEncodedGuidePosts() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; + } + private void initFields() { guidePosts_ = java.util.Collections.emptyList(); byteCount_ = 0L; rowCount_ = 0L; + maxLength_ = 0; + encodedGuidePostsCount_ = 0; + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -238,6 +334,15 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00000002) == 0x00000002)) { output.writeInt64(3, rowCount_); } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt32(4, maxLength_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt32(5, encodedGuidePostsCount_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeBytes(6, encodedGuidePosts_); + } getUnknownFields().writeTo(output); } @@ -264,6 +369,18 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeInt64Size(3, rowCount_); } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, maxLength_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(5, encodedGuidePostsCount_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(6, encodedGuidePosts_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -299,6 +416,21 @@ public boolean equals(final java.lang.Object obj) { result = result && (getRowCount() == other.getRowCount()); } + result = result && (hasMaxLength() == other.hasMaxLength()); + if (hasMaxLength()) { + result = result && (getMaxLength() + == other.getMaxLength()); + } + result = result && (hasEncodedGuidePostsCount() == other.hasEncodedGuidePostsCount()); + if (hasEncodedGuidePostsCount()) { + result = result && (getEncodedGuidePostsCount() + == other.getEncodedGuidePostsCount()); + } + result = result && (hasEncodedGuidePosts() == other.hasEncodedGuidePosts()); + if (hasEncodedGuidePosts()) { + result = result && getEncodedGuidePosts() + .equals(other.getEncodedGuidePosts()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -324,6 +456,18 @@ public int hashCode() { hash = (37 * hash) + ROWCOUNT_FIELD_NUMBER; hash = (53 * hash) + hashLong(getRowCount()); } + if (hasMaxLength()) { + hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; + hash = (53 * hash) + getMaxLength(); + } + if (hasEncodedGuidePostsCount()) { + hash = (37 * hash) + ENCODEDGUIDEPOSTSCOUNT_FIELD_NUMBER; + hash = (53 * hash) + getEncodedGuidePostsCount(); + } + if (hasEncodedGuidePosts()) { + hash = (37 * hash) + ENCODEDGUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getEncodedGuidePosts().hashCode(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -439,6 +583,12 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00000002); rowCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000004); + maxLength_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + encodedGuidePostsCount_ = 0; + bitField0_ = (bitField0_ & ~0x00000010); + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000020); return this; } @@ -480,6 +630,18 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts bu to_bitField0_ |= 0x00000002; } result.rowCount_ = rowCount_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000004; + } + result.maxLength_ = maxLength_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000008; + } + result.encodedGuidePostsCount_ = encodedGuidePostsCount_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000010; + } + result.encodedGuidePosts_ = encodedGuidePosts_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -512,6 +674,15 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PGuidePostsPro if (other.hasRowCount()) { setRowCount(other.getRowCount()); } + if (other.hasMaxLength()) { + setMaxLength(other.getMaxLength()); + } + if (other.hasEncodedGuidePostsCount()) { + setEncodedGuidePostsCount(other.getEncodedGuidePostsCount()); + } + if (other.hasEncodedGuidePosts()) { + setEncodedGuidePosts(other.getEncodedGuidePosts()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -677,6 +848,108 @@ public Builder clearRowCount() { return this; } + // optional int32 maxLength = 4; + private int maxLength_ ; + /** + * optional int32 maxLength = 4; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional int32 maxLength = 4; + */ + public int getMaxLength() { + return maxLength_; + } + /** + * optional int32 maxLength = 4; + */ + public Builder setMaxLength(int value) { + bitField0_ |= 0x00000008; + maxLength_ = value; + onChanged(); + return this; + } + /** + * optional int32 maxLength = 4; + */ + public Builder clearMaxLength() { + bitField0_ = (bitField0_ & ~0x00000008); + maxLength_ = 0; + onChanged(); + return this; + } + + // optional int32 encodedGuidePostsCount = 5; + private int encodedGuidePostsCount_ ; + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public boolean hasEncodedGuidePostsCount() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public int getEncodedGuidePostsCount() { + return encodedGuidePostsCount_; + } + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public Builder setEncodedGuidePostsCount(int value) { + bitField0_ |= 0x00000010; + encodedGuidePostsCount_ = value; + onChanged(); + return this; + } + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public Builder clearEncodedGuidePostsCount() { + bitField0_ = (bitField0_ & ~0x00000010); + encodedGuidePostsCount_ = 0; + onChanged(); + return this; + } + + // optional bytes encodedGuidePosts = 6; + private com.google.protobuf.ByteString encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; + /** + * optional bytes encodedGuidePosts = 6; + */ + public boolean hasEncodedGuidePosts() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public Builder setEncodedGuidePosts(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + encodedGuidePosts_ = value; + onChanged(); + return this; + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public Builder clearEncodedGuidePosts() { + bitField0_ = (bitField0_ & ~0x00000020); + encodedGuidePosts_ = getDefaultInstance().getEncodedGuidePosts(); + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:PGuidePosts) } @@ -702,11 +975,13 @@ public Builder clearRowCount() { descriptor; static { java.lang.String[] descriptorData = { - "\n\021PGuidePosts.proto\"F\n\013PGuidePosts\022\022\n\ngu" + - "idePosts\030\001 \003(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010row" + - "Count\030\003 \001(\003BE\n(org.apache.phoenix.coproc" + - "essor.generatedB\021PGuidePostsProtosH\001\210\001\001\240" + - "\001\001" + "\n\021PGuidePosts.proto\"\224\001\n\013PGuidePosts\022\022\n\ng" + + "uidePosts\030\001 \003(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010ro" + + "wCount\030\003 \001(\003\022\021\n\tmaxLength\030\004 \001(\005\022\036\n\026encod" + + "edGuidePostsCount\030\005 \001(\005\022\031\n\021encodedGuideP" + + "osts\030\006 \001(\014BE\n(org.apache.phoenix.coproce" + + "ssor.generatedB\021PGuidePostsProtosH\001\210\001\001\240\001" + + "\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -718,7 +993,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PGuidePosts_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PGuidePosts_descriptor, - new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", }); + new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", "MaxLength", "EncodedGuidePostsCount", "EncodedGuidePosts", }); return null; } }; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index 7e71cd9fa8a..11a9049ae44 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -1900,6 +1900,26 @@ public interface PTableStatsOrBuilder * optional .PGuidePosts pGuidePosts = 6; */ org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrBuilder getPGuidePostsOrBuilder(); + + // optional bytes encodedGuidePosts = 7; + /** + * optional bytes encodedGuidePosts = 7; + */ + boolean hasEncodedGuidePosts(); + /** + * optional bytes encodedGuidePosts = 7; + */ + com.google.protobuf.ByteString getEncodedGuidePosts(); + + // optional int32 maxLength = 8; + /** + * optional int32 maxLength = 8; + */ + boolean hasMaxLength(); + /** + * optional int32 maxLength = 8; + */ + int getMaxLength(); } /** * Protobuf type {@code PTableStats} @@ -1993,6 +2013,16 @@ private PTableStats( bitField0_ |= 0x00000010; break; } + case 58: { + bitField0_ |= 0x00000020; + encodedGuidePosts_ = input.readBytes(); + break; + } + case 64: { + bitField0_ |= 0x00000040; + maxLength_ = input.readInt32(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -2145,6 +2175,38 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePosts_; } + // optional bytes encodedGuidePosts = 7; + public static final int ENCODEDGUIDEPOSTS_FIELD_NUMBER = 7; + private com.google.protobuf.ByteString encodedGuidePosts_; + /** + * optional bytes encodedGuidePosts = 7; + */ + public boolean hasEncodedGuidePosts() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional bytes encodedGuidePosts = 7; + */ + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; + } + + // optional int32 maxLength = 8; + public static final int MAXLENGTH_FIELD_NUMBER = 8; + private int maxLength_; + /** + * optional int32 maxLength = 8; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional int32 maxLength = 8; + */ + public int getMaxLength() { + return maxLength_; + } + private void initFields() { key_ = com.google.protobuf.ByteString.EMPTY; values_ = java.util.Collections.emptyList(); @@ -2152,6 +2214,8 @@ private void initFields() { keyBytesCount_ = 0L; guidePostsCount_ = 0; pGuidePosts_ = org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance(); + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; + maxLength_ = 0; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -2187,6 +2251,12 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00000010) == 0x00000010)) { output.writeMessage(6, pGuidePosts_); } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeBytes(7, encodedGuidePosts_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeInt32(8, maxLength_); + } getUnknownFields().writeTo(output); } @@ -2225,6 +2295,14 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeMessageSize(6, pGuidePosts_); } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(7, encodedGuidePosts_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(8, maxLength_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -2275,6 +2353,16 @@ public boolean equals(final java.lang.Object obj) { result = result && getPGuidePosts() .equals(other.getPGuidePosts()); } + result = result && (hasEncodedGuidePosts() == other.hasEncodedGuidePosts()); + if (hasEncodedGuidePosts()) { + result = result && getEncodedGuidePosts() + .equals(other.getEncodedGuidePosts()); + } + result = result && (hasMaxLength() == other.hasMaxLength()); + if (hasMaxLength()) { + result = result && (getMaxLength() + == other.getMaxLength()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -2312,6 +2400,14 @@ public int hashCode() { hash = (37 * hash) + PGUIDEPOSTS_FIELD_NUMBER; hash = (53 * hash) + getPGuidePosts().hashCode(); } + if (hasEncodedGuidePosts()) { + hash = (37 * hash) + ENCODEDGUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getEncodedGuidePosts().hashCode(); + } + if (hasMaxLength()) { + hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; + hash = (53 * hash) + getMaxLength(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -2438,6 +2534,10 @@ public Builder clear() { pGuidePostsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000020); + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000040); + maxLength_ = 0; + bitField0_ = (bitField0_ & ~0x00000080); return this; } @@ -2495,6 +2595,14 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats buildPa } else { result.pGuidePosts_ = pGuidePostsBuilder_.build(); } + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000020; + } + result.encodedGuidePosts_ = encodedGuidePosts_; + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { + to_bitField0_ |= 0x00000040; + } + result.maxLength_ = maxLength_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -2536,6 +2644,12 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasPGuidePosts()) { mergePGuidePosts(other.getPGuidePosts()); } + if (other.hasEncodedGuidePosts()) { + setEncodedGuidePosts(other.getEncodedGuidePosts()); + } + if (other.hasMaxLength()) { + setMaxLength(other.getMaxLength()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -2891,6 +3005,75 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePostsBuilder_; } + // optional bytes encodedGuidePosts = 7; + private com.google.protobuf.ByteString encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; + /** + * optional bytes encodedGuidePosts = 7; + */ + public boolean hasEncodedGuidePosts() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional bytes encodedGuidePosts = 7; + */ + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; + } + /** + * optional bytes encodedGuidePosts = 7; + */ + public Builder setEncodedGuidePosts(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000040; + encodedGuidePosts_ = value; + onChanged(); + return this; + } + /** + * optional bytes encodedGuidePosts = 7; + */ + public Builder clearEncodedGuidePosts() { + bitField0_ = (bitField0_ & ~0x00000040); + encodedGuidePosts_ = getDefaultInstance().getEncodedGuidePosts(); + onChanged(); + return this; + } + + // optional int32 maxLength = 8; + private int maxLength_ ; + /** + * optional int32 maxLength = 8; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + /** + * optional int32 maxLength = 8; + */ + public int getMaxLength() { + return maxLength_; + } + /** + * optional int32 maxLength = 8; + */ + public Builder setMaxLength(int value) { + bitField0_ |= 0x00000080; + maxLength_ = value; + onChanged(); + return this; + } + /** + * optional int32 maxLength = 8; + */ + public Builder clearMaxLength() { + bitField0_ = (bitField0_ & ~0x00000080); + maxLength_ = 0; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:PTableStats) } @@ -3228,6 +3411,16 @@ org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStatsOrBuilder getGu * optional bool transactional = 27; */ boolean getTransactional(); + + // optional int64 updateCacheFrequency = 28; + /** + * optional int64 updateCacheFrequency = 28; + */ + boolean hasUpdateCacheFrequency(); + /** + * optional int64 updateCacheFrequency = 28; + */ + long getUpdateCacheFrequency(); } /** * Protobuf type {@code PTable} @@ -3433,6 +3626,11 @@ private PTable( transactional_ = input.readBool(); break; } + case 224: { + bitField0_ |= 0x00800000; + updateCacheFrequency_ = input.readInt64(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -4011,6 +4209,22 @@ public boolean getTransactional() { return transactional_; } + // optional int64 updateCacheFrequency = 28; + public static final int UPDATECACHEFREQUENCY_FIELD_NUMBER = 28; + private long updateCacheFrequency_; + /** + * optional int64 updateCacheFrequency = 28; + */ + public boolean hasUpdateCacheFrequency() { + return ((bitField0_ & 0x00800000) == 0x00800000); + } + /** + * optional int64 updateCacheFrequency = 28; + */ + public long getUpdateCacheFrequency() { + return updateCacheFrequency_; + } + private void initFields() { schemaNameBytes_ = com.google.protobuf.ByteString.EMPTY; tableNameBytes_ = com.google.protobuf.ByteString.EMPTY; @@ -4039,6 +4253,7 @@ private void initFields() { baseColumnCount_ = 0; rowKeyOrderOptimizable_ = false; transactional_ = false; + updateCacheFrequency_ = 0L; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -4187,6 +4402,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00400000) == 0x00400000)) { output.writeBool(27, transactional_); } + if (((bitField0_ & 0x00800000) == 0x00800000)) { + output.writeInt64(28, updateCacheFrequency_); + } getUnknownFields().writeTo(output); } @@ -4309,6 +4527,10 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeBoolSize(27, transactional_); } + if (((bitField0_ & 0x00800000) == 0x00800000)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(28, updateCacheFrequency_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -4455,6 +4677,11 @@ public boolean equals(final java.lang.Object obj) { result = result && (getTransactional() == other.getTransactional()); } + result = result && (hasUpdateCacheFrequency() == other.hasUpdateCacheFrequency()); + if (hasUpdateCacheFrequency()) { + result = result && (getUpdateCacheFrequency() + == other.getUpdateCacheFrequency()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -4576,6 +4803,10 @@ public int hashCode() { hash = (37 * hash) + TRANSACTIONAL_FIELD_NUMBER; hash = (53 * hash) + hashBoolean(getTransactional()); } + if (hasUpdateCacheFrequency()) { + hash = (37 * hash) + UPDATECACHEFREQUENCY_FIELD_NUMBER; + hash = (53 * hash) + hashLong(getUpdateCacheFrequency()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -4754,6 +4985,8 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x02000000); transactional_ = false; bitField0_ = (bitField0_ & ~0x04000000); + updateCacheFrequency_ = 0L; + bitField0_ = (bitField0_ & ~0x08000000); return this; } @@ -4906,6 +5139,10 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTable buildPartial to_bitField0_ |= 0x00400000; } result.transactional_ = transactional_; + if (((from_bitField0_ & 0x08000000) == 0x08000000)) { + to_bitField0_ |= 0x00800000; + } + result.updateCacheFrequency_ = updateCacheFrequency_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -5081,6 +5318,9 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasTransactional()) { setTransactional(other.getTransactional()); } + if (other.hasUpdateCacheFrequency()) { + setUpdateCacheFrequency(other.getUpdateCacheFrequency()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -6784,6 +7024,39 @@ public Builder clearTransactional() { return this; } + // optional int64 updateCacheFrequency = 28; + private long updateCacheFrequency_ ; + /** + * optional int64 updateCacheFrequency = 28; + */ + public boolean hasUpdateCacheFrequency() { + return ((bitField0_ & 0x08000000) == 0x08000000); + } + /** + * optional int64 updateCacheFrequency = 28; + */ + public long getUpdateCacheFrequency() { + return updateCacheFrequency_; + } + /** + * optional int64 updateCacheFrequency = 28; + */ + public Builder setUpdateCacheFrequency(long value) { + bitField0_ |= 0x08000000; + updateCacheFrequency_ = value; + onChanged(); + return this; + } + /** + * optional int64 updateCacheFrequency = 28; + */ + public Builder clearUpdateCacheFrequency() { + bitField0_ = (bitField0_ & ~0x08000000); + updateCacheFrequency_ = 0L; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:PTable) } @@ -6826,31 +7099,33 @@ public Builder clearTransactional() { "\006 \002(\010\022\020\n\010position\030\007 \002(\005\022\021\n\tsortOrder\030\010 \002" + "(\005\022\021\n\tarraySize\030\t \001(\005\022\024\n\014viewConstant\030\n " + "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" + - "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\232\001\n\013PTab" + + "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\310\001\n\013PTab" + "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + - "idePosts\030\006 \001(\0132\014.PGuidePosts\"\206\005\n\006PTable\022" + - "\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByt" + - "es\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType" + - "\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005" + - " \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameBytes\030\007" + - " \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns\030\t \003(\013" + - "2\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTable\022\027\n\017" + - "isImmutableRows\030\013 \002(\010\022 \n\nguidePosts\030\014 \003(" + - "\0132\014.PTableStats\022\032\n\022dataTableNameBytes\030\r ", - "\001(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\ndisabl" + - "eWAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n\010viewT" + - "ype\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025\n\rphys" + - "icalNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022\023\n\013vie" + - "wIndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016sta" + - "tsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n" + - "\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpt" + - "imizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010*A\n" + - "\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIE" + - "W\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.p", - "hoenix.coprocessor.generatedB\014PTableProt" + - "osH\001\210\001\001\240\001\001" + "idePosts\030\006 \001(\0132\014.PGuidePosts\022\031\n\021encodedG" + + "uidePosts\030\007 \001(\014\022\021\n\tmaxLength\030\010 \001(\005\"\244\005\n\006P" + + "Table\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableN" + + "ameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTab" + + "leType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNu" + + "mber\030\005 \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameB" + + "ytes\030\007 \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns" + + "\030\t \003(\0132\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTab" + + "le\022\027\n\017isImmutableRows\030\013 \002(\010\022 \n\nguidePost", + "s\030\014 \003(\0132\014.PTableStats\022\032\n\022dataTableNameBy" + + "tes\030\r \001(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\n" + + "disableWAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n" + + "\010viewType\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025" + + "\n\rphysicalNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022" + + "\023\n\013viewIndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022" + + "\026\n\016statsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 " + + "\001(\010\022\027\n\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOr" + + "derOptimizable\030\032 \001(\010\022\025\n\rtransactional\030\033 " + + "\001(\010\022\034\n\024updateCacheFrequency\030\034 \001(\003*A\n\nPTa", + "bleType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022" + + "\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.phoen" + + "ix.coprocessor.generatedB\014PTableProtosH\001" + + "\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -6868,13 +7143,13 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PTableStats_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTableStats_descriptor, - new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", }); + new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "EncodedGuidePosts", "MaxLength", }); internal_static_PTable_descriptor = getDescriptor().getMessageTypes().get(2); internal_static_PTable_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTable_descriptor, - new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "GuidePosts", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", }); + new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "GuidePosts", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", "UpdateCacheFrequency", }); return null; } }; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java deleted file mode 100644 index 5008f22a303..00000000000 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java +++ /dev/null @@ -1,1269 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: StatisticsCollect.proto - -package org.apache.phoenix.coprocessor.generated; - -public final class StatCollectorProtos { - private StatCollectorProtos() {} - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistry registry) { - } - public interface StatCollectRequestOrBuilder - extends com.google.protobuf.MessageOrBuilder { - - // optional bytes startRow = 1; - /** - * optional bytes startRow = 1; - */ - boolean hasStartRow(); - /** - * optional bytes startRow = 1; - */ - com.google.protobuf.ByteString getStartRow(); - - // optional bytes stopRow = 2; - /** - * optional bytes stopRow = 2; - */ - boolean hasStopRow(); - /** - * optional bytes stopRow = 2; - */ - com.google.protobuf.ByteString getStopRow(); - } - /** - * Protobuf type {@code StatCollectRequest} - */ - public static final class StatCollectRequest extends - com.google.protobuf.GeneratedMessage - implements StatCollectRequestOrBuilder { - // Use StatCollectRequest.newBuilder() to construct. - private StatCollectRequest(com.google.protobuf.GeneratedMessage.Builder builder) { - super(builder); - this.unknownFields = builder.getUnknownFields(); - } - private StatCollectRequest(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } - - private static final StatCollectRequest defaultInstance; - public static StatCollectRequest getDefaultInstance() { - return defaultInstance; - } - - public StatCollectRequest getDefaultInstanceForType() { - return defaultInstance; - } - - private final com.google.protobuf.UnknownFieldSet unknownFields; - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private StatCollectRequest( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - initFields(); - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - default: { - if (!parseUnknownField(input, unknownFields, - extensionRegistry, tag)) { - done = true; - } - break; - } - case 10: { - bitField0_ |= 0x00000001; - startRow_ = input.readBytes(); - break; - } - case 18: { - bitField0_ |= 0x00000002; - stopRow_ = input.readBytes(); - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e.getMessage()).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.Builder.class); - } - - public static com.google.protobuf.Parser PARSER = - new com.google.protobuf.AbstractParser() { - public StatCollectRequest parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new StatCollectRequest(input, extensionRegistry); - } - }; - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - private int bitField0_; - // optional bytes startRow = 1; - public static final int STARTROW_FIELD_NUMBER = 1; - private com.google.protobuf.ByteString startRow_; - /** - * optional bytes startRow = 1; - */ - public boolean hasStartRow() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * optional bytes startRow = 1; - */ - public com.google.protobuf.ByteString getStartRow() { - return startRow_; - } - - // optional bytes stopRow = 2; - public static final int STOPROW_FIELD_NUMBER = 2; - private com.google.protobuf.ByteString stopRow_; - /** - * optional bytes stopRow = 2; - */ - public boolean hasStopRow() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - /** - * optional bytes stopRow = 2; - */ - public com.google.protobuf.ByteString getStopRow() { - return stopRow_; - } - - private void initFields() { - startRow_ = com.google.protobuf.ByteString.EMPTY; - stopRow_ = com.google.protobuf.ByteString.EMPTY; - } - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized != -1) return isInitialized == 1; - - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - getSerializedSize(); - if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeBytes(1, startRow_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeBytes(2, stopRow_); - } - getUnknownFields().writeTo(output); - } - - private int memoizedSerializedSize = -1; - public int getSerializedSize() { - int size = memoizedSerializedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - size += com.google.protobuf.CodedOutputStream - .computeBytesSize(1, startRow_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - size += com.google.protobuf.CodedOutputStream - .computeBytesSize(2, stopRow_); - } - size += getUnknownFields().getSerializedSize(); - memoizedSerializedSize = size; - return size; - } - - private static final long serialVersionUID = 0L; - @java.lang.Override - protected java.lang.Object writeReplace() - throws java.io.ObjectStreamException { - return super.writeReplace(); - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)) { - return super.equals(obj); - } - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest other = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest) obj; - - boolean result = true; - result = result && (hasStartRow() == other.hasStartRow()); - if (hasStartRow()) { - result = result && getStartRow() - .equals(other.getStartRow()); - } - result = result && (hasStopRow() == other.hasStopRow()); - if (hasStopRow()) { - result = result && getStopRow() - .equals(other.getStopRow()); - } - result = result && - getUnknownFields().equals(other.getUnknownFields()); - return result; - } - - private int memoizedHashCode = 0; - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptorForType().hashCode(); - if (hasStartRow()) { - hash = (37 * hash) + STARTROW_FIELD_NUMBER; - hash = (53 * hash) + getStartRow().hashCode(); - } - if (hasStopRow()) { - hash = (37 * hash) + STOPROW_FIELD_NUMBER; - hash = (53 * hash) + getStopRow().hashCode(); - } - hash = (29 * hash) + getUnknownFields().hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - - public static Builder newBuilder() { return Builder.create(); } - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest prototype) { - return newBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { return newBuilder(this); } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code StatCollectRequest} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessage.Builder - implements org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequestOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.Builder.class); - } - - // Construct using org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { - } - } - private static Builder create() { - return new Builder(); - } - - public Builder clear() { - super.clear(); - startRow_ = com.google.protobuf.ByteString.EMPTY; - bitField0_ = (bitField0_ & ~0x00000001); - stopRow_ = com.google.protobuf.ByteString.EMPTY; - bitField0_ = (bitField0_ & ~0x00000002); - return this; - } - - public Builder clone() { - return create().mergeFrom(buildPartial()); - } - - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_descriptor; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest getDefaultInstanceForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance(); - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest build() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest buildPartial() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest result = new org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest(this); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; - } - result.startRow_ = startRow_; - if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000002; - } - result.stopRow_ = stopRow_; - result.bitField0_ = to_bitField0_; - onBuilt(); - return result; - } - - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest) { - return mergeFrom((org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest other) { - if (other == org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance()) return this; - if (other.hasStartRow()) { - setStartRow(other.getStartRow()); - } - if (other.hasStopRow()) { - setStopRow(other.getStopRow()); - } - this.mergeUnknownFields(other.getUnknownFields()); - return this; - } - - public final boolean isInitialized() { - return true; - } - - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest) e.getUnfinishedMessage(); - throw e; - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - private int bitField0_; - - // optional bytes startRow = 1; - private com.google.protobuf.ByteString startRow_ = com.google.protobuf.ByteString.EMPTY; - /** - * optional bytes startRow = 1; - */ - public boolean hasStartRow() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * optional bytes startRow = 1; - */ - public com.google.protobuf.ByteString getStartRow() { - return startRow_; - } - /** - * optional bytes startRow = 1; - */ - public Builder setStartRow(com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000001; - startRow_ = value; - onChanged(); - return this; - } - /** - * optional bytes startRow = 1; - */ - public Builder clearStartRow() { - bitField0_ = (bitField0_ & ~0x00000001); - startRow_ = getDefaultInstance().getStartRow(); - onChanged(); - return this; - } - - // optional bytes stopRow = 2; - private com.google.protobuf.ByteString stopRow_ = com.google.protobuf.ByteString.EMPTY; - /** - * optional bytes stopRow = 2; - */ - public boolean hasStopRow() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - /** - * optional bytes stopRow = 2; - */ - public com.google.protobuf.ByteString getStopRow() { - return stopRow_; - } - /** - * optional bytes stopRow = 2; - */ - public Builder setStopRow(com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000002; - stopRow_ = value; - onChanged(); - return this; - } - /** - * optional bytes stopRow = 2; - */ - public Builder clearStopRow() { - bitField0_ = (bitField0_ & ~0x00000002); - stopRow_ = getDefaultInstance().getStopRow(); - onChanged(); - return this; - } - - // @@protoc_insertion_point(builder_scope:StatCollectRequest) - } - - static { - defaultInstance = new StatCollectRequest(true); - defaultInstance.initFields(); - } - - // @@protoc_insertion_point(class_scope:StatCollectRequest) - } - - public interface StatCollectResponseOrBuilder - extends com.google.protobuf.MessageOrBuilder { - - // required uint64 rowsScanned = 1; - /** - * required uint64 rowsScanned = 1; - */ - boolean hasRowsScanned(); - /** - * required uint64 rowsScanned = 1; - */ - long getRowsScanned(); - } - /** - * Protobuf type {@code StatCollectResponse} - */ - public static final class StatCollectResponse extends - com.google.protobuf.GeneratedMessage - implements StatCollectResponseOrBuilder { - // Use StatCollectResponse.newBuilder() to construct. - private StatCollectResponse(com.google.protobuf.GeneratedMessage.Builder builder) { - super(builder); - this.unknownFields = builder.getUnknownFields(); - } - private StatCollectResponse(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } - - private static final StatCollectResponse defaultInstance; - public static StatCollectResponse getDefaultInstance() { - return defaultInstance; - } - - public StatCollectResponse getDefaultInstanceForType() { - return defaultInstance; - } - - private final com.google.protobuf.UnknownFieldSet unknownFields; - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private StatCollectResponse( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - initFields(); - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - default: { - if (!parseUnknownField(input, unknownFields, - extensionRegistry, tag)) { - done = true; - } - break; - } - case 8: { - bitField0_ |= 0x00000001; - rowsScanned_ = input.readUInt64(); - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e.getMessage()).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.Builder.class); - } - - public static com.google.protobuf.Parser PARSER = - new com.google.protobuf.AbstractParser() { - public StatCollectResponse parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new StatCollectResponse(input, extensionRegistry); - } - }; - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - private int bitField0_; - // required uint64 rowsScanned = 1; - public static final int ROWSSCANNED_FIELD_NUMBER = 1; - private long rowsScanned_; - /** - * required uint64 rowsScanned = 1; - */ - public boolean hasRowsScanned() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * required uint64 rowsScanned = 1; - */ - public long getRowsScanned() { - return rowsScanned_; - } - - private void initFields() { - rowsScanned_ = 0L; - } - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized != -1) return isInitialized == 1; - - if (!hasRowsScanned()) { - memoizedIsInitialized = 0; - return false; - } - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - getSerializedSize(); - if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeUInt64(1, rowsScanned_); - } - getUnknownFields().writeTo(output); - } - - private int memoizedSerializedSize = -1; - public int getSerializedSize() { - int size = memoizedSerializedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - size += com.google.protobuf.CodedOutputStream - .computeUInt64Size(1, rowsScanned_); - } - size += getUnknownFields().getSerializedSize(); - memoizedSerializedSize = size; - return size; - } - - private static final long serialVersionUID = 0L; - @java.lang.Override - protected java.lang.Object writeReplace() - throws java.io.ObjectStreamException { - return super.writeReplace(); - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse)) { - return super.equals(obj); - } - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse other = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) obj; - - boolean result = true; - result = result && (hasRowsScanned() == other.hasRowsScanned()); - if (hasRowsScanned()) { - result = result && (getRowsScanned() - == other.getRowsScanned()); - } - result = result && - getUnknownFields().equals(other.getUnknownFields()); - return result; - } - - private int memoizedHashCode = 0; - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptorForType().hashCode(); - if (hasRowsScanned()) { - hash = (37 * hash) + ROWSSCANNED_FIELD_NUMBER; - hash = (53 * hash) + hashLong(getRowsScanned()); - } - hash = (29 * hash) + getUnknownFields().hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - - public static Builder newBuilder() { return Builder.create(); } - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse prototype) { - return newBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { return newBuilder(this); } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code StatCollectResponse} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessage.Builder - implements org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponseOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.Builder.class); - } - - // Construct using org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { - } - } - private static Builder create() { - return new Builder(); - } - - public Builder clear() { - super.clear(); - rowsScanned_ = 0L; - bitField0_ = (bitField0_ & ~0x00000001); - return this; - } - - public Builder clone() { - return create().mergeFrom(buildPartial()); - } - - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_descriptor; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse getDefaultInstanceForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(); - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse build() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse buildPartial() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse result = new org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse(this); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; - } - result.rowsScanned_ = rowsScanned_; - result.bitField0_ = to_bitField0_; - onBuilt(); - return result; - } - - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) { - return mergeFrom((org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse other) { - if (other == org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance()) return this; - if (other.hasRowsScanned()) { - setRowsScanned(other.getRowsScanned()); - } - this.mergeUnknownFields(other.getUnknownFields()); - return this; - } - - public final boolean isInitialized() { - if (!hasRowsScanned()) { - - return false; - } - return true; - } - - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) e.getUnfinishedMessage(); - throw e; - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - private int bitField0_; - - // required uint64 rowsScanned = 1; - private long rowsScanned_ ; - /** - * required uint64 rowsScanned = 1; - */ - public boolean hasRowsScanned() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * required uint64 rowsScanned = 1; - */ - public long getRowsScanned() { - return rowsScanned_; - } - /** - * required uint64 rowsScanned = 1; - */ - public Builder setRowsScanned(long value) { - bitField0_ |= 0x00000001; - rowsScanned_ = value; - onChanged(); - return this; - } - /** - * required uint64 rowsScanned = 1; - */ - public Builder clearRowsScanned() { - bitField0_ = (bitField0_ & ~0x00000001); - rowsScanned_ = 0L; - onChanged(); - return this; - } - - // @@protoc_insertion_point(builder_scope:StatCollectResponse) - } - - static { - defaultInstance = new StatCollectResponse(true); - defaultInstance.initFields(); - } - - // @@protoc_insertion_point(class_scope:StatCollectResponse) - } - - /** - * Protobuf service {@code StatCollectService} - */ - public static abstract class StatCollectService - implements com.google.protobuf.Service { - protected StatCollectService() {} - - public interface Interface { - /** - * rpc collectStat(.StatCollectRequest) returns (.StatCollectResponse); - */ - public abstract void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done); - - } - - public static com.google.protobuf.Service newReflectiveService( - final Interface impl) { - return new StatCollectService() { - @java.lang.Override - public void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done) { - impl.collectStat(controller, request, done); - } - - }; - } - - public static com.google.protobuf.BlockingService - newReflectiveBlockingService(final BlockingInterface impl) { - return new com.google.protobuf.BlockingService() { - public final com.google.protobuf.Descriptors.ServiceDescriptor - getDescriptorForType() { - return getDescriptor(); - } - - public final com.google.protobuf.Message callBlockingMethod( - com.google.protobuf.Descriptors.MethodDescriptor method, - com.google.protobuf.RpcController controller, - com.google.protobuf.Message request) - throws com.google.protobuf.ServiceException { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.callBlockingMethod() given method descriptor for " + - "wrong service type."); - } - switch(method.getIndex()) { - case 0: - return impl.collectStat(controller, (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)request); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getRequestPrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getRequestPrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getResponsePrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getResponsePrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - }; - } - - /** - * rpc collectStat(.StatCollectRequest) returns (.StatCollectResponse); - */ - public abstract void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done); - - public static final - com.google.protobuf.Descriptors.ServiceDescriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.getDescriptor().getServices().get(0); - } - public final com.google.protobuf.Descriptors.ServiceDescriptor - getDescriptorForType() { - return getDescriptor(); - } - - public final void callMethod( - com.google.protobuf.Descriptors.MethodDescriptor method, - com.google.protobuf.RpcController controller, - com.google.protobuf.Message request, - com.google.protobuf.RpcCallback< - com.google.protobuf.Message> done) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.callMethod() given method descriptor for wrong " + - "service type."); - } - switch(method.getIndex()) { - case 0: - this.collectStat(controller, (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)request, - com.google.protobuf.RpcUtil.specializeCallback( - done)); - return; - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getRequestPrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getRequestPrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getResponsePrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getResponsePrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public static Stub newStub( - com.google.protobuf.RpcChannel channel) { - return new Stub(channel); - } - - public static final class Stub extends org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectService implements Interface { - private Stub(com.google.protobuf.RpcChannel channel) { - this.channel = channel; - } - - private final com.google.protobuf.RpcChannel channel; - - public com.google.protobuf.RpcChannel getChannel() { - return channel; - } - - public void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done) { - channel.callMethod( - getDescriptor().getMethods().get(0), - controller, - request, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(), - com.google.protobuf.RpcUtil.generalizeCallback( - done, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.class, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance())); - } - } - - public static BlockingInterface newBlockingStub( - com.google.protobuf.BlockingRpcChannel channel) { - return new BlockingStub(channel); - } - - public interface BlockingInterface { - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request) - throws com.google.protobuf.ServiceException; - } - - private static final class BlockingStub implements BlockingInterface { - private BlockingStub(com.google.protobuf.BlockingRpcChannel channel) { - this.channel = channel; - } - - private final com.google.protobuf.BlockingRpcChannel channel; - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request) - throws com.google.protobuf.ServiceException { - return (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) channel.callBlockingMethod( - getDescriptor().getMethods().get(0), - controller, - request, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance()); - } - - } - - // @@protoc_insertion_point(class_scope:StatCollectService) - } - - private static com.google.protobuf.Descriptors.Descriptor - internal_static_StatCollectRequest_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_StatCollectRequest_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_StatCollectResponse_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_StatCollectResponse_fieldAccessorTable; - - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; - } - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - static { - java.lang.String[] descriptorData = { - "\n\027StatisticsCollect.proto\"7\n\022StatCollect" + - "Request\022\020\n\010startRow\030\001 \001(\014\022\017\n\007stopRow\030\002 \001" + - "(\014\"*\n\023StatCollectResponse\022\023\n\013rowsScanned" + - "\030\001 \002(\0042N\n\022StatCollectService\0228\n\013collectS" + - "tat\022\023.StatCollectRequest\032\024.StatCollectRe" + - "sponseBG\n(org.apache.phoenix.coprocessor" + - ".generatedB\023StatCollectorProtosH\001\210\001\001\240\001\001" - }; - com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = - new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { - public com.google.protobuf.ExtensionRegistry assignDescriptors( - com.google.protobuf.Descriptors.FileDescriptor root) { - descriptor = root; - internal_static_StatCollectRequest_descriptor = - getDescriptor().getMessageTypes().get(0); - internal_static_StatCollectRequest_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_StatCollectRequest_descriptor, - new java.lang.String[] { "StartRow", "StopRow", }); - internal_static_StatCollectResponse_descriptor = - getDescriptor().getMessageTypes().get(1); - internal_static_StatCollectResponse_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_StatCollectResponse_descriptor, - new java.lang.String[] { "RowsScanned", }); - return null; - } - }; - com.google.protobuf.Descriptors.FileDescriptor - .internalBuildGeneratedFileFrom(descriptorData, - new com.google.protobuf.Descriptors.FileDescriptor[] { - }, assigner); - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index 9767cbea2ba..b1d8e7db6e9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -131,53 +131,53 @@ public SQLException newException(SQLExceptionInfo info) { PRIMARY_KEY_MISSING(509, "42888", "The table does not have a primary key."), PRIMARY_KEY_ALREADY_EXISTS(510, "42889", "The table already has a primary key."), ORDER_BY_NOT_IN_SELECT_DISTINCT(511, "42890", "All ORDER BY expressions must appear in SELECT DISTINCT:"), - INVALID_PRIMARY_KEY_CONSTRAINT(512, "42891", "Invalid column reference in primary key constraint"), - ARRAY_NOT_ALLOWED_IN_PRIMARY_KEY(513, "42892", "Array type not allowed as primary key constraint"), + INVALID_PRIMARY_KEY_CONSTRAINT(512, "42891", "Invalid column reference in primary key constraint."), + ARRAY_NOT_ALLOWED_IN_PRIMARY_KEY(513, "42892", "Array type not allowed as primary key constraint."), COLUMN_EXIST_IN_DEF(514, "42892", "A duplicate column name was detected in the object definition or ALTER TABLE statement.", new Factory() { @Override public SQLException newException(SQLExceptionInfo info) { return new ColumnAlreadyExistsException(info.getSchemaName(), info.getTableName(), info.getColumnName()); } }), - ORDER_BY_ARRAY_NOT_SUPPORTED(515, "42893", "ORDER BY of an array type is not allowed"), - NON_EQUALITY_ARRAY_COMPARISON(516, "42894", "Array types may only be compared using = or !="), - INVALID_NOT_NULL_CONSTRAINT(517, "42895", "Invalid not null constraint on non primary key column"), + ORDER_BY_ARRAY_NOT_SUPPORTED(515, "42893", "ORDER BY of an array type is not allowed."), + NON_EQUALITY_ARRAY_COMPARISON(516, "42894", "Array types may only be compared using = or !=."), + INVALID_NOT_NULL_CONSTRAINT(517, "42895", "Invalid not null constraint on non primary key column."), /** * Invalid Transaction State (errorcode 05, sqlstate 25) */ READ_ONLY_CONNECTION(518,"25502","Mutations are not permitted for a read-only connection."), - VARBINARY_ARRAY_NOT_SUPPORTED(519, "42896", "VARBINARY ARRAY is not supported"), + VARBINARY_ARRAY_NOT_SUPPORTED(519, "42896", "VARBINARY ARRAY is not supported."), /** * Expression Index exceptions. */ - AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_INDEX(520, "42897", "Aggregate expression not allowed in an index"), - NON_DETERMINISTIC_EXPRESSION_NOT_ALLOWED_IN_INDEX(521, "42898", "Non-deterministic expression not allowed in an index"), - STATELESS_EXPRESSION_NOT_ALLOWED_IN_INDEX(522, "42899", "Stateless expression not allowed in an index"), + AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_INDEX(520, "42897", "Aggregate expression not allowed in an index."), + NON_DETERMINISTIC_EXPRESSION_NOT_ALLOWED_IN_INDEX(521, "42898", "Non-deterministic expression not allowed in an index."), + STATELESS_EXPRESSION_NOT_ALLOWED_IN_INDEX(522, "42899", "Stateless expression not allowed in an index."), /** * Transaction exceptions. */ - TRANSACTION_CONFLICT_EXCEPTION(523, "42900", "Transaction aborted due to conflict with other mutations"), - TRANSACTION_EXCEPTION(524, "42901", "Transaction aborted due to error"), + TRANSACTION_CONFLICT_EXCEPTION(523, "42900", "Transaction aborted due to conflict with other mutations."), + TRANSACTION_EXCEPTION(524, "42901", "Transaction aborted due to error."), /** * Union All related errors */ - SELECT_COLUMN_NUM_IN_UNIONALL_DIFFS(525, "42902", "SELECT column number differs in a Union All query is not allowed"), - SELECT_COLUMN_TYPE_IN_UNIONALL_DIFFS(526, "42903", "SELECT column types differ in a Union All query is not allowed"), + SELECT_COLUMN_NUM_IN_UNIONALL_DIFFS(525, "42902", "SELECT column number differs in a Union All query is not allowed."), + SELECT_COLUMN_TYPE_IN_UNIONALL_DIFFS(526, "42903", "SELECT column types differ in a Union All query is not allowed."), /** * Row timestamp column related errors */ - ROWTIMESTAMP_ONE_PK_COL_ONLY(527, "42904", "Only one column that is part of the primary key can be declared as a ROW_TIMESTAMP"), - ROWTIMESTAMP_PK_COL_ONLY(528, "42905", "Only columns part of the primary key can be declared as a ROW_TIMESTAMP"), - ROWTIMESTAMP_CREATE_ONLY(529, "42906", "A column can be added as ROW_TIMESTAMP only in CREATE TABLE"), - ROWTIMESTAMP_COL_INVALID_TYPE(530, "42907", "A column can be added as ROW_TIMESTAMP only if it is of type DATE, BIGINT, TIME OR TIMESTAMP"), - ROWTIMESTAMP_NOT_ALLOWED_ON_VIEW(531, "42908", "Declaring a column as row_timestamp is not allowed for views"), - INVALID_SCN(532, "42909", "Value of SCN cannot be less than zero"), + ROWTIMESTAMP_ONE_PK_COL_ONLY(527, "42904", "Only one column that is part of the primary key can be declared as a ROW_TIMESTAMP."), + ROWTIMESTAMP_PK_COL_ONLY(528, "42905", "Only columns part of the primary key can be declared as a ROW_TIMESTAMP."), + ROWTIMESTAMP_CREATE_ONLY(529, "42906", "A column can be added as ROW_TIMESTAMP only in CREATE TABLE."), + ROWTIMESTAMP_COL_INVALID_TYPE(530, "42907", "A column can be added as ROW_TIMESTAMP only if it is of type DATE, BIGINT, TIME OR TIMESTAMP."), + ROWTIMESTAMP_NOT_ALLOWED_ON_VIEW(531, "42908", "Declaring a column as row_timestamp is not allowed for views."), + INVALID_SCN(532, "42909", "Value of SCN cannot be less than zero."), /** * HBase and Phoenix specific implementation defined sub-classes. * Column family related exceptions. @@ -241,10 +241,10 @@ public SQLException newException(SQLExceptionInfo info) { INVALID_INDEX_STATE_TRANSITION(1028, "42Y87", "Invalid index state transition."), INVALID_MUTABLE_INDEX_CONFIG(1029, "42Y88", "Mutable secondary indexes must have the " + IndexManagementUtil.WAL_EDIT_CODEC_CLASS_KEY + " property set to " - + IndexManagementUtil.INDEX_WAL_EDIT_CODEC_CLASS_NAME + " in the hbase-sites.xml of every region server"), + + IndexManagementUtil.INDEX_WAL_EDIT_CODEC_CLASS_NAME + " in the hbase-sites.xml of every region server."), - CANNOT_CREATE_TENANT_SPECIFIC_TABLE(1030, "42Y89", "Cannot create table for tenant-specific connection"), + CANNOT_CREATE_TENANT_SPECIFIC_TABLE(1030, "42Y89", "Cannot create table for tenant-specific connection."), DEFAULT_COLUMN_FAMILY_ONLY_ON_CREATE_TABLE(1034, "42Y93", "Default column family may only be specified when creating a table."), INSUFFICIENT_MULTI_TENANT_COLUMNS(1040, "42Y96", "A MULTI_TENANT table must have two or more PK columns with the first column being NOT NULL."), TENANTID_IS_OF_WRONG_TYPE(1041, "42Y97", "The TenantId could not be converted to correct format for this table."), @@ -254,33 +254,33 @@ public SQLException newException(SQLExceptionInfo info) { NO_LOCAL_INDEX_ON_TABLE_WITH_IMMUTABLE_ROWS(1048,"43A05","Local indexes aren't allowed on tables with immutable rows."), COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY(1049, "43A06", "Column family not allowed for table properties."), COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL(1050, "43A07", "Setting TTL for a column family not supported. You can only have TTL for the entire table."), - CANNOT_ALTER_PROPERTY(1051, "43A08", "Property can be specified or changed only when creating a table"), - CANNOT_SET_PROPERTY_FOR_COLUMN_NOT_ADDED(1052, "43A09", "Property cannot be specified for a column family that is not being added or modified"), - CANNOT_SET_TABLE_PROPERTY_ADD_COLUMN(1053, "43A10", "Table level property cannot be set when adding a column"), + CANNOT_ALTER_PROPERTY(1051, "43A08", "Property can be specified or changed only when creating a table."), + CANNOT_SET_PROPERTY_FOR_COLUMN_NOT_ADDED(1052, "43A09", "Property cannot be specified for a column family that is not being added or modified."), + CANNOT_SET_TABLE_PROPERTY_ADD_COLUMN(1053, "43A10", "Table level property cannot be set when adding a column."), NO_LOCAL_INDEXES(1054, "43A11", "Local secondary indexes are not supported for HBase versions " + MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MIN_LOCAL_SI_VERSION_DISALLOW) + " through " + MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MAX_LOCAL_SI_VERSION_DISALLOW) + " inclusive."), UNALLOWED_LOCAL_INDEXES(1055, "43A12", "Local secondary indexes are configured to not be allowed."), - DESC_VARBINARY_NOT_SUPPORTED(1056, "43A13", "Descending VARBINARY columns not supported"), + DESC_VARBINARY_NOT_SUPPORTED(1056, "43A13", "Descending VARBINARY columns not supported."), - DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE(1069, "43A13", "Default column family not allowed on VIEW or shared INDEX"), - ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL(1070, "44A01", "Only tables may be declared as transactional"), - TX_MAY_NOT_SWITCH_TO_NON_TX(1071, "44A02", "A transactional table may not be switched to non transactional"), - STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL(1072, "44A03", "Store nulls must be true when a table is transactional"), - CANNOT_START_TRANSACTION_WITH_SCN_SET(1073, "44A04", "Cannot start a transaction on a connection with SCN set"), - TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE(1074, "44A05", "A transactional table must define VERSION of greater than one"), - CANNOT_SPECIFY_SCN_FOR_TXN_TABLE(1075, "44A06", "Cannot use a connection with SCN set for a transactional table"), - NULL_TRANSACTION_CONTEXT(1076, "44A07", "No Transaction Context available"), + DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE(1069, "43A13", "Default column family not allowed on VIEW or shared INDEX."), + ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL(1070, "44A01", "Only tables may be declared as transactional."), + TX_MAY_NOT_SWITCH_TO_NON_TX(1071, "44A02", "A transactional table may not be switched to non transactional."), + STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL(1072, "44A03", "Store nulls must be true when a table is transactional."), + CANNOT_START_TRANSACTION_WITH_SCN_SET(1073, "44A04", "Cannot start a transaction on a connection with SCN set."), + TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE(1074, "44A05", "A transactional table must define VERSION of greater than one."), + CANNOT_SPECIFY_SCN_FOR_TXN_TABLE(1075, "44A06", "Cannot use a connection with SCN set for a transactional table."), + NULL_TRANSACTION_CONTEXT(1076, "44A07", "No Transaction Context available."), TRANSACTION_FAILED(1077, "44A08", "Transaction Failure "), - CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED(1078, "44A09", "Cannot create a transactional table if transactions are disabled"), - CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED(1079, "44A10", "Cannot alter table to be transactional table if transactions are disabled"), - CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP(1080, "44A11", "Cannot create a transactional table if transactions are disabled"), - CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP(1081, "44A12", "Cannot alter table to be transactional table if transactions are disabled"), - TX_MUST_BE_ENABLED_TO_SET_TX_CONTEXT(1082, "44A13", "Cannot set transaction context if transactions are disabled"), - TX_MUST_BE_ENABLED_TO_SET_AUTO_FLUSH(1083, "44A14", "Cannot set auto flush if transactions are disabled"), - TX_MUST_BE_ENABLED_TO_SET_ISOLATION_LEVEL(1084, "44A15", "Cannot set isolation level to TRANSACTION_READ_COMMITTED or TRANSACTION_SERIALIZABLE if transactions are disabled"), - TX_UNABLE_TO_GET_WRITE_FENCE(1085, "44A16", "Unable to obtain write fence for DDL operation"), + CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED(1078, "44A09", "Cannot create a transactional table if transactions are disabled."), + CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED(1079, "44A10", "Cannot alter table to be transactional table if transactions are disabled."), + CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP(1080, "44A11", "Cannot create a transactional table if transactions are disabled."), + CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP(1081, "44A12", "Cannot alter table to be transactional table if transactions are disabled."), + TX_MUST_BE_ENABLED_TO_SET_TX_CONTEXT(1082, "44A13", "Cannot set transaction context if transactions are disabled."), + TX_MUST_BE_ENABLED_TO_SET_AUTO_FLUSH(1083, "44A14", "Cannot set auto flush if transactions are disabled."), + TX_MUST_BE_ENABLED_TO_SET_ISOLATION_LEVEL(1084, "44A15", "Cannot set isolation level to TRANSACTION_READ_COMMITTED or TRANSACTION_SERIALIZABLE if transactions are disabled."), + TX_UNABLE_TO_GET_WRITE_FENCE(1085, "44A16", "Unable to obtain write fence for DDL operation."), /** Sequence related */ SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() { @@ -300,17 +300,17 @@ public SQLException newException(SQLExceptionInfo info) { CACHE_MUST_BE_NON_NEGATIVE_CONSTANT(1204, "42Z04", "Sequence CACHE value must be a non negative integer constant."), INVALID_USE_OF_NEXT_VALUE_FOR(1205, "42Z05", "NEXT VALUE FOR may only be used as in a SELECT or an UPSERT VALUES expression."), CANNOT_CALL_CURRENT_BEFORE_NEXT_VALUE(1206, "42Z06", "NEXT VALUE FOR must be called before CURRENT VALUE FOR is called."), - EMPTY_SEQUENCE_CACHE(1207, "42Z07", "No more cached sequence values"), + EMPTY_SEQUENCE_CACHE(1207, "42Z07", "No more cached sequence values."), MINVALUE_MUST_BE_CONSTANT(1208, "42Z08", "Sequence MINVALUE must be an integer or long constant."), MAXVALUE_MUST_BE_CONSTANT(1209, "42Z09", "Sequence MAXVALUE must be an integer or long constant."), MINVALUE_MUST_BE_LESS_THAN_OR_EQUAL_TO_MAXVALUE(1210, "42Z10", "Sequence MINVALUE must be less than or equal to MAXVALUE."), STARTS_WITH_MUST_BE_BETWEEN_MIN_MAX_VALUE(1211, "42Z11", - "STARTS WITH value must be greater than or equal to MINVALUE and less than or equal to MAXVALUE"), - SEQUENCE_VAL_REACHED_MAX_VALUE(1212, "42Z12", "Reached MAXVALUE of sequence"), - SEQUENCE_VAL_REACHED_MIN_VALUE(1213, "42Z13", "Reached MINVALUE of sequence"), - INCREMENT_BY_MUST_NOT_BE_ZERO(1214, "42Z14", "Sequence INCREMENT BY value cannot be zero"), + "STARTS WITH value must be greater than or equal to MINVALUE and less than or equal to MAXVALUE."), + SEQUENCE_VAL_REACHED_MAX_VALUE(1212, "42Z12", "Reached MAXVALUE of sequence."), + SEQUENCE_VAL_REACHED_MIN_VALUE(1213, "42Z13", "Reached MINVALUE of sequence."), + INCREMENT_BY_MUST_NOT_BE_ZERO(1214, "42Z14", "Sequence INCREMENT BY value cannot be zero."), NUM_SEQ_TO_ALLOCATE_MUST_BE_CONSTANT(1215, "42Z15", "Sequence NEXT n VALUES FOR must be a positive integer or constant." ), - NUM_SEQ_TO_ALLOCATE_NOT_SUPPORTED(1216, "42Z16", "Sequence NEXT n VALUES FOR is not supported for Sequences with the CYCLE flag" ), + NUM_SEQ_TO_ALLOCATE_NOT_SUPPORTED(1216, "42Z16", "Sequence NEXT n VALUES FOR is not supported for Sequences with the CYCLE flag." ), /** Parser error. (errorcode 06, sqlState 42P) */ PARSER_ERROR(601, "42P00", "Syntax error.", Factory.SYNTAX_ERROR), @@ -323,7 +323,7 @@ public SQLException newException(SQLExceptionInfo info) { * Implementation defined class. Execution exceptions (errorcode 11, sqlstate XCL). */ RESULTSET_CLOSED(1101, "XCL01", "ResultSet is closed."), - GET_TABLE_REGIONS_FAIL(1102, "XCL02", "Cannot get all table regions"), + GET_TABLE_REGIONS_FAIL(1102, "XCL02", "Cannot get all table regions."), EXECUTE_QUERY_NOT_APPLICABLE(1103, "XCL03", "executeQuery may not be used."), EXECUTE_UPDATE_NOT_APPLICABLE(1104, "XCL04", "executeUpdate may not be used."), SPLIT_POINT_NOT_CONSTANT(1105, "XCL05", "Split points must be constants."), @@ -335,22 +335,22 @@ public SQLException newException(SQLExceptionInfo info) { return new StaleRegionBoundaryCacheException(info.getSchemaName(), info.getTableName()); } }), - CANNOT_SPLIT_LOCAL_INDEX(1109,"XCL09", "Local index may not be pre-split"), - CANNOT_SALT_LOCAL_INDEX(1110,"XCL10", "Local index may not be salted"), + CANNOT_SPLIT_LOCAL_INDEX(1109,"XCL09", "Local index may not be pre-split."), + CANNOT_SALT_LOCAL_INDEX(1110,"XCL10", "Local index may not be salted."), /** * Implementation defined class. Phoenix internal error. (errorcode 20, sqlstate INT). */ CANNOT_CALL_METHOD_ON_TYPE(2001, "INT01", "Cannot call method on the argument type."), - CLASS_NOT_UNWRAPPABLE(2002, "INT03", "Class not unwrappable"), + CLASS_NOT_UNWRAPPABLE(2002, "INT03", "Class not unwrappable."), PARAM_INDEX_OUT_OF_BOUND(2003, "INT04", "Parameter position is out of range."), - PARAM_VALUE_UNBOUND(2004, "INT05", "Parameter value unbound"), + PARAM_VALUE_UNBOUND(2004, "INT05", "Parameter value unbound."), INTERRUPTED_EXCEPTION(2005, "INT07", "Interrupted exception."), INCOMPATIBLE_CLIENT_SERVER_JAR(2006, "INT08", "Incompatible jars detected between client and server."), OUTDATED_JARS(2007, "INT09", "Outdated jars."), INDEX_METADATA_NOT_FOUND(2008, "INT10", "Unable to find cached index metadata. "), - UNKNOWN_ERROR_CODE(2009, "INT11", "Unknown error code"), - OPERATION_TIMED_OUT(6000, "TIM01", "Operation timed out", new Factory() { + UNKNOWN_ERROR_CODE(2009, "INT11", "Unknown error code."), + OPERATION_TIMED_OUT(6000, "TIM01", "Operation timed out.", new Factory() { @Override public SQLException newException(SQLExceptionInfo info) { return new SQLTimeoutException(OPERATION_TIMED_OUT.getMessage(), diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index 35a36e60141..f1a9c0234f7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -37,18 +37,6 @@ import javax.annotation.Nonnull; import javax.annotation.concurrent.Immutable; -import co.cask.tephra.Transaction; -import co.cask.tephra.Transaction.VisibilityLevel; -import co.cask.tephra.TransactionAware; -import co.cask.tephra.TransactionCodec; -import co.cask.tephra.TransactionConflictException; -import co.cask.tephra.TransactionContext; -import co.cask.tephra.TransactionFailureException; -import co.cask.tephra.TransactionSystemClient; -import co.cask.tephra.hbase11.TransactionAwareHTable; -import co.cask.tephra.visibility.FenceWait; -import co.cask.tephra.visibility.VisibilityFence; - import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HTableInterface; @@ -110,6 +98,18 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import co.cask.tephra.Transaction; +import co.cask.tephra.Transaction.VisibilityLevel; +import co.cask.tephra.TransactionAware; +import co.cask.tephra.TransactionCodec; +import co.cask.tephra.TransactionConflictException; +import co.cask.tephra.TransactionContext; +import co.cask.tephra.TransactionFailureException; +import co.cask.tephra.TransactionSystemClient; +import co.cask.tephra.hbase11.TransactionAwareHTable; +import co.cask.tephra.visibility.FenceWait; +import co.cask.tephra.visibility.VisibilityFence; + /** * * Tracks the uncommitted state @@ -121,6 +121,7 @@ public class MutationState implements SQLCloseable { private static final Logger logger = LoggerFactory.getLogger(MutationState.class); private static final TransactionCodec CODEC = new TransactionCodec(); private static final int[] EMPTY_STATEMENT_INDEX_ARRAY = new int[0]; + private static final int MAX_COMMIT_RETRIES = 3; private final PhoenixConnection connection; private final long maxSize; @@ -134,7 +135,7 @@ public class MutationState implements SQLCloseable { private int numRows = 0; private int[] uncommittedStatementIndexes = EMPTY_STATEMENT_INDEX_ARRAY; private boolean isExternalTxContext = false; - private Map> txMutations; + private Map> txMutations = Collections.emptyMap(); private final MutationMetricQueue mutationMetricQueue; private ReadMetricQueue readMetricQueue; @@ -160,37 +161,37 @@ private MutationState(long maxSize, PhoenixConnection connection, Transaction tx } private MutationState(long maxSize, PhoenixConnection connection, Transaction tx, TransactionContext txContext, long sizeOffset) { - this(maxSize, connection, Maps.>newHashMapWithExpectedSize(5), tx, txContext); + this(maxSize, connection, Maps.>newHashMapWithExpectedSize(5), tx, txContext); this.sizeOffset = sizeOffset; } - MutationState(long maxSize, PhoenixConnection connection, - Map> mutations, - Transaction tx, TransactionContext txContext) { - this.maxSize = maxSize; - this.connection = connection; - this.mutations = mutations; - boolean isMetricsEnabled = connection.isRequestLevelMetricsEnabled(); - this.mutationMetricQueue = isMetricsEnabled ? new MutationMetricQueue() - : NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE; - this.tx = tx; - if (tx == null) { + MutationState(long maxSize, PhoenixConnection connection, + Map> mutations, + Transaction tx, TransactionContext txContext) { + this.maxSize = maxSize; + this.connection = connection; + this.mutations = mutations; + boolean isMetricsEnabled = connection.isRequestLevelMetricsEnabled(); + this.mutationMetricQueue = isMetricsEnabled ? new MutationMetricQueue() + : NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE; + this.tx = tx; + if (tx == null) { this.txAwares = Collections.emptyList(); - if (txContext == null) { - TransactionSystemClient txServiceClient = this.connection - .getQueryServices().getTransactionSystemClient(); - this.txContext = new TransactionContext(txServiceClient); - } else { - isExternalTxContext = true; - this.txContext = txContext; - } - } else { - // this code path is only used while running child scans, we can't pass the txContext to child scans - // as it is not thread safe, so we use the tx member variable - this.txAwares = Lists.newArrayList(); - this.txContext = null; - } - } + if (txContext == null) { + TransactionSystemClient txServiceClient = this.connection + .getQueryServices().getTransactionSystemClient(); + this.txContext = new TransactionContext(txServiceClient); + } else { + isExternalTxContext = true; + this.txContext = txContext; + } + } else { + // this code path is only used while running child scans, we can't pass the txContext to child scans + // as it is not thread safe, so we use the tx member variable + this.txAwares = Lists.newArrayList(); + this.txContext = null; + } + } public MutationState(TableRef table, Map mutations, long sizeOffset, long maxSize, PhoenixConnection connection) { this(maxSize, connection, null, null, sizeOffset); @@ -209,17 +210,19 @@ public long getMaxSize() { * when a data table transaction is started before the create index * but completes after it. In this case, we need to rerun the data * table transaction after the index creation so that the index rows - * are generated. See {@link #addReadFence(PTable)} and TEPHRA-157 + * are generated. See {@link #addDMLFence(PTable)} and TEPHRA-157 * for more information. * @param dataTable the data table upon which an index is being added * @throws SQLException */ - public void commitWriteFence(PTable dataTable) throws SQLException { + public void commitDDLFence(PTable dataTable) throws SQLException { if (dataTable.isTransactional()) { byte[] key = SchemaUtil.getTableKey(dataTable); + boolean success = false; try { FenceWait fenceWait = VisibilityFence.prepareWait(key, connection.getQueryServices().getTransactionSystemClient()); fenceWait.await(10000, TimeUnit.MILLISECONDS); + success = true; } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SQLExceptionInfo.Builder(SQLExceptionCode.INTERRUPTED_EXCEPTION).setRootCause(e).build().buildException(); @@ -235,6 +238,7 @@ public void commitWriteFence(PTable dataTable) throws SQLException { // TODO: seems like an autonomous tx capability in Tephra would be useful here. try { txContext.start(); + if (logger.isInfoEnabled() && success) logger.info("Added write fence at ~" + getTransaction().getReadPointer()); } catch (TransactionFailureException e) { throw TransactionUtil.getTransactionFailureException(e); } @@ -245,12 +249,12 @@ public void commitWriteFence(PTable dataTable) throws SQLException { /** * Add an entry to the change set representing the DML operation that is starting. * These entries will not conflict with each other, but they will conflict with a - * DDL operation of creating an index. See {@link #addReadFence(PTable)} and TEPHRA-157 + * DDL operation of creating an index. See {@link #addDMLFence(PTable)} and TEPHRA-157 * for more information. * @param dataTable the table which is doing DML * @throws SQLException */ - public void addReadFence(PTable dataTable) throws SQLException { + public void addDMLFence(PTable dataTable) throws SQLException { if (this.txContext == null) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.NULL_TRANSACTION_CONTEXT).build().buildException(); } @@ -306,18 +310,18 @@ public boolean checkpointIfNeccessary(MutationPlan plan) throws SQLException { } if (hasUncommittedData) { try { - if (txContext == null) { - currentTx = tx = connection.getQueryServices().getTransactionSystemClient().checkpoint(currentTx); - } else { - txContext.checkpoint(); - currentTx = tx = txContext.getCurrentTransaction(); - } + if (txContext == null) { + currentTx = tx = connection.getQueryServices().getTransactionSystemClient().checkpoint(currentTx); + } else { + txContext.checkpoint(); + currentTx = tx = txContext.getCurrentTransaction(); + } // Since we've checkpointed, we can clear out uncommitted set, since a statement run afterwards // should see all this data. uncommittedPhysicalNames.clear(); } catch (TransactionFailureException e) { throw new SQLException(e); - } + } } // Since we're querying our own table while mutating it, we must exclude // see our current mutations, otherwise we can get erroneous results (for DELETE) @@ -356,7 +360,7 @@ public HTableInterface getHTable(PTable table) throws SQLException { } public PhoenixConnection getConnection() { - return connection; + return connection; } // Kept private as the Transaction may change when check pointed. Keeping it private ensures @@ -366,12 +370,12 @@ private Transaction getTransaction() { } public boolean isTransactionStarted() { - return getTransaction() != null; + return getTransaction() != null; } - public long getReadPointer() { - Transaction tx = getTransaction(); - return tx == null ? HConstants.LATEST_TIMESTAMP : tx.getReadPointer(); + public long getInitialWritePointer() { + Transaction tx = getTransaction(); + return tx == null ? HConstants.LATEST_TIMESTAMP : tx.getTransactionId(); // First write pointer - won't change with checkpointing } // For testing @@ -391,11 +395,11 @@ public boolean startTransaction() throws SQLException { throw new SQLExceptionInfo.Builder(SQLExceptionCode.NULL_TRANSACTION_CONTEXT).build().buildException(); } - if (connection.getSCN() != null) { - throw new SQLExceptionInfo.Builder( - SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET) - .build().buildException(); - } + if (connection.getSCN() != null) { + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET) + .build().buildException(); + } try { if (!isTransactionStarted()) { @@ -431,6 +435,59 @@ public long getUpdateCount() { return sizeOffset + numRows; } + private void joinMutationState(TableRef tableRef, Map srcRows, + Map> dstMutations) { + PTable table = tableRef.getTable(); + boolean isIndex = table.getType() == PTableType.INDEX; + boolean incrementRowCount = dstMutations == this.mutations; + Map existingRows = dstMutations.put(tableRef, srcRows); + if (existingRows != null) { // Rows for that table already exist + // Loop through new rows and replace existing with new + for (Map.Entry rowEntry : srcRows.entrySet()) { + // Replace existing row with new row + RowMutationState existingRowMutationState = existingRows.put(rowEntry.getKey(), rowEntry.getValue()); + if (existingRowMutationState != null) { + Map existingValues = existingRowMutationState.getColumnValues(); + if (existingValues != PRow.DELETE_MARKER) { + Map newRow = rowEntry.getValue().getColumnValues(); + // if new row is PRow.DELETE_MARKER, it means delete, and we don't need to merge it with existing row. + if (newRow != PRow.DELETE_MARKER) { + // Merge existing column values with new column values + existingRowMutationState.join(rowEntry.getValue()); + // Now that the existing row has been merged with the new row, replace it back + // again (since it was merged with the new one above). + existingRows.put(rowEntry.getKey(), existingRowMutationState); + } + } + } else { + if (incrementRowCount && !isIndex) { // Don't count index rows in row count + numRows++; + } + } + } + // Put the existing one back now that it's merged + dstMutations.put(tableRef, existingRows); + } else { + // Size new map at batch size as that's what it'll likely grow to. + Map newRows = Maps.newHashMapWithExpectedSize(connection.getMutateBatchSize()); + newRows.putAll(srcRows); + dstMutations.put(tableRef, newRows); + if (incrementRowCount && !isIndex) { + numRows += srcRows.size(); + } + } + } + + private void joinMutationState(Map> srcMutations, + Map> dstMutations) { + // Merge newMutation with this one, keeping state from newMutation for any overlaps + for (Map.Entry> entry : srcMutations.entrySet()) { + // Replace existing entries for the table with new entries + TableRef tableRef = entry.getKey(); + Map srcRows = entry.getValue(); + joinMutationState(tableRef, srcRows, dstMutations); + } + } /** * Combine a newer mutation with this one, where in the event of overlaps, the newer one will take precedence. * Combine any metrics collected for the newer mutation. @@ -449,48 +506,12 @@ public void join(MutationState newMutationState) { txAwares.addAll(newMutationState.txAwares); } this.sizeOffset += newMutationState.sizeOffset; - // Merge newMutation with this one, keeping state from newMutation for any overlaps - for (Map.Entry> entry : newMutationState.mutations.entrySet()) { - // Replace existing entries for the table with new entries - TableRef tableRef = entry.getKey(); - PTable table = tableRef.getTable(); - boolean isIndex = table.getType() == PTableType.INDEX; - Map existingRows = this.mutations.put(tableRef, entry.getValue()); - if (existingRows != null) { // Rows for that table already exist - // Loop through new rows and replace existing with new - for (Map.Entry rowEntry : entry.getValue().entrySet()) { - // Replace existing row with new row - RowMutationState existingRowMutationState = existingRows.put(rowEntry.getKey(), rowEntry.getValue()); - if (existingRowMutationState != null) { - Map existingValues = existingRowMutationState.getColumnValues(); - if (existingValues != PRow.DELETE_MARKER) { - Map newRow = rowEntry.getValue().getColumnValues(); - // if new row is PRow.DELETE_MARKER, it means delete, and we don't need to merge it with existing row. - if (newRow != PRow.DELETE_MARKER) { - // Merge existing column values with new column values - existingRowMutationState.join(rowEntry.getValue()); - // Now that the existing row has been merged with the new row, replace it back - // again (since it was merged with the new one above). - existingRows.put(rowEntry.getKey(), existingRowMutationState); - } - } - } else { - if (!isIndex) { // Don't count index rows in row count - numRows++; - } - } - } - // Put the existing one back now that it's merged - this.mutations.put(entry.getKey(), existingRows); - } else { - // Size new map at batch size as that's what it'll likely grow to. - Map newRows = Maps.newHashMapWithExpectedSize(connection.getMutateBatchSize()); - newRows.putAll(entry.getValue()); - this.mutations.put(tableRef, newRows); - if (!isIndex) { - numRows += entry.getValue().size(); - } + joinMutationState(newMutationState.mutations, this.mutations); + if (!newMutationState.txMutations.isEmpty()) { + if (txMutations.isEmpty()) { + txMutations = Maps.newHashMapWithExpectedSize(mutations.size()); } + joinMutationState(newMutationState.txMutations, this.txMutations); } mutationMetricQueue.combineMetricQueues(newMutationState.mutationMetricQueue); if (readMetricQueue == null) { @@ -502,7 +523,7 @@ public void join(MutationState newMutationState) { } - private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr ptr, long rowTimestamp, PTable table) { + private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr ptr, long rowTimestamp, PTable table) { RowKeySchema schema = table.getRowKeySchema(); int rowTimestampColPos = table.getRowTimestampColPos(); Field rowTimestampField = schema.getField(rowTimestampColPos); @@ -523,7 +544,7 @@ private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr return ptr; } - private Iterator>> addRowMutations(final TableRef tableRef, final Map values, final long timestamp, boolean includeMutableIndexes, final boolean sendAll) { + private Iterator>> addRowMutations(final TableRef tableRef, final Map values, final long timestamp, boolean includeMutableIndexes, final boolean sendAll) { final PTable table = tableRef.getTable(); final Iterator indexes = // Only maintain tables with immutable rows through this client-side mechanism (table.isImmutableRows() || includeMutableIndexes) ? @@ -554,13 +575,13 @@ public Pair> next() { connection.getKeyValueBuilder(), connection); // we may also have to include delete mutations for immutable tables if we are not processing all the tables in the mutations map if (!sendAll) { - TableRef key = new TableRef(index); - Map rowToColumnMap = mutations.remove(key); - if (rowToColumnMap!=null) { - final List deleteMutations = Lists.newArrayList(); - generateMutations(tableRef, timestamp, rowToColumnMap, deleteMutations, null); - indexMutations.addAll(deleteMutations); - } + TableRef key = new TableRef(index); + Map rowToColumnMap = mutations.remove(key); + if (rowToColumnMap!=null) { + final List deleteMutations = Lists.newArrayList(); + generateMutations(tableRef, timestamp, rowToColumnMap, deleteMutations, null); + indexMutations.addAll(deleteMutations); + } } } catch (SQLException e) { throw new IllegalDataException(e); @@ -676,32 +697,32 @@ public void remove() { }; } - /** - * Validates that the meta data is valid against the server meta data if we haven't yet done so. - * Otherwise, for every UPSERT VALUES call, we'd need to hit the server to see if the meta data - * has changed. - * @param connection - * @return the server time to use for the upsert - * @throws SQLException if the table or any columns no longer exist - */ - private long[] validateAll() throws SQLException { - int i = 0; - long[] timeStamps = new long[this.mutations.size()]; - for (Map.Entry> entry : mutations.entrySet()) { - TableRef tableRef = entry.getKey(); - timeStamps[i++] = validate(tableRef, entry.getValue()); - } - return timeStamps; - } - - private long validate(TableRef tableRef, Map rowKeyToColumnMap) throws SQLException { - Long scn = connection.getSCN(); - MetaDataClient client = new MetaDataClient(connection); - long serverTimeStamp = tableRef.getTimeStamp(); - // If we're auto committing, we've already validated the schema when we got the ColumnResolver, - // so no need to do it again here. - if (!connection.getAutoCommit()) { - PTable table = tableRef.getTable(); + /** + * Validates that the meta data is valid against the server meta data if we haven't yet done so. + * Otherwise, for every UPSERT VALUES call, we'd need to hit the server to see if the meta data + * has changed. + * @param connection + * @return the server time to use for the upsert + * @throws SQLException if the table or any columns no longer exist + */ + private long[] validateAll() throws SQLException { + int i = 0; + long[] timeStamps = new long[this.mutations.size()]; + for (Map.Entry> entry : mutations.entrySet()) { + TableRef tableRef = entry.getKey(); + timeStamps[i++] = validate(tableRef, entry.getValue()); + } + return timeStamps; + } + + private long validate(TableRef tableRef, Map rowKeyToColumnMap) throws SQLException { + Long scn = connection.getSCN(); + MetaDataClient client = new MetaDataClient(connection); + long serverTimeStamp = tableRef.getTimeStamp(); + // If we're auto committing, we've already validated the schema when we got the ColumnResolver, + // so no need to do it again here. + if (!connection.getAutoCommit()) { + PTable table = tableRef.getTable(); MetaDataMutationResult result = client.updateCache(table.getSchemaName().getString(), table.getTableName().getString()); PTable resolvedTable = result.getTable(); if (resolvedTable == null) { @@ -717,14 +738,14 @@ private long validate(TableRef tableRef, Map rowEntry : rowKeyToColumnMap.entrySet()) { - RowMutationState valueEntry = rowEntry.getValue(); + RowMutationState valueEntry = rowEntry.getValue(); if (valueEntry != null) { - Map colValues = valueEntry.getColumnValues(); - if (colValues != PRow.DELETE_MARKER) { + Map colValues = valueEntry.getColumnValues(); + if (colValues != PRow.DELETE_MARKER) { for (PColumn column : colValues.keySet()) { columns[column.getPosition()] = column; } - } + } } } for (PColumn column : columns) { @@ -735,8 +756,8 @@ private long validate(TableRef tableRef, Map mutations) { long byteSize = 0; @@ -845,74 +866,75 @@ private void send(Iterator tableRefIterator) throws SQLException { // add tracing for this operation try (TraceScope trace = Tracing.startNewSpan(connection, "Committing mutations to tables")) { Span span = trace.getSpan(); - ImmutableBytesWritable indexMetaDataPtr = new ImmutableBytesWritable(); - boolean isTransactional; - while (tableRefIterator.hasNext()) { - // at this point we are going through mutations for each table - final TableRef tableRef = tableRefIterator.next(); - valuesMap = mutations.get(tableRef); - if (valuesMap == null || valuesMap.isEmpty()) { - continue; - } + ImmutableBytesWritable indexMetaDataPtr = new ImmutableBytesWritable(); + boolean isTransactional; + while (tableRefIterator.hasNext()) { + // at this point we are going through mutations for each table + final TableRef tableRef = tableRefIterator.next(); + valuesMap = mutations.get(tableRef); + if (valuesMap == null || valuesMap.isEmpty()) { + continue; + } // Validate as we go if transactional since we can undo if a problem occurs (which is unlikely) long serverTimestamp = serverTimeStamps == null ? validate(tableRef, valuesMap) : serverTimeStamps[i++]; - final PTable table = tableRef.getTable(); - // Track tables to which we've sent uncommitted data - if (isTransactional = table.isTransactional()) { - txTableRefs.add(tableRef); - uncommittedPhysicalNames.add(table.getPhysicalName().getString()); - } - boolean isDataTable = true; + final PTable table = tableRef.getTable(); + // Track tables to which we've sent uncommitted data + if (isTransactional = table.isTransactional()) { + txTableRefs.add(tableRef); + uncommittedPhysicalNames.add(table.getPhysicalName().getString()); + } + boolean isDataTable = true; table.getIndexMaintainers(indexMetaDataPtr, connection); - Iterator>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false, sendAll); - while (mutationsIterator.hasNext()) { - Pair> pair = mutationsIterator.next(); - byte[] htableName = pair.getFirst(); - List mutationList = pair.getSecond(); - - //create a span per target table - //TODO maybe we can be smarter about the table name to string here? - Span child = Tracing.child(span,"Writing mutation batch for table: "+Bytes.toString(htableName)); - - int retryCount = 0; - boolean shouldRetry = false; - do { - final ServerCache cache = isDataTable ? setMetaDataOnMutations(tableRef, mutationList, indexMetaDataPtr) : null; - - // If we haven't retried yet, retry for this case only, as it's possible that - // a split will occur after we send the index metadata cache to all known - // region servers. - shouldRetry = cache != null; - SQLException sqlE = null; - HTableInterface hTable = connection.getQueryServices().getTable(htableName); - try { - if (isTransactional) { - // If we have indexes, wrap the HTable in a delegate HTable that - // will attach the necessary index meta data in the event of a - // rollback - if (!table.getIndexes().isEmpty()) { - hTable = new MetaDataAwareHTable(hTable, tableRef); - } - TransactionAwareHTable txnAware = TransactionUtil.getTransactionAwareHTable(hTable, table); - // Don't add immutable indexes (those are the only ones that would participate - // during a commit), as we don't need conflict detection for these. - if (isDataTable) { - // Even for immutable, we need to do this so that an abort has the state - // necessary to generate the rows to delete. - addTransactionParticipant(txnAware); - } else { - txnAware.startTx(getTransaction()); - } - hTable = txnAware; - } - long numMutations = mutationList.size(); + Iterator>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false, sendAll); + while (mutationsIterator.hasNext()) { + Pair> pair = mutationsIterator.next(); + byte[] htableName = pair.getFirst(); + List mutationList = pair.getSecond(); + + //create a span per target table + //TODO maybe we can be smarter about the table name to string here? + Span child = Tracing.child(span,"Writing mutation batch for table: "+Bytes.toString(htableName)); + + int retryCount = 0; + boolean shouldRetry = false; + do { + final ServerCache cache = isDataTable ? setMetaDataOnMutations(tableRef, mutationList, indexMetaDataPtr) : null; + + // If we haven't retried yet, retry for this case only, as it's possible that + // a split will occur after we send the index metadata cache to all known + // region servers. + shouldRetry = cache != null; + SQLException sqlE = null; + HTableInterface hTable = connection.getQueryServices().getTable(htableName); + try { + if (isTransactional) { + // If we have indexes, wrap the HTable in a delegate HTable that + // will attach the necessary index meta data in the event of a + // rollback + if (!table.getIndexes().isEmpty()) { + hTable = new MetaDataAwareHTable(hTable, tableRef); + } + TransactionAwareHTable txnAware = TransactionUtil.getTransactionAwareHTable(hTable, table); + // Don't add immutable indexes (those are the only ones that would participate + // during a commit), as we don't need conflict detection for these. + if (isDataTable) { + // Even for immutable, we need to do this so that an abort has the state + // necessary to generate the rows to delete. + addTransactionParticipant(txnAware); + } else { + txnAware.startTx(getTransaction()); + } + hTable = txnAware; + } + long numMutations = mutationList.size(); GLOBAL_MUTATION_BATCH_SIZE.update(numMutations); long startTime = System.currentTimeMillis(); child.addTimelineAnnotation("Attempt " + retryCount); - hTable.batch(mutationList); - child.stop(); - child.stop(); + hTable.batch(mutationList); + if (logger.isDebugEnabled()) logger.debug("Sent batch of " + numMutations + " for " + Bytes.toString(htableName)); + child.stop(); + child.stop(); shouldRetry = false; long mutationCommitTime = System.currentTimeMillis() - startTime; GLOBAL_MUTATION_COMMIT_TIME.update(mutationCommitTime); @@ -920,80 +942,82 @@ private void send(Iterator tableRefIterator) throws SQLException { long mutationSizeBytes = calculateMutationSize(mutationList); MutationMetric mutationsMetric = new MutationMetric(numMutations, mutationSizeBytes, mutationCommitTime); mutationMetricQueue.addMetricsForTable(Bytes.toString(htableName), mutationsMetric); - } catch (Exception e) { - SQLException inferredE = ServerUtil.parseServerExceptionOrNull(e); - if (inferredE != null) { - if (shouldRetry && retryCount == 0 && inferredE.getErrorCode() == SQLExceptionCode.INDEX_METADATA_NOT_FOUND.getErrorCode()) { - // Swallow this exception once, as it's possible that we split after sending the index metadata - // and one of the region servers doesn't have it. This will cause it to have it the next go around. - // If it fails again, we don't retry. - String msg = "Swallowing exception and retrying after clearing meta cache on connection. " + inferredE; - logger.warn(LogUtil.addCustomAnnotations(msg, connection)); - connection.getQueryServices().clearTableRegionCache(htableName); - - // add a new child span as this one failed - child.addTimelineAnnotation(msg); - child.stop(); - child = Tracing.child(span,"Failed batch, attempting retry"); - - continue; - } - e = inferredE; - } - // Throw to client an exception that indicates the statements that - // were not committed successfully. - sqlE = new CommitException(e, getUncommittedStatementIndexes()); - } finally { - try { - if (cache != null) { - cache.close(); - } - } finally { - try { - hTable.close(); - } - catch (IOException e) { - if (sqlE != null) { - sqlE.setNextException(ServerUtil.parseServerException(e)); - } else { - sqlE = ServerUtil.parseServerException(e); - } - } - if (sqlE != null) { - throw sqlE; - } - } - } - } while (shouldRetry && retryCount++ < 1); - isDataTable = false; - } - if (tableRef.getTable().getType() != PTableType.INDEX) { - numRows -= valuesMap.size(); - } - // For transactions, track the statement indexes as we send data - // over because our CommitException should include all statements - // involved in the transaction since none of them would have been - // committed in the event of a failure. - if (isTransactional) { - addUncommittedStatementIndexes(valuesMap.values()); - if (txMutations == null) { - txMutations = Maps.newHashMapWithExpectedSize(mutations.size()); - } - // Keep all mutations we've encountered until a commit or rollback. - // This is not ideal, but there's not good way to get the values back - // in the event that we need to replay the commit. - txMutations.put(tableRef, valuesMap); - } + } catch (Exception e) { + SQLException inferredE = ServerUtil.parseServerExceptionOrNull(e); + if (inferredE != null) { + if (shouldRetry && retryCount == 0 && inferredE.getErrorCode() == SQLExceptionCode.INDEX_METADATA_NOT_FOUND.getErrorCode()) { + // Swallow this exception once, as it's possible that we split after sending the index metadata + // and one of the region servers doesn't have it. This will cause it to have it the next go around. + // If it fails again, we don't retry. + String msg = "Swallowing exception and retrying after clearing meta cache on connection. " + inferredE; + logger.warn(LogUtil.addCustomAnnotations(msg, connection)); + connection.getQueryServices().clearTableRegionCache(htableName); + + // add a new child span as this one failed + child.addTimelineAnnotation(msg); + child.stop(); + child = Tracing.child(span,"Failed batch, attempting retry"); + + continue; + } + e = inferredE; + } + // Throw to client an exception that indicates the statements that + // were not committed successfully. + sqlE = new CommitException(e, getUncommittedStatementIndexes()); + } finally { + try { + if (cache != null) { + cache.close(); + } + } finally { + try { + hTable.close(); + } + catch (IOException e) { + if (sqlE != null) { + sqlE.setNextException(ServerUtil.parseServerException(e)); + } else { + sqlE = ServerUtil.parseServerException(e); + } + } + if (sqlE != null) { + throw sqlE; + } + } + } + } while (shouldRetry && retryCount++ < 1); + isDataTable = false; + } + if (tableRef.getTable().getType() != PTableType.INDEX) { + numRows -= valuesMap.size(); + } + // For transactions, track the statement indexes as we send data + // over because our CommitException should include all statements + // involved in the transaction since none of them would have been + // committed in the event of a failure. + if (isTransactional) { + addUncommittedStatementIndexes(valuesMap.values()); + if (txMutations.isEmpty()) { + txMutations = Maps.newHashMapWithExpectedSize(mutations.size()); + } + // Keep all mutations we've encountered until a commit or rollback. + // This is not ideal, but there's not good way to get the values back + // in the event that we need to replay the commit. + // Copy TableRef so we have the original PTable and know when the + // indexes have changed. + joinMutationState(new TableRef(tableRef), valuesMap, txMutations); + } // Remove batches as we process them - if (sendAll) { - // Iterating through map key set in this case, so we cannot use - // the remove method without getting a concurrent modification - // exception. - tableRefIterator.remove(); - } else { - mutations.remove(tableRef); - } - } + if (sendAll) { + // Iterating through map key set in this case, so we cannot use + // the remove method without getting a concurrent modification + // exception. + tableRefIterator.remove(); + } else { + mutations.remove(tableRef); + } + } } } @@ -1006,7 +1030,7 @@ public byte[] encodeTransaction() throws SQLException { } public static Transaction decodeTransaction(byte[] txnBytes) throws IOException { - return (txnBytes == null || txnBytes.length==0) ? null : CODEC.decode(txnBytes); + return (txnBytes == null || txnBytes.length==0) ? null : CODEC.decode(txnBytes); } private ServerCache setMetaDataOnMutations(TableRef tableRef, List mutations, @@ -1059,10 +1083,10 @@ private void addUncommittedStatementIndexes(Collection rowMuta } private int[] getUncommittedStatementIndexes() { - for (Map rowMutationMap : mutations.values()) { - addUncommittedStatementIndexes(rowMutationMap.values()); - } - return uncommittedStatementIndexes; + for (Map rowMutationMap : mutations.values()) { + addUncommittedStatementIndexes(rowMutationMap.values()); + } + return uncommittedStatementIndexes; } @Override @@ -1078,7 +1102,7 @@ private void resetState() { private void resetTransactionalState() { tx = null; txAwares.clear(); - txMutations = null; + txMutations = Collections.emptyMap(); uncommittedPhysicalNames.clear(); uncommittedStatementIndexes = EMPTY_STATEMENT_INDEX_ARRAY; } @@ -1101,9 +1125,9 @@ public void commit() throws SQLException { Map> txMutations = Collections.emptyMap(); int retryCount = 0; do { - boolean sendSuccessful=false; - boolean retryCommit = false; - SQLException sqlE = null; + boolean sendSuccessful=false; + boolean retryCommit = false; + SQLException sqlE = null; try { send(); txMutations = this.txMutations; @@ -1121,7 +1145,8 @@ public void commit() throws SQLException { finishSuccessful = true; } } catch (TransactionFailureException e) { - retryCommit = (e instanceof TransactionConflictException && retryCount == 0); + if (logger.isInfoEnabled()) logger.info(e.getClass().getName() + " at timestamp " + getInitialWritePointer() + " with retry count of " + retryCount); + retryCommit = (e instanceof TransactionConflictException && retryCount < MAX_COMMIT_RETRIES); txFailure = e; SQLException nextE = TransactionUtil.getTransactionFailureException(e); if (sqlE == null) { @@ -1134,7 +1159,9 @@ public void commit() throws SQLException { if (!finishSuccessful) { try { txContext.abort(txFailure); + if (logger.isInfoEnabled()) logger.info("Abort successful"); } catch (TransactionFailureException e) { + if (logger.isInfoEnabled()) logger.info("Abort failed with " + e); SQLException nextE = TransactionUtil.getTransactionFailureException(e); if (sqlE == null) { sqlE = nextE; @@ -1151,8 +1178,15 @@ public void commit() throws SQLException { } finally { if (retryCommit) { startTransaction(); + // Add back read fences + Set txTableRefs = txMutations.keySet(); + for (TableRef tableRef : txTableRefs) { + PTable dataTable = tableRef.getTable(); + addDMLFence(dataTable); + } try { - retryCommit = wasIndexAdded(txMutations.keySet()); + // Only retry if an index was added + retryCommit = shouldResubmitTransaction(txTableRefs); } catch (SQLException e) { retryCommit = false; if (sqlE == null) { @@ -1182,10 +1216,12 @@ public void commit() throws SQLException { * @return true if indexes were added and false otherwise. * @throws SQLException */ - private boolean wasIndexAdded(Set txTableRefs) throws SQLException { + private boolean shouldResubmitTransaction(Set txTableRefs) throws SQLException { + if (logger.isInfoEnabled()) logger.info("Checking for index updates as of " + getInitialWritePointer()); MetaDataClient client = new MetaDataClient(connection); PMetaData cache = connection.getMetaDataCache(); - boolean addedIndexes = false; + boolean addedAnyIndexes = false; + boolean allImmutableTables = !txTableRefs.isEmpty(); for (TableRef tableRef : txTableRefs) { PTable dataTable = tableRef.getTable(); List oldIndexes; @@ -1194,20 +1230,24 @@ private boolean wasIndexAdded(Set txTableRefs) throws SQLException { MetaDataMutationResult result = client.updateCache(dataTable.getTenantId(), dataTable.getSchemaName().getString(), dataTable.getTableName().getString()); long timestamp = TransactionUtil.getResolvedTime(connection, result); tableRef.setTimeStamp(timestamp); - if (result.getTable() == null) { + PTable updatedDataTable = result.getTable(); + if (updatedDataTable == null) { throw new TableNotFoundException(dataTable.getSchemaName().getString(), dataTable.getTableName().getString()); } - if (!result.wasUpdated()) { - continue; - } - if (!addedIndexes) { + allImmutableTables &= updatedDataTable.isImmutableRows(); + tableRef.setTable(updatedDataTable); + if (!addedAnyIndexes) { // TODO: in theory we should do a deep equals check here, as it's possible // that an index was dropped and recreated with the same name but different // indexed/covered columns. - addedIndexes = (!oldIndexes.equals(result.getTable().getIndexes())); + addedAnyIndexes = (!oldIndexes.equals(updatedDataTable.getIndexes())); + if (logger.isInfoEnabled()) logger.info((addedAnyIndexes ? "Updates " : "No updates ") + "as of " + timestamp + " to " + updatedDataTable.getName().getString() + " with indexes " + updatedDataTable.getIndexes()); } } - return addedIndexes; + if (logger.isInfoEnabled()) logger.info((addedAnyIndexes ? "Updates " : "No updates ") + "to indexes as of " + getInitialWritePointer() + " over " + (allImmutableTables ? " all immutable tables" : " some mutable tables")); + // If all tables are immutable, we know the conflict we got was due to our DDL/DML fence. + // If any indexes were added, then the conflict might be due to DDL/DML fence. + return allImmutableTables || addedAnyIndexes; } /** diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java index 816cd1ba24b..efb00a8fff6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java @@ -122,7 +122,7 @@ private static boolean isSerial(StatementContext context, } else { // Region size estimated based on total number of bytes divided by number of regions long totByteSize = gpsInfo.getByteCount(); - estRegionSize = totByteSize / (gpsInfo.getGuidePosts().size()+1); + estRegionSize = totByteSize / (gpsInfo.getGuidePostsCount()+1); } // TODO: configurable number of bytes? boolean isSerial = (perScanLimit * estRowSize < estRegionSize); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/CeilFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/CeilFunction.java index 706aadf8a2f..b65010a7536 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/CeilFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/CeilFunction.java @@ -46,6 +46,8 @@ public abstract class CeilFunction extends ScalarFunction { public static final String NAME = "CEIL"; + + public CeilFunction() {} public CeilFunction(List children) { super(children); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FloorFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FloorFunction.java index 79383240aee..f6baa091489 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FloorFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FloorFunction.java @@ -32,8 +32,6 @@ * * Base class for built-in FLOOR function. * - * - * @since 3.0.0 */ @BuiltInFunction(name = FloorFunction.NAME, nodeClass = FloorParseNode.class, @@ -47,6 +45,8 @@ public abstract class FloorFunction extends ScalarFunction { public static final String NAME = "FLOOR"; + public FloorFunction() {} + public FloorFunction(List children) { super(children); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/NowFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/NowFunction.java index dc90249f636..83a5e5374f2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/NowFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/NowFunction.java @@ -35,6 +35,8 @@ public abstract class NowFunction extends ScalarFunction { public static final String NAME = "NOW"; + + public NowFunction() {} public NowFunction(List children) throws SQLException { super(children); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundFunction.java index 9277716f3ea..7d0306c6926 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundFunction.java @@ -31,8 +31,6 @@ /** * Base class for RoundFunction. * - * - * @since 0.1 */ @BuiltInFunction(name = RoundFunction.NAME, nodeClass = RoundParseNode.class, @@ -46,6 +44,8 @@ public abstract class RoundFunction extends ScalarFunction { public static final String NAME = "ROUND"; + public RoundFunction() {} + public RoundFunction(List children) { super(children); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToCharFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToCharFunction.java index e0bc68bff25..a14a0cbea98 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToCharFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToCharFunction.java @@ -57,6 +57,10 @@ public class ToCharFunction extends ScalarFunction { public ToCharFunction() { } + public ToCharFunction(List children) throws SQLException { + this(children, null, null, null); + } + public ToCharFunction(List children, FunctionArgumentType type, String formatString, Format formatter) throws SQLException { super(children.subList(0, 1)); Preconditions.checkNotNull(formatString); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToDateFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToDateFunction.java index 01b0dfd5cf4..f41d9593b90 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToDateFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToDateFunction.java @@ -59,6 +59,10 @@ public class ToDateFunction extends ScalarFunction { public ToDateFunction() { } + public ToDateFunction(List children) throws SQLException { + this(children, null, null); + } + public ToDateFunction(List children, String dateFormat, String timeZoneId) throws SQLException { super(children); init(dateFormat, timeZoneId); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToNumberFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToNumberFunction.java index ed70f2da041..e7659a1b2de 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToNumberFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToNumberFunction.java @@ -60,6 +60,10 @@ public class ToNumberFunction extends ScalarFunction { public ToNumberFunction() {} + public ToNumberFunction(List children) throws SQLException { + this(children, null, null, null); + } + public ToNumberFunction(List children, FunctionArgumentType type, String formatString, Format formatter) throws SQLException { super(children.subList(0, 1)); Preconditions.checkNotNull(type); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimeFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimeFunction.java index 3a26584e278..2fa05533d4b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimeFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimeFunction.java @@ -47,6 +47,10 @@ public class ToTimeFunction extends ToDateFunction { public ToTimeFunction() { } + public ToTimeFunction(List children) throws SQLException { + this(children, null, null); + } + public ToTimeFunction(List children, String dateFormat, String timeZoneId) throws SQLException { super(children, dateFormat, timeZoneId); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimestampFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimestampFunction.java index 17643a292fc..5a4828f50a1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimestampFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimestampFunction.java @@ -47,6 +47,10 @@ public class ToTimestampFunction extends ToDateFunction { public ToTimestampFunction() { } + public ToTimestampFunction(List children) throws SQLException { + this(children, null, null); + } + public ToTimestampFunction(List children, String dateFormat, String timeZoneId) throws SQLException { super(children, dateFormat, timeZoneId); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TruncFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TruncFunction.java index a4e79c24473..3b9616cff4e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TruncFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TruncFunction.java @@ -51,6 +51,8 @@ public abstract class TruncFunction extends ScalarFunction { public static final String NAME = "TRUNC"; + + public TruncFunction() {} public TruncFunction(List children) throws SQLException { super(children); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 519b38f171c..b3235e277da 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -22,6 +22,10 @@ import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_QUERY_TIMEOUT_COUNTER; import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; @@ -46,6 +50,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.QueryPlan; @@ -75,7 +80,10 @@ import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.LogUtil; +import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SQLCloseables; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.SchemaUtil; @@ -356,22 +364,18 @@ private static int getIndexContainingExclusive(List boundaries, byte[] e guideIndex = (guideIndex < 0 ? -(guideIndex + 1) : guideIndex); return guideIndex; } - - private List getGuidePosts(Set whereConditions) { + + private GuidePostsInfo getGuidePosts(Set whereConditions) { /* - * Don't use guide posts if: - * 1) We're doing a point lookup, as HBase is fast enough at those - * to not need them to be further parallelized. TODO: pref test to verify - * 2) We're collecting stats, as in this case we need to scan entire - * regions worth of data to track where to put the guide posts. + * Don't use guide posts if: 1) We're doing a point lookup, as HBase is fast enough at those to not need them to + * be further parallelized. TODO: pref test to verify 2) We're collecting stats, as in this case we need to scan + * entire regions worth of data to track where to put the guide posts. */ - if (!useStats()) { - return Collections.emptyList(); - } - - List gps = null; + if (!useStats()) { return GuidePostsInfo.EMPTY_GUIDEPOST; } + + GuidePostsInfo gps = null; PTable table = getTable(); - Map guidePostMap = tableStats.getGuidePosts(); + Map guidePostMap = tableStats.getGuidePosts(); byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable()); if (table.getColumnFamilies().isEmpty()) { // For sure we can get the defaultCF from the table @@ -379,37 +383,35 @@ private List getGuidePosts(Set whereConditions) { } else { byte[] familyInWhere = null; if (!whereConditions.isEmpty()) { - if (whereConditions.contains(defaultCF)) { - gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); - } else { - familyInWhere = whereConditions.iterator().next(); - if(familyInWhere != null) { - GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere); - if (guidePostsInfo != null) { - gps = guidePostsInfo.getGuidePosts(); - } else { - // As there are no guideposts collected for the where family we go with the default CF + if (whereConditions.contains(defaultCF)) { gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); - } + } else { + familyInWhere = whereConditions.iterator().next(); + if (familyInWhere != null) { + GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere); + if (guidePostsInfo != null) { + gps = guidePostsInfo; + } else { + // As there are no guideposts collected for the where family we go with the default CF + gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); + } + } } - } } else { - gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); + gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); } } - if (gps == null) { - return Collections.emptyList(); - } + if (gps == null) { return GuidePostsInfo.EMPTY_GUIDEPOST; } return gps; } - private List getDefaultFamilyGuidePosts(Map guidePostMap, byte[] defaultCF) { - if (guidePostMap.get(defaultCF) != null) { - return guidePostMap.get(defaultCF).getGuidePosts(); - } - return null; + private GuidePostsInfo getDefaultFamilyGuidePosts(Map guidePostMap, byte[] defaultCF) { + if (guidePostMap.get(defaultCF) != null) { + return guidePostMap.get(defaultCF); + } + return null; } - + private static String toString(List gps) { StringBuilder buf = new StringBuilder(gps.size() * 100); buf.append("["); @@ -461,10 +463,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw for(Pair where : context.getWhereConditionColumns()) { whereConditions.add(where.getFirst()); } - List gps = getGuidePosts(whereConditions); - if (logger.isDebugEnabled()) { - logger.debug("Guideposts: " + toString(gps)); - } + GuidePostsInfo gps = getGuidePosts(whereConditions); boolean traverseAllRegions = isSalted || isLocalIndex; if (!traverseAllRegions) { byte[] scanStartRow = scan.getStartRow(); @@ -490,15 +489,35 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } List> parallelScans = Lists.newArrayListWithExpectedSize(stopIndex - regionIndex + 1); - byte[] currentKey = startKey; - int guideIndex = currentKey.length == 0 ? 0 : getIndexContainingInclusive(gps, currentKey); - int gpsSize = gps.size(); + ImmutableBytesWritable currentKey = new ImmutableBytesWritable(startKey); + + int gpsSize = gps.getGuidePostsCount(); int estGuidepostsPerRegion = gpsSize == 0 ? 1 : gpsSize / regionLocations.size() + 1; int keyOffset = 0; + ImmutableBytesWritable currentGuidePost = ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY; List scans = Lists.newArrayListWithExpectedSize(estGuidepostsPerRegion); + ImmutableBytesWritable guidePosts = gps.getGuidePosts(); + ByteArrayInputStream stream = null; + DataInput input = null; + PrefixByteDecoder decoder = null; + int guideIndex = 0; + if (gpsSize > 0) { + stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength()); + input = new DataInputStream(stream); + decoder = new PrefixByteDecoder(gps.getMaxLength()); + try { + while (currentKey.compareTo(currentGuidePost = PrefixByteCodec.decode(decoder, input)) >= 0 + && currentKey.getLength() != 0) { + guideIndex++; + } + } catch (EOFException e) {} + } + byte[] currentKeyBytes = currentKey.copyBytes(); + // Merge bisect with guideposts for all but the last region while (regionIndex <= stopIndex) { - byte[] currentGuidePost, endKey, endRegionKey = EMPTY_BYTE_ARRAY; + byte[] currentGuidePostBytes = currentGuidePost.copyBytes(); + byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY; if (regionIndex == stopIndex) { endKey = stopKey; } else { @@ -510,14 +529,18 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw endRegionKey = regionInfo.getEndKey(); keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey); } - while (guideIndex < gpsSize - && (Bytes.compareTo(currentGuidePost = gps.get(guideIndex), endKey) <= 0 || endKey.length == 0)) { - Scan newScan = scanRanges.intersectScan(scan, currentKey, currentGuidePost, keyOffset, false); - scans = addNewScan(parallelScans, scans, newScan, currentGuidePost, false, regionLocation); - currentKey = currentGuidePost; - guideIndex++; - } - Scan newScan = scanRanges.intersectScan(scan, currentKey, endKey, keyOffset, true); + try { + while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) { + Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset, + false); + scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); + currentKeyBytes = currentGuidePost.copyBytes(); + currentGuidePost = PrefixByteCodec.decode(decoder, input); + currentGuidePostBytes = currentGuidePost.copyBytes(); + guideIndex++; + } + } catch (EOFException e) {} + Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, endKey, keyOffset, true); if (isLocalIndex) { if (newScan != null) { newScan.setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey); @@ -526,15 +549,17 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } } scans = addNewScan(parallelScans, scans, newScan, endKey, true, regionLocation); - currentKey = endKey; + currentKeyBytes = endKey; regionIndex++; } if (!scans.isEmpty()) { // Add any remaining scans parallelScans.add(scans); } + PrefixByteCodec.close(stream); return parallelScans; } + public static List reverseIfNecessary(List list, boolean reverse) { if (!reverse) { return list; @@ -805,8 +830,9 @@ public void explain(List planSteps) { explain(buf.toString(),planSteps); } - @Override - public String toString() { - return "ResultIterators [name=" + getName() + ",id=" + scanId + ",scans=" + scans + "]"; - } + @Override + public String toString() { + return "ResultIterators [name=" + getName() + ",id=" + scanId + ",scans=" + scans + "]"; + } + } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java index 208e874b81e..82bf31ac53f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java @@ -913,11 +913,11 @@ public PMetaData addFunction(PFunction function) throws SQLException { } @Override - public PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) + public PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException { - metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); //Cascade through to connectionQueryServices too - getQueryServices().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + getQueryServices().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); return metaData; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index 869ba19b12c..fabd9498cfb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -285,6 +285,9 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { public static final String TRANSACTIONAL = "TRANSACTIONAL"; public static final byte[] TRANSACTIONAL_BYTES = Bytes.toBytes(TRANSACTIONAL); + public static final String UPDATE_CACHE_FREQUENCY = "UPDATE_CACHE_FREQUENCY"; + public static final byte[] UPDATE_CACHE_FREQUENCY_BYTES = Bytes.toBytes(UPDATE_CACHE_FREQUENCY); + public static final String ASYNC_CREATED_DATE = "ASYNC_CREATED_DATE"; private final PhoenixConnection connection; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index 6bb57224b26..b54ccd5734d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -331,7 +331,7 @@ public Integer call() throws SQLException { MutationPlan plan = stmt.compilePlan(PhoenixStatement.this, Sequence.ValueOp.VALIDATE_SEQUENCE); if (plan.getTargetRef() != null && plan.getTargetRef().getTable() != null && plan.getTargetRef().getTable().isTransactional()) { state.startTransaction(); - state.addReadFence(plan.getTargetRef().getTable()); + state.addDMLFence(plan.getTargetRef().getTable()); } Iterator tableRefs = plan.getSourceRefs().iterator(); state.sendUncommitted(tableRefs); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java index 2df76fcb261..51ac79550cd 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; @@ -35,6 +36,7 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; import org.apache.phoenix.iterate.ConcatResultIterator; import org.apache.phoenix.iterate.LookAheadResultIterator; import org.apache.phoenix.iterate.PeekingResultIterator; @@ -48,6 +50,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.Lists; +import org.apache.phoenix.query.ConnectionQueryServices; /** * {@link RecordReader} implementation that iterates over the the records. @@ -107,8 +110,16 @@ public void initialize(InputSplit split, TaskAttemptContext context) throws IOEx StatementContext ctx = queryPlan.getContext(); ReadMetricQueue readMetrics = ctx.getReadMetricsQueue(); String tableName = queryPlan.getTableRef().getTable().getPhysicalName().getString(); + + // Clear the table region boundary cache to make sure long running jobs stay up to date + byte[] tableNameBytes = queryPlan.getTableRef().getTable().getPhysicalName().getBytes(); + ConnectionQueryServices services = queryPlan.getContext().getConnection().getQueryServices(); + services.clearTableRegionCache(tableNameBytes); + long renewScannerLeaseThreshold = queryPlan.getContext().getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds(); for (Scan scan : scans) { + // For MR, skip the region boundary check exception if we encounter a split. ref: PHOENIX-2599 + scan.setAttribute(BaseScannerRegionObserver.SKIP_REGION_BOUNDARY_CHECK, Bytes.toBytes(true)); final TableResultIterator tableResultIterator = new TableResultIterator(queryPlan.getContext().getConnection().getMutationState(), queryPlan.getTableRef(), scan, readMetrics.allotMetric(SCAN_BYTES, tableName), renewScannerLeaseThreshold); PeekingResultIterator peekingResultIterator = LookAheadResultIterator.wrap(tableResultIterator); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java index c5ff3c6abf7..adc3c2d1464 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java @@ -285,7 +285,7 @@ private static QueryPlan addPlan(PhoenixStatement statement, SelectStatement sel aliasedNodes.add(FACTORY.aliasedNode(null, indexColNode)); nodes.add(new ColumnParseNode(null, '"' + column.getName().getString() + '"')); } - SelectStatement innerSelect = FACTORY.select(indexSelect.getFrom(), indexSelect.getHint(), false, aliasedNodes, where, null, null, null, indexSelect.getLimit(), indexSelect.getBindCount(), false, indexSelect.hasSequence(), Collections.emptyList(), indexSelect.getUdfParseNodes()); + SelectStatement innerSelect = FACTORY.select(indexSelect.getFrom(), indexSelect.getHint(), false, aliasedNodes, where, null, null, null, null, indexSelect.getBindCount(), false, indexSelect.hasSequence(), Collections.emptyList(), indexSelect.getUdfParseNodes()); ParseNode outerWhere = FACTORY.in(nodes.size() == 1 ? nodes.get(0) : FACTORY.rowValueConstructor(nodes), FACTORY.subquery(innerSelect, false), false, true); ParseNode extractedCondition = whereRewriter.getExtractedCondition(); if (extractedCondition != null) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 8ef31617765..a246e63c323 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -630,12 +630,14 @@ private PMetaData metaDataMutated(PName tenantId, String tableName, long tableSe } @Override - public PMetaData addColumn(final PName tenantId, final String tableName, final List columns, final long tableTimeStamp, final long tableSeqNum, final boolean isImmutableRows, final boolean isWalDisabled, final boolean isMultitenant, final boolean storeNulls, final boolean isTransactional, final long resolvedTime) throws SQLException { + public PMetaData addColumn(final PName tenantId, final String tableName, final List columns, final long tableTimeStamp, + final long tableSeqNum, final boolean isImmutableRows, final boolean isWalDisabled, final boolean isMultitenant, + final boolean storeNulls, final boolean isTransactional, final long updateCacheFrequency, final long resolvedTime) throws SQLException { return metaDataMutated(tenantId, tableName, tableSeqNum, new Mutator() { @Override public PMetaData mutate(PMetaData metaData) throws SQLException { try { - return metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + return metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); } catch (TableNotFoundException e) { // The DROP TABLE may have been processed first, so just ignore. return metaData; @@ -2410,17 +2412,20 @@ public Void call() throws Exception { } if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0) { columnsToAdd = PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP + " " + PBoolean.INSTANCE.getSqlTypeName(); - metaConnection = addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd, false); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd); } if(currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) { - columnsToAdd = PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName(); - metaConnection = addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0, columnsToAdd, false); - // Drop old stats table so that new stats - // table + // Add these columns one at a time, each with different timestamps so that if folks have + // run the upgrade code already for a snapshot, we'll still enter this block (and do the + // parts we haven't yet done). + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2, + PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName()); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, + PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " " + PLong.INSTANCE.getSqlTypeName()); + // Drop old stats table so that new stats table is created metaConnection = dropStatsTable(metaConnection, - MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1); + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0); } } @@ -2525,7 +2530,7 @@ public Void call() throws Exception { private PhoenixConnection dropStatsTable(PhoenixConnection oldMetaConnection, long timestamp) throws SQLException, IOException { Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo()); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp-1)); PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props); SQLException sqlE = null; boolean wasCommit = metaConnection.getAutoCommit(); @@ -2555,23 +2560,9 @@ private PhoenixConnection dropStatsTable(PhoenixConnection oldMetaConnection, lo } } - HBaseAdmin admin = null; - try { - admin = getAdmin(); - admin.disableTable(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES); - try { - admin.deleteTable(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES); - } catch (org.apache.hadoop.hbase.TableNotFoundException e) { - logger.debug("Stats table was not found during upgrade!!"); - } - } finally { - if (admin != null) - admin.close(); - } oldMetaConnection = metaConnection; props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo()); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, - Long.toString(MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp)); try { metaConnection = new PhoenixConnection(oldMetaConnection, ConnectionQueryServicesImpl.this, props); } finally { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java index 199b010f94e..6cfb382c071 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -177,8 +177,8 @@ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) t @Override public PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, - long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException { - return metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException { + return metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java index 84f3e7497bc..9b721f8b265 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java @@ -87,8 +87,8 @@ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) t @Override public PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, - long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException { - return getDelegate().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException { + return getDelegate().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java b/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java index 8e7a70d5a94..753b17261a1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java @@ -38,7 +38,7 @@ public interface MetaDataMutated { PMetaData addTable(PTable table, long resolvedTime) throws SQLException; PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException; PMetaData removeTable(PName tenantId, String tableName, String parentTableName, long tableTimeStamp) throws SQLException; - PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException; + PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException; PMetaData removeColumn(PName tenantId, String tableName, List columnsToRemove, long tableTimeStamp, long tableSeqNum, long resolvedTime) throws SQLException; PMetaData addFunction(PFunction function) throws SQLException; PMetaData removeFunction(PName tenantId, String function, long functionTimeStamp) throws SQLException; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index c1cb0c0b79a..63d4e072b5f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -39,9 +39,9 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; @@ -96,6 +96,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; @@ -257,6 +258,7 @@ public enum JoinType {INNER, LEFT_OUTER} // Column metadata (will be null for table row) IS_ROW_TIMESTAMP + " BOOLEAN, " + TRANSACTIONAL + " BOOLEAN," + + UPDATE_CACHE_FREQUENCY + " BIGINT," + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" + HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" + diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java index 7fb90a1d1b9..e7bf961875a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java @@ -242,6 +242,7 @@ public boolean isTransactional() { return delegate.isTransactional(); } + @Override public int getBaseColumnCount() { return delegate.getBaseColumnCount(); } @@ -260,4 +261,9 @@ public int getRowTimestampColPos() { public String toString() { return delegate.toString(); } + + @Override + public long getUpdateCacheFrequency() { + return delegate.getUpdateCacheFrequency(); + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 995be86db77..0b446b397a7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -22,7 +22,7 @@ import static com.google.common.collect.Sets.newLinkedHashSetWithExpectedSize; import static org.apache.hadoop.hbase.HColumnDescriptor.TTL; import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.ANALYZE_TABLE; -import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC; +import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB; import static org.apache.phoenix.exception.SQLExceptionCode.INSUFFICIENT_MULTI_TENANT_COLUMNS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE; @@ -76,6 +76,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; @@ -113,8 +114,6 @@ import java.util.Properties; import java.util.Set; -import co.cask.tephra.TxConstants; - import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -217,6 +216,8 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Ints; +import co.cask.tephra.TxConstants; + public class MetaDataClient { private static final Logger logger = LoggerFactory.getLogger(MetaDataClient.class); @@ -250,8 +251,9 @@ public class MetaDataClient { INDEX_TYPE + "," + STORE_NULLS + "," + BASE_COLUMN_COUNT + "," + - TRANSACTIONAL + - ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + TRANSACTIONAL + "," + + UPDATE_CACHE_FREQUENCY + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static final String CREATE_LINK = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TENANT_ID + "," + @@ -441,7 +443,7 @@ private long getClientTimeStamp() { } private long getCurrentScn() { - Long scn = connection.getSCN(); + Long scn = connection.getSCN(); long currentScn = scn == null ? HConstants.LATEST_TIMESTAMP : scn; return currentScn; } @@ -465,20 +467,20 @@ private MetaDataMutationResult updateCache(PName tenantId, String schemaName, St } boolean defaultTransactional = connection.getQueryServices().getProps().getBoolean( - QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, - QueryServicesOptions.DEFAULT_TRANSACTIONAL); + QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, + QueryServicesOptions.DEFAULT_TRANSACTIONAL); // start a txn if all table are transactional by default or if we found the table in the cache and it is transactional // TODO if system tables become transactional remove the check boolean isTransactional = defaultTransactional || (table!=null && table.isTransactional()); if (!systemTable && isTransactional && !connection.getMutationState().isTransactionStarted()) { - connection.getMutationState().startTransaction(); + connection.getMutationState().startTransaction(); } resolvedTimestamp = resolvedTimestamp==null ? TransactionUtil.getResolvedTimestamp(connection, isTransactional, HConstants.LATEST_TIMESTAMP) : resolvedTimestamp; // Do not make rpc to getTable if // 1. table is a system table // 2. table was already resolved as of that timestamp - if (table != null && !alwaysHitServer - && (systemTable || resolvedTimestamp == tableResolvedTimestamp)) { + if (table != null && !alwaysHitServer + && (systemTable || resolvedTimestamp == tableResolvedTimestamp || connection.getMetaDataCache().getAge(tableRef) < table.getUpdateCacheFrequency())) { return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, QueryConstants.UNSET_TIMESTAMP, table); } @@ -490,11 +492,11 @@ private MetaDataMutationResult updateCache(PName tenantId, String schemaName, St final byte[] schemaBytes = PVarchar.INSTANCE.toBytes(schemaName); final byte[] tableBytes = PVarchar.INSTANCE.toBytes(tableName); ConnectionQueryServices queryServices = connection.getQueryServices(); - result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, resolvedTimestamp); - // if the table was assumed to be transactional, but is actually not transactional then re-resolve as of the right timestamp (and vice versa) - if (table==null && result.getTable()!=null && result.getTable().isTransactional()!=isTransactional) { - result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, TransactionUtil.getResolvedTimestamp(connection, result.getTable().isTransactional(), HConstants.LATEST_TIMESTAMP)); - } + result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, resolvedTimestamp); + // if the table was assumed to be transactional, but is actually not transactional then re-resolve as of the right timestamp (and vice versa) + if (table==null && result.getTable()!=null && result.getTable().isTransactional()!=isTransactional) { + result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, TransactionUtil.getResolvedTimestamp(connection, result.getTable().isTransactional(), HConstants.LATEST_TIMESTAMP)); + } if (SYSTEM_CATALOG_SCHEMA.equals(schemaName)) { return result; @@ -520,18 +522,18 @@ private MetaDataMutationResult updateCache(PName tenantId, String schemaName, St if (table != null) { // Ensures that table in result is set to table found in our cache. if (code == MutationCode.TABLE_ALREADY_EXISTS) { - result.setTable(table); - // Although this table is up-to-date, the parent table may not be. - // In this case, we update the parent table which may in turn pull - // in indexes to add to this table. - long resolvedTime = TransactionUtil.getResolvedTime(connection, result); - if (addIndexesFromPhysicalTable(result, resolvedTimestamp)) { - connection.addTable(result.getTable(), resolvedTime); - } - else { - // if we aren't adding the table, we still need to update the resolved time of the table - connection.updateResolvedTimestamp(table, resolvedTime); - } + result.setTable(table); + // Although this table is up-to-date, the parent table may not be. + // In this case, we update the parent table which may in turn pull + // in indexes to add to this table. + long resolvedTime = TransactionUtil.getResolvedTime(connection, result); + if (addIndexesFromPhysicalTable(result, resolvedTimestamp)) { + connection.addTable(result.getTable(), resolvedTime); + } + else { + // if we aren't adding the table, we still need to update the resolved time of the table + connection.updateResolvedTimestamp(table, resolvedTime); + } return result; } // If table was not found at the current time stamp and we have one cached, remove it. @@ -628,12 +630,12 @@ private MetaDataMutationResult updateCache(PName tenantId, List function * @throws SQLException if the physical table cannot be found */ private boolean addIndexesFromPhysicalTable(MetaDataMutationResult result, Long resolvedTimestamp) throws SQLException { - PTable table = result.getTable(); + PTable view = result.getTable(); // If not a view or if a view directly over an HBase table, there's nothing to do - if (table.getType() != PTableType.VIEW || table.getViewType() == ViewType.MAPPED) { + if (view.getType() != PTableType.VIEW || view.getViewType() == ViewType.MAPPED) { return false; } - String physicalName = table.getPhysicalName().getString(); + String physicalName = view.getPhysicalName().getString(); String schemaName = SchemaUtil.getSchemaNameFromFullName(physicalName); String tableName = SchemaUtil.getTableNameFromFullName(physicalName); MetaDataMutationResult parentResult = updateCache(null, schemaName, tableName, false, resolvedTimestamp); @@ -644,84 +646,72 @@ private boolean addIndexesFromPhysicalTable(MetaDataMutationResult result, Long if (!result.wasUpdated() && !parentResult.wasUpdated()) { return false; } - List indexes = physicalTable.getIndexes(); - if (indexes.isEmpty()) { + List physicalTableIndexes = physicalTable.getIndexes(); + if (physicalTableIndexes.isEmpty()) { return false; } // Filter out indexes if column doesn't exist in view - List allIndexes = Lists.newArrayListWithExpectedSize(indexes.size() + table.getIndexes().size()); + List indexesToAdd = Lists.newArrayListWithExpectedSize(physicalTableIndexes.size() + view.getIndexes().size()); if (result.wasUpdated()) { // Table from server never contains inherited indexes - allIndexes.addAll(table.getIndexes()); + indexesToAdd.addAll(view.getIndexes()); } else { // Only add original ones, as inherited ones may have changed - for (PTable index : indexes) { - if (index.getViewIndexId() != null) { - allIndexes.add(index); - } - } - } - for (PTable index : indexes) { - if (index.getViewIndexId() == null) { - boolean containsAllReqdCols = true; - // Ensure that all columns required to create index - // exist in the view too (since view columns may be removed) - IndexMaintainer indexMaintainer = index.getIndexMaintainer(physicalTable, connection); - // check that the columns required for the index pk (not including the pk columns of the data table) - // are present in the view - Set indexColRefs = indexMaintainer.getIndexedColumns(); - for (ColumnReference colRef : indexColRefs) { - try { - byte[] cf= colRef.getFamily(); - byte[] cq= colRef.getQualifier(); - if (cf!=null) { - table.getColumnFamily(cf).getColumn(cq); - } - else { - table.getColumn( Bytes.toString(cq)); - } - } catch (ColumnNotFoundException e) { // Ignore this index and continue with others - containsAllReqdCols = false; - break; + for (PTable index : view.getIndexes()) { + // Original indexes will not have a view statement while inherited ones will + if (index.getViewStatement() == null) { + indexesToAdd.add(index); + } + } + } + for (PTable index : physicalTableIndexes) { + boolean containsAllReqdCols = true; + // Ensure that all columns required to create index exist in the view too, + // since view columns may be removed. + IndexMaintainer indexMaintainer = index.getIndexMaintainer(physicalTable, connection); + // Check that the columns required for the index pk are present in the view + Set indexColRefs = indexMaintainer.getIndexedColumns(); + for (ColumnReference colRef : indexColRefs) { + try { + byte[] cf= colRef.getFamily(); + byte[] cq= colRef.getQualifier(); + if (cf!=null) { + view.getColumnFamily(cf).getColumn(cq); + } + else { + view.getColumn( Bytes.toString(cq)); } + } catch (ColumnNotFoundException e) { // Ignore this index and continue with others + containsAllReqdCols = false; + break; } - // check that pk columns of the data table (which are also present in the index pk) are present in the view - List pkColumns = physicalTable.getPKColumns(); - for (int i = physicalTable.getBucketNum() == null ? 0 : 1; i < pkColumns.size(); i++) { + } + // Ensure that constant columns (i.e. columns matched in the view WHERE clause) + // all exist in the index on the physical table. + for (PColumn col : view.getColumns()) { + if (col.getViewConstant() != null) { try { - PColumn pkColumn = pkColumns.get(i); - table.getColumn(pkColumn.getName().getString()); + // It'd be possible to use a local index that doesn't have all view constants, + // but the WHERE clause for the view statement (which is added to the index below) + // would fail to compile. + String indexColumnName = IndexUtil.getIndexColumnName(col); + index.getColumn(indexColumnName); } catch (ColumnNotFoundException e) { // Ignore this index and continue with others containsAllReqdCols = false; break; } } - // Ensure that constant columns (i.e. columns matched in the view WHERE clause) - // all exist in the index on the physical table. - for (PColumn col : table.getColumns()) { - if (col.getViewConstant() != null) { - try { - // TODO: it'd be possible to use a local index that doesn't have all view constants - String indexColumnName = IndexUtil.getIndexColumnName(col); - index.getColumn(indexColumnName); - } catch (ColumnNotFoundException e) { // Ignore this index and continue with others - containsAllReqdCols = false; - break; - } - } - } - if (containsAllReqdCols) { - // Tack on view statement to index to get proper filtering for view - String viewStatement = IndexUtil.rewriteViewStatement(connection, index, physicalTable, table.getViewStatement()); - index = PTableImpl.makePTable(index, viewStatement); - allIndexes.add(index); - } + } + if (containsAllReqdCols) { + // Tack on view statement to index to get proper filtering for view + String viewStatement = IndexUtil.rewriteViewStatement(connection, index, physicalTable, view.getViewStatement()); + index = PTableImpl.makePTable(index, viewStatement); + indexesToAdd.add(index); } } - PTable allIndexesTable = PTableImpl.makePTable(table, table.getTimeStamp(), allIndexes); + PTable allIndexesTable = PTableImpl.makePTable(view, view.getTimeStamp(), indexesToAdd); result.setTable(allIndexesTable); return true; } - private void addColumnMutation(String schemaName, String tableName, PColumn column, PreparedStatement colUpsert, String parentTableName, String pkName, Short keySeq, boolean isSalted) throws SQLException { colUpsert.setString(1, connection.getTenantId() == null ? null : connection.getTenantId().getString()); colUpsert.setString(2, schemaName); @@ -955,8 +945,8 @@ public boolean isTransactional() { Scan scan = plan.getContext().getScan(); scan.setCacheBlocks(false); scan.setAttribute(ANALYZE_TABLE, TRUE_BYTES); - boolean runUpdateStatsAsync = props.getBoolean(RUN_UPDATE_STATS_ASYNC, DEFAULT_RUN_UPDATE_STATS_ASYNC); - scan.setAttribute(RUN_UPDATE_STATS_ASYNC, runUpdateStatsAsync ? TRUE_BYTES : FALSE_BYTES); + boolean runUpdateStatsAsync = props.getBoolean(QueryServices.RUN_UPDATE_STATS_ASYNC, DEFAULT_RUN_UPDATE_STATS_ASYNC); + scan.setAttribute(RUN_UPDATE_STATS_ASYNC_ATTRIB, runUpdateStatsAsync ? TRUE_BYTES : FALSE_BYTES); if (statsProps != null) { Object gp_width = statsProps.get(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB); if (gp_width != null) { @@ -1084,7 +1074,7 @@ private MutationState buildIndex(PTable index, TableRef dataTableRef) throws SQL @Override public MutationState execute() throws SQLException { - connection.getMutationState().commitWriteFence(dataTable); + connection.getMutationState().commitDDLFence(dataTable); Cell kv = plan.iterator().next().getValue(0); ImmutableBytesWritable tmpPtr = new ImmutableBytesWritable(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()); // A single Cell will be returned with the count(*) - we decode that here @@ -1224,14 +1214,14 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits List pkColumns = dataTable.getPKColumns(); Set unusedPkColumns; if (dataTable.getBucketNum() != null) { // Ignore SALT column - unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size()-1); - posOffset++; + unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size()-1); + posOffset++; } else { - unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size()); + unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size()); } for (int i = posOffset; i < pkColumns.size(); i++) { PColumn column = pkColumns.get(i); - unusedPkColumns.add(new RowKeyColumnExpression(column, new RowKeyValueAccessor(pkColumns, i), "\""+column.getName().getString()+"\"")); + unusedPkColumns.add(new RowKeyColumnExpression(column, new RowKeyValueAccessor(pkColumns, i), "\""+column.getName().getString()+"\"")); } List allPkColumns = Lists.newArrayListWithExpectedSize(unusedPkColumns.size()); List columnDefs = Lists.newArrayListWithExpectedSize(includedColumns.size() + indexParseNodeAndSortOrderList.size()); @@ -1240,7 +1230,7 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits // Add tenant ID column as first column in index PColumn col = dataTable.getPKColumns().get(posOffset); RowKeyColumnExpression columnExpression = new RowKeyColumnExpression(col, new RowKeyValueAccessor(pkColumns, posOffset), col.getName().getString()); - unusedPkColumns.remove(columnExpression); + unusedPkColumns.remove(columnExpression); PDataType dataType = IndexUtil.getIndexColumnDataType(col); ColumnName colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(col)); allPkColumns.add(new ColumnDefInPkConstraint(colName, col.getSortOrder(), false)); @@ -1265,7 +1255,7 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits IndexExpressionCompiler expressionIndexCompiler = new IndexExpressionCompiler(context); Set indexedColumnNames = Sets.newHashSetWithExpectedSize(indexParseNodeAndSortOrderList.size()); for (Pair pair : indexParseNodeAndSortOrderList) { - ParseNode parseNode = pair.getFirst(); + ParseNode parseNode = pair.getFirst(); // normalize the parse node parseNode = StatementNormalizer.normalize(parseNode, resolver); // compile the parseNode to get an expression @@ -1291,22 +1281,22 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits ColumnName colName = null; ColumnRef colRef = expressionIndexCompiler.getColumnRef(); - boolean isRowTimestamp = false; + boolean isRowTimestamp = false; if (colRef!=null) { - // if this is a regular column - PColumn column = colRef.getColumn(); - String columnFamilyName = column.getFamilyName()!=null ? column.getFamilyName().getString() : null; - colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(columnFamilyName, column.getName().getString())); - isRowTimestamp = column.isRowTimestamp(); - } - else { - // if this is an expression - // TODO column names cannot have double quotes, remove this once this PHOENIX-1621 is fixed - String name = expressionStr.replaceAll("\"", "'"); + // if this is a regular column + PColumn column = colRef.getColumn(); + String columnFamilyName = column.getFamilyName()!=null ? column.getFamilyName().getString() : null; + colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(columnFamilyName, column.getName().getString())); + isRowTimestamp = column.isRowTimestamp(); + } + else { + // if this is an expression + // TODO column names cannot have double quotes, remove this once this PHOENIX-1621 is fixed + String name = expressionStr.replaceAll("\"", "'"); colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(null, name)); - } - indexedColumnNames.add(colName); - PDataType dataType = IndexUtil.getIndexColumnDataType(expression.isNullable(), expression.getDataType()); + } + indexedColumnNames.add(colName); + PDataType dataType = IndexUtil.getIndexColumnDataType(expression.isNullable(), expression.getDataType()); allPkColumns.add(new ColumnDefInPkConstraint(colName, pair.getSecond(), isRowTimestamp)); columnDefs.add(FACTORY.columnDef(colName, dataType.getSqlTypeName(), expression.isNullable(), expression.getMaxLength(), expression.getScale(), false, pair.getSecond(), expressionStr, isRowTimestamp)); } @@ -1393,6 +1383,7 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits return new MutationState(0,connection); } + if (logger.isInfoEnabled()) logger.info("Created index " + table.getName().getString() + " at " + table.getTimeStamp()); // In async process, we return immediately as the MR job needs to be triggered . if(statement.isAsync()) { return new MutationState(0, connection); @@ -1593,7 +1584,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli // table instead of only a view? We don't have anywhere to put the link // from the table to the index, though. if (indexType == IndexType.LOCAL || (parent.getType() == PTableType.VIEW && parent.getViewType() != ViewType.MAPPED)) { - PName physicalName = parent.getPhysicalName(); + PName physicalName = parent.getPhysicalName(); saltBucketNum = parent.getBucketNum(); addSaltColumn = (saltBucketNum != null && indexType != IndexType.LOCAL); defaultFamilyName = parent.getDefaultFamilyName() == null ? null : parent.getDefaultFamilyName().getString(); @@ -1661,7 +1652,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli // Although unusual, it's possible to set a mapped VIEW as having immutable rows. // This tells Phoenix that you're managing the index maintenance yourself. if (tableType != PTableType.INDEX && (tableType != PTableType.VIEW || viewType == ViewType.MAPPED)) { - Boolean isImmutableRowsProp = (Boolean) tableProps.get(PTable.IS_IMMUTABLE_ROWS_PROP_NAME); + Boolean isImmutableRowsProp = (Boolean) TableProperty.IMMUTABLE_ROWS.getValue(tableProps); if (isImmutableRowsProp == null) { isImmutableRows = connection.getQueryServices().getProps().getBoolean(QueryServices.IMMUTABLE_ROWS_ATTRIB, QueryServicesOptions.DEFAULT_IMMUTABLE_ROWS); } else { @@ -1671,7 +1662,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli // Can't set any of these on views or shared indexes on views if (tableType != PTableType.VIEW && indexId == null) { - saltBucketNum = (Integer) tableProps.get(PhoenixDatabaseMetaData.SALT_BUCKETS); + saltBucketNum = (Integer) TableProperty.SALT_BUCKETS.getValue(tableProps); if (saltBucketNum != null) { if (saltBucketNum < 0 || saltBucketNum > SaltingUtil.MAX_BUCKET_NUM) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_BUCKET_NUM).build().buildException(); @@ -1692,16 +1683,21 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli if (tableType != PTableType.INDEX && (tableType != PTableType.VIEW || viewType == ViewType.MAPPED)) { Boolean multiTenantProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.MULTI_TENANT); multiTenant = Boolean.TRUE.equals(multiTenantProp); - defaultFamilyName = (String)tableProps.get(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME); + defaultFamilyName = (String)TableProperty.DEFAULT_COLUMN_FAMILY.getValue(tableProps); } boolean disableWAL = false; - Boolean disableWALProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.DISABLE_WAL); + Boolean disableWALProp = (Boolean) TableProperty.DISABLE_WAL.getValue(tableProps); if (disableWALProp != null) { disableWAL = disableWALProp; } + long updateCacheFrequency = 0; + Long updateCacheFrequencyProp = (Long) TableProperty.UPDATE_CACHE_FREQUENCY.getValue(tableProps); + if (updateCacheFrequencyProp != null) { + updateCacheFrequency = updateCacheFrequencyProp; + } - Boolean storeNullsProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.STORE_NULLS); + Boolean storeNullsProp = (Boolean) TableProperty.STORE_NULLS.getValue(tableProps); if (storeNullsProp == null) { if (parent == null) { storeNulls = connection.getQueryServices().getProps().getBoolean( @@ -1712,7 +1708,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } else { storeNulls = storeNullsProp; } - Boolean transactionalProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.TRANSACTIONAL); + Boolean transactionalProp = (Boolean) TableProperty.TRANSACTIONAL.getValue(tableProps); if (transactionalProp != null && parent != null) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL) .setSchemaName(schemaName).setTableName(tableName) @@ -1728,8 +1724,8 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } } boolean transactionsEnabled = connection.getQueryServices().getProps().getBoolean( - QueryServices.TRANSACTIONS_ENABLED, - QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED); + QueryServices.TRANSACTIONS_ENABLED, + QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED); // can't create a transactional table if transactions are not enabled if (!transactionsEnabled && transactional) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED) @@ -1738,12 +1734,13 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } // can't create a transactional table if it has a row timestamp column if (pkConstraint.getNumColumnsWithRowTimestamp()>0 && transactional) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP) + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP) .setSchemaName(schemaName).setTableName(tableName) .build().buildException(); } - + // Put potentially inferred value into tableProps as it's used by the createTable call below + // to determine which coprocessors to install on the new table. tableProps.put(PhoenixDatabaseMetaData.TRANSACTIONAL, transactional); if (transactional) { // If TTL set, use Tephra TTL property name instead @@ -1762,16 +1759,16 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli // on the server, but this can mess up our secondary index code as the changes get // committed prior to the // maintenance code being able to see the prior state to update the rows correctly. - if (Boolean.FALSE.equals(storeNullsProp)) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL) - .setSchemaName(schemaName).setTableName(tableName) - .build().buildException(); - } - // Force STORE_NULLS to true when transactional as Tephra cannot deal with column deletes - storeNulls = true; - tableProps.put(PhoenixDatabaseMetaData.STORE_NULLS, Boolean.TRUE); - - if (!sharedTable) { + if (Boolean.FALSE.equals(storeNullsProp)) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL) + .setSchemaName(schemaName).setTableName(tableName) + .build().buildException(); + } + // Force STORE_NULLS to true when transactional as Tephra cannot deal with column deletes + storeNulls = true; + tableProps.put(PhoenixDatabaseMetaData.STORE_NULLS, Boolean.TRUE); + + if (!sharedTable) { Integer maxVersionsProp = (Integer) commonFamilyProps.get(HConstants.VERSIONS); if (maxVersionsProp == null) { if (parent != null) { @@ -1787,7 +1784,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } commonFamilyProps.put(HConstants.VERSIONS, maxVersionsProp); } - } + } } timestamp = timestamp==null ? TransactionUtil.getTableTimestamp(connection, transactional) : timestamp; @@ -2016,8 +2013,8 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } if (familyNames.isEmpty()) { - //if there are no family names, use the default column family name. This also takes care of the case when - //the table ddl has only PK cols present (which means familyNames is empty). + //if there are no family names, use the default column family name. This also takes care of the case when + //the table ddl has only PK cols present (which means familyNames is empty). byte[] cf = defaultFamilyName == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : Bytes.toBytes(defaultFamilyName); familyPropList.add(new Pair>(cf, commonFamilyProps)); } @@ -2032,7 +2029,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli Collections.emptyList(), isImmutableRows, Collections.emptyList(), defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), null, - Boolean.TRUE.equals(disableWAL), false, false, null, indexId, indexType, true, false); + Boolean.TRUE.equals(disableWAL), false, false, null, indexId, indexType, true, false, 0); connection.addTable(table, MetaDataProtocol.MIN_TABLE_TIMESTAMP); } else if (tableType == PTableType.INDEX && indexId == null) { if (tableProps.get(HTableDescriptor.MAX_FILESIZE) == null) { @@ -2135,6 +2132,7 @@ public boolean isViewReferenced() { tableUpsert.setInt(20, BASE_TABLE_BASE_COLUMN_COUNT); } tableUpsert.setBoolean(21, transactional); + tableUpsert.setLong(22, updateCacheFrequency); tableUpsert.execute(); if (asyncCreatedDate != null) { @@ -2197,7 +2195,7 @@ public boolean isViewReferenced() { PTable.INITIAL_SEQ_NUM, pkName == null ? null : PNameFactory.newName(pkName), saltBucketNum, columns, dataTableName == null ? null : newSchemaName, dataTableName == null ? null : PNameFactory.newName(dataTableName), Collections.emptyList(), isImmutableRows, physicalNames, defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), viewStatement, Boolean.TRUE.equals(disableWAL), multiTenant, storeNulls, viewType, - indexId, indexType, rowKeyOrderOptimizable, transactional); + indexId, indexType, rowKeyOrderOptimizable, transactional, updateCacheFrequency); result = new MetaDataMutationResult(code, result.getMutationTime(), table, true); addTableToCache(result); return table; @@ -2353,7 +2351,6 @@ private MutationState dropTable(String schemaName, String tableName, String pare MetaDataMutationResult result = connection.getQueryServices().dropTable(tableMetaData, tableType, cascade); MutationCode code = result.getMutationCode(); PTable table = result.getTable(); - boolean transactional = table!=null && table.isTransactional(); switch (code) { case TABLE_NOT_FOUND: if (!ifExists) { throw new TableNotFoundException(schemaName, tableName); } @@ -2365,54 +2362,64 @@ private MutationState dropTable(String schemaName, String tableName, String pare .setSchemaName(schemaName).setTableName(tableName).build().buildException(); default: - connection.removeTable(tenantId, SchemaUtil.getTableName(schemaName, tableName), parentTableName, result.getMutationTime()); + connection.removeTable(tenantId, SchemaUtil.getTableName(schemaName, tableName), parentTableName, result.getMutationTime()); - if (result.getTable() != null && tableType != PTableType.VIEW) { - connection.setAutoCommit(true); - boolean dropMetaData = result.getTable().getViewIndexId() == null && - connection.getQueryServices().getProps().getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA); + if (result.getTable() != null) { + boolean dropMetaData = false; long ts = (scn == null ? result.getMutationTime() : scn); - // Create empty table and schema - they're only used to get the name from - // PName name, PTableType type, long timeStamp, long sequenceNumber, List columns List tableRefs = Lists.newArrayListWithExpectedSize(2 + table.getIndexes().size()); - // All multi-tenant tables have a view index table, so no need to check in that case - if (tableType == PTableType.TABLE - && (table.isMultiTenant() || hasViewIndexTable || hasLocalIndexTable)) { - - MetaDataUtil.deleteViewIndexSequences(connection, table.getPhysicalName()); - if (hasViewIndexTable) { - String viewIndexSchemaName = null; - String viewIndexTableName = null; - if (schemaName != null) { - viewIndexSchemaName = MetaDataUtil.getViewIndexTableName(schemaName); - viewIndexTableName = tableName; - } else { - viewIndexTableName = MetaDataUtil.getViewIndexTableName(tableName); + connection.setAutoCommit(true); + if (tableType == PTableType.VIEW) { + for (PTable index : table.getIndexes()) { + // Don't drop index data from indexes on physical table + if (index.getViewStatement() == null) { + tableRefs.add(new TableRef(null, index, ts, false)); } - PTable viewIndexTable = new PTableImpl(null, viewIndexSchemaName, viewIndexTableName, ts, - table.getColumnFamilies()); - tableRefs.add(new TableRef(null, viewIndexTable, ts, false)); } - if (hasLocalIndexTable) { - String localIndexSchemaName = null; - String localIndexTableName = null; - if (schemaName != null) { - localIndexSchemaName = MetaDataUtil.getLocalIndexTableName(schemaName); - localIndexTableName = tableName; - } else { - localIndexTableName = MetaDataUtil.getLocalIndexTableName(tableName); + } else { + dropMetaData = result.getTable().getViewIndexId() == null && + connection.getQueryServices().getProps().getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA); + // Create empty table and schema - they're only used to get the name from + // PName name, PTableType type, long timeStamp, long sequenceNumber, List columns + // All multi-tenant tables have a view index table, so no need to check in that case + if (tableType == PTableType.TABLE + && (table.isMultiTenant() || hasViewIndexTable || hasLocalIndexTable)) { + + MetaDataUtil.deleteViewIndexSequences(connection, table.getPhysicalName()); + if (hasViewIndexTable) { + String viewIndexSchemaName = null; + String viewIndexTableName = null; + if (schemaName != null) { + viewIndexSchemaName = MetaDataUtil.getViewIndexTableName(schemaName); + viewIndexTableName = tableName; + } else { + viewIndexTableName = MetaDataUtil.getViewIndexTableName(tableName); + } + PTable viewIndexTable = new PTableImpl(null, viewIndexSchemaName, viewIndexTableName, ts, + table.getColumnFamilies()); + tableRefs.add(new TableRef(null, viewIndexTable, ts, false)); + } + if (hasLocalIndexTable) { + String localIndexSchemaName = null; + String localIndexTableName = null; + if (schemaName != null) { + localIndexSchemaName = MetaDataUtil.getLocalIndexTableName(schemaName); + localIndexTableName = tableName; + } else { + localIndexTableName = MetaDataUtil.getLocalIndexTableName(tableName); + } + PTable localIndexTable = new PTableImpl(null, localIndexSchemaName, localIndexTableName, + ts, Collections. emptyList()); + tableRefs.add(new TableRef(null, localIndexTable, ts, false)); } - PTable localIndexTable = new PTableImpl(null, localIndexSchemaName, localIndexTableName, - ts, Collections. emptyList()); - tableRefs.add(new TableRef(null, localIndexTable, ts, false)); } + tableRefs.add(new TableRef(null, table, ts, false)); + // TODO: Let the standard mutable secondary index maintenance handle this? + for (PTable index : table.getIndexes()) { + tableRefs.add(new TableRef(null, index, ts, false)); + } + deleteFromStatsTable(tableRefs, ts); } - tableRefs.add(new TableRef(null, table, ts, false)); - // TODO: Let the standard mutable secondary index maintenance handle this? - for (PTable index : table.getIndexes()) { - tableRefs.add(new TableRef(null, index, ts, false)); - } - deleteFromStatsTable(tableRefs, ts); if (!dropMetaData) { MutationPlan plan = new PostDDLCompiler(connection).compile(tableRefs, null, null, Collections. emptyList(), ts); @@ -2511,12 +2518,12 @@ private MutationCode processMutationResult(String schemaName, String tableName, return mutationCode; } - private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional) throws SQLException { - return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, null, null, null, null); + private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional, Long updateCacheFrequency) throws SQLException { + return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null); } private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, - Boolean isTransactional, Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls) + Boolean isTransactional, Long updateCacheFrequency, Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls) throws SQLException { String schemaName = table.getSchemaName().getString(); String tableName = table.getTableName().getString(); @@ -2551,6 +2558,9 @@ private long incrementTableSeqNum(PTable table, PTableType expectedType, int col if (isTransactional != null) { mutateBooleanProperty(tenantId, schemaName, tableName, TRANSACTIONAL, isTransactional); } + if (updateCacheFrequency != null) { + mutateLongProperty(tenantId, schemaName, tableName, UPDATE_CACHE_FREQUENCY, updateCacheFrequency); + } return seqNum; } @@ -2570,6 +2580,22 @@ private void mutateBooleanProperty(String tenantId, String schemaName, String ta tableBoolUpsert.execute(); } + private void mutateLongProperty(String tenantId, String schemaName, String tableName, + String propertyName, long propertyValue) throws SQLException { + String updatePropertySql = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + + TENANT_ID + "," + + TABLE_SCHEM + "," + + TABLE_NAME + "," + + propertyName + + ") VALUES (?, ?, ?, ?)"; + PreparedStatement tableBoolUpsert = connection.prepareStatement(updatePropertySql); + tableBoolUpsert.setString(1, tenantId); + tableBoolUpsert.setString(2, schemaName); + tableBoolUpsert.setString(3, tableName); + tableBoolUpsert.setLong(4, propertyValue); + tableBoolUpsert.execute(); + } + public MutationState addColumn(AddColumnStatement statement) throws SQLException { connection.rollback(); boolean wasAutoCommit = connection.getAutoCommit(); @@ -2585,11 +2611,12 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException Boolean disableWALProp = null; Boolean storeNullsProp = null; Boolean isTransactionalProp = null; + Long updateCacheFrequencyProp = null; ListMultimap> stmtProperties = statement.getProps(); Map>> properties = new HashMap<>(stmtProperties.size()); TableRef tableRef = FromCompiler.getResolver(statement, connection).getTables().get(0); - PTable table = tableRef.getTable(); + PTable table = tableRef.getTable(); List columnDefs = statement.getColumnDefs(); if (columnDefs == null) { columnDefs = Collections.emptyList(); @@ -2599,17 +2626,21 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException for (Pair prop : propsList) { String propName = prop.getFirst(); if (TableProperty.isPhoenixTableProperty(propName)) { - TableProperty.valueOf(propName).validate(true, !family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY), table.getType()); + TableProperty tableProp = TableProperty.valueOf(propName); + tableProp.validate(true, !family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY), table.getType()); + Object value = tableProp.getValue(prop.getSecond()); if (propName.equals(PTable.IS_IMMUTABLE_ROWS_PROP_NAME)) { - isImmutableRowsProp = (Boolean)prop.getSecond(); + isImmutableRowsProp = (Boolean)value; } else if (propName.equals(PhoenixDatabaseMetaData.MULTI_TENANT)) { - multiTenantProp = (Boolean)prop.getSecond(); + multiTenantProp = (Boolean)value; } else if (propName.equals(DISABLE_WAL)) { - disableWALProp = (Boolean)prop.getSecond(); + disableWALProp = (Boolean)value; } else if (propName.equals(STORE_NULLS)) { - storeNullsProp = (Boolean)prop.getSecond(); + storeNullsProp = (Boolean)value; } else if (propName.equals(TRANSACTIONAL)) { - isTransactionalProp = (Boolean)prop.getSecond(); + isTransactionalProp = (Boolean)value; + } else if (propName.equals(UPDATE_CACHE_FREQUENCY)) { + updateCacheFrequencyProp = (Long)value; } } } @@ -2665,6 +2696,13 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException changingPhoenixTableProperty = true; } } + Long updateCacheFrequency = null; + if (updateCacheFrequencyProp != null) { + if (updateCacheFrequencyProp.longValue() != table.getUpdateCacheFrequency()) { + updateCacheFrequency = updateCacheFrequencyProp; + changingPhoenixTableProperty = true; + } + } Boolean storeNulls = null; if (storeNullsProp != null) { if (storeNullsProp.booleanValue() != table.getStoreNulls()) { @@ -2686,15 +2724,15 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException } // cannot create a transactional table if transactions are disabled boolean transactionsEnabled = connection.getQueryServices().getProps().getBoolean( - QueryServices.TRANSACTIONS_ENABLED, - QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED); + QueryServices.TRANSACTIONS_ENABLED, + QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED); if (!transactionsEnabled) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED) + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED) .setSchemaName(schemaName).setTableName(tableName).build().buildException(); } // cannot make a table transactional if it has a row timestamp column if (SchemaUtil.hasRowTimestampColumn(table)) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP) + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP) .setSchemaName(schemaName).setTableName(tableName) .build().buildException(); } @@ -2748,20 +2786,20 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException // Add any new PK columns to end of index PK if (numPkColumnsAdded>0) { - // create PK column list that includes the newly created columns - List pkColumns = Lists.newArrayListWithExpectedSize(table.getPKColumns().size()+numPkColumnsAdded); - pkColumns.addAll(table.getPKColumns()); - for (int i=0; i pkColumns = Lists.newArrayListWithExpectedSize(table.getPKColumns().size()+numPkColumnsAdded); + pkColumns.addAll(table.getPKColumns()); + for (int i=0; i0 || nonTxToTx)) { for (PTable index : table.getIndexes()) { - incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, nonTxToTx ? Boolean.TRUE : null); + incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, nonTxToTx ? Boolean.TRUE : null, updateCacheFrequency); } tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); } long seqNum = table.getSequenceNumber(); if (changingPhoenixTableProperty || columnDefs.size() > 0) { - seqNum = incrementTableSeqNum(table, statement.getTableType(), columnDefs.size(), isTransactional, isImmutableRows, disableWAL, multiTenant, storeNulls); + seqNum = incrementTableSeqNum(table, statement.getTableType(), columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows, disableWAL, multiTenant, storeNulls); tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); } @@ -2847,19 +2885,26 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException // Only update client side cache if we aren't adding a PK column to a table with indexes or // transitioning a table from non transactional to transactional. // We could update the cache manually then too, it'd just be a pain. + String fullTableName = SchemaUtil.getTableName(schemaName, tableName); + long resolvedTimeStamp = TransactionUtil.getResolvedTime(connection, result); if (table.getIndexes().isEmpty() || (numPkColumnsAdded==0 && !nonTxToTx)) { - connection.addColumn( - tenantId, - SchemaUtil.getTableName(schemaName, tableName), - columns, - result.getMutationTime(), - seqNum, - isImmutableRows == null ? table.isImmutableRows() : isImmutableRows, - disableWAL == null ? table.isWALDisabled() : disableWAL, - multiTenant == null ? table.isMultiTenant() : multiTenant, - storeNulls == null ? table.getStoreNulls() : storeNulls, - isTransactional == null ? table.isTransactional() : isTransactional, - TransactionUtil.getResolvedTime(connection, result)); + connection.addColumn( + tenantId, + fullTableName, + columns, + result.getMutationTime(), + seqNum, + isImmutableRows == null ? table.isImmutableRows() : isImmutableRows, + disableWAL == null ? table.isWALDisabled() : disableWAL, + multiTenant == null ? table.isMultiTenant() : multiTenant, + storeNulls == null ? table.getStoreNulls() : storeNulls, + isTransactional == null ? table.isTransactional() : isTransactional, + updateCacheFrequency == null ? table.getUpdateCacheFrequency() : updateCacheFrequency, + resolvedTimeStamp); + } else if (updateCacheFrequency != null) { + // Force removal from cache as the update cache frequency has changed + // Note that clients outside this JVM won't be affected. + connection.removeTable(tenantId, fullTableName, null, resolvedTimeStamp); } // Delete rows in view index if we haven't dropped it already // We only need to do this if the multiTenant transitioned to false @@ -2991,7 +3036,7 @@ public MutationState dropColumn(DropColumnStatement statement) throws SQLExcepti while (true) { final ColumnResolver resolver = FromCompiler.getResolver(statement, connection); TableRef tableRef = resolver.getTables().get(0); - PTable table = tableRef.getTable(); + PTable table = tableRef.getTable(); List columnRefs = statement.getColumnRefs(); if(columnRefs == null) { columnRefs = Lists.newArrayListWithCapacity(0); @@ -3022,7 +3067,7 @@ public MutationState dropColumn(DropColumnStatement statement) throws SQLExcepti dropColumnMutations(table, tableColumnsToDrop, tableMetaData); for (PTable index : table.getIndexes()) { - IndexMaintainer indexMaintainer = index.getIndexMaintainer(table, connection); + IndexMaintainer indexMaintainer = index.getIndexMaintainer(table, connection); // get the columns required for the index pk Set indexColumns = indexMaintainer.getIndexedColumns(); // get the covered columns @@ -3030,16 +3075,16 @@ public MutationState dropColumn(DropColumnStatement statement) throws SQLExcepti List indexColumnsToDrop = Lists.newArrayListWithExpectedSize(columnRefs.size()); for(PColumn columnToDrop : tableColumnsToDrop) { ColumnReference columnToDropRef = new ColumnReference(columnToDrop.getFamilyName().getBytes(), columnToDrop.getName().getBytes()); - if (indexColumns.contains(columnToDropRef)) { + if (indexColumns.contains(columnToDropRef)) { indexesToDrop.add(new TableRef(index)); } else if (coveredColumns.contains(columnToDropRef)) { - String indexColumnName = IndexUtil.getIndexColumnName(columnToDrop); + String indexColumnName = IndexUtil.getIndexColumnName(columnToDrop); indexColumnsToDrop.add(index.getColumn(indexColumnName)); } } if(!indexColumnsToDrop.isEmpty()) { - incrementTableSeqNum(index, index.getType(), -indexColumnsToDrop.size(), null); + incrementTableSeqNum(index, index.getType(), -indexColumnsToDrop.size(), null, null); dropColumnMutations(index, indexColumnsToDrop, tableMetaData); } @@ -3048,7 +3093,7 @@ else if (coveredColumns.contains(columnToDropRef)) { tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); - long seqNum = incrementTableSeqNum(table, statement.getTableType(), -tableColumnsToDrop.size(), null); + long seqNum = incrementTableSeqNum(table, statement.getTableType(), -tableColumnsToDrop.size(), null, null); tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); // Force table header to be first in list @@ -3186,7 +3231,7 @@ public MutationState alterIndex(AlterIndexStatement statement) throws SQLExcepti } } Long timeStamp = indexRef.getTable().isTransactional() ? indexRef.getTimeStamp() : null; - List tableMetadata = connection.getMutationState().toMutations(timeStamp).next().getSecond(); + List tableMetadata = connection.getMutationState().toMutations(timeStamp).next().getSecond(); connection.rollback(); MetaDataMutationResult result = connection.getQueryServices().updateIndexState(tableMetadata, dataTableName); @@ -3256,7 +3301,7 @@ public PTableStats getTableStats(PTable table) throws SQLException { */ boolean isSharedIndex = table.getViewIndexId() != null; if (isSharedIndex) { - // we are assuming the stats table is not transactional + // we are assuming the stats table is not transactional return connection.getQueryServices().getTableStats(table.getPhysicalName().getBytes(), getCurrentScn()); } boolean isView = table.getType() == PTableType.VIEW; @@ -3297,7 +3342,7 @@ private boolean isLastPKVariableLength(PTable table) { } private PTable getParentOfView(PTable view) throws SQLException { - //TODO just use view.getParentName().getString() after implementing https://issues.apache.org/jira/browse/PHOENIX-2114 + //TODO just use view.getParentName().getString() after implementing https://issues.apache.org/jira/browse/PHOENIX-2114 SelectStatement select = new SQLParser(view.getViewStatement()).parseQuery(); String parentName = SchemaUtil.normalizeFullTableName(select.getFrom().toString().trim()); return connection.getTable(new PTableKey(view.getTenantId(), parentName)); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java index a3103bff9a4..3adcb7e7aff 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java @@ -17,8 +17,6 @@ */ package org.apache.phoenix.schema; -import java.sql.SQLException; - import org.apache.phoenix.parse.PFunction; import org.apache.phoenix.query.MetaDataMutated; @@ -34,4 +32,5 @@ public static interface Pruner { public PMetaData pruneTables(Pruner pruner); public PFunction getFunction(PTableKey key) throws FunctionNotFoundException; public PMetaData pruneFunctions(Pruner pruner); + public long getAge(PTableRef ref); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java index 9e4460d9a08..66b4af3a8d1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java @@ -170,6 +170,10 @@ public PTable putDuplicate(PTableKey key, PTable value, long resolvedTime) { return put(key, new PTableRef(value, timeKeeper.getCurrentTime(), 0, resolvedTime)); } + public long getAge(PTableRef ref) { + return timeKeeper.getCurrentTime() - ref.getCreateTime(); + } + public PTable remove(PTableKey key) { PTableRef value = this.tables.remove(key); if (value == null) { @@ -313,7 +317,7 @@ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException { } @Override - public PMetaData addColumn(PName tenantId, String tableName, List columnsToAdd, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException { + public PMetaData addColumn(PName tenantId, String tableName, List columnsToAdd, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException { PTableRef oldTableRef = metaData.get(new PTableKey(tenantId, tableName)); if (oldTableRef == null) { return this; @@ -327,7 +331,9 @@ public PMetaData addColumn(PName tenantId, String tableName, List colum newColumns.addAll(oldColumns); newColumns.addAll(columnsToAdd); } - PTable newTable = PTableImpl.makePTable(oldTableRef.getTable(), tableTimeStamp, tableSeqNum, newColumns, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional); + PTable newTable = PTableImpl.makePTable(oldTableRef.getTable(), + tableTimeStamp, tableSeqNum, newColumns, isImmutableRows, + isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency); return addTable(newTable, resolvedTime); } @@ -472,4 +478,9 @@ public PMetaData pruneFunctions(Pruner pruner) { return new PMetaDataImpl(clone); } + + @Override + public long getAge(PTableRef ref) { + return this.metaData.getAge(ref); + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java index ec97394a2d5..4a338f6bc65 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java @@ -339,4 +339,5 @@ public static LinkType fromSerializedValue(byte serializedValue) { * -1 if there is no such column. */ int getRowTimestampColPos(); + long getUpdateCacheFrequency(); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 1805d94bffd..a7260fb79f9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -75,6 +75,7 @@ import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.protobuf.HBaseZeroCopyByteString; import com.sun.istack.NotNull; import co.cask.tephra.TxConstants; @@ -135,6 +136,7 @@ public class PTableImpl implements PTable { private boolean rowKeyOrderOptimizable; // TODO: remove when required that tables have been upgrade for PHOENIX-2067 private boolean hasColumnsRequiringUpgrade; // TODO: remove when required that tables have been upgrade for PHOENIX-2067 private int rowTimestampColPos; + private long updateCacheFrequency; public PTableImpl() { this.indexes = Collections.emptyList(); @@ -205,7 +207,7 @@ public static PTableImpl makePTable(PTable table, long timeStamp, List i table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), parentSchemaName, table.getParentTableName(), indexes, table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), viewStatement, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), - table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, List columns) throws SQLException { @@ -214,7 +216,7 @@ public static PTableImpl makePTable(PTable table, List columns) throws table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), - table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List columns) throws SQLException { @@ -223,7 +225,7 @@ public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceN sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), - table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List columns, boolean isImmutableRows) throws SQLException { @@ -232,16 +234,17 @@ public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceN sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), - table.getIndexType(), table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getIndexType(), table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } - public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List columns, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional) throws SQLException { + public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List columns, boolean isImmutableRows, boolean isWalDisabled, + boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency) throws SQLException { return new PTableImpl( table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timeStamp, sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), isWalDisabled, isMultitenant, storeNulls, table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), - table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), isTransactional); + table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), isTransactional, updateCacheFrequency); } public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLException { @@ -251,7 +254,7 @@ public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLE table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), - table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, boolean rowKeyOrderOptimizable) throws SQLException { @@ -261,7 +264,7 @@ public static PTableImpl makePTable(PTable table, boolean rowKeyOrderOptimizable table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), - table.getBaseColumnCount(), rowKeyOrderOptimizable, table.isTransactional()); + table.getBaseColumnCount(), rowKeyOrderOptimizable, table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, PTableStats stats) throws SQLException { @@ -271,28 +274,33 @@ public static PTableImpl makePTable(PTable table, PTableStats stats) throws SQLE table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), stats, - table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } - public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber, - PName pkName, Integer bucketNum, List columns, PName dataSchemaName, PName dataTableName, List indexes, - boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, - boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException { + public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, + PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum, + List columns, PName dataSchemaName, PName dataTableName, List indexes, + boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, + boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, + IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency) throws SQLException { return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, - indexType, PTableStats.EMPTY_STATS, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, isTransactional); + indexType, PTableStats.EMPTY_STATS, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, isTransactional, updateCacheFrequency); } public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum, List columns, PName dataSchemaName, PName dataTableName, List indexes, boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, - boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, @NotNull PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) + boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, + IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency, + @NotNull PTableStats stats, int baseColumnCount) throws SQLException { return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames, - defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable, isTransactional); + defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, + indexType, stats, baseColumnCount, rowKeyOrderOptimizable, isTransactional, updateCacheFrequency); } private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, @@ -300,12 +308,18 @@ private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType PName parentSchemaName, PName parentTableName, List indexes, boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, - PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException { + PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency) throws SQLException { init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, stats, schemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, - viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional); + viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, + isTransactional, updateCacheFrequency); } + @Override + public long getUpdateCacheFrequency() { + return updateCacheFrequency; + } + @Override public boolean isMultiTenant() { return multiTenant; @@ -331,7 +345,7 @@ private void init(PName tenantId, PName schemaName, PName tableName, PTableType PName pkName, Integer bucketNum, List columns, PTableStats stats, PName parentSchemaName, PName parentTableName, List indexes, boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, - IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException { + IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency) throws SQLException { Preconditions.checkNotNull(schemaName); Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE + 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE + @@ -363,6 +377,7 @@ private void init(PName tenantId, PName schemaName, PName tableName, PTableType this.isTransactional = isTransactional; this.tableStats = stats; this.rowKeyOrderOptimizable = rowKeyOrderOptimizable; + this.updateCacheFrequency = updateCacheFrequency; List pkColumns; PColumn[] allColumns; @@ -1003,26 +1018,16 @@ public static PTable createFromProto(PTableProtos.PTable table) { boolean isImmutableRows = table.getIsImmutableRows(); SortedMap tableGuidePosts = new TreeMap(Bytes.BYTES_COMPARATOR); for (PTableProtos.PTableStats pTableStatsProto : table.getGuidePostsList()) { - List value = Lists.newArrayListWithExpectedSize(pTableStatsProto.getValuesCount()); - for (int j = 0; j < pTableStatsProto.getValuesCount(); j++) { - value.add(pTableStatsProto.getValues(j).toByteArray()); - } - // No op - pTableStatsProto.getGuidePostsByteCount(); - value = Lists.newArrayListWithExpectedSize(pTableStatsProto.getValuesCount()); PGuidePosts pGuidePosts = pTableStatsProto.getPGuidePosts(); - for(int j = 0; j < pGuidePosts.getGuidePostsCount(); j++) { - value.add(pGuidePosts.getGuidePosts(j).toByteArray()); - } long guidePostsByteCount = pGuidePosts.getByteCount(); long rowCount = pGuidePosts.getRowCount(); - // TODO : Not exposing MIN/MAX key outside to client - GuidePostsInfo info = - new GuidePostsInfo(guidePostsByteCount, value, rowCount); + int maxLength = pGuidePosts.getMaxLength(); + int guidePostsCount = pGuidePosts.getEncodedGuidePostsCount(); + GuidePostsInfo info = new GuidePostsInfo(guidePostsByteCount, + new ImmutableBytesWritable(HBaseZeroCopyByteString.zeroCopyGetBytes(pGuidePosts.getEncodedGuidePosts())), rowCount, maxLength, guidePostsCount); tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), info); - } + } PTableStats stats = new PTableStatsImpl(tableGuidePosts, table.getStatsTimeStamp()); - PName dataTableName = null; if (table.hasDataTableNameBytes()) { dataTableName = PNameFactory.newName(table.getDataTableNameBytes().toByteArray()); @@ -1060,12 +1065,18 @@ public static PTable createFromProto(PTableProtos.PTable table) { if (table.hasRowKeyOrderOptimizable()) { rowKeyOrderOptimizable = table.getRowKeyOrderOptimizable(); } + long updateCacheFrequency = 0; + if (table.hasUpdateCacheFrequency()) { + updateCacheFrequency = table.getUpdateCacheFrequency(); + } + try { PTableImpl result = new PTableImpl(); result.init(tenantId, schemaName, tableName, tableType, indexState, timeStamp, sequenceNumber, pkName, - (bucketNum == NO_SALTING) ? null : bucketNum, columns, stats, schemaName,dataTableName, indexes, - isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL, - multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional); + (bucketNum == NO_SALTING) ? null : bucketNum, columns, stats, schemaName,dataTableName, indexes, + isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL, + multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, + isTransactional, updateCacheFrequency); return result; } catch (SQLException e) { throw new RuntimeException(e); // Impossible @@ -1121,16 +1132,16 @@ public static PTableProtos.PTable toProto(PTable table) { for (Map.Entry entry : table.getTableStats().getGuidePosts().entrySet()) { PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder(); statsBuilder.setKey(ByteStringer.wrap(entry.getKey())); - for (byte[] stat : entry.getValue().getGuidePosts()) { - statsBuilder.addValues(ByteStringer.wrap(stat)); - } + statsBuilder.setEncodedGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); statsBuilder.setGuidePostsByteCount(entry.getValue().getByteCount()); + statsBuilder.setMaxLength(entry.getValue().getMaxLength()); + statsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount()); PGuidePostsProtos.PGuidePosts.Builder guidePstsBuilder = PGuidePostsProtos.PGuidePosts.newBuilder(); - for (byte[] stat : entry.getValue().getGuidePosts()) { - guidePstsBuilder.addGuidePosts(ByteStringer.wrap(stat)); - } + guidePstsBuilder.setEncodedGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); guidePstsBuilder.setByteCount(entry.getValue().getByteCount()); guidePstsBuilder.setRowCount(entry.getValue().getRowCount()); + guidePstsBuilder.setMaxLength(entry.getValue().getMaxLength()); + guidePstsBuilder.setEncodedGuidePostsCount(entry.getValue().getGuidePostsCount()); statsBuilder.setPGuidePosts(guidePstsBuilder); builder.addGuidePosts(statsBuilder.build()); } @@ -1157,6 +1168,7 @@ public static PTableProtos.PTable toProto(PTable table) { } builder.setBaseColumnCount(table.getBaseColumnCount()); builder.setRowKeyOrderOptimizable(table.rowKeyOrderOptimizable()); + builder.setUpdateCacheFrequency(table.getUpdateCacheFrequency()); return builder.build(); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java index 83d0b42f82f..c4bc5107525 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java @@ -20,15 +20,16 @@ public class PTableRef { private final PTable table; private final int estSize; + private final long createTime; + private final long resolvedTimeStamp; private volatile long lastAccessTime; - // timestamp (scn or txn timestamp at which rpc to fetch the table was made) - private long resolvedTimeStamp; public PTableRef(PTable table, long lastAccessTime, int estSize, long resolvedTime) { this.table = table; this.lastAccessTime = lastAccessTime; this.estSize = estSize; this.resolvedTimeStamp = resolvedTime; + this.createTime = lastAccessTime; } public PTableRef(PTable table, long lastAccessTime, long resolvedTime) { @@ -39,6 +40,14 @@ public PTableRef(PTableRef tableRef) { this (tableRef.table, tableRef.lastAccessTime, tableRef.estSize, tableRef.resolvedTimeStamp); } + /** + * Tracks how long this entry has been in the cache + * @return time in milliseconds for how long this entry has been in the cache. + */ + public long getCreateTime() { + return createTime; + } + public PTable getTable() { return table; } @@ -58,8 +67,4 @@ public long getLastAccessTime() { public void setLastAccessTime(long lastAccessTime) { this.lastAccessTime = lastAccessTime; } - - public void setResolvedTimeStamp(long resolvedTimeStamp) { - this.resolvedTimeStamp = resolvedTimeStamp; - } } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java index 091b92961eb..5967d8b54b9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java @@ -26,6 +26,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME; import java.sql.SQLException; +import java.util.Map; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.phoenix.exception.SQLExceptionCode; @@ -49,6 +50,23 @@ public enum TableProperty { STORE_NULLS(PhoenixDatabaseMetaData.STORE_NULLS, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false), TRANSACTIONAL(PhoenixDatabaseMetaData.TRANSACTIONAL, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false), + + UPDATE_CACHE_FREQUENCY(PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY, true, true) { + @Override + public Object getValue(Object value) { + if (value instanceof String) { + String strValue = (String) value; + if ("ALWAYS".equalsIgnoreCase(strValue)) { + return 0L; + } else if ("NEVER".equalsIgnoreCase(strValue)) { + return Long.MAX_VALUE; + } + } else { + return value == null ? null : ((Number) value).longValue(); + } + return value; + } + }, ; @@ -87,6 +105,14 @@ public static boolean isPhoenixTableProperty(String property) { return true; } + public Object getValue(Object value) { + return value; + } + + public Object getValue(Map props) { + return getValue(props.get(this.toString())); + } + // isQualified is true if column family name is specified in property name public void validate(boolean isMutating, boolean isQualified, PTableType tableType) throws SQLException { checkForColumnFamily(isQualified); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java index 35e2f779480..e2313421ae9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java @@ -19,7 +19,6 @@ import java.util.Objects; -import org.apache.hadoop.hbase.HConstants; import org.apache.phoenix.compile.TupleProjectionCompiler; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.util.IndexUtil; @@ -36,16 +35,20 @@ public class TableRef { private final long lowerBoundTimeStamp; private final boolean hasDynamicCols; + public TableRef(TableRef tableRef) { + this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols); + } + public TableRef(TableRef tableRef, long timeStamp) { - this(tableRef.alias, tableRef.table, timeStamp, tableRef.hasDynamicCols); + this(tableRef.alias, tableRef.table, timeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols); } public TableRef(TableRef tableRef, String alias) { - this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.hasDynamicCols); + this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols); } public TableRef(PTable table) { - this(null, table, HConstants.LATEST_TIMESTAMP, false); + this(null, table, QueryConstants.UNSET_TIMESTAMP, false); } public TableRef(PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java index 0f1dbeb490f..da7d3a51358 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java @@ -17,14 +17,8 @@ */ package org.apache.phoenix.schema.stats; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.util.Bytes; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.phoenix.util.ByteUtil; /** * A class that holds the guidePosts of a region and also allows combining the * guidePosts of different regions when the GuidePostsInfo is formed for a table. @@ -34,7 +28,7 @@ public class GuidePostsInfo { /** * the total number of guidePosts for the table combining all the guidePosts per region per cf. */ - private List guidePosts; + private ImmutableBytesWritable guidePosts; /** * The bytecount that is flattened across the total number of guide posts. */ @@ -45,7 +39,19 @@ public class GuidePostsInfo { */ private long rowCount = 0; - private long keyByteSize; // Total number of bytes in keys stored in guidePosts + /** + * Maximum length of a guidePost collected + */ + private int maxLength; + + public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0, + new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0); + + public int getMaxLength() { + return maxLength; + } + + private int guidePostsCount; /** * Constructor that creates GuidePostsInfo per region @@ -53,22 +59,20 @@ public class GuidePostsInfo { * @param guidePosts * @param rowCount */ - public GuidePostsInfo(long byteCount, List guidePosts, long rowCount) { - this.guidePosts = ImmutableList.copyOf(guidePosts); - int size = 0; - for (byte[] key : guidePosts) { - size += key.length; - } - this.keyByteSize = size; + public GuidePostsInfo(long byteCount, ImmutableBytesWritable guidePosts, long rowCount, int maxLength, int guidePostsCount) { + this.guidePosts = new ImmutableBytesWritable(guidePosts); + this.maxLength = maxLength; this.byteCount = byteCount; this.rowCount = rowCount; + this.guidePostsCount = guidePostsCount; } + public long getByteCount() { return byteCount; } - public List getGuidePosts() { + public ImmutableBytesWritable getGuidePosts() { return guidePosts; } @@ -76,70 +80,8 @@ public long getRowCount() { return this.rowCount; } - public void incrementRowCount() { - this.rowCount++; - } - - /** - * Combines the GuidePosts per region into one. - * @param oldInfo - */ - public void combine(GuidePostsInfo oldInfo) { - if (!oldInfo.getGuidePosts().isEmpty()) { - byte[] newFirstKey = oldInfo.getGuidePosts().get(0); - byte[] existingLastKey; - if (!this.getGuidePosts().isEmpty()) { - existingLastKey = this.getGuidePosts().get(this.getGuidePosts().size() - 1); - } else { - existingLastKey = HConstants.EMPTY_BYTE_ARRAY; - } - int size = oldInfo.getGuidePosts().size(); - // If the existing guidePosts is lesser than the new RegionInfo that we are combining - // then add the new Region info to the end of the current GuidePosts. - // If the new region info is smaller than the existing guideposts then add the existing - // guide posts after the new guideposts. - List newTotalGuidePosts = new ArrayList(this.getGuidePosts().size() + size); - if (Bytes.compareTo(existingLastKey, newFirstKey) <= 0) { - newTotalGuidePosts.addAll(this.getGuidePosts()); - newTotalGuidePosts.addAll(oldInfo.getGuidePosts()); - } else { - newTotalGuidePosts.addAll(oldInfo.getGuidePosts()); - newTotalGuidePosts.addAll(this.getGuidePosts()); - } - this.guidePosts = ImmutableList.copyOf(newTotalGuidePosts); - } - this.byteCount += oldInfo.getByteCount(); - this.keyByteSize += oldInfo.keyByteSize; - this.rowCount += oldInfo.getRowCount(); - } - - /** - * The guide posts, rowCount and byteCount are accumulated every time a guidePosts depth is - * reached while collecting stats. - * @param row - * @param byteCount - * @return - */ - public boolean addGuidePost(byte[] row, long byteCount, long rowCount) { - if (guidePosts.isEmpty() || Bytes.compareTo(row, guidePosts.get(guidePosts.size() - 1)) > 0) { - List newGuidePosts = Lists.newArrayListWithExpectedSize(this.getGuidePosts().size() + 1); - newGuidePosts.addAll(guidePosts); - newGuidePosts.add(row); - this.guidePosts = ImmutableList.copyOf(newGuidePosts); - this.byteCount += byteCount; - this.keyByteSize += row.length; - this.rowCount+=rowCount; - return true; - } - return false; - } - - public boolean addGuidePost(byte[] row) { - return addGuidePost(row, 0, 0); - } - - public boolean addGuidePost(byte[] row, long byteCount) { - return addGuidePost(row, byteCount, 0); + public int getGuidePostsCount() { + return guidePostsCount; } } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java new file mode 100644 index 00000000000..a8244da1ab3 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema.stats; + +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.PrefixByteCodec; +import org.apache.phoenix.util.PrefixByteEncoder; +import org.apache.phoenix.util.TrustedByteArrayOutputStream; + +/* + * Builder to help in adding guidePosts and building guidePostInfo. This is used when we are collecting stats or reading stats for a table. + */ + +public class GuidePostsInfoBuilder { + private PrefixByteEncoder encoder; + private byte[] lastRow; + private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + private long byteCount = 0; + private int guidePostsCount; + + /** + * The rowCount that is flattened across the total number of guide posts. + */ + private long rowCount = 0; + + /** + * Maximum length of a guidePost collected + */ + private int maxLength; + private DataOutputStream output; + private TrustedByteArrayOutputStream stream; + + public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0, + new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0); + + public int getMaxLength() { + return maxLength; + } + public GuidePostsInfoBuilder(){ + this.stream = new TrustedByteArrayOutputStream(1); + this.output = new DataOutputStream(stream); + this.encoder=new PrefixByteEncoder(); + lastRow = ByteUtil.EMPTY_BYTE_ARRAY; + } + + /** + * The guide posts, rowCount and byteCount are accumulated every time a guidePosts depth is + * reached while collecting stats. + * @param row + * @param byteCount + * @return + * @throws IOException + */ + public boolean addGuidePosts( byte[] row, long byteCount, long rowCount) { + if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) { + try { + encoder.encode(output, row, 0, row.length); + this.byteCount += byteCount; + this.guidePostsCount++; + this.maxLength = encoder.getMaxLength(); + this.rowCount += rowCount; + lastRow = row; + return true; + } catch (IOException e) { + return false; + } + } + return false; + } + + public boolean addGuidePosts(byte[] row){ + return addGuidePosts(row, 0, 0); + } + + public boolean addGuidePosts(byte[] row, long byteCount){ + return addGuidePosts(row, byteCount, 0); + } + + private void close() { + PrefixByteCodec.close(stream); + } + + public GuidePostsInfo build() { + this.guidePosts.set(stream.getBuffer(), 0, stream.size()); + GuidePostsInfo guidePostsInfo = new GuidePostsInfo(this.byteCount, this.guidePosts, this.rowCount, this.maxLength, this.guidePostsCount); + this.close(); + return guidePostsInfo; + } + public void incrementRowCount() { + this.rowCount++; + } + +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java index dc70e8663ac..dacc213e093 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java @@ -17,13 +17,19 @@ */ package org.apache.phoenix.schema.stats; -import java.util.List; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.MetaDataProtocol; +import org.apache.phoenix.util.PrefixByteCodec; +import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SizedUtil; import com.sun.istack.NotNull; @@ -47,12 +53,10 @@ public PTableStatsImpl(@NotNull SortedMap guidePosts, lo for (Map.Entry entry : guidePosts.entrySet()) { byte[] cf = entry.getKey(); estimatedSize += SizedUtil.ARRAY_SIZE + cf.length; - List keys = entry.getValue().getGuidePosts(); - estimatedSize += SizedUtil.sizeOfArrayList(keys.size()); - for (byte[] key : keys) { - estimatedSize += SizedUtil.ARRAY_SIZE + key.length; - } + estimatedSize += entry.getValue().getGuidePosts().getLength(); estimatedSize += SizedUtil.LONG_SIZE; + estimatedSize += SizedUtil.INT_SIZE; + estimatedSize += SizedUtil.INT_SIZE; } this.estimatedSize = estimatedSize; } @@ -65,19 +69,34 @@ public SortedMap getGuidePosts() { @Override public String toString() { StringBuilder buf = new StringBuilder(); + buf.append("PTableStats ["); for (Map.Entry entry : guidePosts.entrySet()) { buf.append(Bytes.toStringBinary(entry.getKey())); buf.append(":("); - List keys = entry.getValue().getGuidePosts(); - if (!keys.isEmpty()) { - for (byte[] key : keys) { - buf.append(Bytes.toStringBinary(key)); - buf.append(","); + ImmutableBytesWritable keys = entry.getValue().getGuidePosts(); + ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength()); + try { + if (keys.getLength() != 0) { + DataInput input = new DataInputStream(stream); + PrefixByteDecoder decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); + try { + while (true) { + ImmutableBytesWritable ptr = PrefixByteCodec.decode(decoder, input); + buf.append(Bytes.toStringBinary(ptr.get())); + buf.append(","); + } + } catch (EOFException e) { // Ignore as this signifies we're done + + } finally { + PrefixByteCodec.close(stream); + } + buf.setLength(buf.length() - 1); } - buf.setLength(buf.length()-1); + buf.append(")"); + } finally { + PrefixByteCodec.close(stream); } - buf.append(")"); } buf.append("]"); return buf.toString(); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index fc8d8bddb4a..3462f2235ca 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -61,9 +60,9 @@ public class StatisticsCollector { private long guidepostDepth; private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - private Map> guidePostsMap = Maps.newHashMap(); + private Map> guidePostsInfoWriterMap = Maps.newHashMap(); protected StatisticsWriter statsTable; - private Pair cachedGps = null; + private Pair cachedGps = null; public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException { @@ -99,8 +98,8 @@ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, l // in a compaction we know the one family ahead of time if (family != null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family); - cachedGps = new Pair(0l, new GuidePostsInfo(0, Collections. emptyList(), 0l)); - guidePostsMap.put(cfKey, cachedGps); + cachedGps = new Pair(0l, new GuidePostsInfoBuilder()); + guidePostsInfoWriterMap.put(cfKey, cachedGps); } } @@ -131,7 +130,7 @@ private void writeStatsToStatsTable(final Region region, boolean delete, List mutations) throws IOException { */ public void collectStatistics(final List results) { Map famMap = Maps.newHashMap(); - List rowTracker = null; + List rowTracker = null; if (cachedGps == null) { - rowTracker = new ArrayList(); + rowTracker = new ArrayList(); } for (Cell cell : results) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp()); - Pair gps; + Pair gps; if (cachedGps == null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()); - gps = guidePostsMap.get(cfKey); + gps = guidePostsInfoWriterMap.get(cfKey); if (gps == null) { - gps = new Pair(0l, - new GuidePostsInfo(0, Collections. emptyList(), 0l)); - guidePostsMap.put(cfKey, gps); + gps = new Pair(0l, + new GuidePostsInfoBuilder()); + guidePostsInfoWriterMap.put(cfKey, gps); } if (famMap.get(cfKey) == null) { famMap.put(cfKey, true); @@ -191,13 +190,13 @@ public void collectStatistics(final List results) { if (byteCount >= guidepostDepth) { byte[] row = ByteUtil.copyKeyBytesIfNecessary( new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())); - if (gps.getSecond().addGuidePost(row, byteCount)) { + if (gps.getSecond().addGuidePosts(row, byteCount)) { gps.setFirst(0l); } } } if (cachedGps == null) { - for (GuidePostsInfo s : rowTracker) { + for (GuidePostsInfoBuilder s : rowTracker) { s.incrementRowCount(); } } else { @@ -221,24 +220,13 @@ protected InternalScanner getInternalScanner(RegionCoprocessorEnvironment env, S } public void clear() { - this.guidePostsMap.clear(); + this.guidePostsInfoWriterMap.clear(); maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; } - public void addGuidePost(ImmutableBytesPtr cfKey, GuidePostsInfo info, long byteSize, long timestamp, - byte[] minKey) { - Pair newInfo = new Pair(byteSize, info); - Pair oldInfo = guidePostsMap.put(cfKey, newInfo); - if (oldInfo != null) { - info.combine(oldInfo.getSecond()); - newInfo.setFirst(oldInfo.getFirst() + newInfo.getFirst()); - } - maxTimeStamp = Math.max(maxTimeStamp, timestamp); - } - public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { - Pair pair = guidePostsMap.get(fam); - if (pair != null) { return pair.getSecond(); } + Pair pair = guidePostsInfoWriterMap.get(fam); + if (pair != null) { return pair.getSecond().build(); } return null; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java index d6f0bf13571..13e9491e029 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java @@ -134,7 +134,8 @@ public Void call() throws IOException { } finally { try { collectionTracker.removeCompactingRegion(regionInfo); - stats.close(); + stats.close();// close the writer + tracker.close();// close the tracker } catch (IOException e) { if (toThrow == null) toThrow = e; LOG.error("Error while closing the stats table", e); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 16f249c6ecd..5b471041d30 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -21,8 +21,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; +import java.util.SortedMap; import java.util.TreeMap; import org.apache.hadoop.hbase.Cell; @@ -55,10 +55,14 @@ private StatisticsUtil() { /** Number of parts in our complex key */ protected static final int NUM_KEY_PARTS = 3; - + public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, byte[] guidePostStartKey) { + return getRowKey(table, fam, new ImmutableBytesWritable(guidePostStartKey,0,guidePostStartKey.length)); + } + + public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, ImmutableBytesWritable guidePostStartKey) { // always starts with the source table - byte[] rowKey = new byte[table.length + fam.getLength() + guidePostStartKey.length + 2]; + byte[] rowKey = new byte[table.length + fam.getLength() + guidePostStartKey.getLength() + 2]; int offset = 0; System.arraycopy(table, 0, rowKey, offset, table.length); offset += table.length; @@ -66,7 +70,7 @@ public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, byte[] guide System.arraycopy(fam.get(), fam.getOffset(), rowKey, offset, fam.getLength()); offset += fam.getLength(); rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC - System.arraycopy(guidePostStartKey, 0, rowKey, offset, guidePostStartKey.length); + System.arraycopy(guidePostStartKey.get(), 0, rowKey, offset, guidePostStartKey.getLength()); return rowKey; } @@ -126,7 +130,7 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); ResultScanner scanner = null; long timeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); + TreeMap guidePostsInfoWriterPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); try { scanner = statsHTable.getScanner(s); Result result = null; @@ -168,23 +172,33 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab } if (cfName != null) { byte[] newGPStartKey = getGuidePostsInfoFromRowKey(tableNameBytes, cfName, result.getRow()); - GuidePostsInfo guidePosts = guidePostsPerCf.get(cfName); - if (guidePosts == null) { - guidePosts = new GuidePostsInfo(0l, Collections. emptyList(), 0l); - guidePostsPerCf.put(cfName, guidePosts); + GuidePostsInfoBuilder guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cfName); + if (guidePostsInfoWriter == null) { + guidePostsInfoWriter = new GuidePostsInfoBuilder(); + guidePostsInfoWriterPerCf.put(cfName, guidePostsInfoWriter); } - guidePosts.addGuidePost(newGPStartKey, byteCount, rowCount); + guidePostsInfoWriter.addGuidePosts(newGPStartKey, byteCount, rowCount); } } + if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl( + getGuidePostsPerCf(guidePostsInfoWriterPerCf), timeStamp); } } finally { if (scanner != null) { scanner.close(); } } - if (!guidePostsPerCf.isEmpty()) { return new PTableStatsImpl(guidePostsPerCf, timeStamp); } return PTableStats.EMPTY_STATS; } + private static SortedMap getGuidePostsPerCf( + TreeMap guidePostsWriterPerCf) { + TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); + for (byte[] key : guidePostsWriterPerCf.keySet()) { + guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).build()); + } + return guidePostsPerCf; + } + public static long getGuidePostDepth(int guidepostPerRegion, long guidepostWidth, HTableDescriptor tableDesc) { if (guidepostPerRegion > 0) { long maxFileSize = HConstants.DEFAULT_MAX_FILE_SIZE; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index b8bd06450b2..d03af7ab0ab 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -17,7 +17,11 @@ */ package org.apache.phoenix.schema.stats; +import java.io.ByteArrayInputStream; import java.io.Closeable; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; import java.io.IOException; import java.sql.Date; import java.util.List; @@ -45,6 +49,8 @@ import org.apache.phoenix.schema.types.PDate; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.PrefixByteCodec; +import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.TimeKeeper; @@ -103,6 +109,7 @@ private StatisticsWriter(HTableInterface statsReaderTable, HTableInterface stats @Override public void close() throws IOException { statsWriterTable.close(); + statsReaderTable.close(); } /** @@ -134,21 +141,32 @@ public void addStats(StatisticsCollector tracker, ImmutableBytesPtr cfKey, List< GuidePostsInfo gps = tracker.getGuidePosts(cfKey); if (gps != null) { boolean rowColumnAdded = false; - for (byte[] gp : gps.getGuidePosts()) { - byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, gp); - Put put = new Put(prefix); - if (!rowColumnAdded) { - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES, - timeStamp, PLong.INSTANCE.toBytes(gps.getByteCount())); - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, - PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp, - PLong.INSTANCE.toBytes(gps.getRowCount())); - rowColumnAdded = true; + ImmutableBytesWritable keys = gps.getGuidePosts(); + ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength()); + DataInput input = new DataInputStream(stream); + PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength()); + try { + while (true) { + ImmutableBytesWritable ptr = decoder.decode(input); + byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, ptr); + Put put = new Put(prefix); + if (!rowColumnAdded) { + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES, + timeStamp, PLong.INSTANCE.toBytes(gps.getByteCount())); + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, + PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp, + PLong.INSTANCE.toBytes(gps.getRowCount())); + rowColumnAdded = true; + } + // Add our empty column value so queries behave correctly + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp, + ByteUtil.EMPTY_BYTE_ARRAY); + mutations.add(put); } - // Add our empty column value so queries behave correctly - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp, - ByteUtil.EMPTY_BYTE_ARRAY); - mutations.add(put); + } catch (EOFException e) { // Ignore as this signifies we're done + + } finally { + PrefixByteCodec.close(stream); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java index 64d064a75d9..44502a5478e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java @@ -49,6 +49,8 @@ public class ByteUtil { public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; public static final ImmutableBytesPtr EMPTY_BYTE_ARRAY_PTR = new ImmutableBytesPtr( EMPTY_BYTE_ARRAY); + public static final ImmutableBytesWritable EMPTY_IMMUTABLE_BYTE_ARRAY = new ImmutableBytesWritable( + EMPTY_BYTE_ARRAY); public static final Comparator BYTES_PTR_COMPARATOR = new Comparator() { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java new file mode 100644 index 00000000000..5f4a3441a9b --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; + +import com.google.common.collect.Lists; + +public class PrefixByteCodec { + + public static List decodeBytes(ImmutableBytesWritable encodedBytes, int maxLength) throws IOException { + ByteArrayInputStream stream = new ByteArrayInputStream(encodedBytes.get(), encodedBytes.getOffset(), encodedBytes.getLength()); + DataInput input = new DataInputStream(stream); + PrefixByteDecoder decoder = new PrefixByteDecoder(maxLength); + List listOfBytes = Lists.newArrayList(); + try { + while (true) { + ImmutableBytesWritable ptr = decoder.decode(input); + // For this test, copy the bytes, but we wouldn't do this unless + // necessary for non testing + listOfBytes.add(ptr.copyBytes()); + } + } catch (EOFException e) { // Ignore as this signifies we're done + + } + return listOfBytes; + } + + public static int encodeBytes(List listOfBytes, ImmutableBytesWritable ptr) throws IOException { + TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(calculateSize(listOfBytes)); + DataOutput output = new DataOutputStream(stream); + PrefixByteEncoder encoder = new PrefixByteEncoder(); + for (byte[] bytes : listOfBytes) { + encoder.encode(output, bytes, 0, bytes.length); + } + close(stream); + ptr.set(stream.getBuffer(), 0, stream.size()); + return encoder.getMaxLength(); + } + + public static int calculateSize(List listOfBytes) { + int size = 0; + for (byte[] bytes : listOfBytes) { + size += bytes.length; + } + return size; + } + + public static ImmutableBytesWritable decode(PrefixByteDecoder decoder, DataInput input) throws EOFException { + try { + ImmutableBytesWritable val= decoder.decode(input); + return val; + } catch(EOFException eof){ + throw eof; + }catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static void close(ByteArrayInputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static void close(ByteArrayOutputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java new file mode 100644 index 00000000000..565f0bcaa7a --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import java.io.DataInput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.io.WritableUtils; + +/** + * + * Prefix decoder for byte arrays. For encoding, see {@link PrefixByteEncoder}. + * + */ +public class PrefixByteDecoder { + private final int maxLength; + private final ImmutableBytesWritable previous; + + /** + * Used when the maximum length of encoded byte array is not known. Will + * cause a new byte array to be allocated for each call to {@link #decode(DataInput)}. + */ + public PrefixByteDecoder() { + maxLength = -1; + previous = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + } + + /** + * Used when the maximum length of encoded byte array is known in advance. + * Will not allocate new byte array with each call to {@link #decode(DataInput)}. + * @param maxLength maximum length needed for any call to {@link #decode(DataInput)}. + */ + public PrefixByteDecoder(int maxLength) { + if (maxLength > 0) { + this.maxLength = maxLength; + this.previous = new ImmutableBytesWritable(new byte[maxLength], 0, 0); + } else { + this.maxLength = -1; + previous = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + } + } + + /** + * Resets state of decoder if it will be used to decode bytes from a + * different DataInput. + */ + public void reset() { + previous.set(previous.get(),0,0); + } + + /** + * Decodes bytes encoded with {@link PrefixByteEncoder}. + * @param in Input from which bytes are read. + * @return Pointer containing bytes that were decoded. Note that the + * same pointer will be returned with each call, so it must be consumed + * prior to calling decode again. + * @throws IOException + */ + public ImmutableBytesWritable decode(DataInput in) throws IOException { + int prefixLen = WritableUtils.readVInt(in); + int suffixLen = WritableUtils.readVInt(in); + int length = prefixLen + suffixLen; + byte[] b; + if (maxLength == -1) { // Allocate new byte array each time + b = new byte[length]; + System.arraycopy(previous.get(), previous.getOffset(), b, 0, prefixLen); + } else { // Reuse same buffer each time + b = previous.get(); + } + in.readFully(b, prefixLen, suffixLen); + previous.set(b, 0, length); + return previous; + } +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java new file mode 100644 index 00000000000..7a04aa0b4a6 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; + +/** + * + * Prefix encoder for byte arrays. For decoding, see {@link PrefixByteDecoder}. + * + */ +public class PrefixByteEncoder { + private int maxLength; + private final ImmutableBytesWritable previous = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + + public PrefixByteEncoder() { + } + + /** + * Resets the state of the encoder to its initial state (i.e. forgetting + * the previous byte array that may have been encoded). + */ + public void reset() { + previous.set(ByteUtil.EMPTY_BYTE_ARRAY); + } + + /** + * @return the maximum length byte array encountered while encoding + */ + public int getMaxLength() { + return maxLength; + } + + /** + * Prefix encodes the byte array pointed to into the output stream + * @param out output stream to encode into + * @param ptr pointer to byte array to encode. + * @throws IOException + */ + public void encode(DataOutput out, ImmutableBytesWritable ptr) throws IOException { + encode(out, ptr.get(), ptr.getOffset(), ptr.getLength()); + } + + /** + * Prefix encodes the byte array into the output stream + * @param out output stream to encode into + * @param b byte array to encode + * @throws IOException + */ + public void encode(DataOutput out, byte[] b) throws IOException { + encode(out, b, 0, b.length); + } + + /** + * Prefix encodes the byte array from offset to length into output stream. + * Instead of writing the entire byte array, only the portion of the byte array + * that differs from the beginning of the previous byte array written is written. + * + * @param out output stream to encode into + * @param b byte array buffer + * @param offset offset into byte array to start encoding + * @param length length of byte array to encode + * @throws IOException + */ + public void encode(DataOutput out, byte[] b, int offset, int length) throws IOException { + int i = 0; + int prevOffset = previous.getOffset(); + byte[] prevBytes = previous.get(); + int prevLength = previous.getLength(); + int minLength = prevLength < b.length ? prevLength : b.length; + for(i = 0; (i < minLength) && (prevBytes[prevOffset + i] == b[offset + i]); i++); + WritableUtils.writeVInt(out, i); + Bytes.writeByteArray(out, b, offset + i, length - i); + previous.set(b, offset, length); + if (length > maxLength) { + maxLength = length; + } + } +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java index 9f16e52297a..041c12eaa9d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java @@ -19,6 +19,11 @@ import java.sql.SQLException; +import co.cask.tephra.TransactionConflictException; +import co.cask.tephra.TransactionFailureException; +import co.cask.tephra.TxConstants; +import co.cask.tephra.hbase11.TransactionAwareHTable; + import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult; import org.apache.phoenix.exception.SQLExceptionCode; @@ -27,11 +32,6 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.schema.PTable; -import co.cask.tephra.TransactionConflictException; -import co.cask.tephra.TransactionFailureException; -import co.cask.tephra.TxConstants; -import co.cask.tephra.hbase11.TransactionAwareHTable; - public class TransactionUtil { private TransactionUtil() { } @@ -67,7 +67,7 @@ public static TransactionAwareHTable getTransactionAwareHTable(HTableInterface h public static long getResolvedTimestamp(PhoenixConnection connection, boolean isTransactional, long defaultResolvedTimestamp) { MutationState mutationState = connection.getMutationState(); Long scn = connection.getSCN(); - return scn != null ? scn : (isTransactional && mutationState.isTransactionStarted()) ? convertToMilliseconds(mutationState.getReadPointer()) : defaultResolvedTimestamp; + return scn != null ? scn : (isTransactional && mutationState.isTransactionStarted()) ? convertToMilliseconds(mutationState.getInitialWritePointer()) : defaultResolvedTimestamp; } public static long getResolvedTime(PhoenixConnection connection, MetaDataMutationResult result) { @@ -80,7 +80,7 @@ public static long getResolvedTimestamp(PhoenixConnection connection, MetaDataMu PTable table = result.getTable(); MutationState mutationState = connection.getMutationState(); boolean txInProgress = table != null && table.isTransactional() && mutationState.isTransactionStarted(); - return txInProgress ? convertToMilliseconds(mutationState.getReadPointer()) : result.getMutationTime(); + return txInProgress ? convertToMilliseconds(mutationState.getInitialWritePointer()) : result.getMutationTime(); } public static Long getTableTimestamp(PhoenixConnection connection, boolean transactional) throws SQLException { @@ -89,17 +89,10 @@ public static Long getTableTimestamp(PhoenixConnection connection, boolean trans return timestamp; } MutationState mutationState = connection.getMutationState(); - // we need to burn a txn so that we are sure the txn read pointer is close to wall clock time if (!mutationState.isTransactionStarted()) { mutationState.startTransaction(); - connection.commit(); - } - else { - connection.commit(); } - mutationState.startTransaction(); - timestamp = convertToMilliseconds(mutationState.getReadPointer()); - connection.commit(); + timestamp = convertToMilliseconds(mutationState.getInitialWritePointer()); return timestamp; } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index c9318512975..4d6afcea50d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -52,12 +52,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; @@ -100,6 +102,7 @@ public class UpgradeUtil { private static final Logger logger = LoggerFactory.getLogger(UpgradeUtil.class); private static final byte[] SEQ_PREFIX_BYTES = ByteUtil.concat(QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("_SEQ_")); + public static final byte[] UPGRADE_TO_4_7_COLUMN_NAME = Bytes.toBytes("UPGRADE_TO_4_7"); public static String UPSERT_BASE_COLUMN_COUNT_IN_HEADER_ROW = "UPSERT " + "INTO SYSTEM.CATALOG " @@ -1215,4 +1218,67 @@ public static void addRowKeyOrderOptimizableCell(List tableMetadata, b MetaDataEndpointImpl.ROW_KEY_ORDER_OPTIMIZABLE_BYTES, PBoolean.INSTANCE.toBytes(true)); tableMetadata.add(put); } + + public static boolean truncateStats(HTableInterface metaTable, HTableInterface statsTable) + throws IOException, InterruptedException { + List columnCells = metaTable + .get(new Get(SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE))) + .getColumnCells(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); + if (!columnCells.isEmpty() + && columnCells.get(0).getTimestamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) { + + byte[] statsTableKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE); + KeyValue upgradeKV = KeyValueUtil.newKeyValue(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + UPGRADE_TO_4_7_COLUMN_NAME, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, + ByteUtil.EMPTY_BYTE_ARRAY); + Put upgradePut = new Put(statsTableKey); + upgradePut.add(upgradeKV); + + // check for null in UPGRADE_TO_4_7_COLUMN_NAME in checkAndPut so that only single client + // drop the rows of SYSTEM.STATS + if (metaTable.checkAndPut(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + UPGRADE_TO_4_7_COLUMN_NAME, null, upgradePut)) { + List mutations = Lists.newArrayListWithExpectedSize(1000); + Scan scan = new Scan(); + scan.setRaw(true); + scan.setMaxVersions(); + ResultScanner statsScanner = statsTable.getScanner(scan); + Result r; + mutations.clear(); + int count = 0; + while ((r = statsScanner.next()) != null) { + Delete delete = null; + for (KeyValue keyValue : r.raw()) { + if (KeyValue.Type.codeToType(keyValue.getType()) == KeyValue.Type.Put) { + if (delete == null) { + delete = new Delete(keyValue.getRow()); + } + KeyValue deleteKeyValue = new KeyValue(keyValue.getRowArray(), keyValue.getRowOffset(), + keyValue.getRowLength(), keyValue.getFamilyArray(), keyValue.getFamilyOffset(), + keyValue.getFamilyLength(), keyValue.getQualifierArray(), + keyValue.getQualifierOffset(), keyValue.getQualifierLength(), + keyValue.getTimestamp(), KeyValue.Type.Delete, ByteUtil.EMPTY_BYTE_ARRAY, 0, 0); + delete.addDeleteMarker(deleteKeyValue); + } + } + if (delete != null) { + mutations.add(delete); + if (count > 10) { + statsTable.batch(mutations); + mutations.clear(); + count = 0; + } + count++; + } + } + if (!mutations.isEmpty()) { + statsTable.batch(mutations); + } + return true; + } + } + return false; + } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java index 94115492707..6d6dcdff00e 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java @@ -217,7 +217,7 @@ public void testNoPK() throws Exception { statement.execute(); fail(); } catch (SQLException e) { - assertTrue(e.getMessage(), e.getMessage().contains("ERROR 517 (42895): Invalid not null constraint on non primary key column columnName=FOO.PK")); + assertEquals(SQLExceptionCode.INVALID_NOT_NULL_CONSTRAINT.getErrorCode(), e.getErrorCode()); } finally { conn.close(); } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java index 9d5e17ce63f..72f3e015c4d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java @@ -36,12 +36,12 @@ import org.apache.phoenix.compile.ColumnResolver; import org.apache.phoenix.compile.FromCompiler; import org.apache.phoenix.compile.JoinCompiler; +import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.compile.RowProjector; import org.apache.phoenix.compile.SequenceManager; import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.compile.TupleProjectionCompiler; -import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.expression.ComparisonExpression; @@ -233,7 +233,7 @@ private TableRef createProjectedTableFromLiterals(Object[] row) { PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.emptyList(), false, Collections.emptyList(), null, null, false, false, false, null, - null, null, true, false); + null, null, true, false, 0); TableRef sourceTable = new TableRef(pTable); List sourceColumnRefs = Lists. newArrayList(); for (PColumn column : sourceTable.getTable().getColumns()) { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/function/BuiltinFunctionConstructorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/function/BuiltinFunctionConstructorTest.java new file mode 100644 index 00000000000..5258283f72d --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/function/BuiltinFunctionConstructorTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.List; + +import org.apache.phoenix.expression.function.ScalarFunction; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.ExpressionType; +import org.junit.Test; + +public class BuiltinFunctionConstructorTest { + + @Test + public void testChildrenListConstructors() { + ExpressionType[] types = ExpressionType.values(); + List children = new ArrayList<>(); + for(int i = 0; i < types.length; i++) { + try { + if((ScalarFunction.class.isAssignableFrom(types[i].getExpressionClass())) && (types[i].getExpressionClass() != UDFExpression.class)) { + Constructor cons = types[i].getExpressionClass().getDeclaredConstructor(List.class); + cons.setAccessible(true); + cons.newInstance(children); + } + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } catch (Exception e) { + } + } + } + + @Test + public void testNoArgumentConstructors() { + ExpressionType[] types = ExpressionType.values(); + for(int i = 0; i < types.length; i++) { + try { + if(!AggregateFunction.class.isAssignableFrom(types[i].getExpressionClass())) { + Constructor cons = types[i].getExpressionClass().getDeclaredConstructor(); + cons.setAccessible(true); + cons.newInstance(); + } + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } catch (Exception e) { + } + } + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java index a0aca4dd18c..bb5f4084ad8 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -22,12 +22,10 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; -import java.util.Collections; import java.util.Map; import java.util.Properties; import java.util.SortedMap; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.end2end.Shadower; @@ -40,6 +38,7 @@ import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.GuidePostsInfoBuilder; import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; @@ -648,10 +647,11 @@ private void testIntersect(byte[][] regionBoundaries, byte[][] guidePosts) throw stmt.execute(); final PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "PERF.BIG_OLAP_DOC")); - GuidePostsInfo info = new GuidePostsInfo(0,Collections. emptyList(), 0l); + GuidePostsInfoBuilder gpWriter = new GuidePostsInfoBuilder(); for (byte[] gp : guidePosts) { - info.addGuidePost(gp, 1000); + gpWriter.addGuidePosts(gp, 1000); } + GuidePostsInfo info = gpWriter.build(); final SortedMap gpMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); gpMap.put(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, info); PTable tableWithStats = PTableImpl.makePTable(table, new PTableStats() { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java index 35bb8ce4ac1..951bfce2bec 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.query; +import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster; import static org.apache.phoenix.util.PhoenixRuntime.CURRENT_SCN_ATTRIB; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR; @@ -150,6 +151,7 @@ import org.apache.phoenix.hbase.index.balancer.IndexLoadBalancer; import org.apache.phoenix.hbase.index.master.IndexMasterObserver; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.jdbc.PhoenixDriver; import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver; import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo; import org.apache.phoenix.jdbc.PhoenixTestDriver; @@ -200,7 +202,7 @@ public abstract class BaseTest { protected static final String TEST_TABLE_SCHEMA = "(" + " varchar_pk VARCHAR NOT NULL, " + - " char_pk CHAR(6) NOT NULL, " + + " char_pk CHAR(10) NOT NULL, " + " int_pk INTEGER NOT NULL, "+ " long_pk BIGINT NOT NULL, " + " decimal_pk DECIMAL(31, 10) NOT NULL, " + @@ -1805,77 +1807,44 @@ public static void assertValuesEqualsResultSet(ResultSet rs, List> public HBaseTestingUtility getUtility() { return utility; } + + public static void upsertRows(Connection conn, String fullTableName, int numRows) throws SQLException { + for (int i=1; i<=numRows; ++i) { + upsertRow(conn, fullTableName, i, false); + } + } + + public static void upsertRow(Connection conn, String fullTableName, int index, boolean firstRowInBatch) throws SQLException { + String upsert = "UPSERT INTO " + fullTableName + + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + PreparedStatement stmt = conn.prepareStatement(upsert); + stmt.setString(1, firstRowInBatch ? "firstRowInBatch_" : "" + "varchar"+index); + stmt.setString(2, "char"+index); + stmt.setInt(3, index); + stmt.setLong(4, index); + stmt.setBigDecimal(5, new BigDecimal(index)); + Date date = DateUtil.parseDate("2015-01-01 00:00:00"); + stmt.setDate(6, date); + stmt.setString(7, "varchar_a"); + stmt.setString(8, "chara"); + stmt.setInt(9, index+1); + stmt.setLong(10, index+1); + stmt.setBigDecimal(11, new BigDecimal(index+1)); + stmt.setDate(12, date); + stmt.setString(13, "varchar_b"); + stmt.setString(14, "charb"); + stmt.setInt(15, index+2); + stmt.setLong(16, index+2); + stmt.setBigDecimal(17, new BigDecimal(index+2)); + stmt.setDate(18, date); + stmt.executeUpdate(); + } // Populate the test table with data. public static void populateTestTable(String fullTableName) throws SQLException { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); try (Connection conn = DriverManager.getConnection(getUrl(), props)) { - String upsert = "UPSERT INTO " + fullTableName - + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - PreparedStatement stmt = conn.prepareStatement(upsert); - stmt.setString(1, "varchar1"); - stmt.setString(2, "char1"); - stmt.setInt(3, 1); - stmt.setLong(4, 1L); - stmt.setBigDecimal(5, new BigDecimal(1.0)); - Date date = DateUtil.parseDate("2015-01-01 00:00:00"); - stmt.setDate(6, date); - stmt.setString(7, "varchar_a"); - stmt.setString(8, "chara"); - stmt.setInt(9, 2); - stmt.setLong(10, 2L); - stmt.setBigDecimal(11, new BigDecimal(2.0)); - stmt.setDate(12, date); - stmt.setString(13, "varchar_b"); - stmt.setString(14, "charb"); - stmt.setInt(15, 3); - stmt.setLong(16, 3L); - stmt.setBigDecimal(17, new BigDecimal(3.0)); - stmt.setDate(18, date); - stmt.executeUpdate(); - - stmt.setString(1, "varchar2"); - stmt.setString(2, "char2"); - stmt.setInt(3, 2); - stmt.setLong(4, 2L); - stmt.setBigDecimal(5, new BigDecimal(2.0)); - date = DateUtil.parseDate("2015-01-02 00:00:00"); - stmt.setDate(6, date); - stmt.setString(7, "varchar_a"); - stmt.setString(8, "chara"); - stmt.setInt(9, 3); - stmt.setLong(10, 3L); - stmt.setBigDecimal(11, new BigDecimal(3.0)); - stmt.setDate(12, date); - stmt.setString(13, "varchar_b"); - stmt.setString(14, "charb"); - stmt.setInt(15, 4); - stmt.setLong(16, 4L); - stmt.setBigDecimal(17, new BigDecimal(4.0)); - stmt.setDate(18, date); - stmt.executeUpdate(); - - stmt.setString(1, "varchar3"); - stmt.setString(2, "char3"); - stmt.setInt(3, 3); - stmt.setLong(4, 3L); - stmt.setBigDecimal(5, new BigDecimal(3.0)); - date = DateUtil.parseDate("2015-01-03 00:00:00"); - stmt.setDate(6, date); - stmt.setString(7, "varchar_a"); - stmt.setString(8, "chara"); - stmt.setInt(9, 4); - stmt.setLong(10, 4L); - stmt.setBigDecimal(11, new BigDecimal(4.0)); - stmt.setDate(12, date); - stmt.setString(13, "varchar_b"); - stmt.setString(14, "charb"); - stmt.setInt(15, 5); - stmt.setLong(16, 5L); - stmt.setBigDecimal(17, new BigDecimal(5.0)); - stmt.setDate(18, date); - stmt.executeUpdate(); - + upsertRows(conn, fullTableName, 3); conn.commit(); } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java index 14b5f1f4218..9c926797a27 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java @@ -17,29 +17,37 @@ */ package org.apache.phoenix.schema; -import com.google.common.collect.Sets; -import org.apache.hadoop.hbase.HConstants; -import org.apache.phoenix.util.TimeKeeper; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import java.sql.SQLException; import java.util.Set; -import static org.junit.Assert.assertEquals; +import org.apache.hadoop.hbase.HConstants; +import org.apache.phoenix.util.TimeKeeper; +import org.junit.Test; + +import com.google.common.collect.Sets; public class PMetaDataImplTest { - private static PMetaData addToTable(PMetaData metaData, String name, int size) throws SQLException { + private static PMetaData addToTable(PMetaData metaData, String name, int size, TestTimeKeeper timeKeeper) throws SQLException { PTable table = new PSizedTable(new PTableKey(null,name), size); - return metaData.addTable(table, System.currentTimeMillis()); + PMetaData newMetaData = metaData.addTable(table, System.currentTimeMillis()); + timeKeeper.incrementTime(); + return newMetaData; } - private static PMetaData removeFromTable(PMetaData metaData, String name) throws SQLException { - return metaData.removeTable(null, name, null, HConstants.LATEST_TIMESTAMP); + private static PMetaData removeFromTable(PMetaData metaData, String name, TestTimeKeeper timeKeeper) throws SQLException { + PMetaData newMetaData = metaData.removeTable(null, name, null, HConstants.LATEST_TIMESTAMP); + timeKeeper.incrementTime(); + return newMetaData; } - private static PTable getFromTable(PMetaData metaData, String name) throws TableNotFoundException { - return metaData.getTableRef(new PTableKey(null,name)).getTable(); + private static PTable getFromTable(PMetaData metaData, String name, TestTimeKeeper timeKeeper) throws TableNotFoundException { + PTable table = metaData.getTableRef(new PTableKey(null,name)).getTable(); + timeKeeper.incrementTime(); + return table; } private static void assertNames(PMetaData metaData, String... names) { @@ -56,50 +64,54 @@ private static class TestTimeKeeper implements TimeKeeper { @Override public long getCurrentTime() { - return time++; + return time; } + public void incrementTime() { + time++; + } } @Test public void testEviction() throws Exception { long maxSize = 10; - PMetaData metaData = new PMetaDataImpl(5, maxSize, new TestTimeKeeper()); - metaData = addToTable(metaData, "a", 5); + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(5, maxSize, timeKeeper); + metaData = addToTable(metaData, "a", 5, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 4); + metaData = addToTable(metaData, "b", 4, timeKeeper); assertEquals(2, metaData.size()); - metaData = addToTable(metaData, "c", 3); + metaData = addToTable(metaData, "c", 3, timeKeeper); assertEquals(2, metaData.size()); assertNames(metaData, "b","c"); - metaData = addToTable(metaData, "b", 8); + metaData = addToTable(metaData, "b", 8, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "b"); - metaData = addToTable(metaData, "d", 11); + metaData = addToTable(metaData, "d", 11, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "d"); - metaData = removeFromTable(metaData, "d"); + metaData = removeFromTable(metaData, "d", timeKeeper); assertNames(metaData); - metaData = addToTable(metaData, "a", 4); + metaData = addToTable(metaData, "a", 4, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 3); + metaData = addToTable(metaData, "b", 3, timeKeeper); assertEquals(2, metaData.size()); - metaData = addToTable(metaData, "c", 2); + metaData = addToTable(metaData, "c", 2, timeKeeper); assertEquals(3, metaData.size()); assertNames(metaData, "a", "b","c"); - getFromTable(metaData, "a"); - metaData = addToTable(metaData, "d", 3); + getFromTable(metaData, "a", timeKeeper); + metaData = addToTable(metaData, "d", 3, timeKeeper); assertEquals(3, metaData.size()); assertNames(metaData, "c", "a","d"); // Clone maintains insert order metaData = metaData.clone(); - metaData = addToTable(metaData, "e", 6); + metaData = addToTable(metaData, "e", 6, timeKeeper); assertEquals(2, metaData.size()); assertNames(metaData, "d","e"); } @@ -107,18 +119,19 @@ public void testEviction() throws Exception { @Test public void shouldNotEvictMoreEntriesThanNecessary() throws Exception { long maxSize = 5; - PMetaData metaData = new PMetaDataImpl(5, maxSize, new TestTimeKeeper()); - metaData = addToTable(metaData, "a", 1); + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(5, maxSize, timeKeeper); + metaData = addToTable(metaData, "a", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 1); + metaData = addToTable(metaData, "b", 1, timeKeeper); assertEquals(2, metaData.size()); assertNames(metaData, "a", "b"); - metaData = addToTable(metaData, "c", 3); + metaData = addToTable(metaData, "c", 3, timeKeeper); assertEquals(3, metaData.size()); assertNames(metaData, "a", "b", "c"); - getFromTable(metaData, "a"); - getFromTable(metaData, "b"); - metaData = addToTable(metaData, "d", 3); + getFromTable(metaData, "a", timeKeeper); + getFromTable(metaData, "b", timeKeeper); + metaData = addToTable(metaData, "d", 3, timeKeeper); assertEquals(3, metaData.size()); assertNames(metaData, "a", "b", "d"); } @@ -126,19 +139,20 @@ public void shouldNotEvictMoreEntriesThanNecessary() throws Exception { @Test public void shouldAlwaysKeepAtLeastOneEntryEvenIfTooLarge() throws Exception { long maxSize = 5; - PMetaData metaData = new PMetaDataImpl(5, maxSize, new TestTimeKeeper()); - metaData = addToTable(metaData, "a", 1); + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(5, maxSize, timeKeeper); + metaData = addToTable(metaData, "a", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 1); + metaData = addToTable(metaData, "b", 1, timeKeeper); assertEquals(2, metaData.size()); - metaData = addToTable(metaData, "c", 5); + metaData = addToTable(metaData, "c", 5, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "d", 20); + metaData = addToTable(metaData, "d", 20, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "d"); - metaData = addToTable(metaData, "e", 1); + metaData = addToTable(metaData, "e", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "f", 2); + metaData = addToTable(metaData, "f", 2, timeKeeper); assertEquals(2, metaData.size()); assertNames(metaData, "e", "f"); } @@ -146,23 +160,42 @@ public void shouldAlwaysKeepAtLeastOneEntryEvenIfTooLarge() throws Exception { @Test public void shouldAlwaysKeepOneEntryIfMaxSizeIsZero() throws Exception { long maxSize = 0; - PMetaData metaData = new PMetaDataImpl(0, maxSize, new TestTimeKeeper()); - metaData = addToTable(metaData, "a", 1); + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(0, maxSize, timeKeeper); + metaData = addToTable(metaData, "a", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 1); + metaData = addToTable(metaData, "b", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "c", 5); + metaData = addToTable(metaData, "c", 5, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "d", 20); + metaData = addToTable(metaData, "d", 20, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "d"); - metaData = addToTable(metaData, "e", 1); + metaData = addToTable(metaData, "e", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "f", 2); + metaData = addToTable(metaData, "f", 2, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "f"); } + @Test + public void testAge() throws Exception { + long maxSize = 10; + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(5, maxSize, timeKeeper); + String tableName = "a"; + metaData = addToTable(metaData, tableName, 1, timeKeeper); + PTableRef aTableRef = metaData.getTableRef(new PTableKey(null,tableName)); + assertNotNull(aTableRef); + assertEquals(1, metaData.getAge(aTableRef)); + tableName = "b"; + metaData = addToTable(metaData, tableName, 1, timeKeeper); + PTableRef bTableRef = metaData.getTableRef(new PTableKey(null,tableName)); + assertNotNull(bTableRef); + assertEquals(1, metaData.getAge(bTableRef)); + assertEquals(2, metaData.getAge(aTableRef)); + } + private static class PSizedTable extends PTableImpl { private final int size; private final PTableKey key; diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java new file mode 100644 index 00000000000..269ae1fed4a --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.query.QueryConstants; +import org.junit.Test; + + +public class PrefixByteEncoderDecoderTest { + + static final List guideposts = Arrays.asList( + ByteUtil.concat(Bytes.toBytes("aaaaaaaaaa"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("aaaaaaaaaa"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbccccc")), + ByteUtil.concat(Bytes.toBytes("aaaaaaaaaa"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(2000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(2000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(2000L), Bytes.toBytes("c")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("d"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("d"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbbc")), + ByteUtil.concat(Bytes.toBytes("e"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")) + ); + + @Test + public void testEncode() throws IOException { + List listOfBytes = Arrays.asList(Bytes.toBytes("aaaaa"), Bytes.toBytes("aaaabb")); + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + int maxLength = PrefixByteCodec.encodeBytes(listOfBytes, ptr); + assertEquals(6, maxLength); + TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(PrefixByteCodec.calculateSize(listOfBytes)); + DataOutput output = new DataOutputStream(stream); + WritableUtils.writeVInt(output, 0); + WritableUtils.writeVInt(output, 5); + output.write(Bytes.toBytes("aaaaa")); // No space savings on first key + WritableUtils.writeVInt(output, 4); + WritableUtils.writeVInt(output, 2); + output.write(Bytes.toBytes("bb")); // Only writes part of second key that's different + assertArrayEquals(stream.toByteArray(), ptr.copyBytes()); + } + + @Test + public void testEncodeDecodeWithSingleBuffer() throws IOException { + testEncodeDecode(true); + } + + @Test + public void testEncodeDecodeWithNewBuffer() throws IOException { + testEncodeDecode(false); + } + + private void testEncodeDecode(boolean useSingleBuffer) throws IOException { + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + int maxLength = PrefixByteCodec.encodeBytes(guideposts, ptr); + int encodedSize = ptr.getLength(); + int unencodedSize = PrefixByteCodec.calculateSize(guideposts); + assertTrue(encodedSize < unencodedSize); + List listOfBytes = PrefixByteCodec.decodeBytes(ptr, useSingleBuffer ? maxLength : -1); + assertListByteArraysEquals(guideposts, listOfBytes); + } + + private static void assertListByteArraysEquals(List listOfBytes1, List listOfBytes2) { + assertEquals(listOfBytes1.size(), listOfBytes2.size()); + for (int i = 0; i < listOfBytes1.size(); i++) { + assertArrayEquals(listOfBytes1.get(i), listOfBytes2.get(i)); + } + } + +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java index e1258e41a50..66a3c65e46d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java @@ -563,7 +563,7 @@ public static void analyzeTable(Connection conn, String tableName, boolean trans String query = "UPDATE STATISTICS " + tableName; conn.createStatement().execute(query); // if the table is transactional burn a txn in order to make sure the next txn read pointer is close to wall clock time - TransactionUtil.getTableTimestamp(conn.unwrap(PhoenixConnection.class), transactional); + conn.commit(); } public static void analyzeTableIndex(Connection conn, String tableName) throws IOException, SQLException { diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java index c2eae864d7e..71b785b083d 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java @@ -49,6 +49,7 @@ public class RuleGeneratorTest { private static final String matcherScenario = PherfConstants.SCENARIO_ROOT_PATTERN + ".xml"; + @Ignore("Flapping - see PHOENIX-2592") @Test public void testDateGenerator() throws Exception { XMLConfigParser parser = new XMLConfigParser(matcherScenario); diff --git a/phoenix-protocol/src/main/PGuidePosts.proto b/phoenix-protocol/src/main/PGuidePosts.proto index 047a658de3e..14de2ebc9e1 100644 --- a/phoenix-protocol/src/main/PGuidePosts.proto +++ b/phoenix-protocol/src/main/PGuidePosts.proto @@ -27,4 +27,7 @@ message PGuidePosts { repeated bytes guidePosts = 1; optional int64 byteCount = 2; optional int64 rowCount = 3; + optional int32 maxLength = 4; + optional int32 encodedGuidePostsCount = 5; + optional bytes encodedGuidePosts = 6; } \ No newline at end of file diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index 3048a40cd99..0b6b763b2bc 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -57,6 +57,8 @@ message PTableStats { optional int64 keyBytesCount = 4; optional int32 guidePostsCount = 5; optional PGuidePosts pGuidePosts = 6; + optional bytes encodedGuidePosts = 7; + optional int32 maxLength = 8; } message PTable { @@ -87,4 +89,5 @@ message PTable { optional int32 baseColumnCount = 25; optional bool rowKeyOrderOptimizable = 26; optional bool transactional = 27; + optional int64 updateCacheFrequency = 28; } diff --git a/pom.xml b/pom.xml index 72551a46543..2c47ca27dae 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ 1.2 2.5.1 0.13.0 - 1.8.8 + 1.9.2 3.5 1.2.17 1.6.4 @@ -99,7 +99,6 @@ 1.0 1.1.8 13.0.1 - 1.8.8 1.4.0 1.3.2 2.11