PHOENIX-2417 Compress memory used by row key byte[] of guideposts

bin/performance.py

@@ -66,10 +66,40 @@ def createFileWithContent(filename, content):
 # HBase/Phoenix client side property override
 hbase_config_path = os.getenv('HBASE_CONF_DIR', phoenix_utils.current_dir)
 
-execute = ('java -cp "%s%s%s" -Dlog4j.configuration=file:' +
+java_home = os.getenv('JAVA_HOME')
+
+# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR
+hbase_env_path = None
+hbase_env_cmd = None
+if os.name == 'posix':
+ hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh')
+ hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path]
+elif os.name == 'nt':
+ hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
+ hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
+if not hbase_env_path or not hbase_env_cmd:
+ print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name
+ sys.exit(-1)
+
+hbase_env = {}
+if os.path.isfile(hbase_env_path):
+ p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
+ for x in p.stdout:
+ (k, _, v) = x.partition('=')
+ hbase_env[k.strip()] = v.strip()
+
+if hbase_env.has_key('JAVA_HOME'):
+ java_home = hbase_env['JAVA_HOME']
+
+if java_home:
+ java_cmd = os.path.join(java_home, 'bin', 'java')
+else:
+ java_cmd = 'java'
+
+execute = ('%s $PHOENIX_OPTS -cp "%s%s%s" -Dlog4j.configuration=file:' +
  os.path.join(phoenix_utils.current_dir, "log4j.properties") +
  ' org.apache.phoenix.util.PhoenixRuntime -t %s %s ') % \
- (hbase_config_path, os.pathsep, phoenix_utils.phoenix_client_jar, table, zookeeper)
+ (java_cmd, hbase_config_path, os.pathsep, phoenix_utils.phoenix_client_jar, table, zookeeper)
 
 # Create Table DDL
 createtable = "CREATE TABLE IF NOT EXISTS %s (HOST CHAR(2) NOT NULL,\
@@ -98,7 +128,7 @@ def createFileWithContent(filename, content):
 queryex("5 - Filter + Count", "SELECT COUNT(1) FROM %s WHERE CORE<10;" % (table))
 
 print "\nGenerating and upserting data..."
-exitcode = subprocess.call('java -jar %s %s %s' % (phoenix_utils.testjar, data, rowcount),
+exitcode = subprocess.call('%s -jar %s %s %s' % (java_cmd, phoenix_utils.testjar, data, rowcount),
  shell=True)
 if exitcode != 0:
  sys.exit(exitcode)

bin/phoenix_sandbox.py

@@ -43,7 +43,7 @@
 with open(cp_file_path, 'rb') as cp_file:
  cp_components.append(cp_file.read())
 
-java_cmd = ("java -Dlog4j.configuration=file:%s " +
+java_cmd = ("java $PHOENIX_OPTS -Dlog4j.configuration=file:%s " +
  "-cp %s org.apache.phoenix.Sandbox") % (
  logging_config, ":".join(cp_components))
 

bin/psql.py

@@ -62,7 +62,8 @@
 else:
  java = 'java'
 
-java_cmd = java +' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + \
+java_cmd = java + ' $PHOENIX_OPTS ' + \
+ ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + \
  os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \
  os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
  " org.apache.phoenix.util.PhoenixRuntime " + args 

bin/queryserver.py

@@ -119,7 +119,7 @@
 
 # " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=n " + \
 # " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true" + \
-java_cmd = '%(java)s -cp ' + hbase_config_path + os.pathsep + hadoop_config_path + os.pathsep + \
+java_cmd = '%(java)s $PHOENIX_OPTS -cp ' + hbase_config_path + os.pathsep + hadoop_config_path + os.pathsep + \
  phoenix_utils.phoenix_queryserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \
  " -Dproc_phoenixserver" + \
  " -Dlog4j.configuration=file:" + os.path.join(phoenix_utils.current_dir, "log4j.properties") + \

bin/sqlline-thin.py

@@ -145,7 +145,8 @@ def get_serialization():
 else:
  java = 'java'
 
-java_cmd = java + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_thin_client_jar + \
+java_cmd = java + ' $PHOENIX_OPTS ' + \
+ ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_thin_client_jar + \
  os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \
  os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
  " sqlline.SqlLine -d org.apache.phoenix.queryserver.client.Driver " + \

bin/sqlline.py

@@ -87,7 +87,8 @@ def kill_child():
 if os.name == 'nt':
  colorSetting = "false"
 
-java_cmd = java + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + os.pathsep + phoenix_utils.hadoop_common_jar + os.pathsep + phoenix_utils.hadoop_hdfs_jar + \
+java_cmd = java + ' $PHOENIX_OPTS ' + \
+ ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + os.pathsep + phoenix_utils.hadoop_common_jar + os.pathsep + phoenix_utils.hadoop_hdfs_jar + \
  os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \
  os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
  " sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver \

bin/traceserver.py

@@ -116,7 +116,8 @@
 
 # " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=n " + \
 # " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true" + \
-java_cmd = '%(java)s -cp ' + hbase_config_path + os.pathsep + phoenix_utils.phoenix_traceserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \
+java_cmd = '%(java)s $PHOENIX_OPTS ' + \
+ '-cp ' + hbase_config_path + os.pathsep + phoenix_utils.phoenix_traceserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \
  " -Dproc_phoenixtraceserver" + \
  " -Dlog4j.configuration=file:" + os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
  " -Dpsql.root.logger=%(root_logger)s" + \

phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java

@@ -27,11 +27,20 @@
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
+import java.sql.Statement;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.util.QueryUtil;
@@ -81,20 +90,40 @@ public static Collection<Boolean> data() {
  }
 
  protected void testUpdatableViewWithIndex(Integer saltBuckets, boolean localIndex) throws Exception {
- testUpdatableView(saltBuckets);
- testUpdatableViewIndex(saltBuckets, localIndex);
+ String viewName = testUpdatableView(saltBuckets);
+ Pair<String,Scan> pair = testUpdatableViewIndex(saltBuckets, localIndex);
+ Scan scan = pair.getSecond();
+ String tableName = pair.getFirst();
+ // Confirm that dropping the view also deletes the rows in the index
+ if (saltBuckets == null) {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(tableName));
+ ResultScanner scanner = htable.getScanner(scan);
+ Result result = scanner.next();
+ // Confirm index has rows
+ assertTrue(result != null && !result.isEmpty());
+
+ conn.createStatement().execute("DROP VIEW " + viewName);
+
+ // Confirm index has no rows after view is dropped
+ scanner = htable.getScanner(scan);
+ result = scanner.next();
+ assertTrue(result == null || result.isEmpty());
+ }
+ }
  }
 
- protected void testUpdatableView(Integer saltBuckets) throws Exception {
+ protected String testUpdatableView(Integer saltBuckets) throws Exception {
  Connection conn = DriverManager.getConnection(getUrl());
  if (saltBuckets!=null) {
  if (tableDDLOptions.length()!=0)
  tableDDLOptions+=",";
  tableDDLOptions+=(" SALT_BUCKETS="+saltBuckets);
  }
+ String viewName = "V";
  String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))" + tableDDLOptions;
  conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v AS SELECT * FROM " + fullTableName + " WHERE k1 = 1";
+ ddl = "CREATE VIEW " + viewName + " AS SELECT * FROM " + fullTableName + " WHERE k1 = 1";
  conn.createStatement().execute(ddl);
  for (int i = 0; i < 10; i++) {
  conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + (i % 4) + "," + (i+100) + "," + (i > 5 ? 2 : 1) + ")");
@@ -135,13 +164,14 @@ protected void testUpdatableView(Integer saltBuckets) throws Exception {
  assertEquals(121, rs.getInt(2));
  assertFalse(rs.next());
  conn.close();
+ return viewName;
  }
 
- protected void testUpdatableViewIndex(Integer saltBuckets) throws Exception {
- testUpdatableViewIndex(saltBuckets, false);
+ protected Pair<String,Scan> testUpdatableViewIndex(Integer saltBuckets) throws Exception {
+return testUpdatableViewIndex(saltBuckets, false);
  }
 
- protected void testUpdatableViewIndex(Integer saltBuckets, boolean localIndex) throws Exception {
+ protected Pair<String,Scan> testUpdatableViewIndex(Integer saltBuckets, boolean localIndex) throws Exception {
  ResultSet rs;
  Connection conn = DriverManager.getConnection(getUrl());
  if (localIndex) {
@@ -196,28 +226,32 @@ protected void testUpdatableViewIndex(Integer saltBuckets, boolean localIndex) t
 
 
  query = "SELECT k1, k2, s FROM v WHERE s = 'foo'";
- rs = conn.createStatement().executeQuery(query);
+ Statement statement = conn.createStatement();
+ rs = statement.executeQuery(query);
+ Scan scan = statement.unwrap(PhoenixStatement.class).getQueryPlan().getContext().getScan();
  assertTrue(rs.next());
  assertEquals(1, rs.getInt(1));
  assertEquals(120, rs.getInt(2));
  assertEquals("foo", rs.getString(3));
  assertFalse(rs.next());
+ String htableName;
  rs = conn.createStatement().executeQuery("EXPLAIN " + query);
  if (localIndex) {
- assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER _LOCAL_IDX_" + tableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n"
+ htableName = "_LOCAL_IDX_" + tableName;
+ assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER " + htableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n"
  + " SERVER FILTER BY FIRST KEY ONLY\n"
  + "CLIENT MERGE SORT",QueryUtil.getExplainPlan(rs));
  } else {
+ htableName = "_IDX_" + tableName;
  assertEquals(saltBuckets == null
- ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_" + tableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n"
+ ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + htableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n"
  + " SERVER FILTER BY FIRST KEY ONLY"
- : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER _IDX_T" + (transactional ? "_TXN" : "") + " [0," + (Short.MIN_VALUE+1) + ",'foo']\n"
+ : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER " + htableName + " [0," + (Short.MIN_VALUE+1) + ",'foo']\n"
  + " SERVER FILTER BY FIRST KEY ONLY\n"
  + "CLIENT MERGE SORT",
  QueryUtil.getExplainPlan(rs));
  }
  conn.close();
+ return new Pair<>(htableName,scan);
  }
-
-
 }

phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java

@@ -52,6 +52,7 @@
 import java.util.GregorianCalendar;
 
 import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.TestUtil;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -664,4 +665,13 @@ public void testNullDate() throws Exception {
  assertNull(rs.getDate(1, GregorianCalendar.getInstance()));
  assertFalse(rs.next());
  }
+
+ @Test
+ public void testCurrentDateWithNoTable() throws Exception {
+ long expectedTime = System.currentTimeMillis();
+ ResultSet rs = conn.createStatement().executeQuery("SELECT CURRENT_DATE()");
+ assertTrue(rs.next());
+ long actualTime = rs.getDate(1).getTime();
+ assertTrue(Math.abs(actualTime - expectedTime) < TestUtil.MILLIS_IN_DAY);
+ }
 }

phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java

@@ -1376,7 +1376,9 @@ public void testUpsertSelectWithFixedWidthNullByteSizeArray() throws Exception {
 
  @Test
  public void testParallelUpsertSelect() throws Exception {
+ long ts = nextTimestamp();
  Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts));
  props.setProperty(QueryServices.MUTATE_BATCH_SIZE_ATTRIB, Integer.toString(3));
  props.setProperty(QueryServices.SCAN_CACHE_SIZE_ATTRIB, Integer.toString(3));
  props.setProperty(QueryServices.SCAN_RESULT_CHUNK_SIZE, Integer.toString(3));
@@ -1385,11 +1387,17 @@ public void testParallelUpsertSelect() throws Exception {
  conn.createStatement().execute("CREATE SEQUENCE S1");
  conn.createStatement().execute("CREATE TABLE SALTEDT1 (pk INTEGER PRIMARY KEY, val INTEGER) SALT_BUCKETS=4");
  conn.createStatement().execute("CREATE TABLE T2 (pk INTEGER PRIMARY KEY, val INTEGER)");
-
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
  for (int i = 0; i < 100; i++) {
  conn.createStatement().execute("UPSERT INTO SALTEDT1 VALUES (NEXT VALUE FOR S1, " + (i%10) + ")");
  }
  conn.commit();
+ conn.close();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20));
+ conn = DriverManager.getConnection(getUrl(), props);
  conn.setAutoCommit(true);
  int upsertCount = conn.createStatement().executeUpdate("UPSERT INTO T2 SELECT pk, val FROM SALTEDT1");
  assertEquals(100,upsertCount);

phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java

@@ -450,13 +450,26 @@ private void validateViewDoesNotExist(Connection conn, String viewName)	throws S
  }
 	}
 
-	@Test
- public void testViewUsesTableIndex() throws Exception {
+ @Test
+ public void testViewUsesTableGlobalIndex() throws Exception {
+ testViewUsesTableIndex(false);
+ }
+
+ @Test
+ public void testViewUsesTableLocalIndex() throws Exception {
+ testViewUsesTableIndex(true);
+ }
+
+
+ private void testViewUsesTableIndex(boolean localIndex) throws Exception {
  ResultSet rs;
+ // Use unique name for table with local index as otherwise we run into issues
+ // when we attempt to drop the table (with the drop metadata option set to false
+ String fullTableName = this.fullTableName + (localIndex ? "_WITH_LI" : "_WITHOUT_LI");
  Connection conn = DriverManager.getConnection(getUrl());
  String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s1 VARCHAR, s2 VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))" + tableDDLOptions;
  conn.createStatement().execute(ddl);
- conn.createStatement().execute("CREATE INDEX i1 ON " + fullTableName + "(k3, k2) INCLUDE(s1, s2)");
+ conn.createStatement().execute("CREATE " + (localIndex ? "LOCAL " : "") + " INDEX i1 ON " + fullTableName + "(k3, k2) INCLUDE(s1, s2)");
  conn.createStatement().execute("CREATE INDEX i2 ON " + fullTableName + "(k3, k2, s2)");
 
  ddl = "CREATE VIEW v AS SELECT * FROM " + fullTableName + " WHERE s1 = 'foo'";
@@ -481,11 +494,18 @@ public void testViewUsesTableIndex() throws Exception {
  assertFalse(rs.next());
  rs = conn.createStatement().executeQuery("EXPLAIN " + query);
  String queryPlan = QueryUtil.getExplainPlan(rs);
- assertEquals(
- "CLIENT PARALLEL 1-WAY SKIP SCAN ON 4 KEYS OVER I1 [1,100] - [2,109]\n" + 
- " SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')", queryPlan);
- }
-
+ // Assert that in either case (local & global) that index from physical table used for query on view.
+ if (localIndex) {
+ assertEquals("CLIENT PARALLEL 1-WAY SKIP SCAN ON 4 KEYS OVER _LOCAL_IDX_" + fullTableName + " [-32768,1,100] - [-32768,2,109]\n" + 
+ " SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')\n" + 
+ "CLIENT MERGE SORT", queryPlan);
+ } else {
+ assertEquals(
+ "CLIENT PARALLEL 1-WAY SKIP SCAN ON 4 KEYS OVER I1 [1,100] - [2,109]\n" + 
+ " SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')", queryPlan);
+ }
+ } 
+
  @Test
  public void testCreateViewDefinesPKColumn() throws Exception {
  Connection conn = DriverManager.getConnection(getUrl());

phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java

@@ -157,6 +157,29 @@ private void testOptimization(Integer saltBuckets) throws Exception {
  assertEquals("a", rs.getString("v1"));
  assertFalse(rs.next());
 
+ query = "SELECT /*+ INDEX(" + TestUtil.DEFAULT_DATA_TABLE_NAME + " " + TestUtil.DEFAULT_INDEX_TABLE_NAME + ")*/ * FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME +" where v1='a' limit 1";
+ rs = conn1.createStatement().executeQuery("EXPLAIN "+ query);
+
+ expected = 
+ "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + TestUtil.DEFAULT_DATA_TABLE_NAME + "\n" +
+ "CLIENT 1 ROW LIMIT\n" +
+ " SKIP-SCAN-JOIN TABLE 0\n" +
+ " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + TestUtil.DEFAULT_INDEX_TABLE_NAME + " \\['a'\\]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ " DYNAMIC SERVER FILTER BY \\(\"T.T_ID\", \"T.K1\", \"T.K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)\n" +
+ " JOIN-SCANNER 1 ROW LIMIT";
+ actual = QueryUtil.getExplainPlan(rs);
+ assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, Pattern.matches(expected, actual));
+
+ rs = conn1.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("f", rs.getString("t_id"));
+ assertEquals(1, rs.getInt("k1"));
+ assertEquals(2, rs.getInt("k2"));
+ assertEquals(3, rs.getInt("k3"));
+ assertEquals("a", rs.getString("v1"));
+ assertFalse(rs.next());
+
  query = "SELECT /*+ INDEX(" + TestUtil.DEFAULT_DATA_TABLE_NAME + " " + TestUtil.DEFAULT_INDEX_TABLE_NAME + ")*/ t_id, k1, k2, k3, V1 from " + TestUtil.DEFAULT_DATA_TABLE_FULL_NAME + " where v1<='z' and k3 > 1 order by V1,t_id";
  rs = conn1.createStatement().executeQuery("EXPLAIN " + query);