diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java index eb6dc3dfbb3..0247df4afd7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java @@ -46,7 +46,7 @@ import com.google.common.collect.Lists; /** - * {@link RecordReader} implementation that iterates over the the records. + * PhoenixRecordReader implementation that iterates over the the records. */ public class PhoenixRecordReader extends RecordReader { diff --git a/phoenix-hive/pom.xml b/phoenix-hive/pom.xml new file mode 100644 index 00000000000..babc5cd5b7c --- /dev/null +++ b/phoenix-hive/pom.xml @@ -0,0 +1,233 @@ + + + + + 4.0.0 + + org.apache.phoenix + phoenix + 4.4.0-SNAPSHOT + + phoenix-hive + Phoenix - Hive + + + + org.apache.phoenix + phoenix-core + + + org.apache.phoenix + phoenix-core + tests + test + + + joda-time + joda-time + + + org.apache.hive + hive-exec + ${hive.version} + + + org.apache.calcite + * + + + + + org.apache.calcite + calcite-core + ${calcite.version} + + + org.apache.calcite + calcite-avatica + ${calcite.version} + + + org.apache.hive + hive-common + ${hive.version} + + + org.apache.hive + hive-cli + ${hive.version} + + + jline + jline + 0.9.94 + + + commons-lang + commons-lang + ${commons-lang.version} + + + commons-logging + commons-logging + ${commons-logging.version} + + + org.apache.hbase + hbase-testing-util + ${hbase.version} + test + true + + + org.jruby + jruby-complete + + + + + org.apache.hbase + hbase-it + ${hbase.version} + test-jar + test + + + org.jruby + jruby-complete + + + + + org.apache.hbase + hbase-common + ${hbase.version} + + + org.apache.hbase + hbase-protocol + ${hbase.version} + + + org.apache.hbase + hbase-client + ${hbase.version} + + + org.apache.hbase + hbase-hadoop-compat + ${hbase.version} + test + + + org.apache.hbase + hbase-hadoop-compat + ${hbase.version} + test-jar + test + + + org.apache.hbase + hbase-hadoop2-compat + ${hbase.version} + test + + + org.apache.hbase + hbase-hadoop2-compat + ${hbase.version} + test-jar + test + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-annotations + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-minicluster + + + + org.mockito + mockito-all + test + + + junit + junit + test + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + org.apache.maven.plugins + maven-failsafe-plugin + + + maven-dependency-plugin + ${maven-dependency-plugin.version} + + + copy-dependencies + package + + copy-dependencies + + + + + + maven-assembly-plugin + + + jar-with-dependencies + + + + + make-jar-with-dependencies + package + + single + + + + + + + diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/DummyAuthenticator.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/DummyAuthenticator.java new file mode 100644 index 00000000000..623bce34188 --- /dev/null +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/DummyAuthenticator.java @@ -0,0 +1,73 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + *distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you maynot use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicablelaw or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; +import org.apache.hadoop.hive.ql.session.SessionState; + +public class DummyAuthenticator implements HiveAuthenticationProvider { + + private final List groupNames; + private final String userName; + private Configuration conf; + + public DummyAuthenticator() { + this.groupNames = new ArrayList(); + groupNames.add("hive_test_group1"); + groupNames.add("hive_test_group2"); + userName = "hive_test_user"; + } + + @Override + public void destroy() throws HiveException{ + return; + } + + @Override + public List getGroupNames() { + return groupNames; + } + + @Override + public String getUserName() { + return userName; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return this.conf; + } + + @Override + public void setSessionState(SessionState ss) { + //no op + } + +} diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestContext.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestContext.java new file mode 100644 index 00000000000..1b0197d5e60 --- /dev/null +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestContext.java @@ -0,0 +1,629 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.hive; + +import java.io.DataInput; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; + +import org.antlr.runtime.TokenRewriteStream; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.TaskRunner; +import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.lockmgr.HiveLock; +import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; +import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj; +import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; +import org.apache.hadoop.hive.ql.plan.LoadTableDesc; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.util.StringUtils; + +/** + * Context for Semantic Analyzers. Usage: not reusable - construct a new one for + * each query should call clear() at end of use to remove temporary folders + */ +public class HiveTestContext { + private boolean isHDFSCleanup; + private Path resFile; + private Path resDir; + private FileSystem resFs; + private static final Log LOG = LogFactory.getLog("HiveTestContext"); + private Path[] resDirPaths; + private int resDirFilesNum; + boolean initialized; + String originalTracker = null; + private final Map pathToCS = new ConcurrentHashMap(); + + // scratch path to use for all non-local (ie. hdfs) file system tmp folders + private final Path nonLocalScratchPath; + + // scratch directory to use for local file system tmp folders + private final String localScratchDir; + + // the permission to scratch directory (local and hdfs) + private final String scratchDirPermission; + + // Keeps track of scratch directories created for different scheme/authority + private final Map fsScratchDirs = new HashMap(); + + private final Configuration conf; + protected int pathid = 10000; + protected boolean explain = false; + protected boolean explainLogical = false; + protected String cmd = ""; + // number of previous attempts + protected int tryCount = 0; + private TokenRewriteStream tokenRewriteStream; + + String executionId; + + // List of Locks for this query + protected List hiveLocks; + protected HiveLockManager hiveLockMgr; + + // Transaction manager for this query + protected HiveTxnManager hiveTxnManager; + + // Used to track what type of acid operation (insert, update, or delete) we are doing. Useful + // since we want to change where bucket columns are accessed in some operators and + // optimizations when doing updates and deletes. + private AcidUtils.Operation acidOperation = AcidUtils.Operation.NOT_ACID; + + private boolean needLockMgr; + + // Keep track of the mapping from load table desc to the output and the lock + private final Map loadTableOutputMap = + new HashMap(); + private final Map> outputLockObjects = + new HashMap>(); + + public HiveTestContext(Configuration conf) throws IOException { + this(conf, generateExecutionId()); + } + + /** + * Create a Context with a given executionId. ExecutionId, together with + * user name and conf, will determine the temporary directory locations. + */ + public HiveTestContext(Configuration conf, String executionId) { + this.conf = conf; + this.executionId = executionId; + + // local & non-local tmp location is configurable. however it is the same across + // all external file systems + nonLocalScratchPath = new Path(SessionState.getHDFSSessionPath(conf), executionId); + localScratchDir = new Path(SessionState.getLocalSessionPath(conf), executionId).toUri().getPath(); + scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION); + } + + + public Map getLoadTableOutputMap() { + return loadTableOutputMap; + } + + public Map> getOutputLockObjects() { + return outputLockObjects; + } + + /** + * Set the context on whether the current query is an explain query. + * @param value true if the query is an explain query, false if not + */ + public void setExplain(boolean value) { + explain = value; + } + + /** + * Find whether the current query is an explain query + * @return true if the query is an explain query, false if not + */ + public boolean getExplain() { + return explain; + } + + /** + * Find whether the current query is a logical explain query + */ + public boolean getExplainLogical() { + return explainLogical; + } + + /** + * Set the context on whether the current query is a logical + * explain query. + */ + public void setExplainLogical(boolean explainLogical) { + this.explainLogical = explainLogical; + } + + /** + * Set the original query command. + * @param cmd the original query command string + */ + public void setCmd(String cmd) { + this.cmd = cmd; + } + + /** + * Find the original query command. + * @return the original query command string + */ + public String getCmd () { + return cmd; + } + + /** + * Get a tmp directory on specified URI + * + * @param scheme Scheme of the target FS + * @param authority Authority of the target FS + * @param mkdir create the directory if true + * @param scratchDir path of tmp directory + */ + private Path getScratchDir(String scheme, String authority, + boolean mkdir, String scratchDir) { + + String fileSystem = scheme + ":" + authority; + Path dir = fsScratchDirs.get(fileSystem + "-" + TaskRunner.getTaskRunnerID()); + + if (dir == null) { + Path dirPath = new Path(scheme, authority, + scratchDir + "-" + TaskRunner.getTaskRunnerID()); + if (mkdir) { + try { + FileSystem fs = dirPath.getFileSystem(conf); + dirPath = new Path(fs.makeQualified(dirPath).toString()); + FsPermission fsPermission = new FsPermission(scratchDirPermission); + + if (!fs.mkdirs(dirPath, fsPermission)) { + throw new RuntimeException("Cannot make directory: " + + dirPath.toString()); + } + if (isHDFSCleanup) { + fs.deleteOnExit(dirPath); + } + } catch (IOException e) { + throw new RuntimeException (e); + } + } + dir = dirPath; + fsScratchDirs.put(fileSystem + "-" + TaskRunner.getTaskRunnerID(), dir); + + } + + return dir; + } + + + /** + * Create a local scratch directory on demand and return it. + */ + public Path getLocalScratchDir(boolean mkdir) { + try { + FileSystem fs = FileSystem.getLocal(conf); + URI uri = fs.getUri(); + return getScratchDir(uri.getScheme(), uri.getAuthority(), + mkdir, localScratchDir); + } catch (IOException e) { + throw new RuntimeException (e); + } + } + + + /** + * Create a map-reduce scratch directory on demand and return it. + * + */ + public Path getMRScratchDir() { + + // if we are executing entirely on the client side - then + // just (re)use the local scratch directory + if(isLocalOnlyExecutionMode()) { + return getLocalScratchDir(!explain); + } + + try { + Path dir = FileUtils.makeQualified(nonLocalScratchPath, conf); + URI uri = dir.toUri(); + + Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(), + !explain, uri.getPath()); + LOG.info("New scratch dir is " + newScratchDir); + return newScratchDir; + } catch (IOException e) { + throw new RuntimeException(e); + } catch (IllegalArgumentException e) { + throw new RuntimeException("Error while making MR scratch " + + "directory - check filesystem config (" + e.getCause() + ")", e); + } + } + + private Path getExternalScratchDir(URI extURI) { + return getScratchDir(extURI.getScheme(), extURI.getAuthority(), + !explain, nonLocalScratchPath.toUri().getPath()); + } + + /** + * Remove any created scratch directories. + */ + private void removeScratchDir() { + for (Map.Entry entry : fsScratchDirs.entrySet()) { + try { + Path p = entry.getValue(); + p.getFileSystem(conf).delete(p, true); + } catch (Exception e) { + LOG.warn("Error Removing Scratch: " + + StringUtils.stringifyException(e)); + } + } + fsScratchDirs.clear(); + } + + private String nextPathId() { + return Integer.toString(pathid++); + } + + + private static final String MR_PREFIX = "-mr-"; + private static final String EXT_PREFIX = "-ext-"; + private static final String LOCAL_PREFIX = "-local-"; + + /** + * Check if path is for intermediate data + * @return true if a uri is a temporary uri for map-reduce intermediate data, + * false otherwise + */ + public boolean isMRTmpFileURI(String uriStr) { + return (uriStr.indexOf(executionId) != -1) && + (uriStr.indexOf(MR_PREFIX) != -1); + } + + /** + * Get a path to store map-reduce intermediate data in. + * + * @return next available path for map-red intermediate data + */ + public Path getMRTmpPath() { + return new Path(getMRScratchDir(), MR_PREFIX + + nextPathId()); + } + + /** + * Get a tmp path on local host to store intermediate data. + * + * @return next available tmp path on local fs + */ + public Path getLocalTmpPath() { + return new Path(getLocalScratchDir(true), LOCAL_PREFIX + nextPathId()); + } + + /** + * Get a path to store tmp data destined for external URI. + * + * @param path + * external URI to which the tmp data has to be eventually moved + * @return next available tmp path on the file system corresponding extURI + */ + public Path getExternalTmpPath(Path path) { + URI extURI = path.toUri(); + if (extURI.getScheme().equals("viewfs")) { + // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/.. + // to final /user/hive/warehouse/ will fail later, so instead pick tmp dir + // on same namespace as tbl dir. + return getExtTmpPathRelTo(path.getParent()); + } + return new Path(getExternalScratchDir(extURI), EXT_PREFIX + + nextPathId()); + } + + /** + * This is similar to getExternalTmpPath() with difference being this method returns temp path + * within passed in uri, whereas getExternalTmpPath() ignores passed in path and returns temp + * path within /tmp + */ + public Path getExtTmpPathRelTo(Path path) { + URI uri = path.toUri(); + return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain, + uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId()); + } + + /** + * @return the resFile + */ + public Path getResFile() { + return resFile; + } + + /** + * @param resFile + * the resFile to set + */ + public void setResFile(Path resFile) { + this.resFile = resFile; + resDir = null; + resDirPaths = null; + resDirFilesNum = 0; + } + + /** + * @return the resDir + */ + public Path getResDir() { + return resDir; + } + + /** + * @param resDir + * the resDir to set + */ + public void setResDir(Path resDir) { + this.resDir = resDir; + resFile = null; + + resDirFilesNum = 0; + resDirPaths = null; + } + + public void clear() throws IOException { + if (resDir != null) { + try { + FileSystem fs = resDir.getFileSystem(conf); + fs.delete(resDir, true); + } catch (IOException e) { + LOG.info("Context clear error: " + StringUtils.stringifyException(e)); + } + } + + if (resFile != null) { + try { + FileSystem fs = resFile.getFileSystem(conf); + fs.delete(resFile, false); + } catch (IOException e) { + LOG.info("Context clear error: " + StringUtils.stringifyException(e)); + } + } + removeScratchDir(); + originalTracker = null; + setNeedLockMgr(false); + } + + public DataInput getStream() { + try { + if (!initialized) { + initialized = true; + if ((resFile == null) && (resDir == null)) { + return null; + } + + if (resFile != null) { + return resFile.getFileSystem(conf).open(resFile); + } + + resFs = resDir.getFileSystem(conf); + FileStatus status = resFs.getFileStatus(resDir); + assert status.isDir(); + FileStatus[] resDirFS = resFs.globStatus(new Path(resDir + "/*")); + resDirPaths = new Path[resDirFS.length]; + int pos = 0; + for (FileStatus resFS : resDirFS) { + if (!resFS.isDir()) { + resDirPaths[pos++] = resFS.getPath(); + } + } + if (pos == 0) { + return null; + } + + return resFs.open(resDirPaths[resDirFilesNum++]); + } else { + return getNextStream(); + } + } catch (FileNotFoundException e) { + LOG.info("getStream error: " + StringUtils.stringifyException(e)); + return null; + } catch (IOException e) { + LOG.info("getStream error: " + StringUtils.stringifyException(e)); + return null; + } + } + + private DataInput getNextStream() { + try { + if (resDir != null && resDirFilesNum < resDirPaths.length + && (resDirPaths[resDirFilesNum] != null)) { + return resFs.open(resDirPaths[resDirFilesNum++]); + } + } catch (FileNotFoundException e) { + LOG.info("getNextStream error: " + StringUtils.stringifyException(e)); + return null; + } catch (IOException e) { + LOG.info("getNextStream error: " + StringUtils.stringifyException(e)); + return null; + } + + return null; + } + + public void resetStream() { + if (initialized) { + resDirFilesNum = 0; + initialized = false; + } + } + + /** + * Little abbreviation for StringUtils. + */ + private static boolean strEquals(String str1, String str2) { + return org.apache.commons.lang.StringUtils.equals(str1, str2); + } + + /** + * Set the token rewrite stream being used to parse the current top-level SQL + * statement. Note that this should not be used for other parsing + * activities; for example, when we encounter a reference to a view, we switch + * to a new stream for parsing the stored view definition from the catalog, + * but we don't clobber the top-level stream in the context. + * + * @param tokenRewriteStream + * the stream being used + */ + public void setTokenRewriteStream(TokenRewriteStream tokenRewriteStream) { + assert (this.tokenRewriteStream == null); + this.tokenRewriteStream = tokenRewriteStream; + } + + /** + * @return the token rewrite stream being used to parse the current top-level + * SQL statement, or null if it isn't available (e.g. for parser + * tests) + */ + public TokenRewriteStream getTokenRewriteStream() { + return tokenRewriteStream; + } + + /** + * Generate a unique executionId. An executionId, together with user name and + * the configuration, will determine the temporary locations of all intermediate + * files. + * + * In the future, users can use the executionId to resume a query. + */ + public static String generateExecutionId() { + Random rand = new Random(); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss_SSS"); + String executionId = "hive_" + format.format(new Date()) + "_" + + Math.abs(rand.nextLong()); + return executionId; + } + + /** + * Does Hive wants to run tasks entirely on the local machine + * (where the query is being compiled)? + * + * Today this translates into running hadoop jobs locally + */ + public boolean isLocalOnlyExecutionMode() { + return ShimLoader.getHadoopShims().isLocalMode(conf); + } + + public List getHiveLocks() { + return hiveLocks; + } + + public void setHiveLocks(List hiveLocks) { + this.hiveLocks = hiveLocks; + } + + public HiveTxnManager getHiveTxnManager() { + return hiveTxnManager; + } + + public void setHiveTxnManager(HiveTxnManager txnMgr) { + hiveTxnManager = txnMgr; + } + + public void setOriginalTracker(String originalTracker) { + this.originalTracker = originalTracker; + } + + public void restoreOriginalTracker() { + if (originalTracker != null) { + ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, originalTracker); + originalTracker = null; + } + } + + public void addCS(String path, ContentSummary cs) { + pathToCS.put(path, cs); + } + + public ContentSummary getCS(Path path) { + return getCS(path.toString()); + } + + public ContentSummary getCS(String path) { + return pathToCS.get(path); + } + + public Map getPathToCS() { + return pathToCS; + } + + public Configuration getConf() { + return conf; + } + + /** + * @return the isHDFSCleanup + */ + public boolean isHDFSCleanup() { + return isHDFSCleanup; + } + + /** + * @param isHDFSCleanup the isHDFSCleanup to set + */ + public void setHDFSCleanup(boolean isHDFSCleanup) { + this.isHDFSCleanup = isHDFSCleanup; + } + + public boolean isNeedLockMgr() { + return needLockMgr; + } + + public void setNeedLockMgr(boolean needLockMgr) { + this.needLockMgr = needLockMgr; + } + + public int getTryCount() { + return tryCount; + } + + public void setTryCount(int tryCount) { + this.tryCount = tryCount; + } + + public void setAcidOperation(AcidUtils.Operation op) { + acidOperation = op; + } + + public AcidUtils.Operation getAcidOperation() { + return acidOperation; + } +} diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java new file mode 100644 index 00000000000..962d0e55abd --- /dev/null +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java @@ -0,0 +1,1298 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + *distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you maynot use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicablelaw or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintStream; +import java.io.StringWriter; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Deque; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import junit.framework.Assert; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hive.cli.CliDriver; +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.common.io.CachingPrintStream; +import org.apache.hadoop.hive.common.io.DigestPrintStream; +import org.apache.hadoop.hive.common.io.SortAndDigestPrintStream; +import org.apache.hadoop.hive.common.io.SortPrintStream; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.ParseDriver; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.HadoopShims; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.util.Shell; +import org.apache.hive.common.util.StreamPrinter; +import org.apache.tools.ant.BuildException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; + +import com.google.common.collect.ImmutableList; + +/** + * HiveTestUtil cloned from Hive QTestUtil + * + */ +public class HiveTestUtil { + + public static final String UTF_8 = "UTF-8"; + private static final Log LOG = LogFactory.getLog("HiveTestUtil"); + private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; + public static final String DEFAULT_DATABASE_NAME = "default"; + + private String testWarehouse; + private final String testFiles; + protected final String outDir; + protected final String logDir; + private final TreeMap qMap; + private final Set qSkipSet; + private final Set qSortSet; + private final Set qSortQuerySet; + private final Set qHashQuerySet; + private final Set qSortNHashQuerySet; + private final Set qJavaVersionSpecificOutput; + private static final String SORT_SUFFIX = ".sorted"; + private static MiniClusterType clusterType = MiniClusterType.none; + private ParseDriver pd; + protected Hive db; + protected HiveConf conf; + private BaseSemanticAnalyzer sem; + protected final boolean overWrite; + private CliDriver cliDriver; + private HadoopShims.MiniMrShim mr = null; + private HadoopShims.MiniDFSShim dfs = null; + private String hadoopVer = null; + private HiveTestSetup setup = null; + private boolean isSessionStateStarted = false; + private static final String javaVersion = getJavaVersion(); + + private String initScript=""; + private String cleanupScript=""; + + public HiveConf getConf() { + return conf; + } + + public boolean deleteDirectory(File path) { + if (path.exists()) { + File[] files = path.listFiles(); + for (File file : files) { + if (file.isDirectory()) { + deleteDirectory(file); + } else { + file.delete(); + } + } + } + return (path.delete()); + } + + public void copyDirectoryToLocal(Path src, Path dest) throws Exception { + + FileSystem srcFs = src.getFileSystem(conf); + FileSystem destFs = dest.getFileSystem(conf); + if (srcFs.exists(src)) { + FileStatus[] files = srcFs.listStatus(src); + for (FileStatus file : files) { + String name = file.getPath().getName(); + Path dfs_path = file.getPath(); + Path local_path = new Path(dest, name); + + if (file.isDir()) { + if (!destFs.exists(local_path)) { + destFs.mkdirs(local_path); + } + copyDirectoryToLocal(dfs_path, local_path); + } else { + srcFs.copyToLocalFile(dfs_path, local_path); + } + } + } + } + + static Pattern mapTok = Pattern.compile("(\\.?)(.*)_map_(.*)"); + static Pattern reduceTok = Pattern.compile("(.*)(reduce_[^\\.]*)((\\..*)?)"); + + public void normalizeNames(File path) throws Exception { + if (path.isDirectory()) { + File[] files = path.listFiles(); + for (File file : files) { + normalizeNames(file); + } + } else { + Matcher m = reduceTok.matcher(path.getName()); + if (m.matches()) { + String name = m.group(1) + "reduce" + m.group(3); + path.renameTo(new File(path.getParent(), name)); + } else { + m = mapTok.matcher(path.getName()); + if (m.matches()) { + String name = m.group(1) + "map_" + m.group(3); + path.renameTo(new File(path.getParent(), name)); + } + } + } + } + + public String getOutputDirectory() { + return outDir; + } + + public String getLogDirectory() { + return logDir; + } + + private String getHadoopMainVersion(String input) { + if (input == null) { + return null; + } + Pattern p = Pattern.compile("^(\\d+\\.\\d+).*"); + Matcher m = p.matcher(input); + if (m.matches()) { + return m.group(1); + } + return null; + } + + public void initConf() throws Exception { + // Plug verifying metastore in for testing. + conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, + "org.apache.hadoop.hive.metastore.VerifyingObjectStore"); + + if (mr != null) { + assert dfs != null; + + mr.setupConfiguration(conf); + + // set fs.default.name to the uri of mini-dfs + String dfsUriString = WindowsPathUtil.getHdfsUriString(dfs.getFileSystem().getUri().toString()); + conf.setVar(HiveConf.ConfVars.HADOOPFS, dfsUriString); + // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs + conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, + (new Path(dfsUriString, + "/build/ql/test/data/warehouse/")).toString()); + } + + // Windows paths should be converted after MiniMrShim.setupConfiguration() + // since setupConfiguration may overwrite configuration values. + if (Shell.WINDOWS) { + WindowsPathUtil.convertPathsFromWindowsToHdfs(conf); + } + } + + public enum MiniClusterType { + mr, + tez, + none; + + public static MiniClusterType valueForString(String type) { + if (type.equals("miniMR")) { + return mr; + } else if (type.equals("tez")) { + return tez; + } else { + return none; + } + } + } + + public HiveTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer) + throws Exception { + this(outDir, logDir, clusterType, null, hadoopVer); + } + + public HiveTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, + String hadoopVer) + throws Exception { + this.outDir = outDir; + this.logDir = logDir; + if (confDir != null && !confDir.isEmpty()) { + HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml")); + System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation()); + } + conf = new HiveConf(); + String tmpBaseDir = System.getProperty("test.tmp.dir"); + if(tmpBaseDir == null || tmpBaseDir == "") { + tmpBaseDir = System.getProperty("java.io.tmpdir"); + } + String metaStoreURL = "jdbc:derby:" + tmpBaseDir + File.separator + "metastore_dbtest;create=true"; + conf.set(ConfVars.METASTORECONNECTURLKEY.varname, metaStoreURL); + System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, metaStoreURL); + + //set where derby logs + File derbyLogFile = new File(tmpBaseDir + "/derby.log"); + derbyLogFile.createNewFile(); + System.setProperty("derby.stream.error.file", derbyLogFile.getPath()); + + this.hadoopVer = getHadoopMainVersion(hadoopVer); + qMap = new TreeMap(); + qSkipSet = new HashSet(); + qSortSet = new HashSet(); + qSortQuerySet = new HashSet(); + qHashQuerySet = new HashSet(); + qSortNHashQuerySet = new HashSet(); + qJavaVersionSpecificOutput = new HashSet(); + this.clusterType = clusterType; + + HadoopShims shims = ShimLoader.getHadoopShims(); + int numberOfDataNodes = 4; + + if (clusterType != MiniClusterType.none) { + dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); + FileSystem fs = dfs.getFileSystem(); + String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); + if (clusterType == MiniClusterType.tez) { + mr = shims.getMiniTezCluster(conf, 4, uriString, 1); + } else { + mr = shims.getMiniMrCluster(conf, 4, uriString, 1); + } + } + + initConf(); + + // Use the current directory if it is not specified + String dataDir = conf.get("test.data.files"); + if (dataDir == null) { + dataDir = new File(".").getAbsolutePath() + "/data/files"; + } + + testFiles = dataDir; + + // Use the current directory if it is not specified + String scriptsDir = conf.get("test.data.scripts"); + if (scriptsDir == null) { + scriptsDir = new File(".").getAbsolutePath() + "/data/scripts"; + } + if (!initScript.isEmpty()) { + this.initScript = scriptsDir + "/" + initScript; + } + if (!cleanupScript.isEmpty()) { + this.cleanupScript = scriptsDir + "/" + cleanupScript; + } + + overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite")); + + setup = new HiveTestSetup(); + setup.preTest(conf); + init(); + } + + public void shutdown() throws Exception { + cleanUp(); + setup.tearDown(); + if (mr != null) { + mr.shutdown(); + mr = null; + } + FileSystem.closeAll(); + if (dfs != null) { + dfs.shutdown(); + dfs = null; + } + } + + public String readEntireFileIntoString(File queryFile) throws IOException { + InputStreamReader isr = new InputStreamReader( + new BufferedInputStream(new FileInputStream(queryFile)), HiveTestUtil.UTF_8); + StringWriter sw = new StringWriter(); + try { + IOUtils.copy(isr, sw); + } finally { + if (isr != null) { + isr.close(); + } + } + return sw.toString(); + } + + public void addFile(String queryFile) throws IOException { + addFile(queryFile, false); + } + + public void addFile(String queryFile, boolean partial) throws IOException { + addFile(new File(queryFile)); + } + + public void addFile(File qf) throws IOException { + addFile(qf, false); + } + + public void addFile(File qf, boolean partial) throws IOException { + String query = readEntireFileIntoString(qf); + qMap.put(qf.getName(), query); + if (partial) return; + + if (matches(SORT_BEFORE_DIFF, query)) { + qSortSet.add(qf.getName()); + } else if (matches(SORT_QUERY_RESULTS, query)) { + qSortQuerySet.add(qf.getName()); + } else if (matches(HASH_QUERY_RESULTS, query)) { + qHashQuerySet.add(qf.getName()); + } else if (matches(SORT_AND_HASH_QUERY_RESULTS, query)) { + qSortNHashQuerySet.add(qf.getName()); + } + } + + private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF"); + private static final Pattern SORT_QUERY_RESULTS = Pattern.compile("-- SORT_QUERY_RESULTS"); + private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS"); + private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS"); + + private boolean matches(Pattern pattern, String query) { + Matcher matcher = pattern.matcher(query); + if (matcher.find()) { + return true; + } + return false; + } + + /** + * Get formatted Java version to include minor version, but + * exclude patch level. + * + * @return Java version formatted as major_version.minor_version + */ + private static String getJavaVersion() { + String version = System.getProperty("java.version"); + if (version == null) { + throw new NullPointerException("No java version could be determined " + + "from system properties"); + } + + // "java version" system property is formatted + // major_version.minor_version.patch_level. + // Find second dot, instead of last dot, to be safe + int pos = version.indexOf('.'); + pos = version.indexOf('.', pos + 1); + return version.substring(0, pos); + } + + /** + * Clear out any side effects of running tests + */ + public void clearPostTestEffects() throws Exception { + setup.postTest(conf); + } + + /** + * Clear out any side effects of running tests + */ + public void clearTablesCreatedDuringTests() throws Exception { + if (System.getenv(QTEST_LEAVE_FILES) != null) { + return; + } + + // Delete any tables other than the source tables + // and any databases other than the default database. + for (String dbName : db.getAllDatabases()) { + SessionState.get().setCurrentDatabase(dbName); + for (String tblName : db.getAllTables()) { + if (!DEFAULT_DATABASE_NAME.equals(dbName)) { + Table tblObj = db.getTable(tblName); + // dropping index table can not be dropped directly. Dropping the base + // table will automatically drop all its index table + if(tblObj.isIndexTable()) { + continue; + } + db.dropTable(dbName, tblName); + } else { + // this table is defined in srcTables, drop all indexes on it + List indexes = db.getIndexes(dbName, tblName, (short)-1); + if (indexes != null && indexes.size() > 0) { + for (Index index : indexes) { + db.dropIndex(dbName, tblName, index.getIndexName(), true); + } + } + } + } + if (!DEFAULT_DATABASE_NAME.equals(dbName)) { + // Drop cascade, may need to drop functions + db.dropDatabase(dbName, true, true, true); + } + } + + // delete remaining directories for external tables (can affect stats for following tests) + try { + Path p = new Path(testWarehouse); + FileSystem fileSystem = p.getFileSystem(conf); + if (fileSystem.exists(p)) { + for (FileStatus status : fileSystem.listStatus(p)) { + if (status.isDir()) { + fileSystem.delete(status.getPath(), true); + } + } + } + } catch (IllegalArgumentException e) { + // ignore.. provides invalid url sometimes intentionally + } + SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME); + + List roleNames = db.getAllRoleNames(); + for (String roleName : roleNames) { + if (!"PUBLIC".equalsIgnoreCase(roleName) && !"ADMIN".equalsIgnoreCase(roleName)) { + db.dropRole(roleName); + } + } + } + + /** + * Clear out any side effects of running tests + */ + public void clearTestSideEffects() throws Exception { + if (System.getenv(QTEST_LEAVE_FILES) != null) { + return; + } + + clearTablesCreatedDuringTests(); + } + + public void cleanUp() throws Exception { + if (!isSessionStateStarted) { + startSessionState(); + } + if (System.getenv(QTEST_LEAVE_FILES) != null) { + return; + } + + clearTablesCreatedDuringTests(); + + SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true); + + if(cleanupScript != "") { + String cleanupCommands = readEntireFileIntoString(new File(cleanupScript)); + LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands); + if (cliDriver == null) { + cliDriver = new CliDriver(); + } + cliDriver.processLine(cleanupCommands); + } + + SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false); + + // delete any contents in the warehouse dir + Path p = new Path(testWarehouse); + FileSystem fs = p.getFileSystem(conf); + + try { + FileStatus[] ls = fs.listStatus(p); + for (int i = 0; (ls != null) && (i < ls.length); i++) { + fs.delete(ls[i].getPath(), true); + } + } catch (FileNotFoundException e) { + // Best effort + } + + FunctionRegistry.unregisterTemporaryUDF("test_udaf"); + FunctionRegistry.unregisterTemporaryUDF("test_error"); + } + + public void createSources() throws Exception { + if(!isSessionStateStarted) { + startSessionState(); + } + conf.setBoolean("hive.test.init.phase", true); + + if(cliDriver == null) { + cliDriver = new CliDriver(); + } + cliDriver.processLine("set test.data.dir=" + testFiles + ";"); + + conf.setBoolean("hive.test.init.phase", false); + } + + public void init() throws Exception { + // System.out.println(conf.toString()); + testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); + // conf.logVars(System.out); + // System.out.flush(); + + String execEngine = conf.get("hive.execution.engine"); + conf.set("hive.execution.engine", "mr"); + SessionState.start(conf); + conf.set("hive.execution.engine", execEngine); + db = Hive.get(conf); + pd = new ParseDriver(); + sem = new SemanticAnalyzer(conf); + } + + public void init(String tname) throws Exception { + cleanUp(); + createSources(); + cliDriver.processCmd("set hive.cli.print.header=true;"); + } + + public void cliInit(String tname) throws Exception { + cliInit(tname, true); + } + + public String cliInit(String tname, boolean recreate) throws Exception { + if (recreate) { + cleanUp(); + createSources(); + } + + HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, + "org.apache.phoenix.hive.DummyAuthenticator"); + Utilities.clearWorkMap(); + CliSessionState ss = new CliSessionState(conf); + assert ss != null; + ss.in = System.in; + + String outFileExtension = getOutFileExtension(tname); + String stdoutName = null; + if (outDir != null) { + File qf = new File(outDir, tname); + stdoutName = qf.getName().concat(outFileExtension); + } else { + stdoutName = tname + outFileExtension; + } + + File outf = new File(logDir, stdoutName); + OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf)); + if (qSortQuerySet.contains(tname)) { + ss.out = new SortPrintStream(fo, "UTF-8"); + } else if (qHashQuerySet.contains(tname)) { + ss.out = new DigestPrintStream(fo, "UTF-8"); + } else if (qSortNHashQuerySet.contains(tname)) { + ss.out = new SortAndDigestPrintStream(fo, "UTF-8"); + } else { + ss.out = new PrintStream(fo, true, "UTF-8"); + } + ss.err = new CachingPrintStream(fo, true, "UTF-8"); + ss.setIsSilent(true); + SessionState oldSs = SessionState.get(); + + if (oldSs != null && clusterType == MiniClusterType.tez) { + oldSs.close(); + } + + if (oldSs != null && oldSs.out != null && oldSs.out != System.out) { + oldSs.out.close(); + } + SessionState.start(ss); + + cliDriver = new CliDriver(); + cliDriver.processInitFiles(ss); + + return outf.getAbsolutePath(); + } + + private CliSessionState startSessionState() + throws IOException { + + HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, + "org.apache.phoenix.hive.DummyAuthenticator"); + + String execEngine = conf.get("hive.execution.engine"); + conf.set("hive.execution.engine", "mr"); + CliSessionState ss = new CliSessionState(conf); + assert ss != null; + ss.in = System.in; + ss.out = System.out; + ss.err = System.out; + + SessionState oldSs = SessionState.get(); + if (oldSs != null && clusterType == MiniClusterType.tez) { + oldSs.close(); + } + if (oldSs != null && oldSs.out != null && oldSs.out != System.out) { + oldSs.out.close(); + } + SessionState.start(ss); + + isSessionStateStarted = true; + + conf.set("hive.execution.engine", execEngine); + return ss; + } + + public int executeOne(String tname) { + String q = qMap.get(tname); + + if (q.indexOf(";") == -1) { + return -1; + } + + String q1 = q.substring(0, q.indexOf(";") + 1); + String qrest = q.substring(q.indexOf(";") + 1); + qMap.put(tname, qrest); + + System.out.println("Executing " + q1); + return cliDriver.processLine(q1); + } + + public static final String CRLF = System.getProperty("line.separator"); + public int executeClient(String tname1, String tname2) { + String commands = getCommands(tname1) + CRLF + getCommands(tname2); + return cliDriver.processLine(commands); + } + + public int executeClient(String tname) { + return cliDriver.processLine(getCommands(tname)); + } + + private String getCommands(String tname) { + String commands = qMap.get(tname); + StringBuilder newCommands = new StringBuilder(commands.length()); + int lastMatchEnd = 0; + Matcher commentMatcher = Pattern.compile("^--.*$", Pattern.MULTILINE).matcher(commands); + while (commentMatcher.find()) { + newCommands.append(commands.substring(lastMatchEnd, commentMatcher.start())); + newCommands.append(commentMatcher.group().replaceAll("(? configs = new ArrayList(); + configs.add(this.hadoopVer); + + Deque stack = new LinkedList(); + StringBuilder sb = new StringBuilder(); + sb.append(testName); + stack.push(sb.toString()); + + // example file names are input1.q.out_0.20.0_minimr or input2.q.out_0.17 + for (String s : configs) { + sb.append('_'); + sb.append(s); + stack.push(sb.toString()); + } + while (stack.size() > 0) { + String fileName = stack.pop(); + File f = new File(outDir, fileName); + if (f.exists()) { + ret = f.getPath(); + break; + } + } + return ret; + } + + private Pattern[] toPattern(String[] patternStrs) { + Pattern[] patterns = new Pattern[patternStrs.length]; + for (int i = 0; i < patternStrs.length; i++) { + patterns[i] = Pattern.compile(patternStrs[i]); + } + return patterns; + } + + private void maskPatterns(Pattern[] patterns, String fname) throws Exception { + String maskPattern = "#### A masked pattern was here ####"; + + String line; + BufferedReader in; + BufferedWriter out; + + File file = new File(fname); + File fileOrig = new File(fname + ".orig"); + FileUtils.copyFile(file, fileOrig); + + in = new BufferedReader(new InputStreamReader(new FileInputStream(fileOrig), "UTF-8")); + out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8")); + + boolean lastWasMasked = false; + while (null != (line = in.readLine())) { + for (Pattern pattern : patterns) { + line = pattern.matcher(line).replaceAll(maskPattern); + } + + if (line.equals(maskPattern)) { + // We're folding multiple masked lines into one. + if (!lastWasMasked) { + out.write(line); + out.write("\n"); + lastWasMasked = true; + } + } else { + out.write(line); + out.write("\n"); + lastWasMasked = false; + } + } + + in.close(); + out.close(); + } + + private final Pattern[] planMask = toPattern(new String[] { + ".*file:.*", + ".*pfile:.*", + ".*hdfs:.*", + ".*/tmp/.*", + ".*invalidscheme:.*", + ".*lastUpdateTime.*", + ".*lastAccessTime.*", + ".*lastModifiedTime.*", + ".*[Oo]wner.*", + ".*CreateTime.*", + ".*LastAccessTime.*", + ".*Location.*", + ".*LOCATION '.*", + ".*transient_lastDdlTime.*", + ".*last_modified_.*", + ".*at org.*", + ".*at sun.*", + ".*at java.*", + ".*at junit.*", + ".*Caused by:.*", + ".*LOCK_QUERYID:.*", + ".*LOCK_TIME:.*", + ".*grantTime.*", + ".*[.][.][.] [0-9]* more.*", + ".*job_[0-9_]*.*", + ".*job_local[0-9_]*.*", + ".*USING 'java -cp.*", + "^Deleted.*", + ".*DagName:.*", + ".*Input:.*/data/files/.*", + ".*Output:.*/data/files/.*", + ".*total number of created files now is.*" + }); + + public int checkCliDriverResults(String tname) throws Exception { + assert(qMap.containsKey(tname)); + + String outFileExtension = getOutFileExtension(tname); + String outFileName = outPath(outDir, tname + outFileExtension); + + File f = new File(logDir, tname + outFileExtension); + + maskPatterns(planMask, f.getPath()); + int exitVal = executeDiffCommand(f.getPath(), + outFileName, false, + qSortSet.contains(tname)); + + if (exitVal != 0 && overWrite) { + exitVal = overwriteResults(f.getPath(), outFileName); + } + + return exitVal; + } + + + public int checkCompareCliDriverResults(String tname, List outputs) throws Exception { + assert outputs.size() > 1; + maskPatterns(planMask, outputs.get(0)); + for (int i = 1; i < outputs.size(); ++i) { + maskPatterns(planMask, outputs.get(i)); + int ecode = executeDiffCommand( + outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname)); + if (ecode != 0) { + System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i)); + return ecode; + } + } + return 0; + } + + private static int overwriteResults(String inFileName, String outFileName) throws Exception { + // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING) + // once Hive uses JAVA 7. + System.out.println("Overwriting results " + inFileName + " to " + outFileName); + return executeCmd(new String[] { + "cp", + getQuotedString(inFileName), + getQuotedString(outFileName) + }); + } + + private static int executeDiffCommand(String inFileName, + String outFileName, + boolean ignoreWhiteSpace, + boolean sortResults + ) throws Exception { + + int result = 0; + + if (sortResults) { + // sort will try to open the output file in write mode on windows. We need to + // close it first. + SessionState ss = SessionState.get(); + if (ss != null && ss.out != null && ss.out != System.out) { + ss.out.close(); + } + + String inSorted = inFileName + SORT_SUFFIX; + String outSorted = outFileName + SORT_SUFFIX; + + result = sortFiles(inFileName, inSorted); + result |= sortFiles(outFileName, outSorted); + if (result != 0) { + System.err.println("ERROR: Could not sort files before comparing"); + return result; + } + inFileName = inSorted; + outFileName = outSorted; + } + + ArrayList diffCommandArgs = new ArrayList(); + diffCommandArgs.add("diff"); + + // Text file comparison + diffCommandArgs.add("-a"); + + // Ignore changes in the amount of white space + if (ignoreWhiteSpace || Shell.WINDOWS) { + diffCommandArgs.add("-b"); + } + + // Files created on Windows machines have different line endings + // than files created on Unix/Linux. Windows uses carriage return and line feed + // ("\r\n") as a line ending, whereas Unix uses just line feed ("\n"). + // Also StringBuilder.toString(), Stream to String conversions adds extra + // spaces at the end of the line. + if (Shell.WINDOWS) { + diffCommandArgs.add("--strip-trailing-cr"); // Strip trailing carriage return on input + diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank + } + // Add files to compare to the arguments list + diffCommandArgs.add(getQuotedString(inFileName)); + diffCommandArgs.add(getQuotedString(outFileName)); + + result = executeCmd(diffCommandArgs); + + if (sortResults) { + new File(inFileName).delete(); + new File(outFileName).delete(); + } + + return result; + } + + private static int sortFiles(String in, String out) throws Exception { + return executeCmd(new String[] { + "sort", + getQuotedString(in), + }, out, null); + } + + private static int executeCmd(Collection args) throws Exception { + return executeCmd(args, null, null); + } + + private static int executeCmd(String[] args) throws Exception { + return executeCmd(args, null, null); + } + + private static int executeCmd(Collection args, String outFile, String errFile) throws Exception { + String[] cmdArray = args.toArray(new String[args.size()]); + return executeCmd(cmdArray, outFile, errFile); + } + + private static int executeCmd(String[] args, String outFile, String errFile) throws Exception { + System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' ')); + + PrintStream out = outFile == null ? + SessionState.getConsole().getChildOutStream() : + new PrintStream(new FileOutputStream(outFile), true); + PrintStream err = errFile == null ? + SessionState.getConsole().getChildErrStream() : + new PrintStream(new FileOutputStream(errFile), true); + + Process executor = Runtime.getRuntime().exec(args); + + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out); + + outPrinter.start(); + errPrinter.start(); + + int result = executor.waitFor(); + + outPrinter.join(); + errPrinter.join(); + + if (outFile != null) { + out.close(); + } + + if (errFile != null) { + err.close(); + } + + return result; + } + + private static String getQuotedString(String str){ + return Shell.WINDOWS ? String.format("\"%s\"", str) : str; + } + + public ASTNode parseQuery(String tname) throws Exception { + return pd.parse(qMap.get(tname)); + } + + public void resetParser() throws SemanticException { + pd = new ParseDriver(); + sem = new SemanticAnalyzer(conf); + } + + public TreeMap getQMap() { + return qMap; + } + + /** + * HiveTestSetup defines test fixtures which are reused across testcases, + * and are needed before any test can be run + */ + public static class HiveTestSetup + { + private MiniZooKeeperCluster zooKeeperCluster = null; + private int zkPort; + private ZooKeeper zooKeeper; + + public HiveTestSetup() { + } + + public void preTest(HiveConf conf) throws Exception { + + if (zooKeeperCluster == null) { + //create temp dir + String tmpBaseDir = System.getProperty("test.tmp.dir"); + File tmpDir = Utilities.createTempDir(tmpBaseDir); + + zooKeeperCluster = new MiniZooKeeperCluster(); + zkPort = zooKeeperCluster.startup(tmpDir); + } + + if (zooKeeper != null) { + zooKeeper.close(); + } + + int sessionTimeout = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT); + zooKeeper = new ZooKeeper("localhost:" + zkPort, sessionTimeout, new Watcher() { + @Override + public void process(WatchedEvent arg0) { + } + }); + + String zkServer = "localhost"; + conf.set("hive.zookeeper.quorum", zkServer); + conf.set("hive.zookeeper.client.port", "" + zkPort); + } + + public void postTest(HiveConf conf) throws Exception { + if (zooKeeperCluster == null) { + return; + } + + if (zooKeeper != null) { + zooKeeper.close(); + } + + ZooKeeperHiveLockManager.releaseAllLocks(conf); + } + + public void tearDown() throws Exception { + if (zooKeeperCluster != null) { + zooKeeperCluster.shutdown(); + zooKeeperCluster = null; + } + } + } + + /** + * QTRunner: Runnable class for running a a single query file. + * + **/ + public static class HiveTestRunner implements Runnable { + private final HiveTestUtil qt; + private final String fname; + + public HiveTestRunner(HiveTestUtil qt, String fname) { + this.qt = qt; + this.fname = fname; + } + + @Override + public void run() { + try { + // assumption is that environment has already been cleaned once globally + // hence each thread does not call cleanUp() and createSources() again + qt.cliInit(fname, false); + qt.executeClient(fname); + } catch (Throwable e) { + System.err.println("Query file " + fname + " failed with exception " + + e.getMessage()); + e.printStackTrace(); + outputTestFailureHelpMessage(); + } + } + } + + /** + * Setup to execute a set of query files. Uses HiveTestUtil to do so. + * + * @param qfiles + * array of input query files containing arbitrary number of hive + * queries + * @param resDir + * output directory + * @param logDir + * log directory + * @return one HiveTestUtil for each query file + */ + public static HiveTestUtil[] queryListRunnerSetup(File[] qfiles, String resDir, + String logDir) throws Exception + { + HiveTestUtil[] qt = new HiveTestUtil[qfiles.length]; + for (int i = 0; i < qfiles.length; i++) { + qt[i] = new HiveTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20"); + qt[i].addFile(qfiles[i]); + qt[i].clearTestSideEffects(); + } + + return qt; + } + + /** + * Executes a set of query files in sequence. + * + * @param qfiles + * array of input query files containing arbitrary number of hive + * queries + * @param qt + * array of HiveTestUtils, one per qfile + * @return true if all queries passed, false otw + */ + public static boolean queryListRunnerSingleThreaded(File[] qfiles, HiveTestUtil[] qt) + throws Exception + { + boolean failed = false; + qt[0].cleanUp(); + qt[0].createSources(); + for (int i = 0; i < qfiles.length && !failed; i++) { + qt[i].clearTestSideEffects(); + qt[i].cliInit(qfiles[i].getName(), false); + qt[i].executeClient(qfiles[i].getName()); + int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); + if (ecode != 0) { + failed = true; + System.err.println("Test " + qfiles[i].getName() + + " results check failed with error code " + ecode); + outputTestFailureHelpMessage(); + } + qt[i].clearPostTestEffects(); + } + return (!failed); + } + + public static void outputTestFailureHelpMessage() { + System.err.println("See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."); + System.err.flush(); + } + + public static String ensurePathEndsInSlash(String path) { + if(path == null) { + throw new NullPointerException("Path cannot be null"); + } + if(path.endsWith(File.separator)) { + return path; + } else { + return path + File.separator; + } + } + + private static String[] cachedQvFileList = null; + private static ImmutableList cachedDefaultQvFileList = null; + private static Pattern qvSuffix = Pattern.compile("_[0-9]+.qv$", Pattern.CASE_INSENSITIVE); + + public static List getVersionFiles(String queryDir, String tname) { + ensureQvFileList(queryDir); + List result = getVersionFilesInternal(tname); + if (result == null) { + result = cachedDefaultQvFileList; + } + return result; + } + + private static void ensureQvFileList(String queryDir) { + if (cachedQvFileList != null) return; + // Not thread-safe. + System.out.println("Getting versions from " + queryDir); + cachedQvFileList = (new File(queryDir)).list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.toLowerCase().endsWith(".qv"); + } + }); + if (cachedQvFileList == null) return; // no files at all + Arrays.sort(cachedQvFileList, String.CASE_INSENSITIVE_ORDER); + List defaults = getVersionFilesInternal("default"); + cachedDefaultQvFileList = (defaults != null) + ? ImmutableList.copyOf(defaults) : ImmutableList.of(); + } + + private static List getVersionFilesInternal(String tname) { + if (cachedQvFileList == null) { + return new ArrayList(); + } + int pos = Arrays.binarySearch(cachedQvFileList, tname, String.CASE_INSENSITIVE_ORDER); + if (pos >= 0) { + throw new BuildException("Unexpected file list element: " + cachedQvFileList[pos]); + } + List result = null; + for (pos = (-pos - 1); pos < cachedQvFileList.length; ++pos) { + String candidate = cachedQvFileList[pos]; + if (candidate.length() <= tname.length() + || !tname.equalsIgnoreCase(candidate.substring(0, tname.length())) + || !qvSuffix.matcher(candidate.substring(tname.length())).matches()) { + break; + } + if (result == null) { + result = new ArrayList(); + } + result.add(candidate); + } + return result; + } + + public void failed(int ecode, String fname, String debugHint) { + String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null; + Assert.fail("Client Execution failed with error code = " + ecode + + (command != null ? " running " + command : "") + (debugHint != null ? debugHint : "")); + } + + // for negative tests, which is succeeded.. no need to print the query string + public void failed(String fname, String debugHint) { + Assert.fail("Client Execution was expected to fail, but succeeded with error code 0 " + + (debugHint != null ? debugHint : "")); + } + + public void failedDiff(int ecode, String fname, String debugHint) { + Assert.fail("Client Execution results failed with error code = " + ecode + + (debugHint != null ? debugHint : "")); + } + + public void failed(Throwable e, String fname, String debugHint) { + String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null; + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.println("Failed query: " + fname); + System.err.flush(); + Assert.fail("Unexpected exception " + + org.apache.hadoop.util.StringUtils.stringifyException(e) + "\n" + + (command != null ? " running " + command : "") + + (debugHint != null ? debugHint : "")); + } + + public static class WindowsPathUtil { + + public static void convertPathsFromWindowsToHdfs(HiveConf conf){ + // Following local paths are used as HDFS paths in unit tests. + // It works well in Unix as the path notation in Unix and HDFS is more or less same. + // But when it comes to Windows, drive letter separator ':' & backslash '\" are invalid + // characters in HDFS so we need to converts these local paths to HDFS paths before using them + // in unit tests. + + String orgWarehouseDir = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); + conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, getHdfsUriString(orgWarehouseDir)); + + String orgTestTempDir = System.getProperty("test.tmp.dir"); + System.setProperty("test.tmp.dir", getHdfsUriString(orgTestTempDir)); + + String orgTestWarehouseDir = System.getProperty("test.warehouse.dir"); + System.setProperty("test.warehouse.dir", getHdfsUriString(orgTestWarehouseDir)); + + String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR); + conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir)); + } + + public static String getHdfsUriString(String uriStr) { + assert uriStr != null; + if(Shell.WINDOWS) { + // If the URI conversion is from Windows to HDFS then replace the '\' with '/' + // and remove the windows single drive letter & colon from absolute path. + return uriStr.replace('\\', '/') + .replaceFirst("/[c-zC-Z]:", "/") + .replaceFirst("^[c-zC-Z]:", ""); + } + return uriStr; + } + } +} diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/PhoenixHiveCreate.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/PhoenixHiveCreate.java new file mode 100644 index 00000000000..29ada56de07 --- /dev/null +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/PhoenixHiveCreate.java @@ -0,0 +1,295 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + *distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you maynot use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicablelaw or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringWriter; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.Properties; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.jdbc.PhoenixDriver; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.StringUtil; +import org.apache.phoenix.util.TestUtil; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.base.Preconditions; + +/** + * + * Test class to run all Hive Phoenix integration tests against a MINI Map-Reduce cluster. + */ +@Category(NeedsOwnMiniClusterTest.class) +public class PhoenixHiveCreate { + + private static final Log LOG = LogFactory.getLog(PhoenixHiveCreate.class); + private static final String SCHEMA_NAME = "T"; + private static final String TABLE_NAME = "HIVE_TEST"; + private static Path TEST_ROOT; + private static final String TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, TABLE_NAME); + private static HBaseTestingUtility hbaseTestUtil; + private static String zkQuorum; + private static Connection conn; + private static Configuration conf; + private static HiveTestUtil qt; + private static String hiveOutputDir; + private static String hiveLogDir; + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + hbaseTestUtil = new HBaseTestingUtility(); + conf = hbaseTestUtil.getConfiguration(); + setUpConfigForMiniCluster(conf); + conf.set(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true)); + hbaseTestUtil.startMiniCluster(3); + + Class.forName(PhoenixDriver.class.getName()); + zkQuorum = "localhost:" + hbaseTestUtil.getZkCluster().getClientPort(); + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + props.put(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true)); + conn = DriverManager.getConnection(PhoenixRuntime.JDBC_PROTOCOL + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum,props); + + // Setup Hive Output Folder + hiveOutputDir = new Path(hbaseTestUtil.getDataTestDir(), "hive_output").toString(); + File outputDir = new File(hiveOutputDir); + outputDir.mkdirs(); + hiveLogDir = new Path(hbaseTestUtil.getDataTestDir(), "hive_log").toString(); + File logDir = new File(hiveLogDir); + logDir.mkdirs(); + + // Setup Hive mini Server + Path testRoot = hbaseTestUtil.getDataTestDir(); + System.setProperty("test.tmp.dir", testRoot.toString()); + System.setProperty("test.warehouse.dir", (new Path(testRoot, "warehouse")).toString()); + + HiveTestUtil.MiniClusterType miniMR = HiveTestUtil.MiniClusterType.mr; + try { + qt = new HiveTestUtil(hiveOutputDir, hiveLogDir, miniMR, null); + } catch (Exception e) { + LOG.error("Unexpected exception in setup", e); + fail("Unexpected exception in setup"); + } + + } + + + /** + * Datatype Test + * @throws Exception + */ + @Test + public void dataTypeTest() throws Exception { + String testName = "dataTypeTest"; + // create a dummy outfile under log folder + hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out")); + createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString()); + createFile(StringUtil.EMPTY_STRING, new Path(hiveOutputDir, testName + ".out").toString()); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE EXTERNAL TABLE IF NOT EXISTS phoenix_datatype(ID int, description STRING, ts TIMESTAMP, db DOUBLE,fl FLOAT, us INT)" + HiveTestUtil.CRLF + + " STORED BY \"org.apache.phoenix.hive.PhoenixStorageHandler\"" +HiveTestUtil.CRLF+ + " TBLPROPERTIES(" + HiveTestUtil.CRLF+ + " 'phoenix.hbase.table.name'='phoenix_datatype'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.znode.parent'='hbase'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.quorum'='localhost:" + hbaseTestUtil.getZkCluster().getClientPort() + "'," +HiveTestUtil.CRLF+ + " 'phoenix.rowkeys'='id'," + HiveTestUtil.CRLF+ + " 'autocreate'='true'," + HiveTestUtil.CRLF+ + " 'autodrop'='true'," + HiveTestUtil.CRLF+ + " 'phoenix.column.mapping'='description:B.description');" + HiveTestUtil.CRLF); + sb.append("INSERT INTO TABLE phoenix_datatype" + HiveTestUtil.CRLF+ + "VALUES (10, \"foodesc\",\"2013-01-05 01:01:01\",200,2.0,-1);" + HiveTestUtil.CRLF); + String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString(); + createFile(sb.toString(), fullPath); + runTest(testName, fullPath); + + String phoenixQuery = "SELECT * FROM phoenix_datatype"; + PreparedStatement statement = conn.prepareStatement(phoenixQuery); + ResultSet rs = statement.executeQuery(); + assert(rs.getMetaData().getColumnCount() == 6); + while(rs.next()){ + assert(rs.getInt(1) == 10); + assert(rs.getString(2).equalsIgnoreCase("foodesc")); + assert(rs.getTimestamp(3).equals(Timestamp.valueOf("2013-01-05 02:01:01"))); + assert(rs.getDouble(4) == 200); + assert(rs.getFloat(5) == 2.0); + assert(rs.getInt(6) == -1); + } + } + + /** + * Datatype Test + * @throws Exception + */ + @Test + public void MultiKey() throws Exception { + String testName = "MultiKey"; + // create a dummy outfile under log folder + hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out")); + createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString()); + createFile(StringUtil.EMPTY_STRING, new Path(hiveOutputDir, testName + ".out").toString()); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE EXTERNAL TABLE IF NOT EXISTS phoenix_MultiKey(ID int, ID2 String,description STRING, ts TIMESTAMP, db DOUBLE,fl FLOAT, us INT)" + HiveTestUtil.CRLF + + " STORED BY \"org.apache.phoenix.hive.PhoenixStorageHandler\"" +HiveTestUtil.CRLF+ + " TBLPROPERTIES(" + HiveTestUtil.CRLF+ + " 'phoenix.hbase.table.name'='phoenix_MultiKey'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.znode.parent'='hbase'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.quorum'='localhost:" + hbaseTestUtil.getZkCluster().getClientPort() + "'," +HiveTestUtil.CRLF+ + " 'phoenix.rowkeys'='id,id2'," + HiveTestUtil.CRLF+ + " 'autocreate'='true'," + HiveTestUtil.CRLF+ + " 'autodrop'='true'," + HiveTestUtil.CRLF+ + " 'phoenix.column.mapping'='description:B.description');" + HiveTestUtil.CRLF); + sb.append("INSERT INTO TABLE phoenix_MultiKey" + HiveTestUtil.CRLF+ + "VALUES (10, \"part2\",\"foodesc\",\"2013-01-05 01:01:01\",200,2.0,-1);" + HiveTestUtil.CRLF); + String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString(); + createFile(sb.toString(), fullPath); + runTest(testName, fullPath); + + String phoenixQuery = "SELECT * FROM phoenix_MultiKey"; + PreparedStatement statement = conn.prepareStatement(phoenixQuery); + ResultSet rs = statement.executeQuery(); + assert(rs.getMetaData().getColumnCount() == 7); + while(rs.next()){ + assert(rs.getInt(1) == 10); + assert(rs.getString(2).equalsIgnoreCase("part2")); + assert(rs.getString(3).equalsIgnoreCase("foodesc")); + assert(rs.getTimestamp(4).equals(Timestamp.valueOf("2013-01-05 02:01:01"))); + assert(rs.getDouble(5) == 200); + assert(rs.getFloat(6) == 2.0); + assert(rs.getInt(7) == -1); + } + } + + + private void runTest(String fname, String fpath) throws Exception { + long startTime = System.currentTimeMillis(); + try { + LOG.info("Begin query: " + fname); + System.err.println("Begin query: " + fname); + + qt.addFile(fpath); + + if (qt.shouldBeSkipped(fname)) { + LOG.error("Test " + fname + " skipped"); + return; + } + + qt.cliInit(fname); + qt.clearTestSideEffects(); + int ecode = qt.executeClient(fname); + if (ecode != 0) { + qt.failed(ecode, fname, null); + } + + ecode = qt.checkCliDriverResults(fname); + if (ecode != 0) { + qt.failedDiff(ecode, fname, null); + } + qt.clearPostTestEffects(); + + } catch (Throwable e) { + qt.failed(e, fname, null); + } + + long elapsedTime = System.currentTimeMillis() - startTime; + System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s"); + assertTrue("Test passed", true); + } + + private void createFile(String content, String fullName) throws IOException { + FileUtils.write(new File(fullName), content); + } + + + private void dropTable(String tableFullName) throws SQLException { + Preconditions.checkNotNull(conn); + conn.createStatement().execute(String.format("DROP TABLE IF EXISTS %s",tableFullName)); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if(qt == null) return; + try { + qt.shutdown(); + } + catch (Exception e) { + LOG.error("Unexpected exception in setup", e); + fail("Unexpected exception in tearDown"); + } + try { + conn.close(); + } finally { + try { + PhoenixDriver.INSTANCE.close(); + } finally { + try { + DriverManager.deregisterDriver(PhoenixDriver.INSTANCE); + } finally { + hbaseTestUtil.shutdownMiniCluster(); + } + } + } + } + + public String read(File queryFile) throws IOException { + InputStreamReader isr = new InputStreamReader( + new BufferedInputStream(new FileInputStream(queryFile)), HiveTestUtil.UTF_8); + StringWriter sw = new StringWriter(); + try { + IOUtils.copy(isr, sw); + } finally { + if (isr != null) { + isr.close(); + } + } + return sw.toString(); + } +} \ No newline at end of file diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/PhoenixHiveNullTest.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/PhoenixHiveNullTest.java new file mode 100644 index 00000000000..e16aa4a7fc2 --- /dev/null +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/PhoenixHiveNullTest.java @@ -0,0 +1,346 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + *distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you maynot use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicablelaw or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringWriter; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.Properties; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.jdbc.PhoenixDriver; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.StringUtil; +import org.apache.phoenix.util.TestUtil; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.base.Preconditions; + +/** + * + * Test class to run all Hive Phoenix integration tests against a MINI Map-Reduce cluster. + */ +@Category(NeedsOwnMiniClusterTest.class) +public class PhoenixHiveNullTest { + + private static final Log LOG = LogFactory.getLog(PhoenixHiveCreate.class); + private static final String SCHEMA_NAME = "T"; + private static final String TABLE_NAME = "HIVE_TEST"; + private static Path TEST_ROOT; + private static final String TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, TABLE_NAME); + private static HBaseTestingUtility hbaseTestUtil; + private static String zkQuorum; + private static Connection conn; + private static Configuration conf; + private static HiveTestUtil qt; + private static String hiveOutputDir; + private static String hiveLogDir; + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + hbaseTestUtil = new HBaseTestingUtility(); + conf = hbaseTestUtil.getConfiguration(); + setUpConfigForMiniCluster(conf); + conf.set(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true)); + hbaseTestUtil.startMiniCluster(3); + + Class.forName(PhoenixDriver.class.getName()); + zkQuorum = "localhost:" + hbaseTestUtil.getZkCluster().getClientPort(); + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + props.put(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true)); + conn = DriverManager.getConnection(PhoenixRuntime.JDBC_PROTOCOL + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum,props); + + // Setup Hive Output Folder + hiveOutputDir = new Path(hbaseTestUtil.getDataTestDir(), "hive_output").toString(); + File outputDir = new File(hiveOutputDir); + outputDir.mkdirs(); + hiveLogDir = new Path(hbaseTestUtil.getDataTestDir(), "hive_log").toString(); + File logDir = new File(hiveLogDir); + logDir.mkdirs(); + + // Setup Hive mini Server + Path testRoot = hbaseTestUtil.getDataTestDir(); + System.setProperty("test.tmp.dir", testRoot.toString()); + System.setProperty("test.warehouse.dir", (new Path(testRoot, "warehouse")).toString()); + + HiveTestUtil.MiniClusterType miniMR = HiveTestUtil.MiniClusterType.mr; + try { + qt = new HiveTestUtil(hiveOutputDir, hiveLogDir, miniMR, null); + } catch (Exception e) { + LOG.error("Unexpected exception in setup", e); + fail("Unexpected exception in setup"); + } + + } + + + /** + * Datatype Test + * @throws Exception + */ + @Test + public void NullKeyTest() throws Exception { + String testName = "NullTest"; + // create a dummy outfile under log folder + hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out")); + createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString()); + createFile(StringUtil.EMPTY_STRING, new Path(hiveOutputDir, testName + ".out").toString()); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE EXTERNAL TABLE IF NOT EXISTS phoenix_null(ID int, description STRING,fl FLOAT, us INT)" + HiveTestUtil.CRLF + + " STORED BY \"org.apache.phoenix.hive.PhoenixStorageHandler\"" +HiveTestUtil.CRLF+ + " TBLPROPERTIES(" + HiveTestUtil.CRLF+ + " 'phoenix.hbase.table.name'='phoenix_null'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.znode.parent'='hbase'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.quorum'='localhost:" + hbaseTestUtil.getZkCluster().getClientPort() + "'," +HiveTestUtil.CRLF+ + " 'phoenix.rowkeys'='id'," + HiveTestUtil.CRLF+ + " 'autocreate'='true'," + HiveTestUtil.CRLF+ + " 'autodrop'='true'," + HiveTestUtil.CRLF+ + " 'phoenix.column.mapping'='description:B.description');" + HiveTestUtil.CRLF); + sb.append("INSERT INTO TABLE phoenix_null" + HiveTestUtil.CRLF+ + "VALUES (NULL, \"foodesc\",2.0,-1);" + HiveTestUtil.CRLF); + String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString(); + createFile(sb.toString(), fullPath); + runTest(testName, fullPath); + + String phoenixQuery = "SELECT * FROM phoenix_null"; + PreparedStatement statement = conn.prepareStatement(phoenixQuery); + ResultSet rs = statement.executeQuery(); + assert(rs.getMetaData().getColumnCount() == 4); + } + + /** + * Datatype Test + * @throws Exception + */ + @Test + public void NullStringTest() throws Exception { + String testName = "NullStringTest"; + // create a dummy outfile under log folder + hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out")); + createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString()); + createFile(StringUtil.EMPTY_STRING, new Path(hiveOutputDir, testName + ".out").toString()); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE EXTERNAL TABLE IF NOT EXISTS phoenix_nullString(ID int, description STRING,fl FLOAT, us INT)" + HiveTestUtil.CRLF + + " STORED BY \"org.apache.phoenix.hive.PhoenixStorageHandler\"" +HiveTestUtil.CRLF+ + " TBLPROPERTIES(" + HiveTestUtil.CRLF+ + " 'phoenix.hbase.table.name'='phoenix_nullString'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.znode.parent'='hbase'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.quorum'='localhost:" + hbaseTestUtil.getZkCluster().getClientPort() + "'," +HiveTestUtil.CRLF+ + " 'phoenix.rowkeys'='id'," + HiveTestUtil.CRLF+ + " 'autocreate'='true'," + HiveTestUtil.CRLF+ + " 'autodrop'='true'," + HiveTestUtil.CRLF+ + " 'phoenix.column.mapping'='description:B.description');" + HiveTestUtil.CRLF); + sb.append("INSERT INTO TABLE phoenix_nullString" + HiveTestUtil.CRLF+ + "VALUES (10, NULL,2.0,-1);" + HiveTestUtil.CRLF); + String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString(); + createFile(sb.toString(), fullPath); + runTest(testName, fullPath); + + String phoenixQuery = "SELECT * FROM phoenix_nullString"; + PreparedStatement statement = conn.prepareStatement(phoenixQuery); + ResultSet rs = statement.executeQuery(); + assert(rs.getMetaData().getColumnCount() == 4); + } + + /** + * Datatype Test + * @throws Exception + */ + @Test + public void NullIntTest() throws Exception { + String testName = "NullIntTest"; + // create a dummy outfile under log folder + hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out")); + createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString()); + createFile(StringUtil.EMPTY_STRING, new Path(hiveOutputDir, testName + ".out").toString()); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE EXTERNAL TABLE IF NOT EXISTS phoenix_nullInt(ID int, description STRING,fl FLOAT, us INT)" + HiveTestUtil.CRLF + + " STORED BY \"org.apache.phoenix.hive.PhoenixStorageHandler\"" +HiveTestUtil.CRLF+ + " TBLPROPERTIES(" + HiveTestUtil.CRLF+ + " 'phoenix.hbase.table.name'='phoenix_nullInt'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.znode.parent'='hbase'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.quorum'='localhost:" + hbaseTestUtil.getZkCluster().getClientPort() + "'," +HiveTestUtil.CRLF+ + " 'phoenix.rowkeys'='id'," + HiveTestUtil.CRLF+ + " 'autocreate'='true'," + HiveTestUtil.CRLF+ + " 'autodrop'='true'," + HiveTestUtil.CRLF+ + " 'phoenix.column.mapping'='description:B.description');" + HiveTestUtil.CRLF); + sb.append("INSERT INTO TABLE phoenix_nullInt" + HiveTestUtil.CRLF+ + "VALUES (10, \"foodesc\",2.0,NULL);" + HiveTestUtil.CRLF); + String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString(); + createFile(sb.toString(), fullPath); + runTest(testName, fullPath); + + String phoenixQuery = "SELECT * FROM phoenix_nullString"; + PreparedStatement statement = conn.prepareStatement(phoenixQuery); + ResultSet rs = statement.executeQuery(); + assert(rs.getMetaData().getColumnCount() == 4); + } + + /** + * Datatype Test + * @throws Exception + */ + @Test + public void NullMultiKey() throws Exception { + String testName = "MultiKey"; + // create a dummy outfile under log folder + hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out")); + createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString()); + createFile(StringUtil.EMPTY_STRING, new Path(hiveOutputDir, testName + ".out").toString()); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE EXTERNAL TABLE IF NOT EXISTS phoenix_NullMultiKey(ID int, ID2 String,description STRING,fl FLOAT, us INT)" + HiveTestUtil.CRLF + + " STORED BY \"org.apache.phoenix.hive.PhoenixStorageHandler\"" +HiveTestUtil.CRLF+ + " TBLPROPERTIES(" + HiveTestUtil.CRLF+ + " 'phoenix.hbase.table.name'='phoenix_NullMultiKey'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.znode.parent'='hbase'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.quorum'='localhost:" + hbaseTestUtil.getZkCluster().getClientPort() + "'," +HiveTestUtil.CRLF+ + " 'phoenix.rowkeys'='id,id2'," + HiveTestUtil.CRLF+ + " 'autocreate'='true'," + HiveTestUtil.CRLF+ + " 'autodrop'='true'," + HiveTestUtil.CRLF+ + " 'phoenix.column.mapping'='description:B.description');" + HiveTestUtil.CRLF); + sb.append("INSERT INTO TABLE phoenix_NullMultiKey" + HiveTestUtil.CRLF+ + "VALUES (10, NULL,\"foodesc\",2.0,-1);" + HiveTestUtil.CRLF); + String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString(); + createFile(sb.toString(), fullPath); + runTest(testName, fullPath); + + String phoenixQuery = "SELECT * FROM phoenix_NullMultiKey"; + PreparedStatement statement = conn.prepareStatement(phoenixQuery); + ResultSet rs = statement.executeQuery(); + assert(rs.getMetaData().getColumnCount() == 5); + } + + + private void runTest(String fname, String fpath) throws Exception { + long startTime = System.currentTimeMillis(); + try { + LOG.info("Begin query: " + fname); + System.err.println("Begin query: " + fname); + + qt.addFile(fpath); + + if (qt.shouldBeSkipped(fname)) { + LOG.error("Test " + fname + " skipped"); + return; + } + + qt.cliInit(fname); + qt.clearTestSideEffects(); + int ecode = qt.executeClient(fname); + if (ecode != 0) { + qt.failed(ecode, fname, null); + } + + ecode = qt.checkCliDriverResults(fname); + if (ecode != 0) { + qt.failedDiff(ecode, fname, null); + } + qt.clearPostTestEffects(); + + } catch (Throwable e) { + qt.failed(e, fname, null); + } + + long elapsedTime = System.currentTimeMillis() - startTime; + System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s"); + assertTrue("Test passed", true); + } + + private void createFile(String content, String fullName) throws IOException { + FileUtils.write(new File(fullName), content); + } + + + private void dropTable(String tableFullName) throws SQLException { + Preconditions.checkNotNull(conn); + conn.createStatement().execute(String.format("DROP TABLE IF EXISTS %s",tableFullName)); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if(qt == null) return; + try { + qt.shutdown(); + } + catch (Exception e) { + LOG.error("Unexpected exception in setup", e); + fail("Unexpected exception in tearDown"); + } + try { + conn.close(); + } finally { + try { + PhoenixDriver.INSTANCE.close(); + } finally { + try { + DriverManager.deregisterDriver(PhoenixDriver.INSTANCE); + } finally { + hbaseTestUtil.shutdownMiniCluster(); + } + } + } + } + + public String read(File queryFile) throws IOException { + InputStreamReader isr = new InputStreamReader( + new BufferedInputStream(new FileInputStream(queryFile)), HiveTestUtil.UTF_8); + StringWriter sw = new StringWriter(); + try { + IOUtils.copy(isr, sw); + } finally { + if (isr != null) { + isr.close(); + } + } + return sw.toString(); + } +} \ No newline at end of file diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/PhoenixHiveStoreIT.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/PhoenixHiveStoreIT.java new file mode 100644 index 00000000000..a7a58d3b037 --- /dev/null +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/PhoenixHiveStoreIT.java @@ -0,0 +1,254 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + *distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you maynot use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicablelaw or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringWriter; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.jdbc.PhoenixDriver; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.StringUtil; +import org.apache.phoenix.util.TestUtil; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.base.Preconditions; + +/** + * + * Test class to run all Hive Phoenix integration tests against a MINI Map-Reduce cluster. + */ +@Category(NeedsOwnMiniClusterTest.class) +public class PhoenixHiveStoreIT { + + private static final Log LOG = LogFactory.getLog(PhoenixHiveStoreIT.class); + private static final String SCHEMA_NAME = "T"; + private static final String TABLE_NAME = "HIVE_TEST"; + private static Path TEST_ROOT; + private static final String TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, TABLE_NAME); + private static HBaseTestingUtility hbaseTestUtil; + private static String zkQuorum; + private static Connection conn; + private static Configuration conf; + private static HiveTestUtil qt; + private static String hiveOutputDir; + private static String hiveLogDir; + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + hbaseTestUtil = new HBaseTestingUtility(); + conf = hbaseTestUtil.getConfiguration(); + setUpConfigForMiniCluster(conf); + conf.set(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true)); + hbaseTestUtil.startMiniCluster(3); + + Class.forName(PhoenixDriver.class.getName()); + zkQuorum = "localhost:" + hbaseTestUtil.getZkCluster().getClientPort(); + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + props.put(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true)); + conn = DriverManager.getConnection(PhoenixRuntime.JDBC_PROTOCOL + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum,props); + + // Setup Hive Output Folder + hiveOutputDir = new Path(hbaseTestUtil.getDataTestDir(), "hive_output").toString(); + File outputDir = new File(hiveOutputDir); + outputDir.mkdirs(); + hiveLogDir = new Path(hbaseTestUtil.getDataTestDir(), "hive_log").toString(); + File logDir = new File(hiveLogDir); + logDir.mkdirs(); + + } + + @Before + public void setUp() throws Exception { + Path testRoot = hbaseTestUtil.getDataTestDir(); + System.setProperty("test.tmp.dir", testRoot.toString()); + System.setProperty("test.warehouse.dir", (new Path(testRoot, "warehouse")).toString()); + + HiveTestUtil.MiniClusterType miniMR = HiveTestUtil.MiniClusterType.mr; + try { + qt = new HiveTestUtil(hiveOutputDir, hiveLogDir, miniMR, null); + } catch (Exception e) { + LOG.error("Unexpected exception in setup", e); + fail("Unexpected exception in setup"); + } + + } + + /** + * Check if Hive Mini Cluster Starts Correctly + * First simple test of connector creation + * @throws Exception + */ + @Test + public void simpleTest() throws Exception { + String testName = "simpleTest"; + // create a dummy outfile under log folder + hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out")); + createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString()); + createFile(StringUtil.EMPTY_STRING, new Path(hiveOutputDir, testName + ".out").toString()); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE EXTERNAL TABLE IF NOT EXISTS phoenix_table(ID int, SALARY INT)" + HiveTestUtil.CRLF + + " STORED BY \"org.apache.phoenix.hive.PhoenixStorageHandler\"" +HiveTestUtil.CRLF+ + " TBLPROPERTIES(" + HiveTestUtil.CRLF+ + " 'phoenix.hbase.table.name'='phoenix_table'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.znode.parent'='hbase'," + HiveTestUtil.CRLF+ + " 'phoenix.zookeeper.quorum'='localhost:" + hbaseTestUtil.getZkCluster().getClientPort() + "'," +HiveTestUtil.CRLF+ + " 'phoenix.rowkeys'='id'," + HiveTestUtil.CRLF+ + " 'autocreate'='true'," + HiveTestUtil.CRLF+ + " 'autodrop'='true'," + HiveTestUtil.CRLF+ + " 'phoenix.column.mapping'='salary:B.salary');" + HiveTestUtil.CRLF); + sb.append("INSERT INTO TABLE phoenix_table" + HiveTestUtil.CRLF+ + "VALUES (10, 1000);" + HiveTestUtil.CRLF); + String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString(); + createFile(sb.toString(), fullPath); + runTest(testName, fullPath); + + String phoenixQuery = "SELECT * FROM phoenix_table"; + PreparedStatement statement = conn.prepareStatement(phoenixQuery); + ResultSet rs = statement.executeQuery(); + assert(rs.getMetaData().getColumnCount() == 2); + + } + + private void runTest(String fname, String fpath) throws Exception { + long startTime = System.currentTimeMillis(); + try { + LOG.info("Begin query: " + fname); + System.err.println("Begin query: " + fname); + + qt.addFile(fpath); + + if (qt.shouldBeSkipped(fname)) { + LOG.error("Test " + fname + " skipped"); + return; + } + + qt.cliInit(fname); + qt.clearTestSideEffects(); + int ecode = qt.executeClient(fname); + if (ecode != 0) { + qt.failed(ecode, fname, null); + } + + ecode = qt.checkCliDriverResults(fname); + if (ecode != 0) { + qt.failedDiff(ecode, fname, null); + } + qt.clearPostTestEffects(); + + } catch (Throwable e) { + qt.failed(e, fname, null); + } + + long elapsedTime = System.currentTimeMillis() - startTime; + System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s"); + //String phoenixQuery = "SELECT * FROM phoenix_table"; + //PreparedStatement statement = conn.prepareStatement(phoenixQuery); + //ResultSet rs = statement.executeQuery(); + //assert(rs.getMetaData().getColumnCount() == 2); + assertTrue("Test passed", true); + } + + private void createFile(String content, String fullName) throws IOException { + FileUtils.write(new File(fullName), content); + } + + @After + public void tearDown() throws Exception { + dropTable(TABLE_FULL_NAME); + if(qt == null) return; + try { + qt.shutdown(); + } + catch (Exception e) { + LOG.error("Unexpected exception in setup", e); + fail("Unexpected exception in tearDown"); + } + } + + private void dropTable(String tableFullName) throws SQLException { + Preconditions.checkNotNull(conn); + conn.createStatement().execute(String.format("DROP TABLE IF EXISTS %s",tableFullName)); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + try { + conn.close(); + } finally { + try { + PhoenixDriver.INSTANCE.close(); + } finally { + try { + DriverManager.deregisterDriver(PhoenixDriver.INSTANCE); + } finally { + hbaseTestUtil.shutdownMiniCluster(); + } + } + } + } + + public String read(File queryFile) throws IOException { + InputStreamReader isr = new InputStreamReader( + new BufferedInputStream(new FileInputStream(queryFile)), HiveTestUtil.UTF_8); + StringWriter sw = new StringWriter(); + try { + IOUtils.copy(isr, sw); + } finally { + if (isr != null) { + isr.close(); + } + } + return sw.toString(); + } +} diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixInputFormat.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixInputFormat.java new file mode 100644 index 00000000000..1502ec17651 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixInputFormat.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.Statement; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.hive.util.HiveConnectionUtil; +import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.mapreduce.*; +import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; +import org.apache.phoenix.query.KeyRange; +import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.util.ScanUtil; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + + +/** +* HivePhoenixInputFormat +* Custom Phoenix InputFormat to feed into Hive +*/ + +public class HivePhoenixInputFormat extends org.apache.phoenix.mapreduce.PhoenixInputFormat implements org.apache.hadoop.mapred.InputFormat{ + private static final Log LOG = LogFactory.getLog(HivePhoenixInputFormat.class); + private Configuration configuration; + private Connection connection; + private QueryPlan queryPlan; + + + public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { + setConf(job); + QueryPlan queryPlan = getQueryPlan(); + List allSplits = queryPlan.getSplits(); + Path path = new Path(job.get("location")); + List splits = generateSplits(queryPlan, allSplits, path); + FileSplit[] asplits = new FileSplit[splits.size()]; + LOG.debug("Splits size " + splits.size()); + splits.toArray(asplits); + return asplits; + } + + public RecordReader getRecordReader(InputSplit split, JobConf job, + Reporter reporter) throws IOException { + setConf(job); + QueryPlan queryPlan = getQueryPlan(); + + Class inputClass = PhoenixConfigurationUtil.getInputClass(this.configuration); + HivePhoenixRecordReader r = new HivePhoenixRecordReader(inputClass, this.configuration, queryPlan); + try { + r.init(split); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return r; + } + + public void setConf(Configuration configuration) { + this.configuration = configuration; + } + + public Configuration getConf() { + return this.configuration; + } + + + private QueryPlan getQueryPlan() throws IOException { + try { + LOG.debug("PhoenixInputFormat getQueryPlan statement " + + this.configuration.get("phoenix.select.stmt")); + Connection connection = getConnection(); + String selectStatement = + PhoenixConfigurationUtil.getSelectStatement(this.configuration); + Preconditions.checkNotNull(selectStatement); + Statement statement = connection.createStatement(); + PhoenixStatement pstmt = (PhoenixStatement) statement.unwrap(PhoenixStatement.class); + this.queryPlan = pstmt.compileQuery(selectStatement); + this.queryPlan.iterator(); + } catch (Exception exception) { + LOG.error(String.format("Failed to get the query plan with error [%s]", + new Object[] { exception.getMessage() })); + throw new RuntimeException(exception); + } + return this.queryPlan; + } + + private List generateSplits(QueryPlan qplan, List splits, Path path) + throws IOException { + Preconditions.checkNotNull(qplan); + Preconditions.checkNotNull(splits); + List psplits = Lists.newArrayListWithExpectedSize(splits.size()); + StatementContext context = qplan.getContext(); + TableRef tableRef = qplan.getTableRef(); + for (KeyRange split : splits) { + Scan splitScan = new Scan(context.getScan()); + + if (tableRef.getTable().getBucketNum() != null) { + LOG.error("Salted/bucketed Tables not yet supported"); + throw new IOException("Salted/bucketed Tables not yet supported"); + } + + if (ScanUtil.intersectScanRange(splitScan, split.getLowerRange(), + split.getUpperRange(), context.getScanRanges().useSkipScanFilter())) { + HivePhoenixInputSplit inputSplit = + new HivePhoenixInputSplit(KeyRange.getKeyRange(splitScan.getStartRow(), + splitScan.getStopRow()), path); + psplits.add(inputSplit); + } + } + return psplits; + } + + private Connection getConnection() { + try { + if (this.connection == null) this.connection = + HiveConnectionUtil.getConnection(this.configuration); + } catch (Exception e) { + throw new RuntimeException(e); + } + return this.connection; + } + + +} diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixInputSplit.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixInputSplit.java new file mode 100755 index 00000000000..115dd71bf77 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixInputSplit.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import com.google.common.base.Preconditions; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.phoenix.mapreduce.PhoenixInputSplit; +import org.apache.phoenix.query.KeyRange; + + +/** +* HivePhoenixInputSplit +*/ + +public class HivePhoenixInputSplit extends FileSplit { + private static final Log LOG = LogFactory.getLog(HivePhoenixInputSplit.class); + private KeyRange keyRange; + private Path path; + + public HivePhoenixInputSplit() { + super((Path) null, 0, 0, (String[]) null); + } + + public HivePhoenixInputSplit(KeyRange keyRange) { + Preconditions.checkNotNull(keyRange); + this.keyRange = keyRange; + } + + public HivePhoenixInputSplit(KeyRange keyRange, Path path) { + Preconditions.checkNotNull(keyRange); + Preconditions.checkNotNull(path); + LOG.debug("path: " + path); + + this.keyRange = keyRange; + this.path = path; + } + + public void readFields(DataInput input) throws IOException { + this.path = new Path(Text.readString(input)); + this.keyRange = new KeyRange(); + this.keyRange.readFields(input); + } + + public void write(DataOutput output) throws IOException { + Preconditions.checkNotNull(this.keyRange); + Text.writeString(output, path.toString()); + this.keyRange.write(output); + } + + public long getLength() { + return 0L; + } + + public String[] getLocations() { + return new String[0]; + } + + public KeyRange getKeyRange() { + return this.keyRange; + } + + @Override + public Path getPath() { + return this.path; + } + + public int hashCode() { + int prime = 31; + int result = 1; + result = 31 * result + (this.keyRange == null ? 0 : this.keyRange.hashCode()); + return result; + } + + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (!(obj instanceof HivePhoenixInputSplit)) return false; + HivePhoenixInputSplit other = (HivePhoenixInputSplit) obj; + if (this.keyRange == null) { + if (other.keyRange != null) return false; + } else if (!this.keyRange.equals(other.keyRange)) return false; + return true; + } +} \ No newline at end of file diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixOutputFormat.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixOutputFormat.java new file mode 100644 index 00000000000..c8ef31e1872 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixOutputFormat.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.apache.hadoop.util.Progressable; +import org.apache.phoenix.hive.util.HiveConnectionUtil; + +/** +* HivePhoenixOutputFormat +* Custom Phoenix OutputFormat to feed into Hive +*/ + + +public class HivePhoenixOutputFormat implements org.apache.hadoop.mapred.OutputFormat { + private static final Log LOG = LogFactory.getLog(HivePhoenixOutputFormat.class); + private Connection connection; + + + public RecordWriter getRecordWriter(FileSystem ignored, JobConf job, + String name, Progressable progress) throws IOException { + try { + return new HivePhoenixRecordWriter(job); + } catch (SQLException e) { + throw new IOException(e); + } + } + + public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { + LOG.debug("checkOutputSpecs"); + + } +} diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixRecordReader.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixRecordReader.java new file mode 100755 index 00000000000..3cf777d20d0 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixRecordReader.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; + +import java.io.IOException; +import java.io.PrintStream; +import java.sql.SQLException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.compile.ScanRanges; +import org.apache.phoenix.compile.SequenceManager; +import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.iterate.ResultIterator; +import org.apache.phoenix.iterate.SequenceResultIterator; +import org.apache.phoenix.iterate.TableResultIterator; +import org.apache.phoenix.jdbc.PhoenixResultSet; +import org.apache.phoenix.mapreduce.PhoenixRecordReader; +import org.apache.phoenix.query.KeyRange; +import org.apache.phoenix.util.ScanUtil; + +/** +* HivePhoenixRecordReader +* +* @version 1.0 +* @since 2015-02-08 +*/ + + +public class HivePhoenixRecordReader extends PhoenixRecordReader implements + org.apache.hadoop.mapred.RecordReader { + private static final Log LOG = LogFactory.getLog(HivePhoenixRecordReader.class); + private final Configuration configuration; + private final QueryPlan queryPlan; + private NullWritable key = NullWritable.get(); + private T value = null; + private Class inputClass; + private ResultIterator resultIterator = null; + private PhoenixResultSet resultSet; + + + public HivePhoenixRecordReader(Class inputClass, Configuration configuration, QueryPlan queryPlan) { + super(inputClass,configuration,queryPlan); + Preconditions.checkNotNull(configuration); + Preconditions.checkNotNull(queryPlan); + this.inputClass = inputClass; + this.configuration = configuration; + this.queryPlan = queryPlan; + } + + public float getProgress() { + return 0.0F; + } + + public void init(InputSplit split) throws IOException, + InterruptedException { + HivePhoenixInputSplit pSplit = (HivePhoenixInputSplit) split; + KeyRange keyRange = pSplit.getKeyRange(); + Scan splitScan = this.queryPlan.getContext().getScan(); + Scan scan = new Scan(splitScan); + ScanUtil.intersectScanRange(scan, keyRange.getLowerRange(), keyRange.getUpperRange(), + this.queryPlan.getContext().getScanRanges().useSkipScanFilter()); + try { + TableResultIterator tableResultIterator = + new TableResultIterator(this.queryPlan.getContext(), + this.queryPlan.getTableRef(), scan); + if (this.queryPlan.getContext().getSequenceManager().getSequenceCount() > 0) this.resultIterator = + new SequenceResultIterator(tableResultIterator, this.queryPlan.getContext() + .getSequenceManager()); + else { + this.resultIterator = tableResultIterator; + } + this.resultSet = + new PhoenixResultSet(this.resultIterator, this.queryPlan.getProjector(), + this.queryPlan.getContext().getStatement()); + } catch (SQLException e) { + LOG.error(String.format(" Error [%s] initializing PhoenixRecordReader. ", + new Object[] { e.getMessage() })); + Throwables.propagate(e); + } + } + + + public boolean next(NullWritable key, T val) throws IOException { + if (key == null) { + key = NullWritable.get(); + } + if (this.value == null) { + this.value = + (T) ((DBWritable) ReflectionUtils.newInstance(this.inputClass, + this.configuration)); + } + Preconditions.checkNotNull(this.resultSet); + try { + if (!this.resultSet.next()) { + return false; + } + this.value.readFields(this.resultSet); + LOG.debug("PhoenixRecordReader resultset size" + this.resultSet.getFetchSize()); + return true; + } catch (SQLException e) { + LOG.error(String.format(" Error [%s] occurred while iterating over the resultset. ", + new Object[] { e.getMessage() })); + Throwables.propagate(e); + } + return false; + } + + public NullWritable createKey() { + return this.key; + } + + public T createValue() { + this.value = + (T) ((DBWritable) ReflectionUtils.newInstance(this.inputClass, this.configuration)); + return this.value; + } + + public long getPos() throws IOException { + return 0L; + } +} \ No newline at end of file diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixRecordWriter.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixRecordWriter.java new file mode 100755 index 00000000000..3f108421c7b --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/HivePhoenixRecordWriter.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.apache.phoenix.hive.util.HiveConnectionUtil; +import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; + +public class HivePhoenixRecordWriter implements RecordWriter { + private static final Log LOG = LogFactory.getLog(HivePhoenixRecordWriter.class); + + private long numRecords = 0L; + private Connection conn; + private final PreparedStatement statement; + private Configuration config; + private final long batchSize; + + public HivePhoenixRecordWriter(Configuration config) throws SQLException, IOException { + this.conn = this.getConnection(config); + this.batchSize = PhoenixConfigurationUtil.getBatchSize(config); + String upsertQuery = PhoenixConfigurationUtil.getUpsertStatement(config); + this.statement = this.conn.prepareStatement(upsertQuery); + } + + public void write(NullWritable n, T record) throws IOException { + try { + record.write(this.statement); + this.numRecords += 1L; + this.statement.addBatch(); + + if (this.numRecords % this.batchSize == 0L) { + LOG.info("log commit called on a batch of size : " + this.batchSize); + this.statement.executeBatch(); + this.conn.commit(); + } + } catch (SQLException e) { + throw new IOException("Exception while committing to database.", e); + } + } + + public void close(Reporter arg0) throws IOException { + try { + this.statement.executeBatch(); + this.conn.commit(); + } catch (SQLException e) { + try { + this.conn.rollback(); + } catch (SQLException ex) { + throw new IOException("Exception while closing the connection", e); + } + throw new IOException(e.getMessage()); + } finally { + try { + this.statement.close(); + this.conn.close(); + } catch (SQLException ex) { + throw new IOException(ex.getMessage()); + } + } + } + + private Connection getConnection(Configuration configuration) throws IOException { + if (this.conn != null) { + return this.conn; + } + + this.config = configuration; + try { + LOG.info("Initializing new Phoenix connection..."); + this.conn = HiveConnectionUtil.getConnection(configuration); + LOG.info("Initialized Phoenix connection, autoCommit=" + + this.conn.getAutoCommit()); + return this.conn; + } catch (SQLException e) { + throw new IOException(e); + } + } +} \ No newline at end of file diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixHiveDBWritable.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixHiveDBWritable.java new file mode 100644 index 00000000000..6ec08514151 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixHiveDBWritable.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import com.google.common.base.Preconditions; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.lib.db.DBWritable; +import org.apache.phoenix.schema.types.PDataType; + + +/** +* PhoenixHiveDBWritable +* PhoenixStorageHandler Serialized Class referenced in the SerDe +*/ + +public class PhoenixHiveDBWritable implements Writable, DBWritable { + private static final Log LOG = LogFactory.getLog(PhoenixHiveDBWritable.class); + + private final List values = new ArrayList(); + private PDataType[] PDataTypes = null; + ResultSet rs; + + public PhoenixHiveDBWritable() { + } + + public PhoenixHiveDBWritable(PDataType[] categories) { + this.PDataTypes = categories; + } + + public void readFields(ResultSet rs) throws SQLException { + Preconditions.checkNotNull(rs); + this.rs = rs; + } + + public List getValues() { + return this.values; + } + + public Object get(String name) { + try { + return this.rs.getObject(name); + } catch (SQLException se) { + se.printStackTrace(); + } + return null; + } + + /** + * adds the Hive Writable values + * @param the PreparedStatement + */ + public void add(Object value) { + this.values.add(value); + } + + public void clear() { + this.values.clear(); + } + + /** + * Writes out the Hive writabke types to PhoenixDatatypes in a prepared statement + * @param the PreparedStatement + */ + + public void write(PreparedStatement statement) throws SQLException { + for (int i = 0; i < this.values.size(); i++) { + Object o = this.values.get(i); + try { + if (o != null) { + LOG.debug(" value " + o.toString() + " type " + + this.PDataTypes[i].getSqlTypeName() + " int value " + + this.PDataTypes[i].getSqlType()); + statement.setObject(i + 1, PDataType + .fromTypeId(this.PDataTypes[i].getSqlType()).toObject(o.toString())); + } else { + LOG.debug(" value NULL type " + this.PDataTypes[i].getSqlTypeName() + + " int value " + this.PDataTypes[i].getSqlType()); + statement.setNull(i + 1, this.PDataTypes[i].getSqlType()); + } + } catch (RuntimeException re) { + throw new RuntimeException(String.format( + "Unable to process column %s, innerMessage=%s", + new Object[] { re.getMessage() }), re); + } + } + } + + public void readFields(DataInput in) throws IOException { + } + + public void write(DataOutput arg0) throws IOException { + } +} \ No newline at end of file diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixMetaHook.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixMetaHook.java new file mode 100644 index 00000000000..5fabd4da8fc --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixMetaHook.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import com.google.common.base.Splitter; +import com.google.common.base.Splitter.MapSplitter; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.metastore.HiveMetaHook; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.phoenix.hive.util.HiveConnectionUtil; +import org.apache.phoenix.hive.util.HiveConfigurationUtil; +import org.apache.phoenix.hive.util.HiveTypeUtil; +import org.apache.phoenix.hive.util.PhoenixUtil; + +/** +* PhoenixMetaHook +* This class captures all create and delete Hive queries and passes them to phoenix +* +* @version 1.0 +* @since 2015-02-08 +*/ + +public class PhoenixMetaHook + implements HiveMetaHook +{ + static Log LOG = LogFactory.getLog(PhoenixMetaHook.class.getName()); + + /** + *commitCreateTable creates a Phoenix table after the hive table has been created + * incoming hive types. + * @param tbl the table properties + * + */ + //Too much logic in this function must revisit and dispatch + public void commitCreateTable(Table tbl) + throws MetaException + { + LOG.debug("PhoenixMetaHook commitCreateTable "); + Map fields = new LinkedHashMap(); + Map mps = tbl.getParameters(); + + String tablename = mps.get("phoenix.hbase.table.name") != null ? (String)mps.get("phoenix.hbase.table.name") : tbl.getTableName(); + + String mapping = (String)mps.get(HiveConfigurationUtil.COLUMN_MAPPING); + Map mappings = null; + if ((mapping != null) && (mapping.length() > 0)) { + mapping = mapping.toLowerCase(); + mappings = Splitter.on(",").omitEmptyStrings().trimResults().withKeyValueSeparator(":").split(mapping); + } + + for (FieldSchema fs : tbl.getSd().getCols()) { + try { + String fname = fs.getName().toLowerCase(); + if (mappings != null) { + fname = mappings.get(fname) == null ? fs.getName().toLowerCase() : (String)mappings.get(fname); + } + + fields.put(fname, HiveTypeUtil.HiveType2PDataType(fs.getType()).toString()); + } catch (SerDeException e) { + e.printStackTrace(); + } + } + + String pk = (String)mps.get(HiveConfigurationUtil.PHOENIX_ROWKEYS); + if ((pk == null) || (pk.length() == 0)) { + throw new MetaException("Phoenix Table no Rowkeys specified in phoenix.rowkeys"); + } + + int salt_buckets = 0; + String salting = (String)mps.get(HiveConfigurationUtil.SALT_BUCKETS); + + if ((salting != null) && (salting.length() > 0)) { + try { + salt_buckets = Integer.parseInt(salting); + if (salt_buckets > 256) { + LOG.warn("Salt Buckets should be between 1-256 we will cap at 256"); + salt_buckets = 256; + } + if (salt_buckets < 0) { + LOG.warn("Salt Buckets should be between 1-256 we will undercap at 0"); + salt_buckets = 0; + } + } catch (NumberFormatException nfe) { + salt_buckets = 0; + } + } + String version = (String)mps.get(HiveConfigurationUtil.VERSIONS); + int version_num = 0 ; + if ((version != null) && (version.length() > 0)) { + version_num = Integer.parseInt(version); + if (version_num <0) { + LOG.warn("Versions should be > 0 ignoring the property"); + version_num = 0; + } + if (version_num > 5) { + LOG.warn("Versions should be between 0-5 we will cap at 5"); + salt_buckets = 256; + } + } + + String compression = (String)mps.get(HiveConfigurationUtil.COMPRESSION); + if ((compression != null) && (compression.equalsIgnoreCase("gz"))) + compression = "GZ"; + else { + compression = null; + } + + try + { + Connection conn = HiveConnectionUtil.getConnection(tbl); + + if (tbl.getTableType().equals(TableType.MANAGED_TABLE.name())) { + if (PhoenixUtil.findTable(conn, tablename)) { + throw new MetaException(" Phoenix table already exists cannot create use EXTERNAL"); + } + + PhoenixUtil.createTable(conn, tablename, fields, pk.split(","), false, salt_buckets, compression,version_num); + } + else if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.name())) { + if (PhoenixUtil.findTable(conn, tablename)) { + LOG.info("CREATE External table table already exists"); + PhoenixUtil.testTable(conn, tablename, fields); + } else if ((tbl.getParameters().get("autocreate") != null) && (((String)tbl.getParameters().get("autocreate")).equalsIgnoreCase("true"))) { + PhoenixUtil.createTable(conn, tablename, fields, pk.split(","), false, salt_buckets, compression,version_num); + } + } else { + throw new MetaException(" Phoenix Unsupported table Type: " + tbl.getTableType()); + } + } catch (SQLException e) { + e.printStackTrace(); + throw new MetaException(" Phoenix table creation SQLException: " + e.getMessage()); + } + } + + /** + *commitDropTable requests a phoenix drop table when deleting a Hive tablem this only happens if this is a managed table + * should not drop an external table unless autodrop is set. + * @param tbl the table properties + * + */ + + public void commitDropTable(Table tbl, boolean bool) + throws MetaException + { + Map mps = tbl.getParameters(); + + String tablename = mps.get("phoenix.hbase.table.name") != null ? (String)mps.get("phoenix.hbase.table.name") : tbl.getTableName(); + try + { + if (tbl.getTableType().equals(TableType.MANAGED_TABLE.name())) { + Connection conn = HiveConnectionUtil.getConnection(tbl); + PhoenixUtil.dropTable(conn, tablename); + } + if ((tbl.getTableType().equals(TableType.EXTERNAL_TABLE.name())) && (tbl.getParameters().get("autodrop") != null) && (((String)tbl.getParameters().get("autodrop")).equalsIgnoreCase("true"))) + { + Connection conn = HiveConnectionUtil.getConnection(tbl); + PhoenixUtil.dropTable(conn, tablename); + } + } catch (SQLException e) { + e.printStackTrace(); + throw new MetaException("Phoenix table drop SQLException: " + e.getMessage()); + } + } + + public void preCreateTable(Table tbl) + throws MetaException + { + } + + public void preDropTable(Table tbl) + throws MetaException + { + } + + public void rollbackCreateTable(Table tbl) + throws MetaException + { + } + + public void rollbackDropTable(Table tbl) + throws MetaException + { + } +} \ No newline at end of file diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixSerde.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixSerde.java new file mode 100644 index 00000000000..1f9a5ba29c9 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixSerde.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.lib.db.DBWritable; +import org.apache.phoenix.hive.util.HiveConstants; +import org.apache.phoenix.hive.util.HiveTypeUtil; +import org.apache.phoenix.schema.types.PDataType; + +/** +* PhoenixSerDe +* Hive SerializerDeserializer Class for Phoenix connection +*/ + +public class PhoenixSerde implements SerDe { + static Log LOG = LogFactory.getLog(PhoenixSerde.class.getName()); + private PhoenixHiveDBWritable phrecord; + private List columnNames; + private List columnTypes; + private ObjectInspector ObjectInspector; + private int fieldCount; + private List row; + private List fieldOIs; + + + /** + * This method initializes the Hive SerDe + * incoming hive types. + * @param conf conf job configuration + * @param tblProps table properties + */ + public void initialize(Configuration conf, Properties tblProps) throws SerDeException { + if (conf != null) { + conf.setClass("phoenix.input.class", PhoenixHiveDBWritable.class, DBWritable.class); + } + this.columnNames = Arrays.asList(tblProps.getProperty(HiveConstants.COLUMNS).split(",")); + this.columnTypes = + TypeInfoUtils.getTypeInfosFromTypeString(tblProps + .getProperty(HiveConstants.COLUMNS_TYPES)); + LOG.debug("columnNames: " + this.columnNames); + LOG.debug("columnTypes: " + this.columnTypes); + this.fieldCount = this.columnTypes.size(); + PDataType[] types = HiveTypeUtil.hiveTypesToSqlTypes(this.columnTypes); + this.phrecord = new PhoenixHiveDBWritable(types); + this.fieldOIs = new ArrayList(this.columnNames.size()); + + for (TypeInfo typeInfo : this.columnTypes) { + this.fieldOIs.add(TypeInfoUtils + .getStandardWritableObjectInspectorFromTypeInfo(typeInfo)); + } + this.ObjectInspector = + ObjectInspectorFactory.getStandardStructObjectInspector(this.columnNames, + this.fieldOIs); + this.row = new ArrayList(this.columnNames.size()); + } + + + /** + * This Deserializes a result from Phoenix to a Hive result + * @param wr the phoenix writable Object here PhoenixHiveDBWritable + * @return Object for Hive + */ + + public Object deserialize(Writable wr) throws SerDeException { + if (!(wr instanceof PhoenixHiveDBWritable)) throw new SerDeException( + "Serialized Object is not of type PhoenixHiveDBWritable"); + try { + this.row.clear(); + PhoenixHiveDBWritable phdbw = (PhoenixHiveDBWritable) wr; + for (int i = 0; i < this.fieldCount; i++) { + Object value = phdbw.get((String) this.columnNames.get(i)); + if (value != null) this.row.add(HiveTypeUtil.SQLType2Writable( + ((TypeInfo) this.columnTypes.get(i)).getTypeName(), value)); + else { + this.row.add(null); + } + } + return this.row; + } catch (Exception e) { + e.printStackTrace(); + throw new SerDeException(e.getCause()); + } + } + + public ObjectInspector getObjectInspector() throws SerDeException { + return this.ObjectInspector; + } + + public SerDeStats getSerDeStats() { + return null; + } + + /** + * This is a getter for the serialized class to use with this SerDE + * @return The class PhoenixHiveDBWritable + */ + + public Class getSerializedClass() { + return PhoenixHiveDBWritable.class; + } + + + /** + * This serializes a Hive row to a Phoenix entry + * incoming hive types. + * @param row Hive row + * @param inspector inspector for the Hive row + */ + + public Writable serialize(Object row, ObjectInspector inspector) throws SerDeException { + final StructObjectInspector structInspector = (StructObjectInspector) inspector; + final List fields = structInspector.getAllStructFieldRefs(); + + if (fields.size() != fieldCount) { + throw new SerDeException(String.format("Required %d columns, received %d.", fieldCount, + fields.size())); + } + phrecord.clear(); + for (int i = 0; i < fieldCount; i++) { + StructField structField = fields.get(i); + if (structField != null) { + Object field = structInspector.getStructFieldData(row, structField); + ObjectInspector fieldOI = structField.getFieldObjectInspector(); + switch (fieldOI.getCategory()) { + case PRIMITIVE: + Writable value = + (Writable) ((PrimitiveObjectInspector) fieldOI) + .getPrimitiveWritableObject(field); + phrecord.add(value); + break; + default: + // TODO add support for Array + new SerDeException("Phoenix Unsupported column type: " + fieldOI.getCategory()); + } + } + } + + return phrecord; + } + +} \ No newline at end of file diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java new file mode 100644 index 00000000000..bbd6c0625d2 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java @@ -0,0 +1,147 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + *distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you maynot use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicablelaw or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive; + +import java.sql.SQLException; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.HiveMetaHook; +import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler; +import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.phoenix.hive.util.HiveConfigurationUtil; + + +/** +* PhoenixStorageHandler +* This class manages all the Phoenix/Hive table initial configurations and SerDe Election +*/ + +public class PhoenixStorageHandler extends DefaultStorageHandler implements + HiveStoragePredicateHandler { + static Log LOG = LogFactory.getLog(PhoenixStorageHandler.class.getName()); + + private Configuration conf = null; + + public PhoenixStorageHandler() { + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public HiveMetaHook getMetaHook() { + return new PhoenixMetaHook(); + } + + @Override + public void configureInputJobProperties(TableDesc tableDesc, Map jobProperties) { + configureJobProperties(tableDesc, jobProperties); + } + + @Override + public void + configureOutputJobProperties(TableDesc tableDesc, Map jobProperties) { + configureJobProperties(tableDesc, jobProperties); + } + + @Override + public void configureTableJobProperties(TableDesc tableDesc, Map jobProperties) { + configureJobProperties(tableDesc, jobProperties); + } + + /** + * Extract all job properties to configure this job + * parameter tableDesc tabledescription, jobProperties + * TODO this avoids any pushdown must revisit + */ + private void configureJobProperties(TableDesc tableDesc, Map jobProperties) + { + Properties tblProps = tableDesc.getProperties(); + tblProps.getProperty("phoenix.hbase.table.name"); + HiveConfigurationUtil.setProperties(tblProps, jobProperties); + + //TODO this avoids any pushdown must revisit and extract meaningful parts + jobProperties.put("phoenix.select.stmt", "select * from " + (String)jobProperties.get("phoenix.hbase.table.name")); + if(((String)jobProperties.get("phoenix.hbase.table.name")).contains("limit")==true){ + + } + LOG.debug("ConfigurationUtil.SELECT_STATEMENT " + (String)jobProperties.get("phoenix.select.stmt")); + } + + /** + * Getter for the class serializing data from Phoenix to Hive + */ + @Override + public Class getInputFormatClass() { + return HivePhoenixInputFormat.class; + } + + /** + * Getter for the class serializing data from Hive to Phoenix + */ + @Override + public Class getOutputFormatClass() { + return HivePhoenixOutputFormat.class; + //return PhoenixOutputFormat.class; + } + + /** + * Getter for the Phoenix Serde + */ + @Override + public Class getSerDeClass() { + return PhoenixSerde.class; + } + + /** + * Class t access Hive query in a case of a select statement + * parameters: jobConf jobconfiguration, ExprNodeDesc contains details on the expression + */ + public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer arg1, + ExprNodeDesc exprn) { + + //TODO revisit the whole logic some information is not consistent + return null; + } + +} \ No newline at end of file diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveConfigurationUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveConfigurationUtil.java new file mode 100755 index 00000000000..d17f188b24f --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveConfigurationUtil.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive.util; + +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; +import org.apache.phoenix.schema.types.PBinary; +import org.apache.phoenix.schema.types.PBoolean; +import org.apache.phoenix.schema.types.PChar; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PDate; +import org.apache.phoenix.schema.types.PDecimal; +import org.apache.phoenix.schema.types.PDouble; +import org.apache.phoenix.schema.types.PFloat; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PSmallint; +import org.apache.phoenix.schema.types.PTime; +import org.apache.phoenix.schema.types.PTimestamp; +import org.apache.phoenix.schema.types.PTinyint; +import org.apache.phoenix.schema.types.PVarchar; +import org.apache.phoenix.util.PhoenixRuntime; + +/** + * + */ +public class HiveConfigurationUtil { + static Log LOG = LogFactory.getLog(HiveConfigurationUtil.class.getName()); + + public static final String TABLE_NAME = "phoenix.hbase.table.name"; + public static final String ZOOKEEPER_QUORUM = "phoenix.zookeeper.quorum"; + public static final String ZOOKEEPER_PORT = "phoenix.zookeeper.client.port"; + public static final String ZOOKEEPER_PARENT = "phoenix.zookeeper.znode.parent"; + public static final String ZOOKEEPER_QUORUM_DEFAULT = "localhost"; + public static final String ZOOKEEPER_PORT_DEFAULT = "2181"; + public static final String ZOOKEEPER_PARENT_DEFAULT = "/hbase-unsecure"; + + public static final String COLUMN_MAPPING = "phoenix.column.mapping"; + public static final String AUTOCREATE = "autocreate"; + public static final String AUTODROP = "autodrop"; + public static final String AUTOCOMMIT = "autocommit"; + public static final String PHOENIX_ROWKEYS = "phoenix.rowkeys"; + public static final String SALT_BUCKETS = "saltbuckets"; + public static final String COMPRESSION = "compression"; + public static final String VERSIONS = "versions"; + public static final int VERSIONS_NUM = 5; + public static final String SPLIT = "split"; + public static final String REDUCE_SPECULATIVE_EXEC = + "mapred.reduce.tasks.speculative.execution"; + public static final String MAP_SPECULATIVE_EXEC = "mapred.map.tasks.speculative.execution"; + + public static void setProperties(Properties tblProps, Map jobProperties) { + String quorum = tblProps.getProperty(HiveConfigurationUtil.ZOOKEEPER_QUORUM) != null ? + tblProps.getProperty(HiveConfigurationUtil.ZOOKEEPER_QUORUM) : + HiveConfigurationUtil.ZOOKEEPER_QUORUM_DEFAULT; + String znode = tblProps.getProperty(HiveConfigurationUtil.ZOOKEEPER_PARENT) != null ? + tblProps.getProperty(HiveConfigurationUtil.ZOOKEEPER_PARENT) : + HiveConfigurationUtil.ZOOKEEPER_PARENT_DEFAULT; + String port = tblProps.getProperty(HiveConfigurationUtil.ZOOKEEPER_PORT) != null ? + tblProps.getProperty(HiveConfigurationUtil.ZOOKEEPER_PORT) : + HiveConfigurationUtil.ZOOKEEPER_PORT_DEFAULT; + if (!znode.startsWith("/")) { + znode = "/" + znode; + } + LOG.debug("quorum:" + quorum); + LOG.debug("port:" + port); + LOG.debug("parent:" +znode); + LOG.debug("table:" + tblProps.getProperty(HiveConfigurationUtil.TABLE_NAME)); + LOG.debug("batch:" + tblProps.getProperty(PhoenixConfigurationUtil.UPSERT_BATCH_SIZE)); + + jobProperties.put(HiveConfigurationUtil.ZOOKEEPER_QUORUM, quorum); + jobProperties.put(HiveConfigurationUtil.ZOOKEEPER_PORT, port); + jobProperties.put(HiveConfigurationUtil.ZOOKEEPER_PARENT, znode); + String tableName = tblProps.getProperty(HiveConfigurationUtil.TABLE_NAME); + if (tableName == null) { + tableName = tblProps.get("name").toString(); + tableName = tableName.split(".")[1]; + } + // TODO this is synch with common Phoenix mechanism revisit to make wiser decisions + jobProperties.put(HConstants.ZOOKEEPER_QUORUM,quorum+PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + + port + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR+znode); + jobProperties.put(HiveConfigurationUtil.TABLE_NAME, tableName); + // TODO this is synch with common Phoenix mechanism revisit to make wiser decisions + jobProperties.put(PhoenixConfigurationUtil.OUTPUT_TABLE_NAME, tableName); + jobProperties.put(PhoenixConfigurationUtil.INPUT_TABLE_NAME, tableName); + } +} diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveConnectionUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveConnectionUtil.java new file mode 100755 index 00000000000..f67a064213d --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveConnectionUtil.java @@ -0,0 +1,161 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + *distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you maynot use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicablelaw or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive.util; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.hive.util.HiveConfigurationUtil; + +import com.google.common.base.Preconditions; + +/** + * Describe your class here. + * + * @since 138 + */ +public final class HiveConnectionUtil { + private static final Log LOG = LogFactory.getLog(HiveConnectionUtil.class); + private static Connection connection = null; + + /** + * Returns the {#link Connection} from Configuration + * @param configuration + * @return + * @throws SQLException + */ + public static Connection getConnection(final Configuration configuration) throws SQLException { + Preconditions.checkNotNull(configuration); + if (connection == null) { + final Properties props = new Properties(); + String quorum = + configuration + .get(HiveConfigurationUtil.ZOOKEEPER_QUORUM != null ? HiveConfigurationUtil.ZOOKEEPER_QUORUM + : configuration.get(HConstants.ZOOKEEPER_QUORUM)); + String znode = + configuration + .get(HiveConfigurationUtil.ZOOKEEPER_PARENT != null ? HiveConfigurationUtil.ZOOKEEPER_PARENT + : configuration.get(HConstants.ZOOKEEPER_ZNODE_PARENT)); + String port = + configuration + .get(HiveConfigurationUtil.ZOOKEEPER_PORT != null ? HiveConfigurationUtil.ZOOKEEPER_PORT + : configuration.get(HConstants.ZOOKEEPER_CLIENT_PORT)); + if (!znode.startsWith("/")) { + znode = "/" + znode; + } + + try { + // Not necessary shoud pick it up + Class.forName("org.apache.phoenix.jdbc.PhoenixDriver"); + + LOG.info("Connection info: " + PhoenixRuntime.JDBC_PROTOCOL + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + quorum + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + port + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + znode); + + final Connection conn = + DriverManager.getConnection(PhoenixRuntime.JDBC_PROTOCOL + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + quorum + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + port + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + znode); + String autocommit = configuration.get(HiveConfigurationUtil.AUTOCOMMIT); + if (autocommit != null && autocommit.equalsIgnoreCase("true")) { + conn.setAutoCommit(true); + } else { + conn.setAutoCommit(false); + } + connection = conn; + } catch (ClassNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + return connection; + } + + /** + * Returns the {#link Connection} from Configuration + * @param configuration + * @return + * @throws SQLException + */ + //TODO redundant + public static Connection getConnection(final Table tbl) throws SQLException { + Preconditions.checkNotNull(tbl); + Map TblParams = tbl.getParameters(); + String quorum = + TblParams.get(HiveConfigurationUtil.ZOOKEEPER_QUORUM) != null ? TblParams.get( + HiveConfigurationUtil.ZOOKEEPER_QUORUM).trim() + : HiveConfigurationUtil.ZOOKEEPER_QUORUM_DEFAULT; + String port = + TblParams.get(HiveConfigurationUtil.ZOOKEEPER_PORT) != null ? TblParams.get( + HiveConfigurationUtil.ZOOKEEPER_PORT).trim() + : HiveConfigurationUtil.ZOOKEEPER_PORT_DEFAULT; + String znode = + TblParams.get(HiveConfigurationUtil.ZOOKEEPER_PARENT) != null ? TblParams.get( + HiveConfigurationUtil.ZOOKEEPER_PARENT).trim() + : HiveConfigurationUtil.ZOOKEEPER_PARENT_DEFAULT; + if (!znode.startsWith("/")) { + znode = "/" + znode; + } + try { + Class.forName("org.apache.phoenix.jdbc.PhoenixDriver"); + final Connection conn = + DriverManager.getConnection((PhoenixRuntime.JDBC_PROTOCOL + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + quorum + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + port + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + znode)); + String autocommit = TblParams.get(HiveConfigurationUtil.AUTOCOMMIT); + if(autocommit!=null && autocommit.equalsIgnoreCase("true")){ + conn.setAutoCommit(true); + }else{ + conn.setAutoCommit(false); + } + + return conn; + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + + + return null; + } + + + /** + * Close the connection. + * @param conn + * @throws SQLException + */ + public static void closeConnection(final Connection conn) throws SQLException { + if(conn != null) { + conn.close(); + } + } +} diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveConstants.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveConstants.java new file mode 100644 index 00000000000..bd2ddab0288 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveConstants.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive.util; + +/** +* HiveConstants +* Class to store alL generic Hive patterns +* +* @version 1.0 +* @since 2015-02-08 +*/ + +public class HiveConstants { + public static String COLUMNS = "columns"; + public static String COLUMNS_TYPES = "columns.types"; + public static String HIVE_TABLE = "name"; + public static String HIVE_NAMESPACE_SEP = "\\."; +} \ No newline at end of file diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveTypeUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveTypeUtil.java new file mode 100644 index 00000000000..0fb397bb060 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/HiveTypeUtil.java @@ -0,0 +1,160 @@ +/* + * Copyright 2010 The Apache Software Foundation Licensed to the Apache Software Foundation (ASF) + * under one or more contributor license agreements. See the NOTICE filedistributed with this work + * for additional information regarding copyright ownership. The ASF licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you maynot use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicablelaw or agreed to in + * writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ +package org.apache.phoenix.hive.util; + +import java.sql.Date; +import java.sql.Timestamp; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.phoenix.schema.types.PBinary; +import org.apache.phoenix.schema.types.PBoolean; +import org.apache.phoenix.schema.types.PChar; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PDate; +import org.apache.phoenix.schema.types.PDouble; +import org.apache.phoenix.schema.types.PFloat; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PSmallint; +import org.apache.phoenix.schema.types.PTime; +import org.apache.phoenix.schema.types.PTimestamp; +import org.apache.phoenix.schema.types.PVarchar; +import org.apache.phoenix.schema.types.PDecimal; + +/** + * HiveTypeUtil + * Utility Class to convert Hive Type to Pheonix and vise versa + * + */ + +public class HiveTypeUtil { + private static final Log LOG = LogFactory.getLog(HiveTypeUtil.class); + + private HiveTypeUtil() { + } + + /** + * This method returns an array of most appropriates PDataType associated with a list of + * incoming hive types. + * @param List of TypeInfo + * @return Array PDataType + */ + public static PDataType[] hiveTypesToSqlTypes(List columnTypes) throws SerDeException { + final PDataType[] result = new PDataType[columnTypes.size()]; + for (int i = 0; i < columnTypes.size(); i++) { + result[i] = HiveType2PDataType(columnTypes.get(i)); + } + return result; + } + + /** + * This method returns the most appropriate PDataType associated with the incoming primitive + * hive type. + * @param hiveType + * @return PDataType + */ + public static PDataType HiveType2PDataType(TypeInfo hiveType) throws SerDeException { + switch (hiveType.getCategory()) { + /* Integrate Complex types like Array */ + case PRIMITIVE: + return HiveType2PDataType(hiveType.getTypeName()); + default: + throw new SerDeException("Phoenix unsupported column type: " + + hiveType.getCategory().name()); + } + } + + /** + * This method returns the most appropriate PDataType associated with the incoming hive type + * name. + * @param hiveType + * @return PDataType + */ + public static PDataType HiveType2PDataType(String hiveType) throws SerDeException { + final String lctype = hiveType.toLowerCase(); + if ("string".equals(lctype)) { + return PVarchar.INSTANCE; + }else if ("varchar".equals(lctype)) { + return PVarchar.INSTANCE; + }else if ("char".equals(lctype)) { + return PChar.INSTANCE; + } else if ("float".equals(lctype)) { + return PFloat.INSTANCE; + } else if ("double".equals(lctype)) { + return PDouble.INSTANCE; + } else if ("boolean".equals(lctype)) { + return PBoolean.INSTANCE; + } else if ("tinyint".equals(lctype)) { + return PSmallint.INSTANCE; + } else if ("smallint".equals(lctype)) { + return PSmallint.INSTANCE; + } else if ("int".equals(lctype)) { + return PInteger.INSTANCE; + } else if ("bigint".equals(lctype)) { + return PLong.INSTANCE; + } else if ("timestamp".equals(lctype)) { + return PTimestamp.INSTANCE; + } else if ("binary".equals(lctype)) { + return PBinary.INSTANCE; + } else if ("decimal".equals(lctype)) { + return PDecimal.INSTANCE; + } else if ("date".equals(lctype)) { + return PDate.INSTANCE; + } + + throw new SerDeException("Phoenix unrecognized column type: " + hiveType); + } + + /** + * This method returns the most appropriate Writable associated with the incoming sql type name. + * @param hiveType,Object + * @return Writable + */ + // TODO awkward logic revisit + public static Writable SQLType2Writable(String hiveType, Object o) throws SerDeException { + String lctype = hiveType.toLowerCase(); + if ("string".equals(lctype)) return new Text(o.toString()); + if ("varchar".equals(lctype)) return new HiveVarcharWritable(new HiveVarchar(o.toString(),o.toString().length())); + if ("char".equals(lctype)) return new HiveCharWritable(new HiveChar(o.toString(),o.toString().length())); + if ("float".equals(lctype)) return new FloatWritable(((Float) o).floatValue()); + if ("double".equals(lctype)) return new DoubleWritable(((Double) o).doubleValue()); + if ("boolean".equals(lctype)) return new BooleanWritable(((Boolean) o).booleanValue()); + if ("tinyint".equals(lctype)) return new ShortWritable(((Integer) o).shortValue()); + if ("smallint".equals(lctype)) return new ShortWritable(((Integer) o).shortValue()); + if ("int".equals(lctype)) return new IntWritable(((Integer) o).intValue()); + if ("bigint".equals(lctype)) return new LongWritable(((Long) o).longValue()); + if ("timestamp".equals(lctype)) return new TimestampWritable((Timestamp)o); + if ("binary".equals(lctype)) return new Text(o.toString()); + if ("date".equals(lctype)) return new DateWritable(new Date((long) o)); + if ("array".equals(lctype)) + ; + throw new SerDeException("Phoenix unrecognized column type: " + hiveType); + } +} diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixHiveConfiguration.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixHiveConfiguration.java new file mode 100644 index 00000000000..18f2a4e22ee --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixHiveConfiguration.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.hive.util; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; +import org.apache.phoenix.util.QueryUtil; + +import com.google.common.base.Preconditions; + +public class PhoenixHiveConfiguration { + private static final Log LOG = LogFactory.getLog(PhoenixHiveConfiguration.class); + private PhoenixHiveConfigurationUtil util; + private final Configuration conf = null; + + private String Quorum = HiveConfigurationUtil.ZOOKEEPER_QUORUM_DEFAULT; + private String Port = HiveConfigurationUtil.ZOOKEEPER_PORT_DEFAULT; + private String Parent = HiveConfigurationUtil.ZOOKEEPER_PARENT_DEFAULT; + private String TableName; + private String DbName; + private long BatchSize = PhoenixConfigurationUtil.DEFAULT_UPSERT_BATCH_SIZE; + + public PhoenixHiveConfiguration(Configuration conf) { + // this.conf = conf; + this.util = new PhoenixHiveConfigurationUtil(); + } + + public PhoenixHiveConfiguration(Table tbl) { + Map mps = tbl.getParameters(); + String quorum = + mps.get(HiveConfigurationUtil.ZOOKEEPER_QUORUM) != null ? mps + .get(HiveConfigurationUtil.ZOOKEEPER_QUORUM) + : HiveConfigurationUtil.ZOOKEEPER_QUORUM_DEFAULT; + String port = + mps.get(HiveConfigurationUtil.ZOOKEEPER_PORT) != null ? mps + .get(HiveConfigurationUtil.ZOOKEEPER_PORT) + : HiveConfigurationUtil.ZOOKEEPER_PORT_DEFAULT; + String parent = + mps.get(HiveConfigurationUtil.ZOOKEEPER_PARENT) != null ? mps + .get(HiveConfigurationUtil.ZOOKEEPER_PARENT) + : HiveConfigurationUtil.ZOOKEEPER_PARENT_DEFAULT; + String pk = mps.get(HiveConfigurationUtil.PHOENIX_ROWKEYS); + if (!parent.startsWith("/")) { + parent = "/" + parent; + } + String tablename = + (mps.get(HiveConfigurationUtil.TABLE_NAME) != null) ? mps + .get(HiveConfigurationUtil.TABLE_NAME) : tbl.getTableName(); + + String mapping = mps.get(HiveConfigurationUtil.COLUMN_MAPPING); + } + + public void configure(String server, String tableName, long batchSize) { + // configure(server, tableName, batchSize, null); + } + + public void configure(String quorum, String port, String parent, long batchSize, + String tableName, String dbname, String columns) { + Quorum = quorum != null ? quorum : HiveConfigurationUtil.ZOOKEEPER_QUORUM_DEFAULT; + Port = port != null ? port : HiveConfigurationUtil.ZOOKEEPER_PORT_DEFAULT; + Parent = parent != null ? parent : HiveConfigurationUtil.ZOOKEEPER_PARENT_DEFAULT; + // BatchSize = batchSize!=null?batchSize:ConfigurationUtil.DEFAULT_UPSERT_BATCH_SIZE; + } + + static class PhoenixHiveConfigurationUtil { + + public Connection getConnection(final Configuration configuration) throws SQLException { + Preconditions.checkNotNull(configuration); + Properties props = new Properties(); + // final Connection conn = + // DriverManager.getConnection(QueryUtil.getUrl(configuration.get(SERVER_NAME)),props).unwrap(PhoenixConnection.class); + // conn.setAutoCommit(false); + return null; + } + } + +} \ No newline at end of file diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixUtil.java new file mode 100644 index 00000000000..c34fc5c8b76 --- /dev/null +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixUtil.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hive.util; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.metastore.api.MetaException; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; + +public class PhoenixUtil { + static Log LOG = LogFactory.getLog(PhoenixUtil.class.getName()); + + public static boolean createTable(Connection conn, String TableName, + Map fields, String[] pks, boolean addIfNotExists, int salt_buckets, + String compression,int versions_num) throws SQLException, MetaException { + Preconditions.checkNotNull(conn); + if (pks == null || pks.length == 0) { + throw new SQLException("Phoenix Table no Rowkeys specified in " + + HiveConfigurationUtil.PHOENIX_ROWKEYS); + } + for (String pk : pks) { + String val = fields.get(pk.toLowerCase()); + if (val == null) { + throw new MetaException("Phoenix Table rowkey " + pk + + " does not belong to listed fields "); + } + val += " not null"; + fields.put(pk, val); + } + + StringBuffer query = new StringBuffer("CREATE TABLE "); + if (addIfNotExists) { + query.append("IF NOT EXISTS "); + } + query.append(TableName + " ( "); + Joiner.MapJoiner mapJoiner = Joiner.on(',').withKeyValueSeparator(" "); + query.append(" " + mapJoiner.join(fields)); + if (pks != null && pks.length > 0) { + query.append(" CONSTRAINT pk PRIMARY KEY ("); + Joiner joiner = Joiner.on(" , "); + query.append(" " + joiner.join(pks) + " )"); + } + query.append(" )"); + if (salt_buckets > 0) { + query.append(" SALT_BUCKETS = " + salt_buckets); + } + if (compression != null) { + query.append(" ,COMPRESSION='GZ'"); + } + if (versions_num > 0) { + query.append(" ,VERSIONS="+versions_num); + } + System.out.println("CREATED QUERY " +query.toString()); + LOG.info("Create table query statement " + query.toString()); + return createTable(conn, query.toString()); + } + + public static boolean createTable(Connection conn, String query) throws SQLException { + Preconditions.checkNotNull(conn); + return conn.createStatement().execute(query); + } + + public static boolean findTable(Connection conn, String name) throws SQLException { + Preconditions.checkNotNull(conn); + Preconditions.checkNotNull(name); + DatabaseMetaData dbm = conn.getMetaData(); + ResultSet rs = dbm.getTables(null, null, name, null); + LOG.info("looking for table"); + if (rs.next()) { + LOG.info("found the table " + rs.getString("TABLE_NAME")); + while (rs.next()) { + LOG.info("found the table " + rs.getString("TABLE_NAME")); + } + return true; + } + return false; + } + + public static boolean testTable(Connection conn, String name, Map fields) + throws SQLException, MetaException { + Preconditions.checkNotNull(conn); + Preconditions.checkNotNull(name); + DatabaseMetaData dbm = conn.getMetaData(); + ResultSet rs = dbm.getTables(null, null, name, null); + ResultSet cols = dbm.getColumns(null, null, name, null); + Map columns = new LinkedHashMap(); + while (cols.next()) { + columns.put(cols.getString("COLUMN_NAME"), cols.getString("TYPE_NAME")); + } + if (columns.size() != fields.size()) { + throw new MetaException("Rowcount mismatch between Hive and Phoenix tables"); + } + if (PhoenixUtil.compareColumns(columns, fields)) { + throw new MetaException("Row order mismatch between Hive and Phoenix tables phoenix cols "+columns.toString()+" hive fields "+fields.toString()); + } + + if (columns.equals(fields)) { + throw new MetaException("Row type mismatch between Hive and Phoenix tables"); + } + return true; + } + + private static Boolean compareColumns(Map col1, Map col2) { + assert col1.size() == col2.size() : " size mismatch"; + Iterator keys1 = col1.keySet().iterator(); + Iterator keys2 = col2.keySet().iterator(); + boolean result = true; + while (keys1.hasNext()) { + String k1 = keys1.next(); + String k2 = keys2.next(); + if (!k1.toLowerCase().equals(k2.toLowerCase())) { + result = false; + } + } + return result; + } + + public static boolean dropTable(Connection conn, String TableName) throws SQLException { + Preconditions.checkNotNull(conn); + return conn.createStatement().execute("DROP TABLE IF EXISTS " + TableName); + } + + public ResultSet getAll(Connection conn, String TableName, String predicate) + throws SQLException { + Preconditions.checkNotNull(conn); + String query = "SELECT * FROM " + TableName; + if (!predicate.isEmpty()) { + query += " where " + predicate; + } + ResultSet rs = conn.createStatement().executeQuery(query); + ; + return rs; + } + +} diff --git a/pom.xml b/pom.xml index 54fabbe4a5a..742da90bff5 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ phoenix-assembly phoenix-pherf phoenix-spark + phoenix-hive @@ -85,6 +86,7 @@ 1.2 2.5.1 0.12.0 + 0.14.0 1.8.8 3.5 1.2.17