Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREWAREHOUSE;
import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM;
import static org.apache.paimon.CoreOptions.DATA_FILE_PATH_DIRECTORY;
import static org.apache.paimon.CoreOptions.FILE_FORMAT;
Expand Down Expand Up @@ -189,7 +188,7 @@ public HiveCatalog(
locationHelper = new TBPropertiesLocationHelper();
} else {
// set the warehouse location to the hiveConf
hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehouse);
hiveConf.set("hive.metastore.warehouse.dir", warehouse);
locationHelper = new StorageLocationHelper();
}
}
Expand Down Expand Up @@ -1964,7 +1963,7 @@ public static HiveConf createHiveConf(
try (InputStream inputStream = hiveSite.getFileSystem(hadoopConf).open(hiveSite)) {
hiveConf.addResource(inputStream, hiveSite.toString());
// trigger a read from the conf to avoid input stream is closed
hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS);
hiveConf.getVar(HiveConf.getConfVars("hive.metastore.uris"));
} catch (IOException e) {
throw new RuntimeException(
"Failed to load hive-site.xml from specified path:" + hiveSite, e);
Expand All @@ -1990,8 +1989,7 @@ public static Catalog createHiveCatalog(CatalogContext context) {
Options options = context.options();
String warehouseStr = options.get(CatalogOptions.WAREHOUSE);
if (warehouseStr == null) {
warehouseStr =
hiveConf.get(METASTOREWAREHOUSE.varname, METASTOREWAREHOUSE.defaultStrVal);
warehouseStr = hiveConf.getVar(HiveConf.getConfVars("hive.metastore.warehouse.dir"));
}
Path warehouse = new Path(warehouseStr);
Path uri =
Expand Down Expand Up @@ -2024,10 +2022,10 @@ public static HiveConf createHiveConf(CatalogContext context) {
// always using user-set parameters overwrite hive-site.xml parameters
context.options().toMap().forEach(hiveConf::set);
if (uri != null) {
hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, uri);
hiveConf.set("hive.metastore.uris", uri);
}

if (hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname) == null) {
if (hiveConf.get("hive.metastore.uris") == null) {
LOG.error(
"Can't find hive metastore uri to connect: "
+ " either set "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ public IMetaStoreClient createClient(HiveConf hiveConf, String clientClassName)
Method getProxy = RetryingMetaStoreClient.class.getMethod("getProxy", classes);
HiveMetastoreProxySupplier supplier = entry.getValue();
IMetaStoreClient client = supplier.get(getProxy, hiveConf, clientClassName);
return isNullOrWhitespaceOnly(hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname))
return isNullOrWhitespaceOnly(hiveConf.get("hive.metastore.uris"))
? client
: HiveMetaStoreClient.newSynchronizedClient(client);
} catch (Exception e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import org.apache.paimon.shade.guava30.com.google.common.util.concurrent.ThreadFactoryBuilder;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.TException;
Expand Down Expand Up @@ -146,7 +145,7 @@ static Key extractKey(String clientClassName, String cacheKeys, Configuration co
// generate key elements in a certain order, so that the Key instances are comparable
List<Object> elements = Lists.newArrayList();
elements.add(clientClassName);
elements.add(conf.get(HiveConf.ConfVars.METASTOREURIS.varname, ""));
elements.add(conf.get("hive.metastore.uris", ""));
elements.add(HiveCatalogOptions.IDENTIFIER);
if (cacheKeys == null || cacheKeys.isEmpty()) {
return Key.of(elements);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,10 @@ public IcebergHiveMetadataCommitter(FileStoreTable table) {

table.options().forEach(hiveConf::set);
if (uri != null) {
hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, uri);
hiveConf.set("hive.metastore.uris", uri);
}

if (hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname) == null) {
if (hiveConf.get("hive.metastore.uris") == null) {
LOG.error(
"Can't find hive metastore uri to connect: "
+ "either set {} for paimon table or set hive.metastore.uris "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import org.apache.paimon.schema.TableSchema;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
Expand Down Expand Up @@ -83,7 +82,7 @@ public void preCreateTable(Table table) throws MetaException {
String location = LocationKeyExtractor.getPaimonLocation(conf, table);
Identifier identifier = Identifier.create(table.getDbName(), table.getTableName());
if (location == null) {
String warehouse = conf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname);
String warehouse = conf.get("hive.metastore.warehouse.dir");
org.apache.hadoop.fs.Path hadoopPath =
getDnsPath(new org.apache.hadoop.fs.Path(warehouse), conf);
warehouse = hadoopPath.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,14 @@ private static DataSplit buildDataSplit(DataSplit current, List<DataFileMeta> fi

private static Long computeSplitSize(
JobConf jobConf, List<DataSplit> splits, int numSplits, long openCostInBytes) {
long maxSize = HiveConf.getLongVar(jobConf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE);
long minSize = HiveConf.getLongVar(jobConf, HiveConf.ConfVars.MAPREDMINSPLITSIZE);
long maxSize =
HiveConf.getLongVar(
jobConf,
HiveConf.getConfVars("mapreduce.input.fileinputformat.split.maxsize"));
long minSize =
HiveConf.getLongVar(
jobConf,
HiveConf.getConfVars("mapreduce.input.fileinputformat.split.minsize"));
long avgSize;
long splitSize;
if (numSplits > 0) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
import org.apache.paimon.types.VarCharType;
import org.apache.paimon.utils.TraceableFileIO;

import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
Expand Down Expand Up @@ -118,8 +117,8 @@ public void before() throws Exception {
@Test
public void testPackSplitsForNonBucketTable() throws Exception {
JobConf jobConf = new JobConf();
jobConf.set(HiveConf.ConfVars.MAPREDMAXSPLITSIZE.varname, "268435456"); // 256MB
jobConf.set(HiveConf.ConfVars.MAPREDMINSPLITSIZE.varname, "268435456"); // 256MB
jobConf.set("mapreduce.input.fileinputformat.split.maxsize", "268435456"); // 256MB
jobConf.set("mapreduce.input.fileinputformat.split.minsize", "268435456"); // 256MB

FileStoreTable table = createFileStoreTable(TABLE_SCHEMA);

Expand All @@ -140,8 +139,8 @@ public void testPackSplitsForNonBucketTable() throws Exception {
@Test
public void testPackSplitsForBucketTable() throws Exception {
JobConf jobConf = new JobConf();
jobConf.set(HiveConf.ConfVars.MAPREDMAXSPLITSIZE.varname, "268435456");
jobConf.set(HiveConf.ConfVars.MAPREDMINSPLITSIZE.varname, "268435456");
jobConf.set("mapreduce.input.fileinputformat.split.maxsize", "268435456");
jobConf.set("mapreduce.input.fileinputformat.split.minsize", "268435456");

FileStoreTable table = createFileStoreTable(TABLE_SCHEMA);

Expand Down
Loading