From bb74cc3aa01152fd7b9c0ff8b16b5761f8bc3902 Mon Sep 17 00:00:00 2001 From: leosanqing Date: Wed, 26 Aug 2026 14:17:03 +0800 Subject: [PATCH] [hive] Avoid binding to HiveConf.ConfVars constants renamed in Hive 4 --- .../java/org/apache/paimon/hive/HiveCatalog.java | 12 +++++------- .../paimon/hive/RetryingMetaStoreClientFactory.java | 2 +- .../apache/paimon/hive/pool/CachedClientPool.java | 3 +-- .../paimon/iceberg/IcebergHiveMetadataCommitter.java | 4 ++-- .../java/org/apache/paimon/hive/PaimonMetaHook.java | 3 +-- .../apache/paimon/hive/utils/HiveSplitGenerator.java | 10 ++++++++-- .../apache/paimon/hive/HiveSplitGeneratorTest.java | 9 ++++----- 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 14a4ead094ba..341a1e0d7916 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -101,7 +101,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREWAREHOUSE; import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM; import static org.apache.paimon.CoreOptions.DATA_FILE_PATH_DIRECTORY; import static org.apache.paimon.CoreOptions.FILE_FORMAT; @@ -189,7 +188,7 @@ public HiveCatalog( locationHelper = new TBPropertiesLocationHelper(); } else { // set the warehouse location to the hiveConf - hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehouse); + hiveConf.set("hive.metastore.warehouse.dir", warehouse); locationHelper = new StorageLocationHelper(); } } @@ -1964,7 +1963,7 @@ public static HiveConf createHiveConf( try (InputStream inputStream = hiveSite.getFileSystem(hadoopConf).open(hiveSite)) { hiveConf.addResource(inputStream, hiveSite.toString()); // trigger a read from the conf to avoid input stream is closed - hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS); + hiveConf.getVar(HiveConf.getConfVars("hive.metastore.uris")); } catch (IOException e) { throw new RuntimeException( "Failed to load hive-site.xml from specified path:" + hiveSite, e); @@ -1990,8 +1989,7 @@ public static Catalog createHiveCatalog(CatalogContext context) { Options options = context.options(); String warehouseStr = options.get(CatalogOptions.WAREHOUSE); if (warehouseStr == null) { - warehouseStr = - hiveConf.get(METASTOREWAREHOUSE.varname, METASTOREWAREHOUSE.defaultStrVal); + warehouseStr = hiveConf.getVar(HiveConf.getConfVars("hive.metastore.warehouse.dir")); } Path warehouse = new Path(warehouseStr); Path uri = @@ -2024,10 +2022,10 @@ public static HiveConf createHiveConf(CatalogContext context) { // always using user-set parameters overwrite hive-site.xml parameters context.options().toMap().forEach(hiveConf::set); if (uri != null) { - hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, uri); + hiveConf.set("hive.metastore.uris", uri); } - if (hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname) == null) { + if (hiveConf.get("hive.metastore.uris") == null) { LOG.error( "Can't find hive metastore uri to connect: " + " either set " diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java index 0ac665fa4e11..54c666bc490b 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java @@ -158,7 +158,7 @@ public IMetaStoreClient createClient(HiveConf hiveConf, String clientClassName) Method getProxy = RetryingMetaStoreClient.class.getMethod("getProxy", classes); HiveMetastoreProxySupplier supplier = entry.getValue(); IMetaStoreClient client = supplier.get(getProxy, hiveConf, clientClassName); - return isNullOrWhitespaceOnly(hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname)) + return isNullOrWhitespaceOnly(hiveConf.get("hive.metastore.uris")) ? client : HiveMetaStoreClient.newSynchronizedClient(client); } catch (Exception e) { diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java index ff981532f420..483b21106ba0 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java @@ -33,7 +33,6 @@ import org.apache.paimon.shade.guava30.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.security.UserGroupInformation; import org.apache.thrift.TException; @@ -146,7 +145,7 @@ static Key extractKey(String clientClassName, String cacheKeys, Configuration co // generate key elements in a certain order, so that the Key instances are comparable List elements = Lists.newArrayList(); elements.add(clientClassName); - elements.add(conf.get(HiveConf.ConfVars.METASTOREURIS.varname, "")); + elements.add(conf.get("hive.metastore.uris", "")); elements.add(HiveCatalogOptions.IDENTIFIER); if (cacheKeys == null || cacheKeys.isEmpty()) { return Key.of(elements); diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java index ad99382574ca..362c5d4cc7a3 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java @@ -92,10 +92,10 @@ public IcebergHiveMetadataCommitter(FileStoreTable table) { table.options().forEach(hiveConf::set); if (uri != null) { - hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, uri); + hiveConf.set("hive.metastore.uris", uri); } - if (hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname) == null) { + if (hiveConf.get("hive.metastore.uris") == null) { LOG.error( "Can't find hive metastore uri to connect: " + "either set {} for paimon table or set hive.metastore.uris " diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java index 003f72ce3bf8..ace1d749c1df 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java @@ -33,7 +33,6 @@ import org.apache.paimon.schema.TableSchema; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -83,7 +82,7 @@ public void preCreateTable(Table table) throws MetaException { String location = LocationKeyExtractor.getPaimonLocation(conf, table); Identifier identifier = Identifier.create(table.getDbName(), table.getTableName()); if (location == null) { - String warehouse = conf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname); + String warehouse = conf.get("hive.metastore.warehouse.dir"); org.apache.hadoop.fs.Path hadoopPath = getDnsPath(new org.apache.hadoop.fs.Path(warehouse), conf); warehouse = hadoopPath.toString(); diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java index a6a55f0aa7a9..b121c4e4d7af 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java @@ -231,8 +231,14 @@ private static DataSplit buildDataSplit(DataSplit current, List fi private static Long computeSplitSize( JobConf jobConf, List splits, int numSplits, long openCostInBytes) { - long maxSize = HiveConf.getLongVar(jobConf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE); - long minSize = HiveConf.getLongVar(jobConf, HiveConf.ConfVars.MAPREDMINSPLITSIZE); + long maxSize = + HiveConf.getLongVar( + jobConf, + HiveConf.getConfVars("mapreduce.input.fileinputformat.split.maxsize")); + long minSize = + HiveConf.getLongVar( + jobConf, + HiveConf.getConfVars("mapreduce.input.fileinputformat.split.minsize")); long avgSize; long splitSize; if (numSplits > 0) { diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java index 16aa824c1b7b..8a021d6cd7be 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java @@ -44,7 +44,6 @@ import org.apache.paimon.types.VarCharType; import org.apache.paimon.utils.TraceableFileIO; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; @@ -118,8 +117,8 @@ public void before() throws Exception { @Test public void testPackSplitsForNonBucketTable() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.MAPREDMAXSPLITSIZE.varname, "268435456"); // 256MB - jobConf.set(HiveConf.ConfVars.MAPREDMINSPLITSIZE.varname, "268435456"); // 256MB + jobConf.set("mapreduce.input.fileinputformat.split.maxsize", "268435456"); // 256MB + jobConf.set("mapreduce.input.fileinputformat.split.minsize", "268435456"); // 256MB FileStoreTable table = createFileStoreTable(TABLE_SCHEMA); @@ -140,8 +139,8 @@ public void testPackSplitsForNonBucketTable() throws Exception { @Test public void testPackSplitsForBucketTable() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.MAPREDMAXSPLITSIZE.varname, "268435456"); - jobConf.set(HiveConf.ConfVars.MAPREDMINSPLITSIZE.varname, "268435456"); + jobConf.set("mapreduce.input.fileinputformat.split.maxsize", "268435456"); + jobConf.set("mapreduce.input.fileinputformat.split.minsize", "268435456"); FileStoreTable table = createFileStoreTable(TABLE_SCHEMA);