From 2f3a61c13671de31964ec1c1770dac4bd88d1c42 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 22 Dec 2023 23:17:58 +0800 Subject: [PATCH 1/7] DX-86156: Cherry pick a4a3d3f4825eb025657121e70c9d86e8d6ecff35 for GH-39265 --- .../io/netty/buffer/PooledByteBufAllocatorL.java | 7 ++----- java/pom.xml | 14 +++++++------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java index d0a5a9945ce2..c1e858d013e9 100644 --- a/java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java +++ b/java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java @@ -71,7 +71,7 @@ public UnsafeDirectLittleEndian allocate(long size) { } public int getChunkSize() { - return allocator.chunkSize; + return allocator.chunkSize(); } public long getHugeBufferSize() { @@ -137,7 +137,6 @@ private class InnerAllocator extends PooledByteBufAllocator { private final PoolArena[] directArenas; private final MemoryStatusThread statusThread; - private final int chunkSize; public InnerAllocator() { super(true); @@ -150,8 +149,6 @@ public InnerAllocator() { throw new RuntimeException("Failure while initializing allocator. Unable to retrieve direct arenas field.", e); } - this.chunkSize = directArenas[0].chunkSize; - if (memoryLogger.isTraceEnabled()) { statusThread = new MemoryStatusThread(); statusThread.start(); @@ -166,7 +163,7 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa if (directArena != null) { - if (initialCapacity > directArena.chunkSize) { + if (initialCapacity > chunkSize()) { // This is beyond chunk size so we'll allocate separately. ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity); diff --git a/java/pom.xml b/java/pom.xml index 747320d2f8a4..5087066a290d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -30,13 +30,13 @@ ${project.build.directory}/generated-sources 1.9.0 - 5.9.0 - 1.7.25 - 31.1-jre - 4.1.96.Final - 1.56.0 - 3.21.9 - 2.13.4 + 5.10.1 + 2.0.9 + 32.1.3-jre + 4.1.104.Final + 1.60.0 + 3.23.1 + 2.16.0 2.7.1 1.12.0 1.10.0 From a5db296efd4b6f49c3da3785714a06d9626fd2e2 Mon Sep 17 00:00:00 2001 From: lriggs Date: Thu, 4 Jan 2024 11:58:23 -0800 Subject: [PATCH 2/7] Update README_DREMIO.md --- README_DREMIO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README_DREMIO.md b/README_DREMIO.md index c0fc7d9bf7fe..9a2480fd4ce3 100644 --- a/README_DREMIO.md +++ b/README_DREMIO.md @@ -2,6 +2,7 @@ git checkout -b dremio_24.3_12.0 apache-arrow-12.0.1 Apply starting from the bottom and working upwards. +git cherry-pick a4a3d3f4825eb025657121e70c9d86e8d6ecff35 git cherry-pick a225426e1400a583bcf4dbd90dfd45563ff78a46 git cherry-pick edab273d91adbcbeab812f15eb2344940d2151f4 git cherry-pick f2dbe77d1b320a54afc3c618ec3cb1879bd0d517 From 9b18459b20065b1f36fabf3aac587e0fadb38fd6 Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Fri, 8 Jul 2022 15:31:45 -0500 Subject: [PATCH 3/7] ARROW-16996: [Java] Configure Netty/GRPC/Protobuf base on BOM configuration + upgrade of dependencies by CVE (#13544) - Configure Netty/GRPC/Protobuf base on BOM Bill Of Material configuration to dependencies versions be added by configuration (https://github.com/netty/netty/issues/5994). - Upgrade Netty/GRPC/Protobuf dependencies. Netty [CVE](https://github.com/advisories/GHSA-269q-hmxg-m83q) Authored-by: david dali susanibar arce Signed-off-by: David Li --- docs/source/developers/java/building.rst | 3 --- java/flight/flight-core/pom.xml | 5 ----- java/flight/flight-sql/pom.xml | 12 ++++++++++++ java/pom.xml | 12 +++++------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index 9f4712cea64b..f63f99cee837 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -341,9 +341,6 @@ Arrow repository, and update the following settings: Settings > Build, Execution, Deployment > Compiler > Java Compiler and disable "Use '--release' option for cross-compilation (Java 9 and later)". Otherwise you will get an error like "package sun.misc does not exist". -* You may want to disable error-prone entirely if it gives spurious - warnings (disable both error-prone profiles in the Maven tool window - and "Reload All Maven Projects"). * If using IntelliJ's Maven integration to build, you may need to change ```` to ``false`` in the pom.xml files due to an `IntelliJ bug `__. diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index b35e9a0a93db..eb5ad094a91a 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -98,11 +98,6 @@ io.grpc grpc-api - - io.grpc - grpc-services - test - com.fasterxml.jackson.core diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index d8371e435556..298b1652e6c7 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -57,14 +57,26 @@ arrow-jdbc ${project.version} + + io.grpc + grpc-protobuf + com.google.guava guava + + io.grpc + grpc-stub + com.google.protobuf protobuf-java + + io.grpc + grpc-api + org.apache.arrow arrow-vector diff --git a/java/pom.xml b/java/pom.xml index 5087066a290d..b32b403eec5d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -543,6 +543,11 @@ flatbuffers-java ${dep.fbs.version} + + com.google.guava + guava + ${dep.guava.version} + com.google.code.findbugs jsr305 @@ -582,13 +587,6 @@ pom import - - com.google.guava - guava-bom - ${dep.guava-bom.version} - pom - import - io.netty netty-bom From be4c90159ad72ed8a74451d4813460fa655d6ab5 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Tue, 9 Jan 2024 12:02:35 -0800 Subject: [PATCH 4/7] Revert "ARROW-16996: [Java] Configure Netty/GRPC/Protobuf base on BOM configuration + upgrade of dependencies by CVE (#13544)" This reverts commit 9b18459b20065b1f36fabf3aac587e0fadb38fd6. --- docs/source/developers/java/building.rst | 3 +++ java/flight/flight-core/pom.xml | 5 +++++ java/flight/flight-sql/pom.xml | 12 ------------ java/pom.xml | 12 +++++++----- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index f63f99cee837..9f4712cea64b 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -341,6 +341,9 @@ Arrow repository, and update the following settings: Settings > Build, Execution, Deployment > Compiler > Java Compiler and disable "Use '--release' option for cross-compilation (Java 9 and later)". Otherwise you will get an error like "package sun.misc does not exist". +* You may want to disable error-prone entirely if it gives spurious + warnings (disable both error-prone profiles in the Maven tool window + and "Reload All Maven Projects"). * If using IntelliJ's Maven integration to build, you may need to change ```` to ``false`` in the pom.xml files due to an `IntelliJ bug `__. diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index eb5ad094a91a..b35e9a0a93db 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -98,6 +98,11 @@ io.grpc grpc-api + + io.grpc + grpc-services + test + com.fasterxml.jackson.core diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index 298b1652e6c7..d8371e435556 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -57,26 +57,14 @@ arrow-jdbc ${project.version} - - io.grpc - grpc-protobuf - com.google.guava guava - - io.grpc - grpc-stub - com.google.protobuf protobuf-java - - io.grpc - grpc-api - org.apache.arrow arrow-vector diff --git a/java/pom.xml b/java/pom.xml index b32b403eec5d..5087066a290d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -543,11 +543,6 @@ flatbuffers-java ${dep.fbs.version} - - com.google.guava - guava - ${dep.guava.version} - com.google.code.findbugs jsr305 @@ -587,6 +582,13 @@ pom import + + com.google.guava + guava-bom + ${dep.guava-bom.version} + pom + import + io.netty netty-bom From c55136ce7f0d6394147a25a99d122ed7ff72a139 Mon Sep 17 00:00:00 2001 From: lriggs Date: Tue, 9 Jan 2024 16:33:56 -0800 Subject: [PATCH 5/7] Update pom.xml Remove unused dependency. --- java/flight/flight-core/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index b35e9a0a93db..5c186c80116e 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -58,10 +58,6 @@ io.grpc grpc-core - - io.grpc - grpc-context - io.grpc grpc-protobuf From ccae6e95d09ce62a5134e4d3a868c8d30c609338 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 30 Nov 2023 18:36:00 -0800 Subject: [PATCH 6/7] GH-38996: [Java] Update dependencies and plugins for JPMS modules (#38994) Update dependencies and plugins to versions that work correctly with JPMS modules. * Update several plugins to use module-enabled versions: * maven-compiler-plugin is updated to 3.11.0 * maven-shade-plugin is updated to 3.2.4 * maven-dependency-plugin is updated to 3.1.2 * CycloneDX is updated to 2.7.10 * Update grpc-java to 1.59 for module support * Update jackson to 2.16.0 as 2.15.1 had corrupt module-info.class files that broke module support. Existing tests only Users may need to update their own dependencies if they have the same ones. * Closes: #38996 Authored-by: James Duong Signed-off-by: Sutou Kouhei --- java/algorithm/pom.xml | 1 + java/flight/flight-core/pom.xml | 8 +++++++- java/flight/flight-grpc/pom.xml | 11 ++++++----- java/flight/flight-sql-jdbc-driver/pom.xml | 1 - java/flight/flight-sql/pom.xml | 2 ++ java/performance/pom.xml | 7 ++++++- java/pom.xml | 21 +++++++++++++++++---- java/tools/pom.xml | 1 + java/vector/pom.xml | 8 +++++++- 9 files changed, 47 insertions(+), 13 deletions(-) diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index b52009e8b617..b487b1353039 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -32,6 +32,7 @@ arrow-vector ${project.version} test-jar + test org.apache.arrow diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 5c186c80116e..75eac45097da 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -149,7 +149,13 @@ org.apache.maven.plugins maven-shade-plugin - 3.1.1 + + 3.2.4 shade-main diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml index 3401c32d1b7d..a3a91974e605 100644 --- a/java/flight/flight-grpc/pom.xml +++ b/java/flight/flight-grpc/pom.xml @@ -49,12 +49,13 @@ io.grpc - grpc-core + grpc-stub + + + io.grpc + grpc-inprocess + test - - io.grpc - grpc-stub - org.apache.arrow arrow-memory-core diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml b/java/flight/flight-sql-jdbc-driver/pom.xml index 15c7f737aee4..615da2e8f839 100644 --- a/java/flight/flight-sql-jdbc-driver/pom.xml +++ b/java/flight/flight-sql-jdbc-driver/pom.xml @@ -41,7 +41,6 @@ org.apache.maven.plugins maven-shade-plugin - 3.4.1 package diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index d8371e435556..ef250adb33f9 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -56,6 +56,7 @@ org.apache.arrow arrow-jdbc ${project.version} + test com.google.guava @@ -74,6 +75,7 @@ org.slf4j slf4j-api + test org.apache.derby diff --git a/java/performance/pom.xml b/java/performance/pom.xml index 4e2d66dd3ebc..8049be7bbbed 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -26,6 +26,7 @@ org.openjdk.jmh jmh-core ${jmh.version} + test org.openjdk.jmh @@ -38,11 +39,13 @@ arrow-vector ${project.version} ${arrow.vector.classifier} + test org.apache.arrow arrow-memory-core ${project.version} + test org.apache.arrow @@ -54,11 +57,13 @@ org.apache.avro avro ${dep.avro.version} + test org.apache.arrow arrow-avro ${project.version} + test com.h2database @@ -70,6 +75,7 @@ org.apache.arrow arrow-jdbc ${project.version} + test org.apache.arrow @@ -109,7 +115,6 @@ org.apache.maven.plugins maven-shade-plugin - 2.2 package diff --git a/java/pom.xml b/java/pom.xml index 5087066a290d..c8b5a76fae69 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -44,8 +44,10 @@ 2 true 9+181-r4173-1 - 2.16 - 3.10.1 + 2.22.0 + 3.11.0 + 5.5.0 + 5.2.0 @@ -358,7 +360,7 @@ org.cyclonedx cyclonedx-maven-plugin - 2.7.6 + 2.7.10 package @@ -375,7 +377,12 @@ org.apache.maven.plugins maven-dependency-plugin - 3.0.1 + + 3.1.2 org.apache.rat @@ -392,6 +399,7 @@ maven-compiler-plugin ${maven-compiler-plugin.version} + false org.immutables @@ -405,6 +413,11 @@ maven-enforcer-plugin 3.0.0-M2 + + org.apache.maven.plugins + maven-shade-plugin + 3.5.1 + maven-surefire-plugin 3.0.0-M7 diff --git a/java/tools/pom.xml b/java/tools/pom.xml index d81a61b5fdcc..90df590e1e6c 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -40,6 +40,7 @@ com.google.guava guava + test commons-cli diff --git a/java/vector/pom.xml b/java/vector/pom.xml index bfedd7d3c181..92dcdc228712 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -177,7 +177,13 @@ org.apache.maven.plugins maven-shade-plugin - 3.1.1 + + 3.2.4 package From 52ccd3b5c8367f8a178dddad8f8f35a14881f23b Mon Sep 17 00:00:00 2001 From: lriggs Date: Fri, 12 Jan 2024 09:19:23 -0800 Subject: [PATCH 7/7] Update README_DREMIO.md --- README_DREMIO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README_DREMIO.md b/README_DREMIO.md index 9a2480fd4ce3..fe12e38e792f 100644 --- a/README_DREMIO.md +++ b/README_DREMIO.md @@ -2,6 +2,7 @@ git checkout -b dremio_24.3_12.0 apache-arrow-12.0.1 Apply starting from the bottom and working upwards. +git cherry-pick ccae6e95d09ce62a5134e4d3a868c8d30c609338 git cherry-pick a4a3d3f4825eb025657121e70c9d86e8d6ecff35 git cherry-pick a225426e1400a583bcf4dbd90dfd45563ff78a46 git cherry-pick edab273d91adbcbeab812f15eb2344940d2151f4