GH-38998: [Java] Build memory-core and memory-unsafe as JPMS modules

dev/archery/archery/integration/tester_java.py

@@ -40,7 +40,7 @@ def load_version_from_pom():
 _JAVA_OPTS = [
  "-Dio.netty.tryReflectionSetAccessible=true",
  "-Darrow.struct.conflict.policy=CONFLICT_APPEND",
- "--add-opens=java.base/java.nio=ALL-UNNAMED",
+ "--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED",
  # GH-39113: avoid failures accessing files in `/tmp/hsperfdata_...`
  "-XX:-UsePerfData",
 ]
@@ -247,7 +247,8 @@ def __init__(self, *args, **kwargs):
  if 'Unrecognized option: --add-opens' not in proc.stderr:
  # Java 9+
  self._java_opts.append(
- '--add-opens=java.base/java.nio=ALL-UNNAMED')
+ '--add-opens=java.base/java.nio='
+ 'org.apache.arrow.memory.core,ALL-UNNAMED')
 
  def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
  cmd = (

docs/source/java/install.rst

@@ -33,14 +33,14 @@ Java modules are compatible with JDK 8 and above.
 Currently, JDK 8, 11, 17, and 21 are tested in CI.
 
 When using Java 9 or later, some JDK internals must be exposed by
-adding ``--add-opens=java.base/java.nio=ALL-UNNAMED`` to the ``java`` command:
+adding ``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`` to the ``java`` command:
 
 .. code-block:: shell
 
  # Directly on the command line
- $ java --add-opens=java.base/java.nio=ALL-UNNAMED -jar ...
+ $ java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar ...
  # Indirectly via environment variables
- $ env _JAVA_OPTIONS="--add-opens=java.base/java.nio=ALL-UNNAMED" java -jar ...
+ $ env _JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ...
 
 Otherwise, you may see errors like ``module java.base does not "opens
 java.nio" to unnamed module``.

java/c/pom.xml

@@ -53,6 +53,11 @@
  <artifactId>arrow-memory-unsafe</artifactId>
  <scope>test</scope>
  </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-format</artifactId>
+ <scope>test</scope>
+ </dependency>
  <dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>

java/gandiva/pom.xml

@@ -79,7 +79,7 @@
  <plugin>
  <groupId>org.apache.maven.plugins</groupId>
  <artifactId>maven-javadoc-plugin</artifactId>
- <version>2.9.1</version>
+ <version>3.6.3</version>
  <executions>
  <execution>
  <id>attach-javadocs</id>

java/maven/module-info-compiler-maven-plugin/pom.xml

@@ -125,6 +125,4 @@
  </plugin>
  </plugins>
  </build>
-
-
-</project>
+</project>

java/memory/memory-core/pom.xml

@@ -46,7 +46,6 @@
  <plugin>
  <groupId>org.apache.maven.plugins</groupId>
  <artifactId>maven-surefire-plugin</artifactId>
-
  <configuration>
  <excludes>
  <!-- Test is only useful when NOT running with add-opens -->
@@ -58,6 +57,30 @@
  </build>
 
  <profiles>
+ <profile>
+ <id>jdk11+</id>
+ <activation>
+ <jdk>[11,]</jdk>
+ <property>
+ <name>!m2e.version</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration combine.self="override">
+ <argLine>--add-reads=org.apache.arrow.memory.core=ch.qos.logback.classic --add-opens=java.base/java.lang.reflect=org.apache.arrow.memory.core --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED</argLine>
+ <excludes>
+ <!-- Test is only useful when NOT running with add-opens -->
+ <exclude>**/TestOpens.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
  <profile>
  <id>opens-tests</id>
  <!-- Run tests WITHOUT add-opens to make sure we fail-fast -->

java/memory/memory-core/src/main/java/module-info.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module org.apache.arrow.memory.core {
+ exports org.apache.arrow.memory;
+ exports org.apache.arrow.memory.rounding;
+ exports org.apache.arrow.memory.util;
+ exports org.apache.arrow.memory.util.hash;
+ exports org.apache.arrow.util;
+ requires transitive jdk.unsupported;
+ requires jsr305;
+ requires org.immutables.value;
+ requires org.slf4j;
+}

java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java

@@ -31,20 +31,35 @@
  */
 final class CheckAllocator {
  private static final Logger logger = LoggerFactory.getLogger(CheckAllocator.class);
- private static final String ALLOCATOR_PATH = "org/apache/arrow/memory/DefaultAllocationManagerFactory.class";
+ // unique package names needed by JPMS module naming
+ private static final String ALLOCATOR_PATH_CORE =
+ "org/apache/arrow/memory/DefaultAllocationManagerFactory.class";
+ private static final String ALLOCATOR_PATH_UNSAFE =
+ "org/apache/arrow/memory/unsafe/DefaultAllocationManagerFactory.class";
+ private static final String ALLOCATOR_PATH_NETTY =
+ "org/apache/arrow/memory/netty/DefaultAllocationManagerFactory.class";
 
  private CheckAllocator() {
-
  }
 
  static String check() {
  Set<URL> urls = scanClasspath();
  URL rootAllocator = assertOnlyOne(urls);
  reportResult(rootAllocator);
- return "org.apache.arrow.memory.DefaultAllocationManagerFactory";
+ if (rootAllocator.getPath().contains("memory-core") ||
+ rootAllocator.getPath().contains("/org/apache/arrow/memory/core/")) {
+ return "org.apache.arrow.memory.DefaultAllocationManagerFactory";
+ } else if (rootAllocator.getPath().contains("memory-unsafe") ||
+ rootAllocator.getPath().contains("/org/apache/arrow/memory/unsafe/")) {
+ return "org.apache.arrow.memory.unsafe.DefaultAllocationManagerFactory";
+ } else if (rootAllocator.getPath().contains("memory-netty") ||
+ rootAllocator.getPath().contains("/org/apache/arrow/memory/netty/")) {
+ return "org.apache.arrow.memory.netty.DefaultAllocationManagerFactory";
+ } else {
+ throw new IllegalStateException("Unknown allocation manager type to infer. Current: " + rootAllocator.getPath());
+ }
  }
 
-
  private static Set<URL> scanClasspath() {
  // LinkedHashSet appropriate here because it preserves insertion order
  // during iteration
@@ -53,9 +68,21 @@ private static Set<URL> scanClasspath() {
  ClassLoader allocatorClassLoader = CheckAllocator.class.getClassLoader();
  Enumeration<URL> paths;
  if (allocatorClassLoader == null) {
- paths = ClassLoader.getSystemResources(ALLOCATOR_PATH);
+ paths = ClassLoader.getSystemResources(ALLOCATOR_PATH_CORE);
+ if (!paths.hasMoreElements()) {
+ paths = ClassLoader.getSystemResources(ALLOCATOR_PATH_UNSAFE);
+ }
+ if (!paths.hasMoreElements()) {
+ paths = ClassLoader.getSystemResources(ALLOCATOR_PATH_NETTY);
+ }
  } else {
- paths = allocatorClassLoader.getResources(ALLOCATOR_PATH);
+ paths = allocatorClassLoader.getResources(ALLOCATOR_PATH_CORE);
+ if (!paths.hasMoreElements()) {
+ paths = allocatorClassLoader.getResources(ALLOCATOR_PATH_UNSAFE);
+ }
+ if (!paths.hasMoreElements()) {
+ paths = allocatorClassLoader.getResources(ALLOCATOR_PATH_NETTY);
+ }
  }
  while (paths.hasMoreElements()) {
  URL path = paths.nextElement();

java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java

@@ -19,6 +19,7 @@
 
 import java.lang.reflect.Field;
 
+import org.apache.arrow.util.VisibleForTesting;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 
@@ -64,8 +65,13 @@ public enum AllocationManagerType {
  Unknown,
  }
 
+ /**
+ * Returns the default allocation manager type.
+ * @return the default allocation manager type.
+ */
  @SuppressWarnings("nullness:argument") //enum types valueOf are implicitly non-null
- static AllocationManagerType getDefaultAllocationManagerType() {
+ @VisibleForTesting
+ public static AllocationManagerType getDefaultAllocationManagerType() {
  AllocationManagerType ret = AllocationManagerType.Unknown;
 
  try {
@@ -122,7 +128,7 @@ private static AllocationManager.Factory getFactory(String clazzName) {
 
  private static AllocationManager.Factory getUnsafeFactory() {
  try {
- return getFactory("org.apache.arrow.memory.UnsafeAllocationManager");
+ return getFactory("org.apache.arrow.memory.unsafe.UnsafeAllocationManager");
  } catch (RuntimeException e) {
  throw new RuntimeException("Please add arrow-memory-unsafe to your classpath," +
  " No DefaultAllocationManager found to instantiate an UnsafeAllocationManager", e);
@@ -131,7 +137,7 @@ private static AllocationManager.Factory getUnsafeFactory() {
 
  private static AllocationManager.Factory getNettyFactory() {
  try {
- return getFactory("org.apache.arrow.memory.NettyAllocationManager");
+ return getFactory("org.apache.arrow.memory.netty.NettyAllocationManager");
  } catch (RuntimeException e) {
  throw new RuntimeException("Please add arrow-memory-netty to your classpath," +
  " No DefaultAllocationManager found to instantiate an NettyAllocationManager", e);

java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java

@@ -148,7 +148,7 @@ public Object run() {
  // the static fields above get initialized
  final RuntimeException failure = new RuntimeException(
  "Failed to initialize MemoryUtil. You must start Java with " +
- "`--add-opens=java.base/java.nio=ALL-UNNAMED` " +
+ "`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` " +
  "(See https://arrow.apache.org/docs/java/install.html)", e);
  failure.printStackTrace();
  throw failure;

java/memory/memory-netty/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java → java/memory/memory-core/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java

@@ -17,6 +17,11 @@
 
 package org.apache.arrow.memory;
 
+import org.apache.arrow.memory.AllocationListener;
+import org.apache.arrow.memory.AllocationOutcome;
+import org.apache.arrow.memory.BufferAllocator;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
 /**
  * Counting allocation listener.
  * It counts the number of times it has been invoked, and how much memory allocation it has seen
@@ -30,6 +35,7 @@ final class CountingAllocationListener implements AllocationListener {
  private long totalMem;
  private long currentMem;
  private boolean expandOnFail;
+ @Nullable
  BufferAllocator expandAlloc;
  long expandLimit;
 
@@ -58,6 +64,10 @@ public void onAllocation(long size) {
  @Override
  public boolean onFailedAllocation(long size, AllocationOutcome outcome) {
  if (expandOnFail) {
+ if (expandAlloc == null) {
+ throw new IllegalStateException("expandAlloc must be non-null because this " +
+ "listener is set to expand on failure.");
+ }
  expandAlloc.setLimit(expandLimit);
  return true;
  }

java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestArrowBuf.java

@@ -167,13 +167,15 @@ public void testEnabledHistoricalLog() {
  fieldDebug.set(null, true);
  try (BufferAllocator allocator = new RootAllocator(128)) {
  allocator.buffer(2);
- Exception e = assertThrows(IllegalStateException.class, () -> allocator.close());
- assertTrue(e.getMessage().contains("event log for:")); // JDK8, JDK11
+ Exception e = assertThrows(IllegalStateException.class, allocator::close);
+ assertTrue("Exception had the following message: " + e.getMessage(),
+ e.getMessage().contains("event log for:")); // JDK8, JDK11
  } finally {
  fieldDebug.set(null, false);
  }
  } catch (Exception e) {
- assertTrue(e.toString().contains("java.lang.NoSuchFieldException: modifiers")); // JDK17+
+ assertTrue("Exception had the following toString(): " + e.toString(),
+ e.toString().contains("java.lang.NoSuchFieldException: modifiers")); // JDK17+
  } finally {
  ((Logger) LoggerFactory.getLogger("org.apache.arrow")).setLevel(null);
  }