[java-source-utils] Better support orphaned Javadocs (#757)

Context: #687 (comment)
What happens when there's a "regular" Java comment in between a
Javadoc comment and a member?
/* partial */ class Object {
/** Create and return a copy of this object… */
// BEGIN Android-changed: Use native local helper for clone()
// …
protected Object clone() throws CloneNotSupportedException {…}
}
What happens is that the Javadoc becomes *orphaned*.
Commit 69e1b80 attempted to handle such orphaned Javadocs via
heuristic, using the first orphaned Javadoc comment in the parent
scope. This didn't work reliably, as the parent scope could contain
multiple "*unrelated*" orphaned Javadoc comments:
class Outer {
/** Orphaned #1 */
// cause orphaning
class Inner {}
void m() {}
}
Because containing types are fully processed before contained types,
`Outer.m()` would grab the Javadoc for `Outer.Inner` before
`Outer.Inner` would have a chance to grab it.
Re-work the logic to associate orphaned Javadocs with their members,
by requiring that the Javadoc comment begin *before* the member of
interest, and *after* any preceding members. This should prevent
incorrect correlation of orphaned Javadoc comment blocks.
Additionally, update gradle to use javaparser 3.18.0, from 3.16.1:
* javaparser/javaparser@javaparser-parent-3.16.1...javaparser-parent-3.18.0

Author: jonpryor

tools/java-source-utils/build.gradle

@@ -30,8 +30,8 @@ repositories {
 
 dependencies {
 // This dependency is used by the application.
- implementation 'com.github.javaparser:javaparser-core:3.16.1'
- implementation 'com.github.javaparser:javaparser-symbol-solver-core:3.16.1'
+ implementation 'com.github.javaparser:javaparser-core:3.18.0'
+ implementation 'com.github.javaparser:javaparser-symbol-solver-core:3.18.0'
 
 // Use JUnit test framework
  testImplementation 'junit:junit:4.12'

tools/java-source-utils/src/main/java/com/microsoft/android/JniPackagesInfoFactory.java

@@ -1,6 +1,7 @@
 packagecom.microsoft.android;
 
 importjava.io.File;
+importjava.util.ArrayList;
 importjava.util.Collection;
 importjava.util.List;
 importjava.util.Optional;
@@ -35,8 +36,6 @@
 
 importcom.microsoft.android.ast.*;
 
-importjavassist.compiler.ast.FieldDecl;
-
 importstaticcom.microsoft.android.util.Parameter.*;
 
 publicfinalclassJniPackagesInfoFactory {
@@ -90,6 +89,8 @@ private void parse(final JniPackagesInfo packages, final CompilationUnit unit) t
  : "";
 finalJniPackageInfopackageInfo = packages.getPackage(packageName);
 
+fixJavadocComments(unit, unit.getTypes());
+
 for (finalTypeDeclaration<?> type : unit.getTypes()) {
 if (JavaSourceUtilsOptions.verboseOutput && type.getFullyQualifiedName().isPresent()) {
 System.out.println("Processing: " + type.getFullyQualifiedName().get());
@@ -171,6 +172,7 @@ static ClassOrInterfaceType getTypeParameterBound(TypeParameter typeParameter) {
 	}
 
 privatefinalvoidparseType(finalJniPackageInfopackageInfo, finalJniTypeInfotypeInfo, TypeDeclaration<?> typeDecl) {
+fixJavadocComments(typeDecl, getUndocumentedBodyMembers(typeDecl.getMembers()));
 for (finalBodyDeclaration<?> body : typeDecl.getMembers()) {
 if (body.isAnnotationDeclaration()) {
 finalAnnotationDeclarationannoDecl = body.asAnnotationDeclaration();
@@ -215,6 +217,68 @@ private final void parseType(final JniPackageInfo packageInfo, final JniTypeInfo
  }
 	}
 
+privatefinalvoidfixJavadocComments(finalNodedecl, finalIterable<? extendsBodyDeclaration<?>> bodyMembers) {
+finalList<BodyDeclaration<?>> members = getUndocumentedBodyMembers(bodyMembers);
+finalList<JavadocComment> orphanedComments = getOrphanComments(decl);
+
+if (members.size() == 0)
+return;
+
+finalBodyDeclaration<?> firstMember = members.get(0);
+JavadocCommentcomment = orphanedComments.stream()
+ .filter(c -> c.getBegin().get().isBefore(firstMember.getBegin().get()))
+ .reduce((a, b) -> b)
+ .orElse(null);
+if (comment != null) {
+ ((NodeWithJavadoc<?>) firstMember).setJavadocComment(comment);
+ }
+
+for (inti = 1; i < members.size(); ++i) {
+BodyDeclaration<?> prevMember = members.get(i-1);
+BodyDeclaration<?> member = members.get(i);
+
+Optional<JavadocComment> commentOpt = orphanedComments.stream()
+ .filter(c -> c.getBegin().get().isAfter(prevMember.getEnd().get()) &&
+c.getEnd().get().isBefore(member.getBegin().get()))
+ .findFirst();
+if (!commentOpt.isPresent())
+continue;
+ ((NodeWithJavadoc<?>)member).setJavadocComment(commentOpt.get());
+ }
+	}
+
+privatefinalList<BodyDeclaration<?>> getUndocumentedBodyMembers(Iterable<? extendsBodyDeclaration<?>> bodyMembers) {
+finalList<BodyDeclaration<?>> members = newArrayList<BodyDeclaration<?>> ();
+for (BodyDeclaration<?> member : bodyMembers) {
+if (!(memberinstanceofNodeWithJavadoc<?>)) {
+continue;
+ }
+finalNodeWithJavadoc<?> memberJavadoc = (NodeWithJavadoc<?>) member;
+if (memberJavadoc.getJavadocComment().isPresent())
+continue;
+finalOptional<Position> memberBeginOpt = member.getBegin();
+if (!memberBeginOpt.isPresent())
+continue;
+members.add(member);
+ }
+members.sort((a, b) -> a.getBegin().get().compareTo(b.getBegin().get()));
+returnmembers;
+	}
+
+privatefinalList<JavadocComment> getOrphanComments(Nodedecl) {
+finalList<JavadocComment> orphanedComments = newArrayList<JavadocComment>(decl.getOrphanComments().size());
+for (Commentc : decl.getOrphanComments()) {
+if (!c.isJavadocComment())
+continue;
+finalOptional<Position> commentBeginOpt = c.getBegin();
+if (!commentBeginOpt.isPresent())
+continue;
+orphanedComments.add(c.asJavadocComment());
+ }
+orphanedComments.sort((a, b) -> a.getBegin().get().compareTo(b.getBegin().get()));
+returnorphanedComments;
+	}
+
 privatefinalvoidparseAnnotationMemberDecl(finalJniTypeInfotypeInfo, finalAnnotationMemberDeclarationmemberDecl) {
 finalJniMethodInfomethodInfo = newJniMethodInfo(typeInfo, memberDecl.getNameAsString());
 typeInfo.add(methodInfo);
@@ -266,30 +330,8 @@ private static final void fillJavadoc(final HasJavadocComment member, NodeWithJa
 JavadocCommentjavadoc = null;
 if (nodeWithJavadoc.getJavadocComment().isPresent()) {
 javadoc = nodeWithJavadoc.getJavadocComment().get();
- } else {
-Nodenode = (Node) nodeWithJavadoc;
-if (!node.getParentNode().isPresent())
-return;
-
-/*
- * Sometimes `JavaParser` won't associate a Javadoc comment block with
- * the AST node we expect it to. In such circumstances the Javadoc
- * comment will become an "orphan" comment, unassociated with anything.
- *
- * If `nodeWithJavadoc` has no Javadoc comment, use the *first*
- * orphan Javadoc comment in the *parent* scope, then *remove* that
- * comment so that it doesn't "stick around" for the next member we
- * attempt to grab Javadoc comments for.
- */
-Nodeparent = node.getParentNode().get();
-for (Commentc : parent.getOrphanComments()) {
-if (c.isJavadocComment()) {
-javadoc = c.asJavadocComment();
-c.remove();
-break;
- }
- }
  }
+
 if (javadoc != null) {
 member.setJavadocComment(javadoc.parse().toText());
  }

tools/java-source-utils/src/test/java/com/microsoft/android/JavadocXmlGeneratorTest.java

@@ -4,6 +4,7 @@
 importjava.io.File;
 importjava.io.FileNotFoundException;
 importjava.io.PrintStream;
+importjava.io.FileOutputStream;
 importjava.io.UnsupportedEncodingException;
 importjava.util.Arrays;
 importjavax.xml.parsers.ParserConfigurationException;
@@ -120,6 +121,9 @@ private static void testWritePackages(final String resourceJava, final String re
 finalStringexpected = JniPackagesInfoTest.getResourceContents(resourceXml);
 
 generator.writePackages(packagesInfo);
+// try (FileOutputStream o = new FileOutputStream(resourceXml + "-jonp.xml")) {
+// bytes.writeTo(o);
+// }
 assertEquals(resourceJava + " Javadoc XML", expected, bytes.toString());
 	}
 }

tools/java-source-utils/src/test/resources/com/microsoft/android/Outer.java

@@ -9,7 +9,7 @@
  * 
  * JNI sig: Lexample/Outer;
  */
-
+// Cause the above Javadoc block to be orphaned from the below class declaration
 @Outer.MyAnnotation(keys={"a", "b", "c"})
 publicclassOuter<TextendsObject & Runnable, UextendsError> {