LevelOrder not to call resolveCompileDependencies

dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardTest.java

@@ -332,10 +332,14 @@ public void testComponent_success() throws IOException, ParsingException {
  public void testComponent_failure() throws IOException, ParsingException {
  Document document = parseOutputFile(
  "com.google.api.grpc_grpc-google-common-protos_1.14.0.html");
+
+ // com.google.api.grpc:grpc-google-common-protos:1.14.0 selects the highest version of all
+ // dependencies.
  Nodes greens = document.query("//h3[@style='color: green']");
- Assert.assertEquals(0, greens.size());
+ Assert.assertEquals(1, greens.size());
+
  Nodes reds = document.query("//h3[@style='color: red']");
- Assert.assertEquals(3, reds.size());
+ Assert.assertEquals(2, reds.size());
  Nodes presDependencyMediation =
  document.query("//pre[@class='suggested-dependency-mediation']");
  Assert.assertTrue(
@@ -398,8 +402,16 @@ public void testGlobalUpperBoundUpgradeMessage() throws IOException, ParsingExce
  // Case 2: Dependency needs to be updated
  Nodes globalUpperBoundDependencyUpgradeNodes =
  document.query("//li[@class='global-upper-bound-dependency-upgrade']");
- Truth.assertThat(globalUpperBoundDependencyUpgradeNodes.size()).isEqualTo(2);
- String dependencyUpgradeMessage = globalUpperBoundDependencyUpgradeNodes.get(0).getValue();
+
+ // The artifact report should contain the following 6 global upper bound dependency upgrades:
+ // Upgrade com.google.code.findbugs:jsr305:jar:1.3.9 to version "3.0.2"
+ // Upgrade com.google.errorprone:error_prone_annotations:jar:2.0.2 to version "2.3.2"
+ // Upgrade com.google.guava:guava:jar:19.0 to version "27.1-android"
+ // Upgrade com.google.j2objc:j2objc-annotations:jar:0.1 to version "1.1"
+ // Upgrade com.google.protobuf:protobuf-java:jar:3.6.1 to version "3.7.1"
+ // Upgrade org.codehaus.mojo:animal-sniffer-annotations:jar:1.14 to version "1.17"
+ Truth.assertThat(globalUpperBoundDependencyUpgradeNodes.size()).isEqualTo(6);
+ String dependencyUpgradeMessage = globalUpperBoundDependencyUpgradeNodes.get(2).getValue();
  Truth.assertThat(dependencyUpgradeMessage).contains(
  "Upgrade com.google.guava:guava:jar:19.0 to version \"27.1-android\"");
  }

dependencies/src/main/java/com/google/cloud/tools/opensource/dependencies/CycleBreakerGraphTransformer.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2020 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.tools.opensource.dependencies;
+
+import com.google.common.collect.ImmutableList;
+import java.util.HashSet;
+import java.util.Set;
+import org.eclipse.aether.RepositoryException;
+import org.eclipse.aether.artifact.Artifact;
+import org.eclipse.aether.collection.DependencyGraphTransformationContext;
+import org.eclipse.aether.collection.DependencyGraphTransformer;
+import org.eclipse.aether.graph.DependencyNode;
+
+/**
+ * Transforms a dependency graph so that it will not contain cycles.
+ *
+ * <p>A cycle in a dependency graph is a situation where a path to a node from the root contains the
+ * same node. For example, jaxen 1.1-beta-6 is known to have cycle with dom4j 1.6.1.
+ */
+final class CycleBreakerGraphTransformer implements DependencyGraphTransformer {
+
+ @Override
+ public DependencyNode transformGraph(
+ DependencyNode dependencyNode, DependencyGraphTransformationContext context)
+ throws RepositoryException {
+
+ removeCycle(null, dependencyNode, new HashSet<>());
+ return dependencyNode;
+ }
+
+ private static void removeCycle(
+ DependencyNode parent, DependencyNode node, Set<Artifact> ancestors) {
+ Artifact artifact = node.getArtifact();
+
+ if (ancestors.contains(artifact)) { // Set (rather than List) gives O(1) lookup here
+ // parent is not null when ancestors is not empty
+ removeChildFromParent(parent, node);
+ return;
+ }
+
+ ancestors.add(artifact);
+ for (DependencyNode child : node.getChildren()) {
+ removeCycle(node, child, ancestors);
+ }
+ ancestors.remove(artifact);
+ }
+
+ private static void removeChildFromParent(DependencyNode child, DependencyNode parent) {
+ ImmutableList<DependencyNode> children =
+ parent.getChildren().stream()
+ .filter(node -> node != child)
+ .collect(ImmutableList.toImmutableList());
+ parent.setChildren(children);
+ }
+}

dependencies/src/main/java/com/google/cloud/tools/opensource/dependencies/DependencyGraphBuilder.java

@@ -253,11 +253,8 @@ private DependencyGraphResult buildDependencyGraph(
  }
  }
 
- DependencyGraphResult result = levelOrder(node, traversalOption);
- // Duplicate problems found in resolveDependencyGraph and levelOrder are removed by ImmutableSet
- artifactProblems.addAll(result.getArtifactProblems());
-
- return new DependencyGraphResult(result.getDependencyGraph(), artifactProblems.build());
+ DependencyGraph graph = levelOrder(node);
+ return new DependencyGraphResult(graph, artifactProblems.build());
  }
 
  /**
@@ -316,15 +313,11 @@ private enum GraphTraversalOption {
  * just follows the given dependency tree starting with firstNode.
  *
  * @param firstNode node to start traversal
- * @param graphTraversalOption option to recursively resolve the dependency to build complete
- * dependency tree, with or without dependencies of provided scope
  */
- private DependencyGraphResult levelOrder(
- DependencyNode firstNode, GraphTraversalOption graphTraversalOption) {
+ private DependencyGraph levelOrder(DependencyNode firstNode) {
 
  DependencyGraph graph = new DependencyGraph();
 
- boolean resolveFullDependency = graphTraversalOption == GraphTraversalOption.FULL;
  Queue<LevelOrderQueueItem> queue = new ArrayDeque<>();
  queue.add(new LevelOrderQueueItem(firstNode, new Stack<>()));
 
@@ -358,29 +351,6 @@ private DependencyGraphResult levelOrder(
  path.add(dependencyNode.getDependency());
  parentNodes.push(dependencyNode);
  graph.addPath(path);
-
- if (resolveFullDependency && !"system".equals(dependencyNode.getDependency().getScope())) {
- try {
- dependencyNode =
- resolveCompileTimeDependencies(
- ImmutableList.of(dependencyNode), resolveFullDependency);
- } catch (DependencyResolutionException resolutionException) {
- // A dependency may be unavailable. For example, com.google.guava:guava-gwt:jar:20.0
- // has a transitive dependency to org.eclipse.jdt.core.compiler:ecj:jar:4.4RC4 (not
- // found in Maven central)
- for (ArtifactResult artifactResult :
- resolutionException.getResult().getArtifactResults()) {
- if (artifactResult.getArtifact() == null) {
- DependencyNode failedDependencyNode =
- artifactResult.getRequest().getDependencyNode();
-
- List<DependencyNode> fullPath = makeFullPath(parentNodes, failedDependencyNode);
-
- artifactProblems.add(new UnresolvableArtifactProblem(fullPath));
- }
- }
- }
- }
  }
 
  for (DependencyNode child : dependencyNode.getChildren()) {
@@ -390,7 +360,7 @@ private DependencyGraphResult levelOrder(
  }
  }
 
- return new DependencyGraphResult(graph, artifactProblems.build());
+ return graph;
  }
 
  private static List<DependencyNode> makeFullPath(

dependencies/src/main/java/com/google/cloud/tools/opensource/dependencies/DependencyPath.java

@@ -98,7 +98,7 @@ public boolean equals(Object o) {
  if (other.path.size() != path.size()) {
  return false;
  }
-
+
  for (int i = 0; i < path.size(); i++) {
  Dependency thisNode = path.get(i);
  Dependency otherNode = other.path.get(i);

dependencies/src/main/java/com/google/cloud/tools/opensource/dependencies/RepositoryUtility.java

@@ -76,8 +76,9 @@
 import org.eclipse.aether.transport.http.HttpTransporterFactory;
 import org.eclipse.aether.util.graph.selector.AndDependencySelector;
 import org.eclipse.aether.util.graph.selector.ExclusionDependencySelector;
-import org.eclipse.aether.util.graph.selector.OptionalDependencySelector;
 import org.eclipse.aether.util.graph.selector.ScopeDependencySelector;
+import org.eclipse.aether.util.graph.transformer.ChainedDependencyGraphTransformer;
+import org.eclipse.aether.util.graph.transformer.JavaDependencyContextRefiner;
 
 /**
  * Aether initialization.
@@ -107,7 +108,8 @@ public static RepositorySystem newRepositorySystem() {
  return locator.getService(RepositorySystem.class);
  }
 
- private static DefaultRepositorySystemSession createDefaultRepositorySystemSession(
+ @VisibleForTesting
+ static DefaultRepositorySystemSession createDefaultRepositorySystemSession(
  RepositorySystem system) {
  DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession();
  LocalRepository localRepository = new LocalRepository(findLocalRepository().getAbsolutePath());
@@ -127,25 +129,37 @@ public static RepositorySystemSession newSession(RepositorySystem system) {
  }
 
  /**
- * Opens a new Maven repository session in the same way as {@link
- * RepositoryUtility#newSession(RepositorySystem)}, with its dependency selector to include
- * dependencies with 'provided' scope and optional dependencies.
+ * Open a new Maven repository session for full dependency graph resolution.
+ *
+ * @see {@link DependencyGraphBuilder}
  */
  static RepositorySystemSession newSessionForFullDependency(RepositorySystem system) {
  DefaultRepositorySystemSession session = createDefaultRepositorySystemSession(system);
 
  // This combination of DependencySelector comes from the default specified in
  // `MavenRepositorySystemUtils.newSession`.
- // LinkageChecker needs to include 'provided'scope.
+ // LinkageChecker needs to include 'provided'-scope and optional dependencies.
  DependencySelector dependencySelector =
  new AndDependencySelector(
  // ScopeDependencySelector takes exclusions. 'Provided' scope is not here to avoid
  // false positive in LinkageChecker.
  new ScopeDependencySelector("test"),
- new OptionalDependencySelector(),
  new ExclusionDependencySelector(),
  new FilteringZipDependencySelector());
  session.setDependencySelector(dependencySelector);
+
+ // By default, Maven's MavenRepositorySystemUtils.newSession() returns a session with
+ // ChainedDependencyGraphTransformer(ConflictResolver(...), JavaDependencyContextRefiner()).
+ // Because the full dependency graph does not resolve conflicts of versions, this session does
+ // not use ConflictResolver.
+ session.setDependencyGraphTransformer(
+ new ChainedDependencyGraphTransformer(
+ new CycleBreakerGraphTransformer(), // Avoids StackOverflowError
+ new JavaDependencyContextRefiner()));
+
+ // No dependency management in the full dependency graph
+ session.setDependencyManager(null);
+
  session.setReadOnly();
 
  return session;

dependencies/src/test/java/com/google/cloud/tools/opensource/dependencies/CycleBreakerGraphTransformerTest.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2020 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.tools.opensource.dependencies;
+
+import com.google.common.collect.ImmutableList;
+import org.eclipse.aether.DefaultRepositorySystemSession;
+import org.eclipse.aether.RepositorySystem;
+import org.eclipse.aether.artifact.DefaultArtifact;
+import org.eclipse.aether.collection.CollectRequest;
+import org.eclipse.aether.graph.Dependency;
+import org.eclipse.aether.resolution.DependencyRequest;
+import org.eclipse.aether.resolution.DependencyResolutionException;
+import org.eclipse.aether.util.graph.selector.ScopeDependencySelector;
+import org.eclipse.aether.util.graph.transformer.ChainedDependencyGraphTransformer;
+import org.eclipse.aether.util.graph.transformer.JavaDependencyContextRefiner;
+import org.junit.Test;
+
+public class CycleBreakerGraphTransformerTest {
+
+ @Test
+ public void testCycleBreaking() throws DependencyResolutionException {
+ RepositorySystem system = RepositoryUtility.newRepositorySystem();
+ DefaultRepositorySystemSession session =
+ RepositoryUtility.createDefaultRepositorySystemSession(system);
+
+ // This dependencySelector selects everything except test scope. This creates a dependency tree
+ // with a cycle of dom4j:dom4j:jar:1.6.1 (optional) and jaxen:jaxen:jar:1.1-beta-6 (optional).
+ session.setDependencySelector(new ScopeDependencySelector("test"));
+
+ session.setDependencyGraphTransformer(
+ new ChainedDependencyGraphTransformer(
+ new CycleBreakerGraphTransformer(), // This prevents StackOverflowError
+ new JavaDependencyContextRefiner()));
+
+ // dom4j:1.6.1 is known to have a cycle
+ CollectRequest collectRequest = new CollectRequest();
+ collectRequest.setRepositories(ImmutableList.of(RepositoryUtility.CENTRAL));
+ collectRequest.setRoot(new Dependency(new DefaultArtifact("dom4j:dom4j:1.6.1"), "compile"));
+ DependencyRequest request = new DependencyRequest(collectRequest, null);
+
+ // This should not raise StackOverflowError
+ system.resolveDependencies(session, request);
+ }
+}

dependencies/src/test/java/com/google/cloud/tools/opensource/dependencies/DependencyGraphBuilderTest.java

@@ -74,9 +74,9 @@ public void testGetCompleteDependencies() throws RepositoryException {
  HashSet<DependencyPath> noDups = new HashSet<>(paths);
  Assert.assertEquals(paths.size(), noDups.size());
 
- // This method should find Guava multiple times.
+ // This method should find Guava multiple times, respecting exclusion elements
  int guavaCount = countGuava(graph);
- Assert.assertEquals(30, guavaCount);
+ Assert.assertEquals(29, guavaCount);
  }
 
  private static int countGuava(DependencyGraph graph) {
@@ -250,4 +250,34 @@ public void testBuildLinkageCheckDependencyGraph_catchRootException() throws Rep
  "xml-apis:xml-apis:jar:2.6.2 was not resolved. Dependency path: ant:ant:jar:1.6.2"
  + " (compile) > xml-apis:xml-apis:jar:2.6.2 (compile?)");
  }
+
+ @Test
+ public void testAlts_exclusionElements() throws RepositoryException {
+ Correspondence<DependencyPath, String> dependencyPathToString =
+ Correspondence.transforming(DependencyPath::toString, "has string representation");
+
+ DefaultArtifact artifact = new DefaultArtifact("io.grpc:grpc-alts:jar:1.27.0");
+ DependencyGraph graph =
+ dependencyGraphBuilder
+ .buildFullDependencyGraph(ImmutableList.of(artifact))
+ .getDependencyGraph();
+ List<DependencyPath> dependencyPaths = graph.list();
+
+ String expectedDependencyPathForOpencensusContribHttpUtil =
+ "io.grpc:grpc-alts:1.27.0 (compile) " // this has exclusion of Guava
+ + "/ com.google.auth:google-auth-library-oauth2-http:0.19.0 (compile) "
+ + "/ com.google.http-client:google-http-client:1.33.0 (compile) "
+ + "/ io.opencensus:opencensus-contrib-http-util:0.24.0 (compile)";
+
+ Truth.assertThat(dependencyPaths)
+ .comparingElementsUsing(dependencyPathToString)
+ .contains(expectedDependencyPathForOpencensusContribHttpUtil);
+
+ String unexpectedDependencyPathForGuava =
+ expectedDependencyPathForOpencensusContribHttpUtil
+ + " / com.google.guava:guava:jar:26.0-android (compile)";
+ Truth.assertThat(dependencyPaths)
+ .comparingElementsUsing(dependencyPathToString)
+ .doesNotContain(unexpectedDependencyPathForGuava);
+ }
 }

dependencies/src/test/java/com/google/cloud/tools/opensource/dependencies/DependencyGraphIntegrationTest.java

@@ -74,8 +74,6 @@ public void testFindUpdates() throws RepositoryException {
  + "upgrade com.google.protobuf:protobuf-java:3.5.1 to 3.6.0",
  "com.google.api.grpc:proto-google-iam-v1:0.12.0 needs to "
  + "upgrade com.google.protobuf:protobuf-java:3.5.1 to 3.6.0",
- "com.google.guava:guava-jdk5:17.0 needs to "
- + "upgrade com.google.code.findbugs:jsr305:1.3.9 to 3.0.2",
  "org.apache.httpcomponents:httpclient:4.0.1 needs to "
  + "upgrade commons-codec:commons-codec:1.3 to 1.6");
  }