Add AST walking mode for dependency analyzer (#967)

* plugin

* lint

Author: Jamie5

third_party/dependency_analyzer/src/main/BUILD

@@ -5,6 +5,7 @@ load("//scala:scala.bzl", "scala_library_for_plugin_bootstrapping")
 scala_library_for_plugin_bootstrapping(
     name = "dependency_analyzer",
     srcs = [
+        "io/bazel/rulesscala/dependencyanalyzer/AstUsedJarFinder.scala",
         "io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzer.scala",
         "io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzerSettings.scala",
         "io/bazel/rulesscala/dependencyanalyzer/HighLevelCrawlUsedJarFinder.scala",

third_party/dependency_analyzer/src/main/io/bazel/rulesscala/dependencyanalyzer/AstUsedJarFinder.scala

@@ -0,0 +1,63 @@
+package third_party.dependency_analyzer.src.main.io.bazel.rulesscala.dependencyanalyzer
+
+import scala.reflect.io.AbstractFile
+import scala.tools.nsc.Global
+
+class AstUsedJarFinder(
+  global: Global
+) {
+  import global._
+
+  def findUsedJars: Set[AbstractFile] = {
+    val jars = collection.mutable.Set[AbstractFile]()
+
+    def handleType(tpe: Type): Unit = {
+      val sym = tpe.typeSymbol
+      val assocFile = sym.associatedFile
+      if (assocFile.path.endsWith(".class"))
+        assocFile.underlyingSource.foreach { source =>
+          jars.add(source)
+        }
+    }
+
+    def exploreType(tpe: Type): Unit = {
+      handleType(tpe)
+      tpe.typeArgs.foreach(exploreType)
+    }
+
+    def fullyExploreTree(tree: Tree): Unit = {
+      exploreTree(tree)
+      tree.foreach(exploreTree)
+    }
+
+    def exploreTree(tree: Tree): Unit = {
+      tree match {
+        case node: TypeTree =>
+          if (node.original != null) {
+            node.original.foreach(fullyExploreTree)
+          }
+        case node: Literal =>
+          node.value.value match {
+            case tpe: Type =>
+              exploreType(tpe)
+            case _ =>
+          }
+        case _ =>
+      }
+
+      if (tree.hasSymbolField) {
+        tree.symbol.annotations.foreach { annot =>
+          annot.tree.foreach(fullyExploreTree)
+        }
+      }
+      if (tree.tpe != null) {
+        exploreType(tree.tpe)
+      }
+    }
+
+    currentRun.units.foreach { unit =>
+      unit.body.foreach(fullyExploreTree)
+    }
+    jars.toSet
+  }
+}

third_party/dependency_analyzer/src/main/io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzer.scala

@@ -14,7 +14,17 @@ class DependencyAnalyzer(val global: Global) extends Plugin {
       "compilation for issues including not directly including " +
       "dependencies which are directly included in the code, or " +
       "including unused dependencies."
-  override val components = List[PluginComponent](Component)
+  override val components =
+    List[PluginComponent](
+      new AnalyzerComponent(
+        runsAfterPhase = "typer",
+        handles = DependencyTrackingMethod.Ast
+      ),
+      new AnalyzerComponent(
+        runsAfterPhase = "jvm",
+        handles = DependencyTrackingMethod.HighLevel
+      )
+    )
 
   private val isWindows: Boolean = System.getProperty("os.name").toLowerCase.contains("windows")
   private var settings: DependencyAnalyzerSettings = null
@@ -27,34 +37,44 @@ class DependencyAnalyzer(val global: Global) extends Plugin {
     true
   }
 
-  private object Component extends PluginComponent {
-    val global: DependencyAnalyzer.this.global.type =
+  private class AnalyzerComponent(
+    // Typer seems to be the better method at least for AST - it seems like
+    // some things get eliminated in later phases. However, due to backwards
+    // compatibility we have to preserve using jvm for the high-level-crawl
+    // dependency tracking method
+    runsAfterPhase: String,
+    handles: DependencyTrackingMethod
+  ) extends PluginComponent {
+    override val global: DependencyAnalyzer.this.global.type =
       DependencyAnalyzer.this.global
 
-    override val runsAfter = List("jvm")
+    override val runsAfter = List(runsAfterPhase)
 
-    val phaseName = DependencyAnalyzer.this.name
+    val phaseName = s"${DependencyAnalyzer.this.name}-post-$runsAfterPhase"
 
     override def newPhase(prev: Phase): StdPhase = new StdPhase(prev) {
       override def run(): Unit = {
-
         super.run()
-
-        val usedJars = findUsedJars
-        val usedJarPaths = if (!isWindows) usedJars.map(_.path) else usedJars.map(_.path.replaceAll("\\\\", "/"))
-
-        if (settings.unusedDepsMode != AnalyzerMode.Off) {
-          reportUnusedDepsFoundIn(usedJarPaths)
-        }
-
-        if (settings.strictDepsMode != AnalyzerMode.Off) {
-          reportIndirectTargetsFoundIn(usedJarPaths)
+        if (settings.dependencyTrackingMethod == handles) {
+          runAnalysis()
         }
       }
 
       override def apply(unit: global.CompilationUnit): Unit = ()
     }
+  }
+
+  private def runAnalysis(): Unit = {
+    val usedJars = findUsedJars
+    val usedJarPaths = if (!isWindows) usedJars.map(_.path) else usedJars.map(_.path.replaceAll("\\\\", "/"))
 
+    if (settings.unusedDepsMode != AnalyzerMode.Off) {
+      reportUnusedDepsFoundIn(usedJarPaths)
+    }
+
+    if (settings.strictDepsMode != AnalyzerMode.Off) {
+      reportIndirectTargetsFoundIn(usedJarPaths)
+    }
   }
 
   private def reportIndirectTargetsFoundIn(usedJarPaths: Set[String]): Unit = {
@@ -112,6 +132,8 @@ class DependencyAnalyzer(val global: Global) extends Plugin {
     settings.dependencyTrackingMethod match {
       case DependencyTrackingMethod.HighLevel =>
         new HighLevelCrawlUsedJarFinder(global).findUsedJars
+      case DependencyTrackingMethod.Ast =>
+        new AstUsedJarFinder(global).findUsedJars
     }
   }
 }

third_party/dependency_analyzer/src/main/io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzerSettings.scala

@@ -18,17 +18,19 @@ object AnalyzerMode {
 sealed trait AnalyzerMode
 
 object DependencyTrackingMethod {
-  case object HighLevel extends DependencyTrackingMethod
+  case object HighLevel extends DependencyTrackingMethod("high-level")
+
+  /**
+   * Discovers dependencies by crawling the AST.
+   */
+  case object Ast extends DependencyTrackingMethod("ast")
 
   def parse(mode: String): Option[DependencyTrackingMethod] = {
-    mode match {
-      case "high-level" => Some(HighLevel)
-      case _ => None
-    }
+    Seq(HighLevel, Ast).find(_.name == mode)
   }
 }
 
-sealed trait DependencyTrackingMethod
+sealed abstract class DependencyTrackingMethod(val name: String)
 
 class TargetSet(
   prefix: String,

third_party/dependency_analyzer/src/test/BUILD

@@ -2,15 +2,36 @@ licenses(["notice"])  # 3-clause BSD
 
 load("//scala:scala.bzl", "scala_junit_test", "scala_test")
 
+common_jvm_flags = [
+    "-Dplugin.jar.location=$(location //third_party/dependency_analyzer/src/main:dependency_analyzer)",
+    "-Dscala.library.location=$(location //external:io_bazel_rules_scala/dependency/scala/scala_library)",
+]
+
+scala_test(
+    name = "ast_used_jar_finder_test",
+    size = "small",
+    srcs = [
+        "io/bazel/rulesscala/dependencyanalyzer/AstUsedJarFinderTest.scala",
+    ],
+    jvm_flags = common_jvm_flags,
+    unused_dependency_checker_mode = "off",
+    deps = [
+        "//external:io_bazel_rules_scala/dependency/scala/scala_compiler",
+        "//external:io_bazel_rules_scala/dependency/scala/scala_library",
+        "//external:io_bazel_rules_scala/dependency/scala/scala_reflect",
+        "//third_party/dependency_analyzer/src/main:dependency_analyzer",
+        "//third_party/utils/src/test:test_util",
+        "@scalac_rules_commons_io//jar",
+    ],
+)
+
 scala_test(
     name = "strict_deps_test",
     size = "small",
     srcs = [
         "io/bazel/rulesscala/dependencyanalyzer/StrictDepsTest.scala",
     ],
-    jvm_flags = [
-        "-Dplugin.jar.location=$(location //third_party/dependency_analyzer/src/main:dependency_analyzer)",
-        "-Dscala.library.location=$(location //external:io_bazel_rules_scala/dependency/scala/scala_library)",
+    jvm_flags = common_jvm_flags + [
         "-Dguava.jar.location=$(location @com_google_guava_guava_21_0_with_file//jar)",
         "-Dapache.commons.jar.location=$(location @org_apache_commons_commons_lang_3_5_without_file//:linkable_org_apache_commons_commons_lang_3_5_without_file)",
     ],
@@ -32,9 +53,7 @@ scala_test(
     srcs = [
         "io/bazel/rulesscala/dependencyanalyzer/UnusedDependencyCheckerTest.scala",
     ],
-    jvm_flags = [
-        "-Dplugin.jar.location=$(location //third_party/dependency_analyzer/src/main:dependency_analyzer)",
-        "-Dscala.library.location=$(location //external:io_bazel_rules_scala/dependency/scala/scala_library)",
+    jvm_flags = common_jvm_flags + [
         "-Dapache.commons.jar.location=$(location @org_apache_commons_commons_lang_3_5_without_file//:linkable_org_apache_commons_commons_lang_3_5_without_file)",
     ],
     unused_dependency_checker_mode = "off",