WIP: use scala sculpt to automatically break targets

WORKSPACE

@@ -246,3 +246,8 @@ scala_maven_import_external(
         "https://repo.maven.apache.org/maven2/",
     ],
 )
+
+# here is the source build for sculpt
+load("@io_bazel_rules_scala//src/scala/io/bazel/rules_scala/target_split:sculpt.bzl", "make_sculpt")
+
+make_sculpt()

scala/private/macros/scala_repositories.bzl

@@ -30,6 +30,10 @@ def _default_scala_extra_jars():
                 "version": "1.0.4",
                 "sha256": "0dfaafce29a9a245b0a9180ec2c1073d2bd8f0330f03a9f1f6a74d1bc83f62d6",
             },
+            "io_spray_spray_json": {
+                "version": "1.3.5",
+                "sha256": "6f341db92de96d4c4a768f478aa44662ecfecdb6d8d94facaf8e23cd00f59a93",
+            },
         },
         "2.12": {
             "scalatest": {
@@ -48,6 +52,10 @@ def _default_scala_extra_jars():
                 "version": "1.0.4",
                 "sha256": "282c78d064d3e8f09b3663190d9494b85e0bb7d96b0da05994fe994384d96111",
             },
+            "io_spray_spray_json": {
+                "version": "1.3.5",
+                "sha256": "0dfaafce29a9a245b0a9180ec2c1073d2bd8f0330f03a9f1f6a74d1bc83f62d6",
+            },
         },
     }
 
@@ -113,6 +121,18 @@ def scala_repositories(
         server_urls = maven_servers,
     )
 
+    _scala_maven_import_external(
+        name = "io_bazel_rules_scala_spray_json",
+        artifact =
+            "io.spray:spray-json_{major_version}:{extra_jar_version}".format(
+                major_version = major_version,
+                extra_jar_version = scala_version_extra_jars["io_spray_spray_json"]["version"],
+            ),
+        artifact_sha256 = scala_version_extra_jars["io_spray_spray_json"]["sha256"],
+        licenses = ["notice"],
+        server_urls = maven_servers,
+    )
+
     # used by ScalacProcessor
     _scala_maven_import_external(
         name = "scalac_rules_commons_io",

src/scala/io/bazel/rules_scala/target_split/BUILD

@@ -0,0 +1,71 @@
+load(":sculpt.bzl", "sculpt_json")
+load("@io_bazel_rules_scala//scala:scala.bzl", "scala_binary", "scala_library")
+
+sculpt_json(
+    name = "example",
+    srcs = [
+        "File0.scala",
+        "File1.scala",
+        "File2.scala",
+    ],
+    deps = [
+        "@io_bazel_rules_scala_scala_compiler",
+        "@io_bazel_rules_scala_scala_library",
+        "@io_bazel_rules_scala_scala_reflect",
+    ],
+)
+
+scala_library(
+    name = "example0_scala",
+    srcs = ["File0.scala"],
+)
+
+scala_library(
+    name = "example_scala",
+    srcs = [
+        "File1.scala",
+        "File2.scala",
+    ],
+    exports = [":example0_scala"],
+    deps = [":example0_scala"],
+)
+
+java_binary(
+    name = "scalac_runner",
+    main_class = "scala.tools.nsc.Main",
+    visibility = ["//visibility:public"],
+    runtime_deps = [
+        "@io_bazel_rules_scala_scala_compiler",
+        "@io_bazel_rules_scala_scala_library",
+        "@io_bazel_rules_scala_scala_reflect",
+        "@io_bazel_rules_scala_spray_json",
+    ],
+)
+
+scala_library(
+    name = "dagify",
+    srcs = ["Dagify.scala"],
+)
+
+scala_library(
+    name = "bazelgen",
+    srcs = ["BazelGen.scala"],
+)
+
+scala_library(
+    name = "sculpt_processor",
+    srcs = ["SculptProcessor.scala"],
+    deps = [
+        ":bazelgen",
+        ":dagify",
+        "@io_bazel_rules_scala_spray_json",
+        "@sculpt",
+    ],
+)
+
+java_binary(
+    name = "sculpt_processor_runner",
+    main_class = "io.bazel.rules_scala.target_split.SculptProcessor",
+    visibility = ["//visibility:public"],
+    runtime_deps = [":sculpt_processor"],
+)

src/scala/io/bazel/rules_scala/target_split/BazelGen.scala

@@ -0,0 +1,28 @@
+package io.bazel.rules_scala.target_split
+
+object BazelGen {
+  final case class ScalaLibrary(
+    name: String,
+    srcs: List[String],
+    deps: List[String],
+    visibility: List[String],
+    exports: List[String]) {
+
+    def render: String = {
+      def q(s: String): String = "\"" + s + "\""
+      def list(s: List[String], indent: String): String =
+        if (s.isEmpty) "[]"
+        else s.iterator.map(q).mkString(s"[\n$indent", s",\n$indent",s",\n$indent]")
+
+      val ident = " " * 8
+      s"""|scala_library(
+          |    name = ${q(name)},
+          |    srcs = ${list(srcs, ident)},
+          |    deps = ${list(deps, ident)},
+          |    visibility = ${list(visibility, ident)},
+          |    exports = ${list(exports, ident)},
+          |    )""".stripMargin
+    }
+  }
+}
+

src/scala/io/bazel/rules_scala/target_split/Dagify.scala

@@ -0,0 +1,123 @@
+package io.bazel.rules_scala.target_split
+
+import scala.collection.immutable.SortedSet
+
+object Graph {
+
+  def dagify[A: Ordering](seed: Set[A])(neighbors: A => Set[A]): Dagify[A] =
+    new Dagify[A](seed, neighbors)
+
+  def dagifyGraph[A: Ordering](graph: Map[A, Iterable[A]]): Dagify[A] =
+    dagify(graph.keySet) { a => graph.getOrElse(a, Nil).toSet }
+
+  /**
+   * Build a DAG by merging individual nodes of type A into merged nodes of type SortedSet[A]
+   */
+  final class Dagify[A: Ordering](seed: Set[A], val neighbors: A => Set[A]) {
+
+    @annotation.tailrec
+    private def allNodes(toCheck: List[A], reached: Set[A], acc: SortedSet[A]): SortedSet[A] =
+      toCheck match {
+        case Nil => acc
+        case h :: tail =>
+          if (reached(h)) allNodes(tail, reached, acc)
+          else allNodes(neighbors(h).toList.sorted ::: tail, reached + h, acc + h)
+      }
+
+    // all the reachable nodes in a sorted order
+    val nodes: SortedSet[A] =
+      allNodes(seed.toList, Set.empty, SortedSet.empty)
+
+    /*
+     * For each node, build the full set of nodes it can reach by the neighbors function
+     */
+    @annotation.tailrec
+    private def reachable(m: List[(A, SortedSet[A])],
+                          acc: List[(A, SortedSet[A])]): Map[A, SortedSet[A]] =
+      if (m.isEmpty) acc.toMap
+      else {
+        // if A -> B, then include all the nodes B can reach
+        val stepped = m.iterator.map {
+          case (src, dest0) =>
+            // expand src + dest0 by looking at all the neighbors of this set
+            val dest1 = dest0.flatMap(neighbors) ++ neighbors(src)
+            (src, (dest0, dest1))
+        }.toList
+
+        // if the expanded set is the same as the initial set, that node has computed its full reach
+        // and is done, else we need to continue to expand
+        val (done, notDone) = stepped.partition { case (_, (d0, d1)) => d0 == d1 }
+        reachable(notDone.map { case (k, (_, d1))                    => (k, d1) }, done.map {
+          case (k, (d0, _))                                          => (k, d0)
+        } ::: acc)
+      }
+
+    private def toSortedSet[T: Ordering](it: Iterator[T]): SortedSet[T] = {
+      val bldr = SortedSet.newBuilder[T]
+      bldr ++= it
+      bldr.result()
+    }
+
+    // all the reachable nodes from a given node
+    val reachableMap: Map[A, SortedSet[A]] =
+      reachable(nodes.iterator.map { a =>
+        (a, SortedSet.empty[A])
+      }.toList, Nil)
+
+    type Cluster = SortedSet[A]
+    implicit val ordCluster: Ordering[Cluster] = Ordering.Iterable[A].on { s: SortedSet[A] =>
+      s
+    }
+
+    // To make a dag, we group nodes together that are mutually reachable, these larger sets
+    // become the new nodes in the bigger graph
+    val clusterMembers: SortedSet[Cluster] =
+      toSortedSet(reachableMap.iterator.map {
+        case (n, reach) =>
+          if (reach(n)) {
+            // we can reach ourselves, so we include everyone in this cluster that can reach us
+            toSortedSet(reach.iterator.collect { case n1 if reachableMap(n1)(n) => n1 })
+          } else SortedSet(n)
+      })
+
+    // which cluster is each node in
+    val clusterMap: Map[A, Cluster] =
+      nodes.iterator.map { n =>
+        n -> clusterMembers.iterator.filter(_(n)).next
+      }.toMap
+
+    // this must form a DAG now by construction
+    val clusterDeps: Map[Cluster, SortedSet[Cluster]] =
+      clusterMembers.iterator.map { c =>
+        val reach = c.flatMap(neighbors)
+        val deps = clusterMembers.filter { c1 =>
+          reach.exists(c1)
+        } - c
+        (c, deps)
+      }.toMap
+
+    lazy val topological: List[List[Cluster]] = {
+      def loop(prev: List[Cluster], rest: List[Cluster], acc: List[List[Cluster]]): List[List[Cluster]] =
+        if (rest.isEmpty) acc.reverse.map(_.sorted)
+        else {
+          // everyone that can reach the prev is in the next group
+          val (next, notyet) = rest.partition { c =>
+            val cdeps = clusterDeps(c)
+            // are all of the deps already seen?
+            prev.forall(cdeps)
+          }
+
+          loop(next reverse_::: prev, notyet, next :: acc)
+        }
+
+        loop(Nil, clusterMembers.toList, Nil)
+    }
+
+    /**
+     * if the original A graph was a DAG, then all the clusters are singletons
+     */
+    lazy val originalIsDag: Boolean =
+      clusterMembers.forall(_.size == 1)
+  }
+}
+

src/scala/io/bazel/rules_scala/target_split/File0.scala

@@ -0,0 +1,5 @@
+package example
+
+object File0 {
+  def ident[A](a: A): A = a
+}

src/scala/io/bazel/rules_scala/target_split/File1.scala

@@ -0,0 +1,5 @@
+package example
+
+object File1 {
+  val message: String = File0.ident("this is file 1")
+}

src/scala/io/bazel/rules_scala/target_split/File2.scala

@@ -0,0 +1,5 @@
+package example
+
+object File2 {
+  val message: String = File1.message + " called from File2"
+}