community build: add automatic dependency override handling

Introduce a new sbt plugin: `sbt-community-build`, which tracks the
modules published during a community build run, and injects dependency
overrides to force version alignment.

Author: griggt

.github/workflows/ci.yaml

@@ -283,7 +283,7 @@ jobs:
         run: cp -vf .github/workflows/repositories /root/.sbt/ ; true
 
       - name: Test
-        run: ./project/scripts/sbt sbt-dotty/scripted
+        run: ./project/scripts/sbt "sbt-dotty/scripted; sbt-community-build/scripted"
 
   test_java8:
     runs-on: [self-hosted, Linux]

.gitignore

@@ -84,6 +84,7 @@ vscode-dotty/.vscode-test
 community-build/scala3-bootstrapped.version
 community-build/sbt-dotty-sbt
 community-build/sbt-scalajs-sbt
+community-build/dotty-community-build-deps
 
 # Vulpix output files
 *.check.out

build.sbt

@@ -26,6 +26,7 @@ val `scaladoc-js` = Build.`scaladoc-js`
 val `scala3-bench-run` = Build.`scala3-bench-run`
 val dist = Build.dist
 val `community-build` = Build.`community-build`
+val `sbt-community-build` = Build.`sbt-community-build`
 
 val sjsSandbox = Build.sjsSandbox
 val sjsJUnitTests = Build.sjsJUnitTests

community-build/src/scala/dotty/communitybuild/projects.scala

@@ -27,27 +27,6 @@ def exec(projectDir: Path, binary: String, arguments: String*): Int =
   exitCode
 
 
-/** Versions of published projects, needs to be updated when a project in the build is updated.
- *
- *  TODO: instead of harcoding these numbers, we could get them from the
- *  projects themselves. This likely requires injecting a custom task in the
- *  projects to output the version number to a file.
- */
-object Versions:
-  val cats = "2.3.1-SNAPSHOT"
-  val catsMtl = "1.1+DOTTY-SNAPSHOT"
-  val coop = "1.0+DOTTY-SNAPSHOT"
-  val discipline = "1.1.3-SNAPSHOT"
-  val disciplineMunit = "1.0.3+DOTTY-SNAPSHOT"
-  val disciplineSpecs2 = "1.1.3-SNAPSHOT"
-  val izumiReflect = "1.0.0-SNAPSHOT"
-  val scalacheck = "1.15.2-SNAPSHOT"
-  val scalatest = "3.2.3"
-  val munit = "0.7.19+DOTTY-SNAPSHOT"
-  val scodecBits = "1.1+DOTTY-SNAPSHOT"
-  val simulacrumScalafix = "0.5.1-SNAPSHOT"
-  val scalaCollectionCompat = "2.3.0+DOTTY-SNAPSHOT"
-
 sealed trait CommunityProject:
   private var published = false
 
@@ -115,56 +94,8 @@ final case class SbtCommunityProject(
   ) extends CommunityProject:
   override val binaryName: String = "sbt"
 
-  // A project in the community build can depend on an arbitrary version of
-  // another project in the build, so we force the use of the version that is
-  // actually in the community build.
-  val dependencyOverrides = List(
-    // dependencyOverrides doesn't seem to understand `%%%`
-    s""""org.scalacheck" %% "scalacheck" % "${Versions.scalacheck}"""",
-    s""""org.scalacheck" %% "scalacheck_sjs1" % "${Versions.scalacheck}"""",
-    s""""org.scalatest" %% "scalatest" % "${Versions.scalatest}"""",
-    s""""org.scalatest" %% "scalatest_sjs1" % "${Versions.scalatest}"""",
-    s""""org.scalatestplus" %% "junit-4-13" % "${Versions.scalatest}.0"""",
-    s""""org.scalameta" %% "munit" % "${Versions.munit}"""",
-    s""""org.scalameta" %% "munit_sjs1" % "${Versions.munit}"""",
-    s""""org.scalameta" %% "munit-scalacheck" % "${Versions.munit}"""",
-    s""""org.scalameta" %% "munit-scalacheck_sjs1" % "${Versions.munit}"""",
-    s""""org.scalameta" %% "junit-interface" % "${Versions.munit}"""",
-    s""""org.scodec" %% "scodec-bits" % "${Versions.scodecBits}"""",
-    s""""org.scodec" %% "scodec-bits_sjs1" % "${Versions.scodecBits}"""",
-    s""""org.typelevel" %% "discipline-core" % "${Versions.discipline}"""",
-    s""""org.typelevel" %% "discipline-core_sjs1" % "${Versions.discipline}"""",
-    s""""org.typelevel" %% "discipline-munit" % "${Versions.disciplineMunit}"""",
-    s""""org.typelevel" %% "discipline-munit_sjs1" % "${Versions.disciplineMunit}"""",
-    s""""org.typelevel" %% "discipline-specs2" % "${Versions.disciplineSpecs2}"""",
-    s""""org.typelevel" %% "discipline-specs2_sjs1" % "${Versions.disciplineSpecs2}"""",
-    s""""org.typelevel" %% "simulacrum-scalafix-annotations" % "${Versions.simulacrumScalafix}"""",
-    s""""org.typelevel" %% "simulacrum-scalafix-annotations_sjs1" % "${Versions.simulacrumScalafix}"""",
-    s""""org.typelevel" %% "cats-core" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-core_sjs1" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-free" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-free_sjs1" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-kernel" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-kernel_sjs1" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-kernel-laws" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-kernel-laws_sjs1" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-laws" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-laws_sjs1" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-testkit" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-testkit_sjs1" % "${Versions.cats}"""",
-    s""""org.typelevel" %% "cats-mtl" % "${Versions.catsMtl}"""",
-    s""""org.typelevel" %% "cats-mtl_sjs1" % "${Versions.catsMtl}"""",
-    s""""org.typelevel" %% "cats-mtl-laws" % "${Versions.catsMtl}"""",
-    s""""org.typelevel" %% "cats-mtl-laws_sjs1" % "${Versions.catsMtl}"""",
-    s""""org.typelevel" %% "coop" % "${Versions.coop}"""",
-    s""""org.typelevel" %% "coop_sjs1" % "${Versions.coop}"""",
-    s""""dev.zio" %% "izumi-reflect" % "${Versions.izumiReflect}"""",
-    s""""org.scala-lang.modules" %% "scala-collection-compat" % "${Versions.scalaCollectionCompat}"""",
-  )
-
   private val baseCommand =
     "clean; set logLevel in Global := Level.Error; set updateOptions in Global ~= (_.withLatestSnapshots(false)); "
-    ++ s"""set dependencyOverrides in ThisBuild ++= ${dependencyOverrides.mkString("Seq(", ", ", ")")}; """
     ++ s"++$compilerVersion!; "
 
   override val testCommand =
@@ -186,7 +117,8 @@ final case class SbtCommunityProject(
       case _ => Nil
     extraSbtArgs ++ sbtProps ++ List(
       "-sbt-version", "1.4.7",
-       "-Dsbt.supershell=false",
+      "-Dsbt.supershell=false",
+      s"-Ddotty.communitybuild.dir=$communitybuildDir",
       s"--addPluginSbtFile=$sbtPluginFilePath"
     )
 
@@ -423,17 +355,15 @@ object projects:
   lazy val munit = SbtCommunityProject(
     project = "munit",
     sbtTestCommand  = "testsJVM/test;testsJS/test;",
-    // Hardcode the version to avoid having to deal with something set by sbt-dynver
-    sbtPublishCommand   = s"""set every version := "${Versions.munit}"; munitJVM/publishLocal; munitJS/publishLocal; munitScalacheckJVM/publishLocal; munitScalacheckJS/publishLocal; junit/publishLocal""",
+    sbtPublishCommand = "munitJVM/publishLocal; munitJS/publishLocal; munitScalacheckJVM/publishLocal; munitScalacheckJS/publishLocal; junit/publishLocal",
     sbtDocCommand   = "junit/doc; munitJVM/doc",
     dependencies = List(scalacheck)
   )
 
   lazy val scodecBits = SbtCommunityProject(
     project          = "scodec-bits",
     sbtTestCommand   = "coreJVM/test;coreJS/test",
-    // Hardcode the version to avoid having to deal with something set by sbt-git
-    sbtPublishCommand = s"""set every version := "${Versions.scodecBits}"; coreJVM/publishLocal;coreJS/publishLocal""",
+    sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal",
     sbtDocCommand   = "coreJVM/doc",
     dependencies = List(munit)
   )
@@ -509,7 +439,7 @@ object projects:
   lazy val scalaCollectionCompat = SbtCommunityProject(
     project        = "scala-collection-compat",
     sbtTestCommand = "compat30/test",
-    sbtPublishCommand = s"""set every version := "${Versions.scalaCollectionCompat}"; compat30/publishLocal""",
+    sbtPublishCommand = "compat30/publishLocal",
   )
 
   lazy val verify = SbtCommunityProject(
@@ -528,7 +458,7 @@ object projects:
   lazy val disciplineMunit = SbtCommunityProject(
     project = "discipline-munit",
     sbtTestCommand = "test",
-    sbtPublishCommand = s"""set every version := "${Versions.disciplineMunit}";coreJVM/publishLocal;coreJS/publishLocal""",
+    sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal",
     dependencies = List(discipline, munit)
   )
 
@@ -555,14 +485,14 @@ object projects:
   lazy val catsMtl = SbtCommunityProject(
     project = "cats-mtl",
     sbtTestCommand = "testsJVM/test;testsJS/test",
-    sbtPublishCommand = s"""set every version := "${Versions.catsMtl}";coreJVM/publishLocal;coreJS/publishLocal;lawsJVM/publishLocal;lawsJS/publishLocal""",
+    sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal;lawsJVM/publishLocal;lawsJS/publishLocal",
     dependencies = List(cats, disciplineMunit)
   )
 
   lazy val coop = SbtCommunityProject(
     project = "coop",
     sbtTestCommand = "test",
-    sbtPublishCommand = s"""set every version := "${Versions.coop}";coreJVM/publishLocal;coreJS/publishLocal""",
+    sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal",
     dependencies = List(cats, catsMtl)
   )
 

project/Build.scala

@@ -101,6 +101,8 @@ object Build {
     if (isRelease) baseSbtDottyVersion else baseSbtDottyVersion + "-SNAPSHOT"
   }
 
+  val sbtCommunityBuildVersion = "0.1.0-SNAPSHOT"
+
   val agentOptions = List(
     // "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
     // "-agentpath:/home/dark/opt/yjp-2013-build-13072/bin/linux-x86-64/libyjpagent.so"
@@ -1272,6 +1274,45 @@ object Build {
       }.dependsOn(compile in Compile).evaluated
     )
 
+  lazy val `sbt-community-build` = project.in(file("sbt-community-build")).
+    enablePlugins(SbtPlugin).
+    settings(commonSettings).
+    settings(
+      name := "sbt-community-build",
+      version := sbtCommunityBuildVersion,
+      organization := "ch.epfl.lamp",
+      sbtTestDirectory := baseDirectory.value / "sbt-test",
+      scriptedLaunchOpts ++= Seq(
+        "-Dplugin.version=" + version.value,
+        "-Dplugin.scalaVersion=" + dottyVersion,
+        "-Dplugin.scalaJSVersion=" + scalaJSVersion,
+        "-Dplugin.sbtDottyVersion=" + sbtDottyVersion,
+        "-Ddotty.communitybuild.dir=" + baseDirectory.value / "target",
+        "-Dsbt.boot.directory=" + ((baseDirectory in ThisBuild).value / ".sbt-scripted").getAbsolutePath // Workaround sbt/sbt#3469
+      ),
+      // Pass along ivy home and repositories settings to sbt instances run from the tests
+      scriptedLaunchOpts ++= {
+        val repositoryPath = (io.Path.userHome / ".sbt" / "repositories").absolutePath
+        s"-Dsbt.repository.config=$repositoryPath" ::
+        ivyPaths.value.ivyHome.map("-Dsbt.ivy.home=" + _.getAbsolutePath).toList
+      },
+      scriptedBufferLog := true,
+      scriptedBatchExecution := true,
+      scripted := scripted.dependsOn(
+        publishLocal in `scala3-sbt-bridge`,
+        publishLocal in `scala3-interfaces`,
+        publishLocal in `scala3-compiler-bootstrapped`,
+        publishLocal in `scala3-library-bootstrapped`,
+        publishLocal in `scala3-library-bootstrappedJS`,
+        publishLocal in `tasty-core-bootstrapped`,
+        publishLocal in `scala3-staging`,
+        publishLocal in `scala3-tasty-inspector`,
+        publishLocal in `scaladoc`,
+        publishLocal in `scala3-bootstrapped`,
+        publishLocal in `sbt-dotty`,
+      ).evaluated
+   )
+
   val prepareCommunityBuild = taskKey[Unit]("Publish local the compiler and the sbt plugin. Also store the versions of the published local artefacts in two files, community-build/{scala3-bootstrapped.version,sbt-dotty-sbt}.")
 
   lazy val `community-build` = project.in(file("community-build")).
@@ -1289,13 +1330,16 @@ object Build {
         (publishLocal in `sbt-dotty`).value
         (publishLocal in `scala3-bootstrapped`).value
         (publishLocal in `scala3-library-bootstrappedJS`).value
+        (publishLocal in `sbt-community-build`).value
         // (publishLocal in `scala3-staging`).value
         val pluginText =
           s"""updateOptions in Global ~= (_.withLatestSnapshots(false))
              |addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "$sbtDottyVersion")
+             |addSbtPlugin("ch.epfl.lamp" % "sbt-community-build" % "$sbtCommunityBuildVersion")
              |addSbtPlugin("org.scala-js" % "sbt-scalajs" % "$scalaJSVersion")""".stripMargin
         IO.write(baseDirectory.value / "sbt-dotty-sbt", pluginText)
         IO.write(baseDirectory.value / "scala3-bootstrapped.version", dottyVersion)
+        IO.delete(baseDirectory.value / "dotty-community-build-deps")  // delete any stale deps file
       },
       testOptions in Test += Tests.Argument(
         TestFrameworks.JUnit,

sbt-community-build/sbt-test/sbt-community-build/inter-project-transitive-dep/build.sbt

@@ -0,0 +1,23 @@
+ThisBuild / scalaVersion := sys.props("plugin.scalaVersion")
+ThisBuild / organization := "org.example"
+
+lazy val a = project
+  .settings(
+    name := "a",
+    version := "0.4.1-SNAPSHOT",
+    libraryDependencies := Seq(),  // don't depend on scala-library
+  )
+
+lazy val b = project
+  .settings(onlyThisTestResolverSettings)
+  .settings(
+    name := "b",
+    libraryDependencies := Seq(organization.value %% "a" % "0.4.0-SNAPSHOT"),
+  )
+
+lazy val c = project
+  .settings(onlyThisTestResolverSettings)
+  .settings(
+    name := "c",
+    libraryDependencies := Seq(),  // don't depend on scala-library
+  ).dependsOn(b)

sbt-community-build/sbt-test/sbt-community-build/inter-project-transitive-dep/project/ThisTestPlugin.scala

@@ -0,0 +1,33 @@
+import sbt._
+import Keys._
+
+object ThisTestPlugin extends AutoPlugin {
+  override def requires = plugins.IvyPlugin
+  override def trigger = allRequirements
+
+  val thisTestIvyHome = settingKey[File]("Ivy home directory for artifacts published by this test")
+  val thisTestResolver = settingKey[Resolver]("Resolver for artifacts published by this test")
+  val deleteDepsFile = taskKey[Unit]("Deletes the dotty-community-build-deps dependency tracking file")
+
+  override val projectSettings = Seq(
+    publishLocalConfiguration := publishLocalConfiguration.value.withResolverName("this-test")
+  )
+
+  override val buildSettings = defaultThisTestSettings ++ Seq(
+    resolvers += thisTestResolver.value
+  )
+
+  def defaultThisTestSettings: Seq[Setting[_]] = {
+    Seq(
+      thisTestIvyHome := (LocalRootProject / target).value / "ivy-cache",
+      thisTestResolver := Resolver.file("this-test", thisTestIvyHome.value / "local")(Resolver.ivyStylePatterns),
+      deleteDepsFile := IO.delete(file(sys.props("dotty.communitybuild.dir")) / "dotty-community-build-deps"),
+    )
+  }
+
+  object autoImport {
+    def onlyThisTestResolverSettings: Seq[Setting[_]] = Seq(
+      externalResolvers := thisTestResolver.value :: Nil
+    )
+  }
+}

sbt-community-build/sbt-test/sbt-community-build/inter-project-transitive-dep/project/plugins.sbt

@@ -0,0 +1,2 @@
+addSbtPlugin("ch.epfl.lamp" % "sbt-community-build" % sys.props("plugin.version"))
+addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % sys.props("plugin.sbtDottyVersion"))

sbt-community-build/sbt-test/sbt-community-build/inter-project-transitive-dep/test

@@ -0,0 +1,10 @@
+> deleteDepsFile
+> reload
+
+> a/publishLocal
+
+-> c/update
+
+> reload
+
+> c/update

sbt-community-build/sbt-test/sbt-community-build/multiple-deps/build.sbt

@@ -0,0 +1,26 @@
+ThisBuild / scalaVersion := sys.props("plugin.scalaVersion")
+ThisBuild / organization := "org.example"
+
+lazy val a = project
+  .settings(
+    name := "a",
+    version := "0.2.1-SNAPSHOT",
+    libraryDependencies := Seq(),  // don't depend on scala-library
+  )
+
+lazy val b = project
+  .settings(
+    name := "b",
+    version := "1.2.1-SNAPSHOT",
+    libraryDependencies := Seq(),  // don't depend on scala-library
+  )
+
+lazy val c = project
+  .settings(onlyThisTestResolverSettings)
+  .settings(
+    name := "c",
+    libraryDependencies := Seq(
+      organization.value %% "a" % "0.2.0-SNAPSHOT",
+      organization.value %% "b" % "1.2.0-SNAPSHOT",
+    ),
+  )

sbt-community-build/sbt-test/sbt-community-build/multiple-deps/project/ThisTestPlugin.scala

@@ -0,0 +1,33 @@
+import sbt._
+import Keys._
+
+object ThisTestPlugin extends AutoPlugin {
+  override def requires = plugins.IvyPlugin
+  override def trigger = allRequirements
+
+  val thisTestIvyHome = settingKey[File]("Ivy home directory for artifacts published by this test")
+  val thisTestResolver = settingKey[Resolver]("Resolver for artifacts published by this test")
+  val deleteDepsFile = taskKey[Unit]("Deletes the dotty-community-build-deps dependency tracking file")
+
+  override val projectSettings = Seq(
+    publishLocalConfiguration := publishLocalConfiguration.value.withResolverName("this-test")
+  )
+
+  override val buildSettings = defaultThisTestSettings ++ Seq(
+    resolvers += thisTestResolver.value
+  )
+
+  def defaultThisTestSettings: Seq[Setting[_]] = {
+    Seq(
+      thisTestIvyHome := (LocalRootProject / target).value / "ivy-cache",
+      thisTestResolver := Resolver.file("this-test", thisTestIvyHome.value / "local")(Resolver.ivyStylePatterns),
+      deleteDepsFile := IO.delete(file(sys.props("dotty.communitybuild.dir")) / "dotty-community-build-deps"),
+    )
+  }
+
+  object autoImport {
+    def onlyThisTestResolverSettings: Seq[Setting[_]] = Seq(
+      externalResolvers := thisTestResolver.value :: Nil
+    )
+  }
+}

sbt-community-build/sbt-test/sbt-community-build/multiple-deps/project/plugins.sbt

@@ -0,0 +1,2 @@
+addSbtPlugin("ch.epfl.lamp" % "sbt-community-build" % sys.props("plugin.version"))
+addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % sys.props("plugin.sbtDottyVersion"))

sbt-community-build/sbt-test/sbt-community-build/multiple-deps/test

@@ -0,0 +1,11 @@
+> deleteDepsFile
+> reload
+
+> a/publishLocal
+> b/publishLocal
+
+-> c/update
+
+> reload
+
+> c/update