From dd11270d7e9c91888eb4cbeb8c02ce055b9e05fc Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Thu, 26 Nov 2020 10:40:42 +0100 Subject: [PATCH 1/4] community-build: Update many projects - We start by updating scalatest (thanks Nicolas), with an extra commit to increase the heapsize otherwise it wouldn't run on the CI. - scalatestplus-scalacheck had to upgraded too. - upickle/ujson do not depend on scalatest when building with dotty currently so we can drop the dependency in projects.scala. - updated scodec/scodec-bits to latest upstream which doesn't depend on scalatest anymore, but does depend on munit, which I also upgraded, a few tweaks to these projects were required. - the upgraded munit required a patch because MacroCompat was in scala-3.0.0-M2 but we need it to work with our snapshots releases, so I copy-pasted it to scala-3. - Also updated scala-parallel-collections to latest upstream since we don't need any special handling anymore. Co-Authored-By: Nicolas Stucki --- community-build/community-projects/munit | 2 +- .../scala-parallel-collections | 2 +- community-build/community-projects/scalatest | 2 +- .../scalatestplus-scalacheck | 2 +- community-build/community-projects/scodec | 2 +- .../community-projects/scodec-bits | 2 +- .../scala/dotty/communitybuild/projects.scala | 43 ++++++++++++++----- .../communitybuild/CommunityBuildTest.scala | 4 +- 8 files changed, 40 insertions(+), 19 deletions(-) diff --git a/community-build/community-projects/munit b/community-build/community-projects/munit index 84325e516d47..3ce72dda2454 160000 --- a/community-build/community-projects/munit +++ b/community-build/community-projects/munit @@ -1 +1 @@ -Subproject commit 84325e516d47401c735a8a5132d1763d75033ce8 +Subproject commit 3ce72dda2454d1a1cd8d1b150d3b9120cd541869 diff --git a/community-build/community-projects/scala-parallel-collections b/community-build/community-projects/scala-parallel-collections index 4f0cc945bc83..0d8cdab0b16c 160000 --- a/community-build/community-projects/scala-parallel-collections +++ b/community-build/community-projects/scala-parallel-collections @@ -1 +1 @@ -Subproject commit 4f0cc945bc83d00104ad88faf36934f94e1658ad +Subproject commit 0d8cdab0b16cc1c17dd2867cb2300b30f1e53cc6 diff --git a/community-build/community-projects/scalatest b/community-build/community-projects/scalatest index d6e75ea20cc2..5098af41b4b8 160000 --- a/community-build/community-projects/scalatest +++ b/community-build/community-projects/scalatest @@ -1 +1 @@ -Subproject commit d6e75ea20cc2d8b2ba5b9d2b08400c118f3556eb +Subproject commit 5098af41b4b8dfca2b0844f83a28bbf7351eca49 diff --git a/community-build/community-projects/scalatestplus-scalacheck b/community-build/community-projects/scalatestplus-scalacheck index fb66f0cfa9ca..010387f5854e 160000 --- a/community-build/community-projects/scalatestplus-scalacheck +++ b/community-build/community-projects/scalatestplus-scalacheck @@ -1 +1 @@ -Subproject commit fb66f0cfa9ca4bf7abe2c32fef83d0b6f1a157d2 +Subproject commit 010387f5854eb473ddc71ecd2d6a0183dfebfbcb diff --git a/community-build/community-projects/scodec b/community-build/community-projects/scodec index 620880b81bff..2aff4c2f11bc 160000 --- a/community-build/community-projects/scodec +++ b/community-build/community-projects/scodec @@ -1 +1 @@ -Subproject commit 620880b81bff1cb365f28ff4cbf93e799265bf4d +Subproject commit 2aff4c2f11bcd9f4d108f82754da2eba02c61297 diff --git a/community-build/community-projects/scodec-bits b/community-build/community-projects/scodec-bits index dc2c5ee6560e..c6dbf213f2e8 160000 --- a/community-build/community-projects/scodec-bits +++ b/community-build/community-projects/scodec-bits @@ -1 +1 @@ -Subproject commit dc2c5ee6560ec0b6430736358a539311ad6cc00f +Subproject commit c6dbf213f2e8b442b0584af702ee830579147993 diff --git a/community-build/src/scala/dotty/communitybuild/projects.scala b/community-build/src/scala/dotty/communitybuild/projects.scala index 6fdc5658cfdd..67413d033635 100644 --- a/community-build/src/scala/dotty/communitybuild/projects.scala +++ b/community-build/src/scala/dotty/communitybuild/projects.scala @@ -30,6 +30,13 @@ def exec(projectDir: Path, binary: String, arguments: String*): Int = exitCode +/** Versions of published projects, needs to be updated when a project in the build is updated. */ +object Versions: + val scalacheck = "1.15.2-SNAPSHOT" + val scalatest = "3.2.3" + val munit = "0.7.19+7-3ce72dda-SNAPSHOT" + val scodecBits = "1.1-17-c6dbf21" + sealed trait CommunityProject: private var published = false @@ -73,10 +80,22 @@ final case class SbtCommunityProject( sbtPublishCommand: String = null) extends CommunityProject: override val binaryName: String = "sbt" + // A project in the community build can depend on an arbitrary version of + // another project in the build, so we force the use of the version that is + // actually in the community build. val dependencyOverrides = List( // dependencyOverrides doesn't seem to understand `%%%` - """"org.scalacheck" %% "scalacheck" % "1.15.2-SNAPSHOT"""", - """"org.scalacheck" %% "scalacheck_sjs1" % "1.15.2-SNAPSHOT"""" + s""""org.scalacheck" %% "scalacheck" % "${Versions.scalacheck}"""", + s""""org.scalacheck" %% "scalacheck_sjs1" % "${Versions.scalacheck}"""", + s""""org.scalatest" %% "scalatest" % "${Versions.scalatest}"""", + s""""org.scalatest" %% "scalatest_sjs1" % "${Versions.scalatest}"""", + s""""org.scalameta" %% "munit" % "${Versions.munit}"""", + s""""org.scalameta" %% "munit_sjs1" % "${Versions.munit}"""", + s""""org.scalameta" %% "munit-scalacheck" % "${Versions.munit}"""", + s""""org.scalameta" %% "munit-scalacheck_sjs1" % "${Versions.munit}"""", + s""""org.scalameta" %% "junit-interface" % "${Versions.munit}"""", + s""""org.scodec" %% "scodec-bits" % "${Versions.scodecBits}"""", + s""""org.scodec" %% "scodec-bits_sjs1" % "${Versions.scodecBits}"""", ) private val baseCommand = @@ -127,19 +146,19 @@ object projects: lazy val ujson = MillCommunityProject( project = "upickle", baseCommand = s"ujson.jvm[$compilerVersion]", - dependencies = List(scalatest, scalacheck, scalatestplusScalacheck, geny) + dependencies = List(geny) ) lazy val upickle = MillCommunityProject( project = "upickle", baseCommand = s"upickle.jvm[$compilerVersion]", - dependencies = List(scalatest, scalacheck, scalatestplusScalacheck, geny, utest) + dependencies = List(geny, utest) ) lazy val upickleCore = MillCommunityProject( project = "upickle", baseCommand = s"core.jvm[$compilerVersion]", - dependencies = List(scalatest, scalacheck, scalatestplusScalacheck, geny, utest) + dependencies = List(geny, utest) ) lazy val geny = MillCommunityProject( @@ -280,21 +299,23 @@ object projects: ) lazy val munit = SbtCommunityProject( - project = "munit", - sbtTestCommand = "testsJVM/test", + project = "munit", + sbtTestCommand = "testsJVM/test;testsJS/test;", + sbtPublishCommand = "munitJVM/publishLocal;munitJS/publishLocal;munitScalacheckJVM/publishLocal;munitScalacheckJS/publishLocal;junit/publishLocal", + dependencies = List(scalacheck) ) lazy val scodecBits = SbtCommunityProject( project = "scodec-bits", - sbtTestCommand = "coreJVM/test", - sbtPublishCommand = "coreJVM/publishLocal", - dependencies = List(scalatest, scalacheck, scalatestplusScalacheck) + sbtTestCommand = "coreJVM/test;coreJS/test", + sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal", + dependencies = List(munit) ) lazy val scodec = SbtCommunityProject( project = "scodec", sbtTestCommand = "unitTests/test", - dependencies = List(scalatest, scalacheck, scalatestplusScalacheck, scodecBits) + dependencies = List(munit, scodecBits) ) lazy val scalaParserCombinators = SbtCommunityProject( diff --git a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala index 747893264b43..bcefd88aa726 100644 --- a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala +++ b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala @@ -93,8 +93,6 @@ class CommunityBuildTestA extends CommunityBuildTest: @Test def scalatest = projects.scalatest.run() @Test def scalatestplusScalacheck = projects.scalatestplusScalacheck.run() @Test def sourcecode = projects.sourcecode.run() - @Test def scodec = projects.scodec.run() - @Test def scodecBits = projects.scodecBits.run() @Test def ujson = projects.ujson.run() @Test def upickle = projects.upickle.run() @Test def utest = projects.utest.run() @@ -114,6 +112,8 @@ class CommunityBuildTestB extends CommunityBuildTest: @Test def intent = projects.intent.run() @Test def minitest = projects.minitest.run() @Test def munit = projects.munit.run() + @Test def scodec = projects.scodec.run() + @Test def scodecBits = projects.scodecBits.run() @Test def scalap = projects.scalap.run() @Test def scalaCollectionCompat = projects.scalaCollectionCompat.run() @Test def scalaParallelCollections = projects.scalaParallelCollections.run() From 0b51303b3bc9305c459bdc3f9118d593316b30ac Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Sun, 29 Nov 2020 01:36:29 +0100 Subject: [PATCH 2/4] community-build: always force sbt-scalajs version This didn't work previously because scalatest dependend on an obselete version of sbt-scalajs but this isn't a problem now that we've upgrade scalatest. --- community-build/sbt-scalajs-sbt | 1 - .../src/scala/dotty/communitybuild/projects.scala | 12 ++---------- project/Build.scala | 6 ++---- 3 files changed, 4 insertions(+), 15 deletions(-) delete mode 100644 community-build/sbt-scalajs-sbt diff --git a/community-build/sbt-scalajs-sbt b/community-build/sbt-scalajs-sbt deleted file mode 100644 index 8f1d0ac9bcc4..000000000000 --- a/community-build/sbt-scalajs-sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.3.1") diff --git a/community-build/src/scala/dotty/communitybuild/projects.scala b/community-build/src/scala/dotty/communitybuild/projects.scala index 67413d033635..86a7ab6d3287 100644 --- a/community-build/src/scala/dotty/communitybuild/projects.scala +++ b/community-build/src/scala/dotty/communitybuild/projects.scala @@ -15,9 +15,6 @@ lazy val sbtPluginFilePath: String = new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs() communitybuildDir.resolve("sbt-dotty-sbt").toAbsolutePath().toString() -lazy val sbtScalaJSPluginFilePath: String = - communitybuildDir.resolve("sbt-scalajs-sbt").toAbsolutePath().toString() - def log(msg: String) = println(Console.GREEN + msg + Console.RESET) /** Executes shell command, returns false in case of error. */ @@ -75,7 +72,6 @@ final case class SbtCommunityProject( project: String, sbtTestCommand: String, extraSbtArgs: List[String] = Nil, - forceUpgradeSbtScalajsPlugin: Boolean = false, dependencies: List[CommunityProject] = Nil, sbtPublishCommand: String = null) extends CommunityProject: override val binaryName: String = "sbt" @@ -111,14 +107,11 @@ final case class SbtCommunityProject( val sbtProps = Option(System.getProperty("sbt.ivy.home")) match case Some(ivyHome) => List(s"-Dsbt.ivy.home=$ivyHome") case _ => Nil - val scalaJSPluginArgs = - if (forceUpgradeSbtScalajsPlugin) List(s"--addPluginSbtFile=$sbtScalaJSPluginFilePath") - else Nil extraSbtArgs ++ sbtProps ++ List( "-sbt-version", "1.4.4", "-Dsbt.supershell=false", s"--addPluginSbtFile=$sbtPluginFilePath" - ) ++ scalaJSPluginArgs + ) object projects: lazy val utest = MillCommunityProject( @@ -341,8 +334,7 @@ object projects: lazy val catsEffect2 = SbtCommunityProject( project = "cats-effect-2", - sbtTestCommand = "test", - forceUpgradeSbtScalajsPlugin = true + sbtTestCommand = "test" ) lazy val catsEffect3 = SbtCommunityProject( diff --git a/project/Build.scala b/project/Build.scala index 614719e8cfa6..916b7f2e4719 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -1309,11 +1309,9 @@ object Build { // (publishLocal in `scala3-staging`).value val pluginText = s"""updateOptions in Global ~= (_.withLatestSnapshots(false)) - |addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "$sbtDottyVersion")""".stripMargin + |addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "$sbtDottyVersion") + |addSbtPlugin("org.scala-js" % "sbt-scalajs" % "$scalaJSVersion")""".stripMargin IO.write(baseDirectory.value / "sbt-dotty-sbt", pluginText) - val scalaJSPluginText = - s"""addSbtPlugin("org.scala-js" % "sbt-scalajs" % "$scalaJSVersion")\n""" - IO.write(baseDirectory.value / "sbt-scalajs-sbt", scalaJSPluginText) IO.write(baseDirectory.value / "scala3-bootstrapped.version", dottyVersion) }, testOptions in Test += Tests.Argument( From 5d27ead241293153d119e8b1b05e238c8aef63bf Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Mon, 30 Nov 2020 20:44:17 +0100 Subject: [PATCH 3/4] Fetch tags of submodules Some projects rely on tag to determine the published version number, so this is needed for the community build to work properly. --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e4b3c2c284ab..a685204371cc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -166,6 +166,7 @@ jobs: run: | git submodule sync git submodule update --init --recursive --jobs 7 + git submodule foreach --recursive 'git fetch -f --tags' ./project/scripts/sbt "community-build/testOnly dotty.communitybuild.CommunityBuildTestA" community_build_b: @@ -201,6 +202,7 @@ jobs: run: | git submodule sync git submodule update --init --recursive --jobs 7 + git submodule foreach --recursive 'git fetch -f --tags' ./project/scripts/sbt "community-build/testOnly dotty.communitybuild.CommunityBuildTestB" test_sbt: From 364d2822e3a22dffc7f5d6875d0eb448eeb346a2 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Mon, 30 Nov 2020 21:18:13 +0100 Subject: [PATCH 4/4] community-build: Hardcode the version of some projects Some projects rely on plugins (sbt-dynver, sbt-git) to compute a version number that includes the current git commit sha, this makes the community build really fragile since we need to hardcode the version numbers of published projects, and that number will change everytime we add a commit to a project. Moreover, `git submodule update` doesn't fetch new tags and sbt-dynver relies on the latest to set the version number, so we can end up in confusing situations where we're not publishing the version we think we're publishing. Hardcoding version numbers avoids this problem. An alternative solution would be to get the version number from the sbt project, but I don't know any good way to do that, parsing the output of `sbt "show version"` to get just the version number doesn't seem easy. I guess we could inject a new task that writes the version number to a file on disk. Revert the previous commit since we no longer need to fetch tags and so that would just slowdown the CI for no benefits. --- .github/workflows/ci.yaml | 2 -- .../scala/dotty/communitybuild/projects.scala | 17 ++++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a685204371cc..e4b3c2c284ab 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -166,7 +166,6 @@ jobs: run: | git submodule sync git submodule update --init --recursive --jobs 7 - git submodule foreach --recursive 'git fetch -f --tags' ./project/scripts/sbt "community-build/testOnly dotty.communitybuild.CommunityBuildTestA" community_build_b: @@ -202,7 +201,6 @@ jobs: run: | git submodule sync git submodule update --init --recursive --jobs 7 - git submodule foreach --recursive 'git fetch -f --tags' ./project/scripts/sbt "community-build/testOnly dotty.communitybuild.CommunityBuildTestB" test_sbt: diff --git a/community-build/src/scala/dotty/communitybuild/projects.scala b/community-build/src/scala/dotty/communitybuild/projects.scala index 86a7ab6d3287..a14eb96e11d2 100644 --- a/community-build/src/scala/dotty/communitybuild/projects.scala +++ b/community-build/src/scala/dotty/communitybuild/projects.scala @@ -27,12 +27,17 @@ def exec(projectDir: Path, binary: String, arguments: String*): Int = exitCode -/** Versions of published projects, needs to be updated when a project in the build is updated. */ +/** Versions of published projects, needs to be updated when a project in the build is updated. + * + * TODO: instead of harcoding these numbers, we could get them from the + * projects themselves. This likely requires injecting a custom task in the + * projects to output the version number to a file. + */ object Versions: val scalacheck = "1.15.2-SNAPSHOT" val scalatest = "3.2.3" - val munit = "0.7.19+7-3ce72dda-SNAPSHOT" - val scodecBits = "1.1-17-c6dbf21" + val munit = "0.7.19+DOTTY-SNAPSHOT" + val scodecBits = "1.1+DOTTY-SNAPSHOT" sealed trait CommunityProject: private var published = false @@ -294,14 +299,16 @@ object projects: lazy val munit = SbtCommunityProject( project = "munit", sbtTestCommand = "testsJVM/test;testsJS/test;", - sbtPublishCommand = "munitJVM/publishLocal;munitJS/publishLocal;munitScalacheckJVM/publishLocal;munitScalacheckJS/publishLocal;junit/publishLocal", + // Hardcode the version to avoid having to deal with something set by sbt-dynver + sbtPublishCommand = s"""set every version := "${Versions.munit}"; munitJVM/publishLocal; munitJS/publishLocal; munitScalacheckJVM/publishLocal; munitScalacheckJS/publishLocal; junit/publishLocal""", dependencies = List(scalacheck) ) lazy val scodecBits = SbtCommunityProject( project = "scodec-bits", sbtTestCommand = "coreJVM/test;coreJS/test", - sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal", + // Hardcode the version to avoid having to deal with something set by sbt-git + sbtPublishCommand = s"""set every version := "${Versions.scodecBits}"; coreJVM/publishLocal;coreJS/publishLocal""", dependencies = List(munit) )