More robust bisect script (#16594)

* Simplify the flow for most common kinds of release/commit validation -
compiling and running sources
* Allow specifying a range of releases to bisect with `--releases` flag
(needed for finding regressions in features introduced after 3.0.0)
* Automatically assert correctness of the validation script - it should
succeed for the first validated release and fail for the last one
* Add dry run mode (`--dry-run` flag) to check the validation script
without running the entire bisection
* Use `--server=false` in scala-cli by default
* this avoids false negatives of validation failing for some compiler
versions for which it should succeed but fails because of some
zinc/scala-cli related errors
* this allows using a non-bootstrapped compiler for testing, which
significantly speeds up `publishLocal`; non-boostrapped is now the
default, bootstrapped can be turned on with `--bootstrapped` flag

Author: KacperFKorban

project/scripts/bisect.scala

@@ -8,58 +8,203 @@ Look at the `usageMessage` below for more details.
 
 import sys.process._
 import scala.io.Source
-import Releases.Release
 import java.io.File
+import java.nio.file.attribute.PosixFilePermissions
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
 
 val usageMessage = """
   |Usage:
-  |  > scala-cli project/scripts/bisect.scala -- <validation-script>
+  |  > scala-cli project/scripts/bisect.scala -- [<bisect-options>] <validation-command>
   |
-  |The validation script should be executable and accept a single parameter, which will be the scala version to validate.
+  |The <validation-command> should be one of:
+  |* compile <arg1> <arg2> ...
+  |* run <arg1> <arg2> ...
+  |* <custom-validation-script-path>
+  |
+  |The arguments for 'compile' and 'run' should be paths to the source file(s) and optionally additional options passed directly to scala-cli.
+  |
+  |A custom validation script should be executable and accept a single parameter, which will be the scala version to validate.
   |Look at bisect-cli-example.sh and bisect-expect-example.exp for reference.
-  |Don't use the example scripts modified in place as they might disappear from the repo during a checkout.
-  |Instead copy them to a different location first.
+  |If you want to use one of the example scripts - use a copy of the file instead of modifying it in place because that might mess up the checkout.
   |
-  |Warning: The bisect script should not be run multiple times in parallel because of a potential race condition while publishing artifacts locally.
+  |The optional <bisect-options> may be any combination of:
+  |* --dry-run
+  |    Don't try to bisect - just make sure the validation command works correctly
+  |* --releases <releases-range>
+  |    Bisect only releases from the given range (defaults to all releases).
+  |    The range format is <first>...<last>, where both <first> and <last> are optional, e.g.
+  |    * 3.1.0-RC1-bin-20210827-427d313-NIGHTLY..3.2.1-RC1-bin-20220716-bb9c8ff-NIGHTLY
+  |    * 3.2.1-RC1-bin-20220620-de3a82c-NIGHTLY..
+  |    * ..3.3.0-RC1-bin-20221124-e25362d-NIGHTLY
+  |    The ranges are treated as inclusive.
+  |* --bootstrapped
+  |    Publish locally and test a bootstrapped compiler rather than a nonboostrapped one
   |
-  |Tip: Before running the bisect script run the validation script manually with some published versions of the compiler to make sure it succeeds and fails as expected.
+  |Warning: The bisect script should not be run multiple times in parallel because of a potential race condition while publishing artifacts locally.
+
 """.stripMargin
 
-@main def dottyCompileBisect(args: String*): Unit =
-  val validationScriptPath = args match
-    case Seq(path) =>
-      (new File(path)).getAbsolutePath.toString
-    case _ =>
-      println("Wrong script parameters.")
-      println()
-      println(usageMessage)
-      System.exit(1)
-      null
-
-  val releaseBisect = ReleaseBisect(validationScriptPath)
-  val bisectedBadRelease = releaseBisect.bisectedBadRelease(Releases.allReleases)
-  println("\nFinished bisecting releases\n")
-
-  bisectedBadRelease match
-    case Some(firstBadRelease) =>
-      firstBadRelease.previous match
-        case Some(lastGoodRelease) =>
-          println(s"Last good release: $lastGoodRelease")
-          println(s"First bad release: $firstBadRelease")
-          val commitBisect = CommitBisect(validationScriptPath)
-          commitBisect.bisect(lastGoodRelease.hash, firstBadRelease.hash)
-        case None =>
-          println(s"No good release found")
-    case None =>
-      println(s"No bad release found")
-
-class ReleaseBisect(validationScriptPath: String):
-  def bisectedBadRelease(releases: Vector[Release]): Option[Release] =
-    Some(bisect(releases: Vector[Release]))
-      .filter(!isGoodRelease(_))
-
-  def bisect(releases: Vector[Release]): Release =
-    assert(releases.length > 1, "Need at least 2 releases to bisect")
+@main def run(args: String*): Unit =
+  val scriptOptions =
+    try ScriptOptions.fromArgs(args)
+    catch
+      case _ =>
+        sys.error(s"Wrong script parameters.\n${usageMessage}")
+
+  val validationScript = scriptOptions.validationCommand.validationScript
+  val releases = Releases.fromRange(scriptOptions.releasesRange)
+  val releaseBisect = ReleaseBisect(validationScript, releases)
+
+  releaseBisect.verifyEdgeReleases()
+
+  if (!scriptOptions.dryRun) then
+    val (lastGoodRelease, firstBadRelease) = releaseBisect.bisectedGoodAndBadReleases()
+    println(s"Last good release: ${lastGoodRelease.version}")
+    println(s"First bad release: ${firstBadRelease.version}")
+    println("\nFinished bisecting releases\n")
+
+    val commitBisect = CommitBisect(validationScript, bootstrapped = scriptOptions.bootstrapped, lastGoodRelease.hash, firstBadRelease.hash)
+    commitBisect.bisect()
+
+
+case class ScriptOptions(validationCommand: ValidationCommand, dryRun: Boolean, bootstrapped: Boolean, releasesRange: ReleasesRange)
+object ScriptOptions:
+  def fromArgs(args: Seq[String]) =
+    val defaultOptions = ScriptOptions(
+      validationCommand = null,
+      dryRun = false,
+      bootstrapped = false,
+      ReleasesRange(first = None, last = None)
+    )
+    parseArgs(args, defaultOptions)
+
+  private def parseArgs(args: Seq[String], options: ScriptOptions): ScriptOptions =
+    args match
+      case "--dry-run" :: argsRest => parseArgs(argsRest, options.copy(dryRun = true))
+      case "--bootstrapped" :: argsRest => parseArgs(argsRest, options.copy(bootstrapped = true))
+      case "--releases" :: argsRest =>
+        val range = ReleasesRange.tryParse(argsRest.head).get
+        parseArgs(argsRest.tail, options.copy(releasesRange = range))
+      case _ =>
+        val command = ValidationCommand.fromArgs(args)
+        options.copy(validationCommand = command)
+
+enum ValidationCommand:
+  case Compile(args: Seq[String])
+  case Run(args: Seq[String])
+  case CustomValidationScript(scriptFile: File)
+
+  def validationScript: File = this match
+    case Compile(args) =>
+      ValidationScript.tmpScalaCliScript(command = "compile", args)
+    case Run(args) =>
+      ValidationScript.tmpScalaCliScript(command = "run", args)
+    case CustomValidationScript(scriptFile) =>
+      ValidationScript.copiedFrom(scriptFile)
+
+object ValidationCommand:
+  def fromArgs(args: Seq[String]) = args match
+    case Seq("compile", commandArgs*) => Compile(commandArgs)
+    case Seq("run", commandArgs*) => Run(commandArgs)
+    case Seq(path) => CustomValidationScript(new File(path))
+
+
+object ValidationScript:
+  def copiedFrom(file: File): File =
+    val fileContent = scala.io.Source.fromFile(file).mkString
+    tmpScript(fileContent)
+
+  def tmpScalaCliScript(command: String, args: Seq[String]): File = tmpScript(s"""
+    |#!/usr/bin/env bash
+    |scala-cli ${command} -S "$$1" --server=false ${args.mkString(" ")}
+    |""".stripMargin
+  )
+
+  private def tmpScript(content: String): File =
+    val executableAttr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x"))
+    val tmpPath = Files.createTempFile("scala-bisect-validator", "", executableAttr)
+    val tmpFile = tmpPath.toFile
+
+    print(s"Bisecting with validation script: ${tmpPath.toAbsolutePath}\n")
+    print("#####################################\n")
+    print(s"${content}\n\n")
+    print("#####################################\n\n")
+
+    tmpFile.deleteOnExit()
+    Files.write(tmpPath, content.getBytes(StandardCharsets.UTF_8))
+    tmpFile
+
+
+case class ReleasesRange(first: Option[String], last: Option[String]):
+  def filter(releases: Seq[Release]) =
+    def releaseIndex(version: String): Int =
+      val index = releases.indexWhere(_.version == version)
+      assert(index > 0, s"${version} matches no nightly compiler release")
+      index
+
+    val startIdx = first.map(releaseIndex(_)).getOrElse(0)
+    val endIdx = last.map(releaseIndex(_) + 1).getOrElse(releases.length)
+    val filtered = releases.slice(startIdx, endIdx).toVector
+    assert(filtered.nonEmpty, "No matching releases")
+    filtered
+
+object ReleasesRange:
+  def all = ReleasesRange(None, None)
+  def tryParse(range: String): Option[ReleasesRange] = range match
+    case s"${first}...${last}" => Some(ReleasesRange(
+      Some(first).filter(_.nonEmpty),
+      Some(last).filter(_.nonEmpty)
+    ))
+    case _ => None
+
+class Releases(val releases: Vector[Release])
+
+object Releases:
+  lazy val allReleases: Vector[Release] =
+    val re = raw"""(?<=title=")(.+-bin-\d{8}-\w{7}-NIGHTLY)(?=/")""".r
+    val html = Source.fromURL("https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/")
+    re.findAllIn(html.mkString).map(Release.apply).toVector
+
+  def fromRange(range: ReleasesRange): Vector[Release] = range.filter(allReleases)
+
+case class Release(version: String):
+  private val re = raw".+-bin-(\d{8})-(\w{7})-NIGHTLY".r
+  def date: String =
+    version match
+      case re(date, _) => date
+      case _ => sys.error(s"Could not extract date from release name: $version")
+  def hash: String =
+    version match
+      case re(_, hash) => hash
+      case _ => sys.error(s"Could not extract hash from release name: $version")
+
+  override def toString: String = version
+
+
+class ReleaseBisect(validationScript: File, allReleases: Vector[Release]):
+  assert(allReleases.length > 1, "Need at least 2 releases to bisect")
+
+  private val isGoodReleaseCache = collection.mutable.Map.empty[Release, Boolean]
+
+  def verifyEdgeReleases(): Unit =
+    println(s"Verifying the first release: ${allReleases.head.version}")
+    assert(isGoodRelease(allReleases.head), s"The evaluation script unexpectedly failed for the first checked release")
+    println(s"Verifying the last release: ${allReleases.last.version}")
+    assert(!isGoodRelease(allReleases.last), s"The evaluation script unexpectedly succeeded for the last checked release")
+
+  def bisectedGoodAndBadReleases(): (Release, Release) =
+    val firstBadRelease = bisect(allReleases)
+    assert(!isGoodRelease(firstBadRelease), s"Bisection error: the 'first bad release' ${firstBadRelease.version} is not a bad release")
+    val lastGoodRelease = firstBadRelease.previous
+    assert(isGoodRelease(lastGoodRelease), s"Bisection error: the 'last good release' ${lastGoodRelease.version} is not a good release")
+    (lastGoodRelease, firstBadRelease)
+
+  extension (release: Release) private def previous: Release =
+    val idx = allReleases.indexOf(release)
+    allReleases(idx - 1)
+
+  private def bisect(releases: Vector[Release]): Release =
     if releases.length == 2 then
       if isGoodRelease(releases.head) then releases.last
       else releases.head
@@ -69,44 +214,24 @@ class ReleaseBisect(validationScriptPath: String):
       else bisect(releases.take(releases.length / 2 + 1))
 
   private def isGoodRelease(release: Release): Boolean =
-    println(s"Testing ${release.version}")
-    val result = Seq(validationScriptPath, release.version).!
-    val isGood = result == 0
-    println(s"Test result: ${release.version} is a ${if isGood then "good" else "bad"} release\n")
-    isGood
-
-object Releases:
-  lazy val allReleases: Vector[Release] =
-    val re = raw"(?<=title=$")(.+-bin-\d{8}-\w{7}-NIGHTLY)(?=/$")".r
-    val html = Source.fromURL("https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/")
-    re.findAllIn(html.mkString).map(Release.apply).toVector
+    isGoodReleaseCache.getOrElseUpdate(release, {
+      println(s"Testing ${release.version}")
+      val result = Seq(validationScript.getAbsolutePath, release.version).!
+      val isGood = result == 0
+      println(s"Test result: ${release.version} is a ${if isGood then "good" else "bad"} release\n")
+      isGood
+    })
 
-  case class Release(version: String):
-    private val re = raw".+-bin-(\d{8})-(\w{7})-NIGHTLY".r
-    def date: String =
-      version match
-        case re(date, _) => date
-        case _ => sys.error(s"Could not extract date from version $version")
-    def hash: String =
-      version match
-        case re(_, hash) => hash
-        case _ => sys.error(s"Could not extract hash from version $version")
-
-    def previous: Option[Release] =
-      val idx = allReleases.indexOf(this)
-      if idx == 0 then None
-      else Some(allReleases(idx - 1))
-
-    override def toString: String = version
-
-class CommitBisect(validationScriptPath: String):
-  def bisect(lastGoodHash: String, fistBadHash: String): Unit =
+class CommitBisect(validationScript: File, bootstrapped: Boolean, lastGoodHash: String, fistBadHash: String):
+  def bisect(): Unit =
     println(s"Starting bisecting commits $lastGoodHash..$fistBadHash\n")
+    val scala3CompilerProject = if bootstrapped then "scala3-compiler-bootstrapped" else "scala3-compiler"
+    val scala3Project = if bootstrapped then "scala3-bootstrapped" else "scala3"
     val bisectRunScript = s"""
-      |scalaVersion=$$(sbt "print scala3-compiler-bootstrapped/version" | tail -n1)
+      |scalaVersion=$$(sbt "print ${scala3CompilerProject}/version" | tail -n1)
       |rm -r out
-      |sbt "clean; scala3-bootstrapped/publishLocal"
-      |$validationScriptPath "$$scalaVersion"
+      |sbt "clean; ${scala3Project}/publishLocal"
+      |${validationScript.getAbsolutePath} "$$scalaVersion"
     """.stripMargin
     "git bisect start".!
     s"git bisect bad $fistBadHash".!

project/scripts/examples/bisect-cli-example.sh

@@ -3,4 +3,4 @@
 # Don't use this example script modified in place as it might disappear from the repo during a checkout.
 # Instead copy it to a different location first.
 
-scala-cli compile -S "$1" file1.scala file2.scala
+scala-cli compile -S "$1" --server=false file1.scala file2.scala

project/scripts/examples/bisect-expect-example.exp

@@ -6,7 +6,7 @@
 set scalaVersion [lindex $argv 0] ;# Get the script argument
 
 set timeout 30 ;# Give scala-cli some time to download the compiler
-spawn scala-cli repl -S "$scalaVersion" ;# Start the REPL
+spawn scala-cli repl -S "$scalaVersion" --server=false ;# Start the REPL
 expect "scala>" ;# REPL has started
 set timeout 5
 send -- "Seq.empty.len\t" ;# Tab pressed to trigger code completion