Skip to content

Add --should-fail flag to the bisect script #16864

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 9, 2023
Merged

Add --should-fail flag to the bisect script #16864

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions project/scripts/bisect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,20 @@ val usageMessage = """
|The optional <bisect-options> may be any combination of:
|* --dry-run
| Don't try to bisect - just make sure the validation command works correctly
|
|* --releases <releases-range>
| Bisect only releases from the given range (defaults to all releases).
| The range format is <first>...<last>, where both <first> and <last> are optional, e.g.
| * 3.1.0-RC1-bin-20210827-427d313-NIGHTLY..3.2.1-RC1-bin-20220716-bb9c8ff-NIGHTLY
| * 3.2.1-RC1-bin-20220620-de3a82c-NIGHTLY..
| * ..3.3.0-RC1-bin-20221124-e25362d-NIGHTLY
| The ranges are treated as inclusive.
|
|* --bootstrapped
| Publish locally and test a bootstrapped compiler rather than a nonboostrapped one
| Publish locally and test a bootstrapped compiler rather than a nonboostrapped one.
|
|* --should-fail
| Expect the validation command to fail rather that succeed. This can be used e.g. to find out when some illegal code started to compile.
|
|Warning: The bisect script should not be run multiple times in parallel because of a potential race condition while publishing artifacts locally.

Expand All @@ -54,7 +59,7 @@ val usageMessage = """

val validationScript = scriptOptions.validationCommand.validationScript
val releases = Releases.fromRange(scriptOptions.releasesRange)
val releaseBisect = ReleaseBisect(validationScript, releases)
val releaseBisect = ReleaseBisect(validationScript, shouldFail = scriptOptions.shouldFail, releases)

releaseBisect.verifyEdgeReleases()

Expand All @@ -64,18 +69,19 @@ val usageMessage = """
println(s"First bad release: ${firstBadRelease.version}")
println("\nFinished bisecting releases\n")

val commitBisect = CommitBisect(validationScript, bootstrapped = scriptOptions.bootstrapped, lastGoodRelease.hash, firstBadRelease.hash)
val commitBisect = CommitBisect(validationScript, shouldFail = scriptOptions.shouldFail, bootstrapped = scriptOptions.bootstrapped, lastGoodRelease.hash, firstBadRelease.hash)
commitBisect.bisect()


case class ScriptOptions(validationCommand: ValidationCommand, dryRun: Boolean, bootstrapped: Boolean, releasesRange: ReleasesRange)
case class ScriptOptions(validationCommand: ValidationCommand, dryRun: Boolean, bootstrapped: Boolean, releasesRange: ReleasesRange, shouldFail: Boolean)
object ScriptOptions:
def fromArgs(args: Seq[String]) =
val defaultOptions = ScriptOptions(
validationCommand = null,
dryRun = false,
bootstrapped = false,
ReleasesRange(first = None, last = None)
ReleasesRange(first = None, last = None),
shouldFail = false
)
parseArgs(args, defaultOptions)

Expand All @@ -86,6 +92,7 @@ object ScriptOptions:
case "--releases" :: argsRest =>
val range = ReleasesRange.tryParse(argsRest.head).get
parseArgs(argsRest.tail, options.copy(releasesRange = range))
case "--should-fail" :: argsRest => parseArgs(argsRest, options.copy(shouldFail = true))
case _ =>
val command = ValidationCommand.fromArgs(args)
options.copy(validationCommand = command)
Expand Down Expand Up @@ -182,7 +189,7 @@ case class Release(version: String):
override def toString: String = version


class ReleaseBisect(validationScript: File, allReleases: Vector[Release]):
class ReleaseBisect(validationScript: File, shouldFail: Boolean, allReleases: Vector[Release]):
assert(allReleases.length > 1, "Need at least 2 releases to bisect")

private val isGoodReleaseCache = collection.mutable.Map.empty[Release, Boolean]
Expand Down Expand Up @@ -217,21 +224,22 @@ class ReleaseBisect(validationScript: File, allReleases: Vector[Release]):
isGoodReleaseCache.getOrElseUpdate(release, {
println(s"Testing ${release.version}")
val result = Seq(validationScript.getAbsolutePath, release.version).!
val isGood = result == 0
val isGood = if(shouldFail) result != 0 else result == 0 // invert the process status if failure was expected
println(s"Test result: ${release.version} is a ${if isGood then "good" else "bad"} release\n")
isGood
})

class CommitBisect(validationScript: File, bootstrapped: Boolean, lastGoodHash: String, fistBadHash: String):
class CommitBisect(validationScript: File, shouldFail: Boolean, bootstrapped: Boolean, lastGoodHash: String, fistBadHash: String):
def bisect(): Unit =
println(s"Starting bisecting commits $lastGoodHash..$fistBadHash\n")
val scala3CompilerProject = if bootstrapped then "scala3-compiler-bootstrapped" else "scala3-compiler"
val scala3Project = if bootstrapped then "scala3-bootstrapped" else "scala3"
val validationCommandStatusModifier = if shouldFail then "! " else "" // invert the process status if failure was expected
val bisectRunScript = s"""
|scalaVersion=$$(sbt "print ${scala3CompilerProject}/version" | tail -n1)
|rm -r out
|sbt "clean; ${scala3Project}/publishLocal"
|${validationScript.getAbsolutePath} "$$scalaVersion"
|${validationCommandStatusModifier}${validationScript.getAbsolutePath} "$$scalaVersion"
""".stripMargin
"git bisect start".!
s"git bisect bad $fistBadHash".!
Expand Down