diff --git a/bench/src/main/scala/Benchmarks.scala b/bench/src/main/scala/Benchmarks.scala
index b3224e6ee97b..f3c7567c3d0d 100644
--- a/bench/src/main/scala/Benchmarks.scala
+++ b/bench/src/main/scala/Benchmarks.scala
@@ -34,10 +34,9 @@ object Bench {
     }
     storeCompileOptions(args2)
 
-    val libs = System.getProperty("BENCH_CLASS_PATH")
-
     val opts = new OptionsBuilder()
-               .jvmArgsPrepend(s"-classpath $libs", "-Xms2G", "-Xmx2G")
+               .shouldFailOnError(true)
+               .jvmArgs("-Xms2G", "-Xmx2G")
                .mode(Mode.AverageTime)
                .timeUnit(TimeUnit.MILLISECONDS)
                .warmupIterations(warmup)
@@ -54,9 +53,13 @@ object Bench {
   def removeCompileOptions: Unit = new File(COMPILE_OPTS_FILE).delete()
 
   def storeCompileOptions(args: Array[String]): Unit = {
+    val libs = System.getProperty("BENCH_CLASS_PATH")
+
     val file = new File(COMPILE_OPTS_FILE)
     val bw = new BufferedWriter(new FileWriter(file))
-    bw.write(args.mkString("\n"))
+    bw.write(args.mkString("", "\n", "\n"))
+    bw.write("-classpath\n")
+    bw.write(libs)
     bw.close()
   }
 
diff --git a/project/Build.scala b/project/Build.scala
index e50bfa73362a..796788f9bedc 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -302,7 +302,7 @@ object Build {
   lazy val commonBenchmarkSettings = Seq(
     outputStrategy := Some(StdoutOutput),
     mainClass in (Jmh, run) := Some("dotty.tools.benchmarks.Bench"), // custom main for jmh:run
-    javaOptions += "-DBENCH_CLASS_PATH=" + Attributed.data((fullClasspath in Compile).value).mkString("", ":", "")
+    javaOptions += "-DBENCH_CLASS_PATH=" + Attributed.data((fullClasspath in (`dotty-library-bootstrapped`, Compile)).value).mkString("", ":", "")
   )
 
   // sbt >= 0.13.12 will automatically rewrite transitive dependencies on