From 45984f5d5c53c4ca0b7235b8acb12274329af31d Mon Sep 17 00:00:00 2001 From: Dmitry Komanov Date: Sun, 1 Apr 2018 03:18:09 +0300 Subject: [PATCH 1/3] Support JMH for scala 2.12 #295 (#465) * Upgrade versions of JMH 1.17.4 -> 1.20, asm 5.0.4 -> 6.1.1 * Add reflection-based generator for JMH (make it default) --- jmh/jmh.bzl | 22 +++-- .../jmh_support/BenchmarkGenerator.scala | 96 ++++++++++++++----- 2 files changed, 86 insertions(+), 32 deletions(-) diff --git a/jmh/jmh.bzl b/jmh/jmh.bzl index 7fe0f49eb..4b4af76fc 100644 --- a/jmh/jmh.bzl +++ b/jmh/jmh.bzl @@ -3,8 +3,8 @@ load("//scala:scala.bzl", "scala_binary", "scala_library") def jmh_repositories(): native.maven_jar( name = "io_bazel_rules_scala_org_openjdk_jmh_jmh_core", - artifact = "org.openjdk.jmh:jmh-core:1.17.4", - sha1 = "126d989b196070a8b3653b5389e602a48fe6bb2f", + artifact = "org.openjdk.jmh:jmh-core:1.20", + sha1 = "5f9f9839bda2332e9acd06ce31ad94afa7d6d447", ) native.bind( name = 'io_bazel_rules_scala/dependency/jmh/jmh_core', @@ -12,8 +12,8 @@ def jmh_repositories(): ) native.maven_jar( name = "io_bazel_rules_scala_org_openjdk_jmh_jmh_generator_asm", - artifact = "org.openjdk.jmh:jmh-generator-asm:1.17.4", - sha1 = "c85c3d8cfa194872b260e89770d41e2084ce2cb6", + artifact = "org.openjdk.jmh:jmh-generator-asm:1.20", + sha1 = "3c43040e08ae68905657a375e669f11a7352f9db", ) native.bind( name = 'io_bazel_rules_scala/dependency/jmh/jmh_generator_asm', @@ -21,8 +21,8 @@ def jmh_repositories(): ) native.maven_jar( name = "io_bazel_rules_scala_org_openjdk_jmh_jmh_generator_reflection", - artifact = "org.openjdk.jmh:jmh-generator-reflection:1.17.4", - sha1 = "f75a7823c9fcf03feed6d74aa44ea61fc70a8439", + artifact = "org.openjdk.jmh:jmh-generator-reflection:1.20", + sha1 = "f2154437b42426a48d5dac0b3df59002f86aed26", ) native.bind( name = 'io_bazel_rules_scala/dependency/jmh/jmh_generator_reflection', @@ -30,8 +30,8 @@ def jmh_repositories(): ) native.maven_jar( name = "io_bazel_rules_scala_org_ows2_asm_asm", - artifact = "org.ow2.asm:asm:5.0.4", - sha1 = "0da08b8cce7bbf903602a25a3a163ae252435795", + artifact = "org.ow2.asm:asm:6.1.1", + sha1 = "264754515362d92acd39e8d40395f6b8dee7bc08", ) native.bind( name = "io_bazel_rules_scala/dependency/jmh/org_ows2_asm_asm", @@ -78,7 +78,7 @@ def _scala_generate_benchmark(ctx): outputs = [ctx.outputs.src_jar, ctx.outputs.resource_jar], inputs = [class_jar] + classpath, executable = ctx.executable._generator, - arguments = [f.path for f in [class_jar, ctx.outputs.src_jar, ctx.outputs.resource_jar] + classpath], + arguments = [ctx.attr.generator_type] + [f.path for f in [class_jar, ctx.outputs.src_jar, ctx.outputs.resource_jar] + classpath], progress_message = "Generating benchmark code for %s" % ctx.label, ) @@ -86,6 +86,7 @@ scala_generate_benchmark = rule( implementation = _scala_generate_benchmark, attrs = { "src": attr.label(allow_single_file=True, mandatory=True), + "generator_type": attr.string(default='reflection', mandatory=False), "_generator": attr.label(executable=True, cfg="host", default=Label("//src/scala/io/bazel/rules_scala/jmh_support:benchmark_generator")) }, outputs = { @@ -98,6 +99,7 @@ def scala_benchmark_jmh(**kw): name = kw["name"] deps = kw.get("deps", []) srcs = kw["srcs"] + generator_type = kw.get("generator_type", "reflection") lib = "%s_generator" % name scalacopts = kw.get("scalacopts", []) main_class = kw.get("main_class", "org.openjdk.jmh.Main") @@ -115,7 +117,7 @@ def scala_benchmark_jmh(**kw): ) codegen = name + "_codegen" - scala_generate_benchmark(name=codegen, src=lib) + scala_generate_benchmark(name=codegen, src=lib, generator_type=generator_type) compiled_lib = name + "_compiled_benchmark_lib" scala_library( name = compiled_lib, diff --git a/src/scala/io/bazel/rules_scala/jmh_support/BenchmarkGenerator.scala b/src/scala/io/bazel/rules_scala/jmh_support/BenchmarkGenerator.scala index 86408f348..579e8ddab 100644 --- a/src/scala/io/bazel/rules_scala/jmh_support/BenchmarkGenerator.scala +++ b/src/scala/io/bazel/rules_scala/jmh_support/BenchmarkGenerator.scala @@ -4,15 +4,15 @@ import java.net.URLClassLoader import scala.annotation.tailrec import scala.collection.JavaConverters._ - -import org.openjdk.jmh.generators.core.{ BenchmarkGenerator => JMHGenerator, FileSystemDestination } +import org.openjdk.jmh.generators.core.{FileSystemDestination, GeneratorSource, BenchmarkGenerator => JMHGenerator} import org.openjdk.jmh.generators.asm.ASMGeneratorSource -import org.openjdk.jmh.runner.{ Runner, RunnerException } -import org.openjdk.jmh.runner.options.{ Options, OptionsBuilder } - +import org.openjdk.jmh.generators.reflection.RFGeneratorSource +import org.openjdk.jmh.runner.{Runner, RunnerException} +import org.openjdk.jmh.runner.options.{Options, OptionsBuilder} import java.net.URI + import scala.collection.JavaConverters._ -import java.nio.file.{Files, FileSystems, Path} +import java.nio.file.{FileSystems, Files, Path, Paths} import io.bazel.rulesscala.jar.JarCreator @@ -27,7 +27,14 @@ import io.bazel.rulesscala.jar.JarCreator */ object BenchmarkGenerator { - case class BenchmarkGeneratorArgs( + private sealed trait GeneratorType + + private case object ReflectionGenerator extends GeneratorType + + private case object AsmGenerator extends GeneratorType + + private case class BenchmarkGeneratorArgs( + generatorType: GeneratorType, inputJar: Path, resultSourceJar: Path, resultResourceJar: Path, @@ -37,6 +44,7 @@ object BenchmarkGenerator { def main(argv: Array[String]): Unit = { val args = parseArgs(argv) generateJmhBenchmark( + args.generatorType, args.resultSourceJar, args.resultResourceJar, args.inputJar, @@ -47,17 +55,18 @@ object BenchmarkGenerator { private def parseArgs(argv: Array[String]): BenchmarkGeneratorArgs = { if (argv.length < 3) { System.err.println( - "Usage: BenchmarkGenerator INPUT_JAR RESULT_JAR RESOURCE_JAR [CLASSPATH_ELEMENT] [CLASSPATH_ELEMENT...]" + "Usage: BenchmarkGenerator GENERATOR_TYPE INPUT_JAR RESULT_JAR RESOURCE_JAR [CLASSPATH_ELEMENT] [CLASSPATH_ELEMENT...]" ) System.exit(1) } val fs = FileSystems.getDefault BenchmarkGeneratorArgs( - fs.getPath(argv(0)), + parseGeneratorType(argv(0)), fs.getPath(argv(1)), fs.getPath(argv(2)), - argv.slice(3, argv.length).map { s => fs.getPath(s) }.toList + fs.getPath(argv(3)), + argv.slice(4, argv.length).map { s => fs.getPath(s) }.toList ) } @@ -88,13 +97,13 @@ object BenchmarkGenerator { } // Courtesy of Doug Tangren (https://groups.google.com/forum/#!topic/simple-build-tool/CYeLHcJjHyA) - private def withClassLoader[A](cp: Seq[Path])(f: => A): A = { + private def withClassLoader[A](cp: Seq[Path])(f: ClassLoader => A): A = { val originalLoader = Thread.currentThread.getContextClassLoader val jmhLoader = classOf[JMHGenerator].getClassLoader val classLoader = new URLClassLoader(cp.map(_.toUri.toURL).toArray, jmhLoader) try { Thread.currentThread.setContextClassLoader(classLoader) - f + f(classLoader) } finally { Thread.currentThread.setContextClassLoader(originalLoader) } @@ -119,6 +128,7 @@ object BenchmarkGenerator { } private def generateJmhBenchmark( + generatorType: GeneratorType, sourceJarOut: Path, resourceJarOut: Path, benchmarkJarPath: Path, @@ -131,17 +141,26 @@ object BenchmarkGenerator { tmpResourceDir.toFile.mkdir() tmpSourceDir.toFile.mkdir() - withClassLoader(benchmarkJarPath :: classpath) { - val source = new ASMGeneratorSource - val destination = new FileSystemDestination(tmpResourceDir.toFile, tmpSourceDir.toFile) - val generator = new JMHGenerator - - collectClassesFromJar(benchmarkJarPath).foreach { path => - // this would fail due to https://github.com/bazelbuild/rules_scala/issues/295 - // let's throw a useful message instead - sys.error("jmh in rules_scala doesn't work with Java 8 bytecode: https://github.com/bazelbuild/rules_scala/issues/295") - source.processClass(Files.newInputStream(path)) + withClassLoader(benchmarkJarPath :: classpath) { isolatedClassLoader => + + val source: GeneratorSource = generatorType match { + case AsmGenerator => + val generatorSource = new ASMGeneratorSource + generatorSource.processClasses(collectClassesFromJar(benchmarkJarPath).map(_.toFile).asJavaCollection) + generatorSource + + case ReflectionGenerator => + val generatorSource = new RFGeneratorSource + generatorSource.processClasses( + collectClassesFromJar(benchmarkJarPath) + .flatMap(classByPath(_, isolatedClassLoader)) + .asJavaCollection + ) + generatorSource } + + val generator = new JMHGenerator + val destination = new FileSystemDestination(tmpResourceDir.toFile, tmpSourceDir.toFile) generator.generate(source, destination) generator.complete(source, destination) if (destination.hasErrors) { @@ -156,6 +175,39 @@ object BenchmarkGenerator { } } + private def classByPath(path: Path, cl: ClassLoader): Option[Class[_]] = { + val separator = path.getFileSystem.getSeparator + var s = path.toString + .stripPrefix(separator) + .stripSuffix(".class") + .replace(separator, ".") + + var index = -1 + do { + s = s.substring(index + 1) + try { + return Some(Class.forName(s, false, cl)) + } catch { + case _: ClassNotFoundException => + // ignore and try next one + index = s.indexOf('.') + } + } while (index != -1) + + log(s"Failed to find class for path $path") + None + } + + private def parseGeneratorType(s: String): GeneratorType = { + if ("asm".equalsIgnoreCase(s)) { + AsmGenerator + } else if ("reflection".equalsIgnoreCase(s)) { + ReflectionGenerator + } else { + throw new IllegalArgumentException(s"unknown generator_type: $s") + } + } + private def log(str: String): Unit = { System.err.println(s"JMH benchmark generation: $str") } From ed9cdd57df600a39cceb2d1a49f57dcb432c39df Mon Sep 17 00:00:00 2001 From: "P. Oscar Boykin" Date: Sun, 1 Apr 2018 11:53:42 -1000 Subject: [PATCH 2/3] reenable the jmh test --- test/jmh/BUILD | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/jmh/BUILD b/test/jmh/BUILD index bf4556866..1c402f9c2 100644 --- a/test/jmh/BUILD +++ b/test/jmh/BUILD @@ -20,9 +20,8 @@ scala_library( visibility = ["//visibility:public"], ) -# Disable the jmh test due to https://github.com/bazelbuild/rules_scala/issues/295 -# scala_benchmark_jmh( -# name = "test_benchmark", -# srcs = ["TestBenchmark.scala"], -# deps = [":add_numbers"], -# ) +scala_benchmark_jmh( + name = "test_benchmark", + srcs = ["TestBenchmark.scala"], + deps = [":add_numbers"], +) From 1cab4ec8a8916cd20595da81778933d2e0c5fc9f Mon Sep 17 00:00:00 2001 From: "P. Oscar Boykin" Date: Sun, 1 Apr 2018 11:57:18 -1000 Subject: [PATCH 3/3] add the new bazel ci to scala 2.12 --- .bazelci/presubmit.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .bazelci/presubmit.yml diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml new file mode 100644 index 000000000..ea06a33e5 --- /dev/null +++ b/.bazelci/presubmit.yml @@ -0,0 +1,17 @@ +--- +platforms: + ubuntu1404: + build_targets: + - "//test/..." + test_targets: + - "//test/..." + ubuntu1604: + build_targets: + - "//test/..." + test_targets: + - "//test/..." + macos: + build_targets: + - "//test/..." + test_targets: + - "//test/..."