diff --git a/.gitignore b/.gitignore index 58f29029c..c44fe6f29 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ bazel-* *.idea hash1 hash2 +.DS_store \ No newline at end of file diff --git a/CONTRIBUTORS b/CONTRIBUTORS index d217d5b4f..b08c0b31d 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -19,3 +19,5 @@ Laurent Le Brun Nathan Harmata Oscar Boykin Justine Alexandra Roberts Tunney +Natan Silnitsky +Nadav Wexler diff --git a/WORKSPACE b/WORKSPACE index 833b8e577..0e5d8dab4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,5 +1,7 @@ workspace(name = "io_bazel_rules_scala") + + load("//scala:scala.bzl", "scala_repositories", "scala_mvn_artifact") scala_repositories() @@ -28,4 +30,15 @@ maven_jar( artifact = scala_mvn_artifact("org.psywerx.hairyfotr:linter:0.1.13"), sha1 = "e5b3e2753d0817b622c32aedcb888bcf39e275b4") +# test of strict deps (scalac plugin UT + E2E) +maven_jar( + name = "com_google_guava_guava_21_0", + artifact = "com.google.guava:guava:21.0", + sha1 = "3a3d111be1be1b745edfa7d91678a12d7ed38709" +) +maven_jar( + name = "org_apache_commons_commons_lang_3_5", + artifact = "org.apache.commons:commons-lang3:3.5", + sha1 = "6c6c702c89bfff3cd9e80b04d668c5e190d588c6" +) \ No newline at end of file diff --git a/scala/scala.bzl b/scala/scala.bzl index fc9a6e789..780cfb3ed 100644 --- a/scala/scala.bzl +++ b/scala/scala.bzl @@ -149,7 +149,7 @@ def _collect_plugin_paths(plugins): return paths -def _compile(ctx, cjars, dep_srcjars, buildijar): +def _compile(ctx, cjars, dep_srcjars, buildijar, transitive_compile_jars=[], labels = {}): ijar_output_path = "" ijar_cmd_path = "" if buildijar: @@ -162,9 +162,40 @@ def _compile(ctx, cjars, dep_srcjars, buildijar): all_srcjars = set(srcjars + list(dep_srcjars)) # look for any plugins: plugins = _collect_plugin_paths(ctx.attr.plugins) + dependency_analyzer_plugin_jars = [] + dependency_analyzer_mode_soon_to_be_removed = "off" + compiler_classpath_jars = cjars + optional_scalac_args = "" + + if not is_dependency_analyzer_off(ctx): + # "off" mode is used as a feature toggle, that preserves original behaviour + dependency_analyzer_mode_soon_to_be_removed = ctx.attr.dependency_analyzer_mode_soon_to_be_removed + dep_plugin = ctx.attr._dependency_analyzer_plugin + plugins += [f.path for f in dep_plugin.files] + dependency_analyzer_plugin_jars = ctx.files._dependency_analyzer_plugin + compiler_classpath_jars = transitive_compile_jars + + direct_jars = ",".join([j.path for j in cjars]) + indirect_jars = ",".join([j.path for j in transitive_compile_jars]) + indirect_targets = ",".join([labels[j.path] for j in transitive_compile_jars]) + current_target = str(ctx.label) + + optional_scalac_args = """ +DirectJars: {direct_jars} +IndirectJars: {indirect_jars} +IndirectTargets: {indirect_targets} +CurrentTarget: {current_target} + """.format( + direct_jars=direct_jars, + indirect_jars=indirect_jars, + indirect_targets=indirect_targets, + current_target = current_target + ) + plugin_arg = ",".join(list(plugins)) - compiler_classpath = ":".join([j.path for j in cjars]) + compiler_classpath = ":".join([j.path for j in compiler_classpath_jars]) + scalac_args = """ Classpath: {cp} @@ -185,6 +216,7 @@ ResourceSrcs: {resource_src} ResourceStripPrefix: {resource_strip_prefix} ScalacOpts: {scala_opts} SourceJars: {srcjars} +DependencyAnalyzerMode: {dependency_analyzer_mode_soon_to_be_removed} """.format( out=ctx.outputs.jar.path, manifest=ctx.outputs.manifest.path, @@ -197,7 +229,7 @@ SourceJars: {srcjars} ijar_out=ijar_output_path, ijar_cmd_path=ijar_cmd_path, srcjars=",".join([f.path for f in all_srcjars]), - javac_opts=" ".join(ctx.attr.javacopts) + + javac_opts=" ".join(ctx.attr.javacopts) + # these are the flags passed to javac, which needs them prefixed by -J " ".join(["-J" + flag for flag in ctx.attr.javac_jvm_flags]), javac_path=ctx.executable._javac.path, @@ -208,12 +240,14 @@ SourceJars: {srcjars} ), resource_strip_prefix=ctx.attr.resource_strip_prefix, resource_jars=",".join([f.path for f in ctx.files.resource_jars]), + dependency_analyzer_mode_soon_to_be_removed = dependency_analyzer_mode_soon_to_be_removed, ) argfile = ctx.new_file( ctx.outputs.jar, "%s_worker_input" % ctx.label.name ) - ctx.file_action(output=argfile, content=scalac_args) + + ctx.file_action(output=argfile, content=scalac_args + optional_scalac_args) outs = [ctx.outputs.jar] if buildijar: @@ -221,12 +255,13 @@ SourceJars: {srcjars} # _jdk added manually since _java doesn't currently setup runfiles # _scalac, as a java_binary, should already have it in its runfiles; however, # adding does ensure _java not orphaned if _scalac ever was not a java_binary - ins = (list(cjars) + + ins = (list(compiler_classpath_jars) + list(dep_srcjars) + list(srcjars) + list(sources) + ctx.files.srcs + ctx.files.plugins + + dependency_analyzer_plugin_jars + ctx.files.resources + ctx.files.resource_jars + ctx.files._jdk + @@ -255,14 +290,14 @@ SourceJars: {srcjars} ) -def _compile_or_empty(ctx, jars, srcjars, buildijar): +def _compile_or_empty(ctx, jars, srcjars, buildijar, transitive_compile_jars, jars2labels): # We assume that if a srcjar is present, it is not empty if len(ctx.files.srcs) + len(srcjars) == 0: _build_nosrc_jar(ctx, buildijar) # no need to build ijar when empty return struct(ijar=ctx.outputs.jar, class_jar=ctx.outputs.jar) else: - _compile(ctx, jars, srcjars, buildijar) + _compile(ctx, jars, srcjars, buildijar, transitive_compile_jars, jars2labels) ijar = None if buildijar: ijar = ctx.outputs.ijar @@ -351,35 +386,113 @@ def collect_srcjars(targets): srcjars += [target.srcjars.srcjar] return srcjars +def add_labels_of_jars_to(jars2labels, dependency, all_jars): + for jar in all_jars: + add_label_of_jar_to(jars2labels, dependency, jar) + + +def add_label_of_jar_to(jars2labels, dependency, jar): + if label_already_exists(jars2labels, jar): + return + + # skylark exposes only labels of direct dependencies. + # to get labels of indirect dependencies we collect them from the providers transitively + if provider_of_dependency_contains_label_of(dependency, jar): + jars2labels[jar.path] = dependency.jars_to_labels[jar.path] + else: + jars2labels[jar.path] = dependency.label + + +def label_already_exists(jars2labels, jar): + return jar.path in jars2labels + +def provider_of_dependency_contains_label_of(dependency, jar): + return hasattr(dependency, "jars_to_labels") and jar.path in dependency.jars_to_labels + +def dep_target_contains_ijar(dep_target): + return hasattr(dep_target, 'scala') and hasattr(dep_target.scala, 'outputs') and hasattr(dep_target.scala.outputs, 'ijar') + +def _collect_jars_when_dependency_analyzer_is_off(dep_targets): + compile_jars = depset() + runtime_jars = depset() + + for dep_target in dep_targets: + if java_common.provider in dep_target: + java_provider = dep_target[java_common.provider] + compile_jars += java_provider.compile_jars + runtime_jars += java_provider.transitive_runtime_jars + else: + # support http_file pointed at a jar. http_jar uses ijar, + # which breaks scala macros + compile_jars += dep_target.files + runtime_jars += dep_target.files + + return struct(compile_jars = compile_jars, transitive_runtime_jars = runtime_jars, jars2labels = {}, transitive_compile_jars = depset()) + +def _collect_jars_when_dependency_analyzer_is_on(dep_targets): + transitive_compile_jars = depset() + jars2labels = {} + compile_jars = depset() + runtime_jars = depset() + + for dep_target in dep_targets: + if dep_target_contains_ijar(dep_target): + transitive_compile_jars += [dep_target.scala.outputs.ijar] + if hasattr(dep_target, 'transitive_compile_jars'): + transitive_compile_jars += dep_target.transitive_compile_jars + if java_common.provider in dep_target: + java_provider = dep_target[java_common.provider] + compile_jars += java_provider.compile_jars + transitive_compile_jars += java_provider.compile_jars + runtime_jars += java_provider.transitive_runtime_jars + else: + # support http_file pointed at a jar. http_jar uses ijar, + # which breaks scala macros + compile_jars += dep_target.files + runtime_jars += dep_target.files + transitive_compile_jars += dep_target.files + + add_labels_of_jars_to(jars2labels, dep_target, transitive_compile_jars) -def _collect_jars(targets): + return struct(compile_jars = compile_jars, transitive_runtime_jars = runtime_jars, jars2labels = jars2labels, transitive_compile_jars = transitive_compile_jars) + +def _collect_jars(dep_targets, dependency_analyzer_is_off = True): """Compute the runtime and compile-time dependencies from the given targets""" # noqa - compile_jars = depset() - runtime_jars = depset() - for target in targets: - if java_common.provider in target: - java_provider = target[java_common.provider] - compile_jars += java_provider.compile_jars - runtime_jars += java_provider.transitive_runtime_jars - else: - # support http_file pointed at a jar. http_jar uses ijar, - # which breaks scala macros - compile_jars += target.files - runtime_jars += target.files - return struct(compile_jars = compile_jars, transitive_runtime_jars = runtime_jars) + if dependency_analyzer_is_off: + return _collect_jars_when_dependency_analyzer_is_off(dep_targets) + else: + return _collect_jars_when_dependency_analyzer_is_on(dep_targets) + +def is_dependency_analyzer_off(ctx): + if not hasattr(ctx.attr, 'dependency_analyzer_mode_soon_to_be_removed'): + return True + + if ctx.attr.dependency_analyzer_mode_soon_to_be_removed not in ["error", "warn", "off"]: + fail("Incorrect mode of dependency analyzer plugin! Mode must be 'error', 'warn' or 'off'") + + return ctx.attr.dependency_analyzer_mode_soon_to_be_removed == "off" + # Extract very common code out from dependency analysis into single place # automatically adds dependency on scala-library and scala-reflect # collects jars from deps, runtime jars from runtime_deps, and def _collect_jars_from_common_ctx(ctx, extra_deps = [], extra_runtime_deps = []): + + dependency_analyzer_is_off = is_dependency_analyzer_off(ctx) + # Get jars from deps auto_deps = [ctx.attr._scalalib, ctx.attr._scalareflect] - deps_jars = _collect_jars(ctx.attr.deps + auto_deps + extra_deps) - (cjars, transitive_rjars) = (deps_jars.compile_jars, deps_jars.transitive_runtime_jars) - transitive_rjars += _collect_jars( - ctx.attr.runtime_deps + extra_runtime_deps).transitive_runtime_jars - return struct(compile_jars = cjars, transitive_runtime_jars = transitive_rjars) + deps_jars = _collect_jars(ctx.attr.deps + auto_deps + extra_deps, dependency_analyzer_is_off) + (cjars, transitive_rjars, jars2labels, transitive_compile_jars) = (deps_jars.compile_jars, deps_jars.transitive_runtime_jars, deps_jars.jars2labels, deps_jars.transitive_compile_jars) + + runtime_dep_jars = _collect_jars(ctx.attr.runtime_deps + extra_runtime_deps, dependency_analyzer_is_off) + transitive_rjars += runtime_dep_jars.transitive_runtime_jars + + if not dependency_analyzer_is_off: + jars2labels.update(runtime_dep_jars.jars2labels) + + return struct(compile_jars = cjars, transitive_runtime_jars = transitive_rjars, jars2labels=jars2labels, transitive_compile_jars = transitive_compile_jars) def _lib(ctx, non_macro_lib): # Build up information from dependency-like attributes @@ -391,7 +504,7 @@ def _lib(ctx, non_macro_lib): (cjars, transitive_rjars) = (jars.compile_jars, jars.transitive_runtime_jars) write_manifest(ctx) - outputs = _compile_or_empty(ctx, cjars, srcjars, non_macro_lib) + outputs = _compile_or_empty(ctx, cjars, srcjars, non_macro_lib, jars.transitive_compile_jars, jars.jars2labels) transitive_rjars += [ctx.outputs.jar] @@ -447,6 +560,8 @@ def _lib(ctx, non_macro_lib): # this information through, and it is up to the new_targets # to filter and make sense of this information. extra_information=_collect_extra_information(ctx.attr.deps), + jars_to_labels = jars.jars2labels, + transitive_compile_jars = jars.transitive_compile_jars, ) @@ -464,9 +579,9 @@ def _scala_macro_library_impl(ctx): return _lib(ctx, False) # don't build the ijar for macros # Common code shared by all scala binary implementations. -def _scala_binary_common(ctx, cjars, rjars): +def _scala_binary_common(ctx, cjars, rjars, transitive_compile_time_jars, jars2labels): write_manifest(ctx) - outputs = _compile_or_empty(ctx, cjars, [], False) # no need to build an ijar for an executable + outputs = _compile_or_empty(ctx, cjars, [], False, transitive_compile_time_jars, jars2labels) # no need to build an ijar for an executable _build_deployable(ctx, list(rjars)) java_wrapper = ctx.new_file(ctx.label.name + "_wrapper.sh") @@ -510,7 +625,7 @@ def _scala_binary_impl(ctx): main_class = ctx.attr.main_class, jvm_flags = ctx.attr.jvm_flags, ) - return _scala_binary_common(ctx, cjars, transitive_rjars) + return _scala_binary_common(ctx, cjars, transitive_rjars, jars.transitive_compile_jars, jars.jars2labels) def _scala_repl_impl(ctx): # need scala-compiler for MainGenericRunner below @@ -541,7 +656,7 @@ trap finish EXIT """, ) - return _scala_binary_common(ctx, cjars, transitive_rjars) + return _scala_binary_common(ctx, cjars, transitive_rjars, jars.transitive_compile_jars, jars.jars2labels) def _scala_test_impl(ctx): if len(ctx.attr.suites) != 0: @@ -551,13 +666,18 @@ def _scala_test_impl(ctx): jars = _collect_jars_from_common_ctx(ctx, extra_runtime_deps = [ctx.attr._scalatest_reporter, ctx.attr._scalatest_runner], ) - (cjars, transitive_rjars) = (jars.compile_jars, jars.transitive_runtime_jars) + (cjars, transitive_rjars, transitive_compile_jars, jars_to_labels) = (jars.compile_jars, jars.transitive_runtime_jars, + jars.transitive_compile_jars, jars.jars2labels) # _scalatest is an http_jar, so its compile jar is run through ijar # however, contains macros, so need to handle separately scalatest_jars = _collect_jars([ctx.attr._scalatest]).transitive_runtime_jars cjars += scalatest_jars transitive_rjars += scalatest_jars + if is_dependency_analyzer_off(ctx): + transitive_compile_jars += scalatest_jars + add_labels_of_jars_to(jars_to_labels, ctx.attr._scalatest, scalatest_jars) + transitive_rjars += [ctx.outputs.jar] args = " ".join([ @@ -573,7 +693,7 @@ def _scala_test_impl(ctx): jvm_flags = ctx.attr.jvm_flags, args = args, ) - return _scala_binary_common(ctx, cjars, transitive_rjars) + return _scala_binary_common(ctx, cjars, transitive_rjars, transitive_compile_jars, jars_to_labels) def _gen_test_suite_flags_based_on_prefixes_and_suffixes(ctx, archive): return struct(testSuiteFlag = "-Dbazel.test_suite=io.bazel.rulesscala.test_discovery.DiscoveredTestSuite", @@ -601,7 +721,7 @@ def _scala_junit_test_impl(ctx): jvm_flags = launcherJvmFlags + ctx.attr.jvm_flags, ) - return _scala_binary_common(ctx, cjars, transitive_rjars) + return _scala_binary_common(ctx, cjars, transitive_rjars, jars.transitive_compile_jars, jars.jars2labels) _launcher_template = { "_java_stub_template": attr.label(default=Label("@java_stub_template//file")), @@ -643,7 +763,7 @@ _junit_resolve_deps = { } # Common attributes reused across multiple rules. -_common_attrs = { +_common_attrs_for_plugin_bootstrapping = { "srcs": attr.label_list( allow_files=_scala_srcjar_filetype), "deps": attr.label_list(), @@ -661,18 +781,40 @@ _common_attrs = { "print_compile_time": attr.bool(default=False, mandatory=False), } +_common_attrs = _common_attrs_for_plugin_bootstrapping + { + # dependency_analyzer_mode_soon_to_be_removed will be replaced by using command line flag called 'strict_java_deps' (https://github.com/bazelbuild/bazel/issues/3295) + # switching mode to "on" means that ANY API change in a target's transitive dependencies will trigger a recompilation of that target, + # on the other hand any internal change (i.e. on code that ijar omits) WON’T trigger recompilation by transitive dependencies + "dependency_analyzer_mode_soon_to_be_removed": attr.string(default="off", mandatory=False), + "_dependency_analyzer_plugin": attr.label(default=Label("@io_bazel_rules_scala//third_party/plugin/src/main:dependency_analyzer"), allow_files=_jar_filetype, mandatory=False), +} + +library_attrs = { + "main_class": attr.string(), + "exports": attr.label_list(allow_files=False), +} + +library_outputs = { + "jar": "%{name}.jar", + "deploy_jar": "%{name}_deploy.jar", + "ijar": "%{name}_ijar.jar", + "manifest": "%{name}_MANIFEST.MF", +} + scala_library = rule( implementation=_scala_library_impl, attrs={ - "main_class": attr.string(), - "exports": attr.label_list(allow_files=False), - } + _implicit_deps + _common_attrs + _resolve_deps, - outputs={ - "jar": "%{name}.jar", - "deploy_jar": "%{name}_deploy.jar", - "ijar": "%{name}_ijar.jar", - "manifest": "%{name}_MANIFEST.MF", - }, + } + _implicit_deps + _common_attrs + library_attrs + _resolve_deps, + outputs=library_outputs, +) + +# the scala compiler plugin used for dependency analysis is compiled using `scala_library`. +# in order to avoid cyclic dependencies `scala_library_for_plugin_bootstrapping` was created for this purpose, +# which does not contain plugin related attributes, and thus avoids the cyclic dependency issue +scala_library_for_plugin_bootstrapping = rule( + implementation=_scala_library_impl, + attrs= _implicit_deps + library_attrs + _resolve_deps + _common_attrs_for_plugin_bootstrapping, + outputs=library_outputs, ) scala_macro_library = rule( diff --git a/src/java/io/bazel/rulesscala/scalac/CompileOptions.java b/src/java/io/bazel/rulesscala/scalac/CompileOptions.java index 41bb4ecb9..e57a334ba 100644 --- a/src/java/io/bazel/rulesscala/scalac/CompileOptions.java +++ b/src/java/io/bazel/rulesscala/scalac/CompileOptions.java @@ -22,6 +22,11 @@ public class CompileOptions { final public Map resourceFiles; final public String resourceStripPrefix; final public String[] resourceJars; + final public String[] directJars; + final public String[] indirectJars; + final public String[] indirectTargets; + final public String dependencyAnalyzerMode; + final public String currentTarget; public CompileOptions(List args) { Map argMap = buildArgMap(args); @@ -52,6 +57,13 @@ public CompileOptions(List args) { resourceFiles = getResources(argMap); resourceStripPrefix = getOrEmpty(argMap, "ResourceStripPrefix"); resourceJars = getCommaList(argMap, "ResourceJars"); + + directJars = getCommaList(argMap, "DirectJars"); + indirectJars = getCommaList(argMap, "IndirectJars"); + indirectTargets = getCommaList(argMap, "IndirectTargets"); + + dependencyAnalyzerMode = getOrElse(argMap, "DependencyAnalyzerMode", "off"); + currentTarget = getOrElse(argMap, "CurrentTarget", "NA"); } private static Map getResources(Map args) { @@ -96,10 +108,14 @@ private static String[] getCommaList(Map m, String k) { } private static String getOrEmpty(Map m, String k) { - if(m.containsKey(k)) { - return m.get(k); + return getOrElse(m, k, ""); + } + + private static String getOrElse(Map attrs, String key, String defaultValue) { + if(attrs.containsKey(key)) { + return attrs.get(key); } else { - return ""; + return defaultValue; } } diff --git a/src/java/io/bazel/rulesscala/scalac/ScalacProcessor.java b/src/java/io/bazel/rulesscala/scalac/ScalacProcessor.java index a83250661..736b76239 100644 --- a/src/java/io/bazel/rulesscala/scalac/ScalacProcessor.java +++ b/src/java/io/bazel/rulesscala/scalac/ScalacProcessor.java @@ -17,11 +17,7 @@ import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Enumeration; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -187,7 +183,45 @@ private static boolean matchesFileExtensions(String fileName, String[] extension return false; } + private static String[] encodeBazelTargets(String[] targets) { + return Arrays.stream(targets) + .map(ScalacProcessor::encodeBazelTarget) + .toArray(String[]::new); + } + + private static String encodeBazelTarget(String target) { + return target.replace(":", ";"); + } + + private static boolean isModeEnabled(String mode) { + return !"off".equals(mode); + } + + private static String[] getPluginParamsFrom(CompileOptions ops) { + String[] pluginParams; + + if (isModeEnabled(ops.dependencyAnalyzerMode)) { + String[] targets = encodeBazelTargets(ops.indirectTargets); + String currentTarget = encodeBazelTarget(ops.currentTarget); + + String[] pluginParamsInUse = { + "-P:dependency-analyzer:direct-jars:" + String.join(":", ops.directJars), + "-P:dependency-analyzer:indirect-jars:" + String.join(":", ops.indirectJars), + "-P:dependency-analyzer:indirect-targets:" + String.join(":", targets), + "-P:dependency-analyzer:mode:" + ops.dependencyAnalyzerMode, + "-P:dependency-analyzer:current-target:" + currentTarget, + }; + pluginParams = pluginParamsInUse; + } else { + pluginParams = new String[0]; + } + return pluginParams; + } + private static void compileScalaSources(CompileOptions ops, String[] scalaSources, Path tmpPath) throws IllegalAccessException { + + String[] pluginParams = getPluginParamsFrom(ops); + String[] constParams = { "-classpath", ops.classpath, @@ -199,6 +233,7 @@ private static void compileScalaSources(CompileOptions ops, String[] scalaSource ops.scalaOpts, ops.pluginArgs, constParams, + pluginParams, scalaSources); MainClass comp = new MainClass(); diff --git a/test/src/main/scala/scala/test/strict_deps/no_recompilation/A.scala b/test/src/main/scala/scala/test/strict_deps/no_recompilation/A.scala new file mode 100644 index 000000000..b673edef8 --- /dev/null +++ b/test/src/main/scala/scala/test/strict_deps/no_recompilation/A.scala @@ -0,0 +1,7 @@ +package scala.test.strict_deps.no_recompilation; + +object A { + def foo = { + B.foo + } +} \ No newline at end of file diff --git a/test/src/main/scala/scala/test/strict_deps/no_recompilation/B.scala b/test/src/main/scala/scala/test/strict_deps/no_recompilation/B.scala new file mode 100644 index 000000000..c5f952ed0 --- /dev/null +++ b/test/src/main/scala/scala/test/strict_deps/no_recompilation/B.scala @@ -0,0 +1,7 @@ +package scala.test.strict_deps.no_recompilation; + +object B { + def foo = { + C.foo + } +} \ No newline at end of file diff --git a/test/src/main/scala/scala/test/strict_deps/no_recompilation/BUILD b/test/src/main/scala/scala/test/strict_deps/no_recompilation/BUILD new file mode 100644 index 000000000..005c7dd47 --- /dev/null +++ b/test/src/main/scala/scala/test/strict_deps/no_recompilation/BUILD @@ -0,0 +1,28 @@ +package(default_visibility = ["//visibility:public"]) +load("//scala:scala.bzl", "scala_library", "scala_test", "scala_binary") + +scala_library( + name="transitive_dependency_user", + srcs=[ + "A.scala", + ], + deps = ["direct_dependency"], + dependency_analyzer_mode_soon_to_be_removed="error", +) + +scala_library( + name="direct_dependency", + srcs=[ + "B.scala", + ], + deps = ["transitive_dependency"], + dependency_analyzer_mode_soon_to_be_removed="error", +) + +scala_library( + name="transitive_dependency", + srcs=[ + "C.scala", + ], + dependency_analyzer_mode_soon_to_be_removed="error", +) \ No newline at end of file diff --git a/test/src/main/scala/scala/test/strict_deps/no_recompilation/C.scala b/test/src/main/scala/scala/test/strict_deps/no_recompilation/C.scala new file mode 100644 index 000000000..4bebb08cc --- /dev/null +++ b/test/src/main/scala/scala/test/strict_deps/no_recompilation/C.scala @@ -0,0 +1,7 @@ +package scala.test.strict_deps.no_recompilation; + +object C { + def foo = { + println("orig") + } +} diff --git a/test_expect_failure/dep_analyzer_modes/A.scala b/test_expect_failure/dep_analyzer_modes/A.scala new file mode 100644 index 000000000..a50dab0f7 --- /dev/null +++ b/test_expect_failure/dep_analyzer_modes/A.scala @@ -0,0 +1,8 @@ +package test_expect_failure.dep_analyzer_modes + +object A { + def foo = { + B.foo + C.foo + } +} \ No newline at end of file diff --git a/test_expect_failure/dep_analyzer_modes/B.scala b/test_expect_failure/dep_analyzer_modes/B.scala new file mode 100644 index 000000000..2f4dcd3d4 --- /dev/null +++ b/test_expect_failure/dep_analyzer_modes/B.scala @@ -0,0 +1,7 @@ +package test_expect_failure.dep_analyzer_modes + +object B { + def foo = { + C.foo + } +} \ No newline at end of file diff --git a/test_expect_failure/dep_analyzer_modes/BUILD b/test_expect_failure/dep_analyzer_modes/BUILD new file mode 100644 index 000000000..ec41ed682 --- /dev/null +++ b/test_expect_failure/dep_analyzer_modes/BUILD @@ -0,0 +1,56 @@ +package(default_visibility = ["//visibility:public"]) +load("//scala:scala.bzl", "scala_library", "scala_binary", "scala_test") + +scala_library( + name="error_mode", + dependency_analyzer_mode_soon_to_be_removed="error", + srcs=[ + "A.scala", + ], + deps = ["direct_dependency"], +) + +scala_library( + name="warn_mode", + dependency_analyzer_mode_soon_to_be_removed="warn", + srcs=[ + "A.scala", + ], + deps = ["direct_dependency"], +) + +scala_library( + name="weird_mode", + dependency_analyzer_mode_soon_to_be_removed="kuki_buki", + srcs=[ + "A.scala", + ], + deps = ["direct_dependency"], +) + +scala_library( + name="off_mode", + dependency_analyzer_mode_soon_to_be_removed="off", + srcs=[ + "A.scala", + ], + deps = ["direct_dependency"], +) + + +scala_library( + name="direct_dependency", + dependency_analyzer_mode_soon_to_be_removed="error", + srcs=[ + "B.scala", + ], + deps = ["transitive_dependency"], +) + +scala_library( + name="transitive_dependency", + dependency_analyzer_mode_soon_to_be_removed="error", + srcs=[ + "C.scala", + ], +) \ No newline at end of file diff --git a/test_expect_failure/dep_analyzer_modes/C.scala b/test_expect_failure/dep_analyzer_modes/C.scala new file mode 100644 index 000000000..a4f0ee834 --- /dev/null +++ b/test_expect_failure/dep_analyzer_modes/C.scala @@ -0,0 +1,7 @@ +package test_expect_failure.dep_analyzer_modes + +object C { + def foo = { + println("in C") + } +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/external_deps/A.scala b/test_expect_failure/missing_direct_deps/external_deps/A.scala new file mode 100644 index 000000000..ffb9e5a43 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/external_deps/A.scala @@ -0,0 +1,8 @@ +package test_expect_failure.missing_direct_deps.external_deps + +object A { + def foo = { + B.foo + com.google.common.base.Strings.commonPrefix("abc", "abcd") + } +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/external_deps/B.scala b/test_expect_failure/missing_direct_deps/external_deps/B.scala new file mode 100644 index 000000000..32f5ae73a --- /dev/null +++ b/test_expect_failure/missing_direct_deps/external_deps/B.scala @@ -0,0 +1,8 @@ +package test_expect_failure.missing_direct_deps.external_deps + +object B { + def foo = { + println("in B") + com.google.common.base.Strings.commonPrefix("abc", "abcd") + } +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/external_deps/BUILD b/test_expect_failure/missing_direct_deps/external_deps/BUILD new file mode 100644 index 000000000..0dc06d9e9 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/external_deps/BUILD @@ -0,0 +1,20 @@ +package(default_visibility = ["//visibility:public"]) +load("//scala:scala.bzl", "scala_library", "scala_test") + +scala_library( + name="transitive_external_dependency_user", + srcs=[ + "A.scala", + ], + deps = ["external_dependency_user"], + dependency_analyzer_mode_soon_to_be_removed="error", +) + +scala_library( + name="external_dependency_user", + srcs=[ + "B.scala", + ], + deps = ["@com_google_guava_guava_21_0//jar"], + dependency_analyzer_mode_soon_to_be_removed="error", +) \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/external_deps_file_group/A.scala b/test_expect_failure/missing_direct_deps/external_deps_file_group/A.scala new file mode 100644 index 000000000..748954c15 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/external_deps_file_group/A.scala @@ -0,0 +1,8 @@ +package test_expect_failure.missing_direct_deps.external_deps_file_group + +object A { + def foo = { + B.foo + com.google.common.base.Strings.commonPrefix("abc", "abcd") + } +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/external_deps_file_group/B.scala b/test_expect_failure/missing_direct_deps/external_deps_file_group/B.scala new file mode 100644 index 000000000..bdf49e8cc --- /dev/null +++ b/test_expect_failure/missing_direct_deps/external_deps_file_group/B.scala @@ -0,0 +1,8 @@ +package test_expect_failure.missing_direct_deps.external_deps_file_group + +object B { + def foo = { + println("in B") + com.google.common.base.Strings.commonPrefix("abc", "abcd") + } +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/external_deps_file_group/BUILD b/test_expect_failure/missing_direct_deps/external_deps_file_group/BUILD new file mode 100644 index 000000000..d5f6e6c32 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/external_deps_file_group/BUILD @@ -0,0 +1,20 @@ +package(default_visibility = ["//visibility:public"]) +load("//scala:scala.bzl", "scala_library", "scala_test") + +scala_library( + name="transitive_external_dependency_user", + srcs=[ + "A.scala", + ], + deps = ["external_dependency_user"], + dependency_analyzer_mode_soon_to_be_removed="error", +) + +scala_library( + name="external_dependency_user", + srcs=[ + "B.scala", + ], + deps = ["@com_google_guava_guava_21_0//jar:file"], + dependency_analyzer_mode_soon_to_be_removed="error", +) \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/internal_deps/A.scala b/test_expect_failure/missing_direct_deps/internal_deps/A.scala new file mode 100644 index 000000000..960467293 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/internal_deps/A.scala @@ -0,0 +1,10 @@ +package test_expect_failure.missing_direct_deps.internal_deps; + +object A { + def foo = { + B.foo + C.foo + } + + def main = foo +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/internal_deps/B.scala b/test_expect_failure/missing_direct_deps/internal_deps/B.scala new file mode 100644 index 000000000..8d0a7ea83 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/internal_deps/B.scala @@ -0,0 +1,7 @@ +package test_expect_failure.missing_direct_deps.internal_deps; + +object B { + def foo = { + C.foo + } +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/internal_deps/BUILD b/test_expect_failure/missing_direct_deps/internal_deps/BUILD new file mode 100644 index 000000000..0854bb49f --- /dev/null +++ b/test_expect_failure/missing_direct_deps/internal_deps/BUILD @@ -0,0 +1,38 @@ +package(default_visibility = ["//visibility:public"]) +load("//scala:scala.bzl", "scala_library", "scala_test", "scala_binary") + +scala_library( + name="transitive_dependency_user", + srcs=[ + "A.scala", + ], + deps = ["direct_dependency"], + dependency_analyzer_mode_soon_to_be_removed="error", +) + +scala_library( + name="direct_dependency", + srcs=[ + "B.scala", + ], + deps = ["transitive_dependency"], + dependency_analyzer_mode_soon_to_be_removed="error", +) + +scala_library( + name="transitive_dependency", + srcs=[ + "C.scala", + ], + dependency_analyzer_mode_soon_to_be_removed="error", +) + +scala_binary( + name="user_binary", + main_class="A", + srcs=[ + "A.scala", + ], + deps = ["direct_dependency"], + dependency_analyzer_mode_soon_to_be_removed="error", +) \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/internal_deps/C.scala b/test_expect_failure/missing_direct_deps/internal_deps/C.scala new file mode 100644 index 000000000..739ab8a46 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/internal_deps/C.scala @@ -0,0 +1,7 @@ +package test_expect_failure.missing_direct_deps.internal_deps; + +object C { + def foo = { + println("in C") + } +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/strict_disabled/A.scala b/test_expect_failure/missing_direct_deps/strict_disabled/A.scala new file mode 100644 index 000000000..5a22b8214 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/strict_disabled/A.scala @@ -0,0 +1,8 @@ +package test_expect_failure.missing_direct_deps.strict_disabled + +object A { + def foo = { + B.foo + C.foo + } +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/strict_disabled/B.scala b/test_expect_failure/missing_direct_deps/strict_disabled/B.scala new file mode 100644 index 000000000..656b69a2d --- /dev/null +++ b/test_expect_failure/missing_direct_deps/strict_disabled/B.scala @@ -0,0 +1,7 @@ +package test_expect_failure.missing_direct_deps.strict_disabled + +object B { + def foo = { + C.foo + } +} \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/strict_disabled/BUILD b/test_expect_failure/missing_direct_deps/strict_disabled/BUILD new file mode 100644 index 000000000..7a8fcc914 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/strict_disabled/BUILD @@ -0,0 +1,28 @@ +package(default_visibility = ["//visibility:public"]) +load("//scala:scala.bzl", "scala_library", "scala_test") + +scala_library( + name="transitive_dependency_user", + srcs=[ + "A.scala", + ], + deps = ["direct_dependency"], + dependency_analyzer_mode_soon_to_be_removed = "off" +) + +scala_library( + name="direct_dependency", + srcs=[ + "B.scala", + ], + deps = ["transitive_dependency"], + dependency_analyzer_mode_soon_to_be_removed="error", +) + +scala_library( + name="transitive_dependency", + srcs=[ + "C.scala", + ], + dependency_analyzer_mode_soon_to_be_removed="error", +) \ No newline at end of file diff --git a/test_expect_failure/missing_direct_deps/strict_disabled/C.scala b/test_expect_failure/missing_direct_deps/strict_disabled/C.scala new file mode 100644 index 000000000..800a62b41 --- /dev/null +++ b/test_expect_failure/missing_direct_deps/strict_disabled/C.scala @@ -0,0 +1,7 @@ +package test_expect_failure.missing_direct_deps.strict_disabled + +object C { + def foo = { + println("in C") + } +} \ No newline at end of file diff --git a/test_run.sh b/test_run.sh index 4a12f3b01..1a8b15194 100755 --- a/test_run.sh +++ b/test_run.sh @@ -63,6 +63,118 @@ test_scala_library_suite() { action_should_fail build test_expect_failure/scala_library_suite:library_suite_dep_on_children } +test_expect_failure_or_warning_on_missing_direct_deps_with_expected_message() { + set +e + + expected_message=$1 + test_target=$2 + operator=${3:-"eq"} + + if [ "${operator}" = "eq" ]; then + error_message="bazel build of scala_library with missing direct deps should have failed." + else + error_message="bazel build of scala_library with missing direct deps should not have failed." + fi + + command="bazel build ${test_target}" + + output=$(${command} 2>&1) + status_code=$? + + echo "$output" + if [ ${status_code} -${operator} 0 ]; then + echo ${error_message} + exit 1 + fi + + echo ${output} | grep "$expected_message" + if [ $? -ne 0 ]; then + echo "'bazel build ${test_target}' should have logged \"${expected_message}\"." + exit 1 + fi + + set -e +} + +test_scala_library_expect_failure_on_missing_direct_deps_when_strict_is_disabled() { + expected_message="not found: value C" + test_target='test_expect_failure/missing_direct_deps/strict_disabled:transitive_dependency_user' + + test_expect_failure_or_warning_on_missing_direct_deps_with_expected_message "$expected_message" $test_target +} + +test_scala_library_expect_failure_on_missing_direct_deps() { + dependenecy_target=$1 + test_target=$2 + + local expected_messages="buildozer 'add deps $dependenecy_target' //$test_target" + + test_expect_failure_or_warning_on_missing_direct_deps_with_expected_message "${expected_message}" $test_target +} + +test_scala_library_expect_failure_on_missing_direct_internal_deps() { + dependenecy_target='//test_expect_failure/missing_direct_deps/internal_deps:transitive_dependency' + test_target='test_expect_failure/missing_direct_deps/internal_deps:transitive_dependency_user' + + test_scala_library_expect_failure_on_missing_direct_deps $dependenecy_target $test_target +} + +test_scala_binary_expect_failure_on_missing_direct_deps() { + dependency_target='//test_expect_failure/missing_direct_deps/internal_deps:transitive_dependency' + test_target='test_expect_failure/missing_direct_deps/internal_deps:user_binary' + + test_scala_library_expect_failure_on_missing_direct_deps ${dependency_target} ${test_target} +} + +test_scala_library_expect_failure_on_missing_direct_external_deps_jar() { + dependenecy_target='@com_google_guava_guava_21_0//jar:jar' + test_target='test_expect_failure/missing_direct_deps/external_deps:transitive_external_dependency_user' + + test_scala_library_expect_failure_on_missing_direct_deps $dependenecy_target $test_target +} + +test_scala_library_expect_failure_on_missing_direct_external_deps_file_group() { + dependenecy_target='@com_google_guava_guava_21_0//jar:file' + test_target='test_expect_failure/missing_direct_deps/external_deps_file_group:transitive_external_dependency_user' + + test_scala_library_expect_failure_on_missing_direct_deps $dependenecy_target $test_target +} + +test_scala_library_expect_failure_on_missing_direct_deps_error_mode() { + dependenecy_target='//test_expect_failure/dep_analyzer_modes:transitive_dependency' + test_target='test_expect_failure/dep_analyzer_modes:error_mode' + + expected_message="error: Target '$dependenecy_target' is used but isn't explicitly declared, please add it to the deps" + + test_expect_failure_or_warning_on_missing_direct_deps_with_expected_message "${expected_message}" ${test_target} +} + +test_scala_library_expect_failure_on_missing_direct_deps_warn_mode() { + # warnings are cached. requires clean build... + bazel clean + + dependenecy_target='//test_expect_failure/dep_analyzer_modes:transitive_dependency' + test_target='test_expect_failure/dep_analyzer_modes:warn_mode' + + expected_message="warning: Target '$dependenecy_target' is used but isn't explicitly declared, please add it to the deps" + + test_expect_failure_or_warning_on_missing_direct_deps_with_expected_message "${expected_message}" ${test_target} "ne" +} + +test_scala_library_expect_failure_on_missing_direct_deps_weird_mode() { + expected_message="Incorrect mode of dependency analyzer plugin! Mode must be 'error', 'warn' or 'off'." + test_target='test_expect_failure/dep_analyzer_modes:weird_mode' + + test_expect_failure_or_warning_on_missing_direct_deps_with_expected_message "${expected_message}" ${test_target} +} + +test_scala_library_expect_failure_on_missing_direct_deps_off_mode() { + expected_message="test_expect_failure/dep_analyzer_modes/A.scala:[0-9+]: error: not found: value C" + test_target='test_expect_failure/dep_analyzer_modes:off_mode' + + test_expect_failure_or_warning_on_missing_direct_deps_with_expected_message "${expected_message}" ${test_target} +} + test_scala_junit_test_can_fail() { action_should_fail test test_expect_failure/scala_junit_test:failing_test } @@ -107,7 +219,7 @@ run_test_ci() { result=$? kill $pulse_printer_pid && wait $pulse_printer_pid 2>/dev/null || true } || return 1 - + DURATION=$SECONDS if [ $result -eq 0 ]; then echo -e "\n${GREEN}Test \"$TEST_ARG\" successful ($DURATION sec) $NC" @@ -217,7 +329,7 @@ junit_generates_xml_logs() { else return 1 fi - test -e + test -e } test_junit_test_must_have_prefix_or_suffix() { @@ -315,6 +427,37 @@ javac_jvm_flags_are_configured(){ action_should_fail build //test_expect_failure/compilers_jvm_flags:can_configure_jvm_flags_for_javac } +revert_internal_change() { + sed -i.bak "s/println(\"altered\")/println(\"orig\")/" $no_recompilation_path/C.scala + rm $no_recompilation_path/C.scala.bak +} + +test_scala_library_expect_no_recompilation_on_internal_change_of_transitive_dependency() { + set +e + no_recompilation_path="test/src/main/scala/scala/test/strict_deps/no_recompilation" + build_command="bazel build //$no_recompilation_path/... --subcommands" + + echo "running initial build" + $build_command + echo "changing internal behaviour of C.scala" + sed -i.bak "s/println(\"orig\")/println(\"altered\")/" ./$no_recompilation_path/C.scala + + echo "running second build" + output=$(${build_command} 2>&1) + + not_expected_recompiled_target="//$no_recompilation_path:transitive_dependency_user" + + echo ${output} | grep "$not_expected_recompiled_target" + if [ $? -eq 0 ]; then + echo "bazel build was executed after change of internal behaviour of 'transitive_dependency' target. compilation of 'transitive_dependency_user' should not have been triggered." + revert_internal_change + exit 1 + fi + + revert_internal_change + set -e +} + if [ "$1" != "ci" ]; then runner="run_test_local" else @@ -323,6 +466,7 @@ fi $runner bazel build test/... $runner bazel test test/... +$runner bazel test third_party/... $runner bazel run test/src/main/scala/scala/test/twitter_scrooge:justscrooges $runner bazel run test:JavaBinary $runner bazel run test:JavaBinary2 @@ -354,3 +498,13 @@ $runner scala_test_test_filters $runner scala_junit_test_test_filter $runner scalac_jvm_flags_are_configured $runner javac_jvm_flags_are_configured +$runner test_scala_library_expect_failure_on_missing_direct_internal_deps +$runner test_scala_library_expect_failure_on_missing_direct_external_deps_jar +$runner test_scala_library_expect_failure_on_missing_direct_external_deps_file_group +$runner test_scala_library_expect_failure_on_missing_direct_deps_when_strict_is_disabled +$runner test_scala_binary_expect_failure_on_missing_direct_deps +$runner test_scala_library_expect_failure_on_missing_direct_deps_error_mode +$runner test_scala_library_expect_failure_on_missing_direct_deps_warn_mode +$runner test_scala_library_expect_failure_on_missing_direct_deps_weird_mode +$runner test_scala_library_expect_failure_on_missing_direct_deps_off_mode +$runner test_scala_library_expect_no_recompilation_on_internal_change_of_transitive_dependency \ No newline at end of file diff --git a/third_party/README.md b/third_party/README.md new file mode 100644 index 000000000..54ee73139 --- /dev/null +++ b/third_party/README.md @@ -0,0 +1,5 @@ +This file lists license and version information of all code we did not author + +# dependency_analyzer +dependency_analyzer scala compiler plugin is based on [classpath-shrinker](https://github.com/scalacenter/classpath-shrinker) plugin. +License: 3-clause revised BSD \ No newline at end of file diff --git a/third_party/plugin/LICENSE b/third_party/plugin/LICENSE new file mode 100644 index 000000000..ec832b9a1 --- /dev/null +++ b/third_party/plugin/LICENSE @@ -0,0 +1,29 @@ +******************************************************************************* +* Classpath Shrinker: a scalac plugin to detect unused classpath entries +* Copyright (c) Scala Center +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* 3. Neither the name of the copyright holders nor the names of its +* contributors may be used to endorse or promote products derived from +* this software without specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +* THE POSSIBILITY OF SUCH DAMAGE. +******************************************************************************* \ No newline at end of file diff --git a/third_party/plugin/src/main/BUILD b/third_party/plugin/src/main/BUILD new file mode 100644 index 000000000..bd9b4ab36 --- /dev/null +++ b/third_party/plugin/src/main/BUILD @@ -0,0 +1,14 @@ +licenses(["notice"]) # 3-clause BSD + +load("//scala:scala.bzl", "scala_library", "scala_library_for_plugin_bootstrapping") + +scala_library_for_plugin_bootstrapping( + name = "dependency_analyzer", + srcs = ["io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzer.scala", + ], + resources = ["resources/scalac-plugin.xml"], + visibility = ["//visibility:public"], + deps = [ + "//external:io_bazel_rules_scala/dependency/scala/scala_compiler" + ], +) \ No newline at end of file diff --git a/third_party/plugin/src/main/io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzer.scala b/third_party/plugin/src/main/io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzer.scala new file mode 100644 index 000000000..65144995f --- /dev/null +++ b/third_party/plugin/src/main/io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzer.scala @@ -0,0 +1,110 @@ +package third_party.plugin.src.main.io.bazel.rulesscala.dependencyanalyzer + +import scala.reflect.io.AbstractFile +import scala.tools.nsc.plugins.{Plugin, PluginComponent} +import scala.tools.nsc.{Global, Phase} + +class DependencyAnalyzer(val global: Global) extends Plugin { + + val name = "dependency-analyzer" + val description = + "Analyzes the used dependencies and fails the compilation " + + "if they are not explicitly used as direct dependencies (only declared transitively)" + val components = List[PluginComponent](Component) + + var indirect: Map[String, String] = Map.empty + var direct: Set[String] = Set.empty + var analyzerMode: String = "error" + var currentTarget: String = "NA" + + override def processOptions(options: List[String], error: (String) => Unit): Unit = { + var indirectJars: Seq[String] = Seq.empty + var indirectTargets: Seq[String] = Seq.empty + + for (option <- options) { + option.split(":").toList match { + case "direct-jars" :: data => direct = data.toSet + case "indirect-jars" :: data => indirectJars = data; + case "indirect-targets" :: data => indirectTargets = data.map(_.replace(";", ":")) + case "current-target" :: target => currentTarget = target.map(_.replace(";", ":")).head + case "mode" :: mode => analyzerMode = mode.head + case unknown :: _ => error(s"unknown param $unknown") + case Nil => + } + } + indirect = indirectJars.zip(indirectTargets).toMap + } + + + private object Component extends PluginComponent { + val global: DependencyAnalyzer.this.global.type = + DependencyAnalyzer.this.global + + import global._ + + override val runsAfter = List("jvm") + + val phaseName = DependencyAnalyzer.this.name + + override def newPhase(prev: Phase): StdPhase = new StdPhase(prev) { + override def run(): Unit = { + + super.run() + + val usedJars = findUsedJars + + warnOnIndirectTargetsFoundIn(usedJars) + } + + private def warnOnIndirectTargetsFoundIn(usedJars: Set[AbstractFile]) = { + for (usedJar <- usedJars; + usedJarPath = usedJar.path; + target <- indirect.get(usedJarPath) if !direct.contains(usedJarPath)) { + val errorMessage = + s"""Target '$target' is used but isn't explicitly declared, please add it to the deps. + |You can use the following buildozer command: + |buildozer 'add deps $target' $currentTarget""".stripMargin + + analyzerMode match { + case "error" => reporter.error(NoPosition, errorMessage) + case "warn" => reporter.warning(NoPosition, errorMessage) + } + } + } + + override def apply(unit: CompilationUnit): Unit = () + } + + } + + import global._ + + private def findUsedJars: Set[AbstractFile] = { + val jars = collection.mutable.Set[AbstractFile]() + + def walkTopLevels(root: Symbol): Unit = { + def safeInfo(sym: Symbol): Type = + if (sym.hasRawInfo && sym.rawInfo.isComplete) sym.info else NoType + + def packageClassOrSelf(sym: Symbol): Symbol = + if (sym.hasPackageFlag && !sym.isModuleClass) sym.moduleClass else sym + + for (x <- safeInfo(packageClassOrSelf(root)).decls) { + if (x == root) () + else if (x.hasPackageFlag) walkTopLevels(x) + else if (x.owner != root) { // exclude package class members + if (x.hasRawInfo && x.rawInfo.isComplete) { + val assocFile = x.associatedFile + if (assocFile.path.endsWith(".class") && assocFile.underlyingSource.isDefined) + assocFile.underlyingSource.foreach(jars += _) + } + } + } + } + + exitingTyper { + walkTopLevels(RootClass) + } + jars.toSet + } +} diff --git a/third_party/plugin/src/main/resources/scalac-plugin.xml b/third_party/plugin/src/main/resources/scalac-plugin.xml new file mode 100644 index 000000000..35b599afe --- /dev/null +++ b/third_party/plugin/src/main/resources/scalac-plugin.xml @@ -0,0 +1,4 @@ + + dependency-analyzer + third_party.plugin.src.main.io.bazel.rulesscala.dependencyanalyzer.DependencyAnalyzer + \ No newline at end of file diff --git a/third_party/plugin/src/test/BUILD b/third_party/plugin/src/test/BUILD new file mode 100644 index 000000000..b5cbd69cb --- /dev/null +++ b/third_party/plugin/src/test/BUILD @@ -0,0 +1,21 @@ +licenses(["notice"]) # 3-clause BSD + +load("//scala:scala.bzl", "scala_junit_test") + +scala_junit_test( + name = "dependency_analyzer_test", + srcs = ["io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzerTest.scala", + "io/bazel/rulesscala/dependencyanalyzer/TestUtil.scala"], + suffixes = ["Test"], + size = "small", + deps = ["//third_party/plugin/src/main:dependency_analyzer", + "//external:io_bazel_rules_scala/dependency/scala/scala_compiler", + "//external:io_bazel_rules_scala/dependency/scala/scala_library", + "@com_google_guava_guava_21_0//jar", + "@org_apache_commons_commons_lang_3_5//jar" + ], + jvm_flags = ["-Dplugin.jar.location=$(location //third_party/plugin/src/main:dependency_analyzer)", + "-Dscala.library.location=$(location //external:io_bazel_rules_scala/dependency/scala/scala_library)", + "-Dguava.jar.location=$(location @com_google_guava_guava_21_0//jar)", + "-Dapache.commons.jar.location=$(location @org_apache_commons_commons_lang_3_5//jar)"], +) \ No newline at end of file diff --git a/third_party/plugin/src/test/io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzerTest.scala b/third_party/plugin/src/test/io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzerTest.scala new file mode 100644 index 000000000..0701fec9b --- /dev/null +++ b/third_party/plugin/src/test/io/bazel/rulesscala/dependencyanalyzer/DependencyAnalyzerTest.scala @@ -0,0 +1,86 @@ +package third_party.plugin.src.test.io.bazel.rulesscala.dependencyanalyzer + +import TestUtil._ +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(classOf[JUnit4]) +class DependencyAnalyzerTest { + + @Test + def `error on indirect dependency target`(): Unit = { + val testCode = + """object Foo { + | org.apache.commons.lang3.ArrayUtils.EMPTY_BOOLEAN_ARRAY.length + |} + """.stripMargin + val commonsTarget = "//commons:Target".encode() + val indirect = Map(apacheCommonsClasspath -> commonsTarget) + run(testCode, withIndirect = indirect).expectErrorOn(indirect(apacheCommonsClasspath).decoded) + } + + @Test + def `error on multiple indirect dependency targets`(): Unit = { + val testCode = + """object Foo { + | org.apache.commons.lang3.ArrayUtils.EMPTY_BOOLEAN_ARRAY.length + | com.google.common.base.Strings.commonPrefix("abc", "abcd") + |} + """.stripMargin + val commonsTarget = "commonsTarget" + + val guavaTarget = "guavaTarget" + + val indirect = Map(apacheCommonsClasspath -> commonsTarget, guavaClasspath -> guavaTarget) + run(testCode, withIndirect = indirect).expectErrorOn(commonsTarget, guavaTarget) + } + + @Test + def `do not give error on direct dependency target`(): Unit = { + val testCode = + """object Foo { + | org.apache.commons.lang3.ArrayUtils.EMPTY_BOOLEAN_ARRAY.length + |} + """.stripMargin + val commonsTarget = "commonsTarget" + + val direct = Seq(apacheCommonsClasspath) + val indirect = Map(apacheCommonsClasspath -> commonsTarget) + run(testCode, withDirect = direct, withIndirect = indirect).noErrorOn(commonsTarget) + } + + + implicit class `nice errors on sequence of strings`(infos: Seq[String]) { + + private def checkErrorContainsMessage(target: String) = { info: String => + info.contains(targetErrorMessage(target)) & + info.contains(buildozerCommand(target)) + } + + private def targetErrorMessage(target: String) = + s"Target '$target' is used but isn't explicitly declared, please add it to the deps" + + private def buildozerCommand(depTarget: String) = + s"buildozer 'add deps $depTarget' $defaultTarget" + + def expectErrorOn(targets: String*) = targets.foreach(target => assert( + infos.exists(checkErrorContainsMessage(target)), + s"expected an error on $target to appear in errors (with buildozer command)!") + ) + + def noErrorOn(target: String) = assert( + !infos.exists(checkErrorContainsMessage(target)), + s"error on $target should not appear in errors!") + } + + implicit class `decode bazel labels`(targetLabel: String) { + def decoded() = { + targetLabel.replace(";", ":") + } + + def encode() = { + targetLabel.replace(":", ";") + } + } +} diff --git a/third_party/plugin/src/test/io/bazel/rulesscala/dependencyanalyzer/TestUtil.scala b/third_party/plugin/src/test/io/bazel/rulesscala/dependencyanalyzer/TestUtil.scala new file mode 100644 index 000000000..dd126e854 --- /dev/null +++ b/third_party/plugin/src/test/io/bazel/rulesscala/dependencyanalyzer/TestUtil.scala @@ -0,0 +1,92 @@ +package third_party.plugin.src.test.io.bazel.rulesscala.dependencyanalyzer + +import java.nio.file.Paths + +import scala.reflect.internal.util.BatchSourceFile +import scala.reflect.io.VirtualDirectory +import scala.tools.cmd.CommandLineParser +import scala.tools.nsc.reporters.StoreReporter +import scala.tools.nsc.{CompilerCommand, Global, Settings} + +object TestUtil { + + import scala.language.postfixOps + + final val defaultTarget = "//..." + + def run(code: String, withDirect: Seq[String] = Seq.empty, withIndirect: Map[String, String] = Map.empty): Seq[String] = { + val compileOptions = Seq( + constructParam("direct-jars", withDirect), + constructParam("indirect-jars", withIndirect.keys), + constructParam("indirect-targets", withIndirect.values), + constructParam("current-target", Seq(defaultTarget)) + ).mkString(" ") + + val extraClasspath = withDirect ++ withIndirect.keys + + val reporter: StoreReporter = runCompilation(code, compileOptions, extraClasspath) + reporter.infos.collect({ case msg if msg.severity == reporter.ERROR => msg.msg }).toSeq + } + + private def runCompilation(code: String, compileOptions: String, extraClasspath: Seq[String]) = { + val fullClasspath: String = { + val extraClasspathString = extraClasspath.mkString(":") + if (toolboxClasspath.isEmpty) extraClasspathString + else s"$toolboxClasspath:$extraClasspathString" + } + val basicOptions = + createBasicCompileOptions(fullClasspath, toolboxPluginOptions) + + eval(code, s"$basicOptions $compileOptions") + } + + /** Evaluate using global instance instead of toolbox because toolbox seems + * to fail to typecheck code that comes from external dependencies. */ + private def eval(code: String, compileOptions: String = ""): StoreReporter = { + // TODO: Optimize and cache global. + val options = CommandLineParser.tokenize(compileOptions) + val reporter = new StoreReporter() + val settings = new Settings(println) + val _ = new CompilerCommand(options, settings) + settings.outputDirs.setSingleOutput(new VirtualDirectory("(memory)", None)) + val global = new Global(settings, reporter) + val run = new global.Run + val toCompile = new BatchSourceFile("", code) + run.compileSources(List(toCompile)) + reporter + } + + lazy val baseDir = System.getProperty("user.dir") + + lazy val toolboxClasspath: String = + pathOf("scala.library.location") + + lazy val toolboxPluginOptions: String = { + val jar = System.getProperty("plugin.jar.location") + val start= jar.indexOf("/third_party/plugin") + // this substring is needed due to issue: https://github.com/bazelbuild/bazel/issues/2475 + val jarInRelationToBaseDir = jar.substring(start, jar.length) + val pluginPath = Paths.get(baseDir, jarInRelationToBaseDir).toAbsolutePath + s"-Xplugin:${pluginPath} -Jdummy=${pluginPath.toFile.lastModified}" + } + + lazy val guavaClasspath: String = + pathOf("guava.jar.location") + + lazy val apacheCommonsClasspath: String = + pathOf("apache.commons.jar.location") + + private def pathOf(jvmFlag: String) = { + val jar = System.getProperty(jvmFlag) + val libPath = Paths.get(baseDir, jar).toAbsolutePath + libPath.toString + } + + private def createBasicCompileOptions(classpath: String, usePluginOptions: String) = + s"-classpath $classpath $usePluginOptions" + + private def constructParam(name: String, values: Iterable[String]) = { + if (values.isEmpty) "" + else s"-P:dependency-analyzer:$name:${values.mkString(":")}" + } +}