diff --git a/src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala b/src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala index 73520dffb925..557385744ac2 100644 --- a/src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala +++ b/src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala @@ -54,19 +54,8 @@ trait MacroRuntimes extends JavaReflectionRuntimes { /** Macro classloader that is used to resolve and run macro implementations. * Loads classes from from -cp (aka the library classpath). * Is also capable of detecting REPL and reusing its classloader. - * - * When -Xmacro-jit is enabled, we sometimes fallback to on-the-fly compilation of macro implementations, - * which compiles implementations into a virtual directory (very much like REPL does) and then conjures - * a classloader mapped to that virtual directory. */ - private lazy val defaultMacroClassloaderCache = { - def attemptClose(loader: ClassLoader): Unit = loader match { - case u: URLClassLoader => debuglog("Closing macro runtime classloader"); u.close() - case afcl: AbstractFileClassLoader => attemptClose(afcl.getParent) - case _ => ??? - } - perRunCaches.newGeneric(findMacroClassLoader, attemptClose _) - } + private lazy val defaultMacroClassloaderCache: () => ClassLoader = perRunCaches.newGeneric(findMacroClassLoader()) def defaultMacroClassloader: ClassLoader = defaultMacroClassloaderCache() /** Abstracts away resolution of macro runtimes. diff --git a/src/compiler/scala/tools/nsc/CloseableRegistry.scala b/src/compiler/scala/tools/nsc/CloseableRegistry.scala new file mode 100644 index 000000000000..9812a2136263 --- /dev/null +++ b/src/compiler/scala/tools/nsc/CloseableRegistry.scala @@ -0,0 +1,34 @@ +/* + * Scala (https://www.scala-lang.org) + * + * Copyright EPFL and Lightbend, Inc. + * + * Licensed under Apache License 2.0 + * (http://www.apache.org/licenses/LICENSE-2.0). + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package scala.tools.nsc + +import scala.util.control.NonFatal + +/** Registry for resources to close when `Global` is closed */ +final class CloseableRegistry { + private[this] var closeables: List[java.io.Closeable] = Nil + final def registerClosable(c: java.io.Closeable): Unit = { + closeables ::= c + } + + def close(): Unit = { + for (c <- closeables) { + try { + c.close() + } catch { + case NonFatal(_) => + } + } + closeables = Nil + } +} diff --git a/src/compiler/scala/tools/nsc/GenericRunnerSettings.scala b/src/compiler/scala/tools/nsc/GenericRunnerSettings.scala index cb26b4d9d666..d7e379b58eb2 100644 --- a/src/compiler/scala/tools/nsc/GenericRunnerSettings.scala +++ b/src/compiler/scala/tools/nsc/GenericRunnerSettings.scala @@ -16,7 +16,14 @@ import java.net.URL import scala.tools.util.PathResolver class GenericRunnerSettings(error: String => Unit) extends Settings(error) { - lazy val classpathURLs: Seq[URL] = new PathResolver(this).resultAsURLs + lazy val classpathURLs: Seq[URL] = { + val registry = new CloseableRegistry + try { + new PathResolver(this, new CloseableRegistry).resultAsURLs + } finally { + registry.close() + } + } val howtorun = ChoiceSetting( diff --git a/src/compiler/scala/tools/nsc/Global.scala b/src/compiler/scala/tools/nsc/Global.scala index 79358c172dfa..6ca1e922e621 100644 --- a/src/compiler/scala/tools/nsc/Global.scala +++ b/src/compiler/scala/tools/nsc/Global.scala @@ -39,9 +39,12 @@ import scala.language.postfixOps import scala.tools.nsc.ast.{TreeGen => AstTreeGen} import scala.tools.nsc.classpath._ import scala.tools.nsc.profile.Profiler +import scala.util.control.NonFatal +import java.io.Closeable class Global(var currentSettings: Settings, reporter0: Reporter) extends SymbolTable + with Closeable with CompilationUnits with Plugins with PhaseAssembly @@ -816,7 +819,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter) /** Extend classpath of `platform` and rescan updated packages. */ def extendCompilerClassPath(urls: URL*): Unit = { - val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings)) + val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings, closeableRegistry)) val newClassPath = AggregateClassPath.createAggregate(platform.classPath +: urlClasspaths : _*) platform.currentClassPath = Some(newClassPath) invalidateClassPathEntries(urls.map(_.getPath): _*) @@ -878,7 +881,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter) } entries(classPath) find matchesCanonical match { case Some(oldEntry) => - Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings)) + Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings, closeableRegistry)) case None => error(s"Error adding entry to classpath. During invalidation, no entry named $path in classpath $classPath") None @@ -1682,6 +1685,13 @@ class Global(var currentSettings: Settings, reporter0: Reporter) } def createJavadoc = false + + final val closeableRegistry: CloseableRegistry = new CloseableRegistry + + def close(): Unit = { + perRunCaches.clearAll() + closeableRegistry.close() + } } object Global { diff --git a/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala b/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala index ff11f434710b..05396fc6ce73 100644 --- a/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala +++ b/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala @@ -27,7 +27,7 @@ trait JavaPlatform extends Platform { private[nsc] var currentClassPath: Option[ClassPath] = None private[nsc] def classPath: ClassPath = { - if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result) + if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings, global.closeableRegistry).result) currentClassPath.get } diff --git a/src/compiler/scala/tools/nsc/classpath/AggregateClassPath.scala b/src/compiler/scala/tools/nsc/classpath/AggregateClassPath.scala index 68fb3000b8c0..ac4970f3cb6b 100644 --- a/src/compiler/scala/tools/nsc/classpath/AggregateClassPath.scala +++ b/src/compiler/scala/tools/nsc/classpath/AggregateClassPath.scala @@ -66,7 +66,6 @@ case class AggregateClassPath(aggregates: Seq[ClassPath]) extends ClassPath { override def asClassPathStrings: Seq[String] = aggregates.map(_.asClassPathString).distinct override def asSourcePathString: String = ClassPath.join(aggregates map (_.asSourcePathString): _*) - override private[nsc] def packages(inPackage: String): Seq[PackageEntry] = { val aggregatedPackages = aggregates.flatMap(_.packages(inPackage)).distinct aggregatedPackages diff --git a/src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala b/src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala index fa9166483594..f2fb2b0224d7 100644 --- a/src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala +++ b/src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala @@ -14,7 +14,7 @@ package scala.tools.nsc.classpath import scala.reflect.io.{AbstractFile, VirtualDirectory} import scala.reflect.io.Path.string2path -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import FileUtils.AbstractFileOps import scala.tools.nsc.util.ClassPath @@ -22,11 +22,11 @@ import scala.tools.nsc.util.ClassPath * Provides factory methods for classpath. When creating classpath instances for a given path, * it uses proper type of classpath depending on a types of particular files containing sources or classes. */ -class ClassPathFactory(settings: Settings) { +class ClassPathFactory(settings: Settings, closeableRegistry: CloseableRegistry) { /** * Create a new classpath based on the abstract file. */ - def newClassPath(file: AbstractFile): ClassPath = ClassPathFactory.newClassPath(file, settings) + def newClassPath(file: AbstractFile): ClassPath = ClassPathFactory.newClassPath(file, settings, closeableRegistry) /** * Creators for sub classpaths which preserve this context. @@ -70,7 +70,7 @@ class ClassPathFactory(settings: Settings) { private def createSourcePath(file: AbstractFile): ClassPath = if (file.isJarOrZip) - ZipAndJarSourcePathFactory.create(file, settings) + ZipAndJarSourcePathFactory.create(file, settings, closeableRegistry) else if (file.isDirectory) DirectorySourcePath(file.file) else @@ -78,11 +78,11 @@ class ClassPathFactory(settings: Settings) { } object ClassPathFactory { - def newClassPath(file: AbstractFile, settings: Settings): ClassPath = file match { + def newClassPath(file: AbstractFile, settings: Settings, closeableRegistry: CloseableRegistry): ClassPath = file match { case vd: VirtualDirectory => VirtualDirectoryClassPath(vd) case _ => if (file.isJarOrZip) - ZipAndJarClassPathFactory.create(file, settings) + ZipAndJarClassPathFactory.create(file, settings, closeableRegistry) else if (file.isDirectory) DirectoryClassPath(file.file) else diff --git a/src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala b/src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala index 9f51672e79a6..c6ab18a1e484 100644 --- a/src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala +++ b/src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala @@ -12,7 +12,7 @@ package scala.tools.nsc.classpath -import java.io.File +import java.io.{Closeable, File} import java.net.{URI, URL} import java.nio.file.{FileSystems, Files, SimpleFileVisitor} import java.util.function.IntFunction @@ -25,6 +25,7 @@ import FileUtils._ import scala.collection.JavaConverters._ import scala.collection.immutable import scala.reflect.internal.JDK9Reflectors +import scala.tools.nsc.CloseableRegistry import scala.tools.nsc.classpath.PackageNameUtils.{packageContains, separatePkgAndClassNames} /** @@ -61,6 +62,7 @@ trait DirectoryLookup[FileEntryType <: ClassRepresentation] extends ClassPath { private[nsc] def packages(inPackage: String): Seq[PackageEntry] = { val dirForPackage = getDirectory(inPackage) + val nestedDirs: Array[F] = dirForPackage match { case None => emptyFiles case Some(directory) => listChildren(directory, Some(isPackage)) @@ -137,7 +139,7 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo object JrtClassPath { import java.nio.file._, java.net.URI - def apply(release: Option[String]): Option[ClassPath] = { + def apply(release: Option[String], closeableRegistry: CloseableRegistry): Option[ClassPath] = { import scala.util.Properties._ if (!isJavaAtLeast("9")) None else { @@ -154,7 +156,11 @@ object JrtClassPath { try { val ctSym = Paths.get(javaHome).resolve("lib").resolve("ct.sym") if (Files.notExists(ctSym)) None - else Some(new CtSymClassPath(ctSym, v.toInt)) + else { + val classPath = new CtSymClassPath(ctSym, v.toInt) + closeableRegistry.registerClosable(classPath) + Some(classPath) + } } catch { case _: Throwable => None } @@ -230,7 +236,7 @@ final class JrtClassPath(fs: java.nio.file.FileSystem) extends ClassPath with No /** * Implementation `ClassPath` based on the $JAVA_HOME/lib/ct.sym backing http://openjdk.java.net/jeps/247 */ -final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths { +final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths with Closeable { import java.nio.file.Path, java.nio.file._ private val fileSystem: FileSystem = FileSystems.newFileSystem(ctSym, null) @@ -276,7 +282,7 @@ final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends Clas def asURLs: Seq[URL] = Nil def asClassPathStrings: Seq[String] = Nil - + override def close(): Unit = fileSystem.close() def findClassFile(className: String): Option[AbstractFile] = { if (!className.contains(".")) None else { diff --git a/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala b/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala index 5b157e9b386e..a4ba7cec2c42 100644 --- a/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala +++ b/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala @@ -37,7 +37,6 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi // mimic the behavior of the old nsc.util.DirectoryClassPath def asURLs: Seq[URL] = Seq(new URL(dir.name)) def asClassPathStrings: Seq[String] = Seq(dir.path) - override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl def findClassFile(className: String): Option[AbstractFile] = { diff --git a/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala b/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala index 6f8b9a55c0cd..318552361502 100644 --- a/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala +++ b/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala @@ -12,15 +12,17 @@ package scala.tools.nsc.classpath -import java.io.File +import java.io.{Closeable, File} import java.net.URL import java.nio.file.Files import java.nio.file.attribute.{BasicFileAttributes, FileTime} +import java.util.{Timer, TimerTask} +import java.util.concurrent.atomic.AtomicInteger import scala.annotation.tailrec import scala.reflect.io.{AbstractFile, FileZipArchive, ManifestResources} import scala.tools.nsc.util.{ClassPath, ClassRepresentation} -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import FileUtils._ /** @@ -29,18 +31,19 @@ import FileUtils._ * when there are a lot of projects having a lot of common dependencies. */ sealed trait ZipAndJarFileLookupFactory { - private val cache = new FileBasedCache[ClassPath] - - def create(zipFile: AbstractFile, settings: Settings): ClassPath = { - if (settings.YdisableFlatCpCaching || zipFile.file == null) createForZipFile(zipFile, settings.releaseValue) - else createUsingCache(zipFile, settings) + private val cache = new FileBasedCache[ClassPath with Closeable] + + def create(zipFile: AbstractFile, settings: Settings, closeableRegistry: CloseableRegistry): ClassPath = { + if (settings.YdisableFlatCpCaching || zipFile.file == null) { + val result: ClassPath with Closeable = createForZipFile(zipFile, settings.releaseValue) + closeableRegistry.registerClosable(result) + result + } else { + cache.getOrCreate(List(zipFile.file.toPath), () => createForZipFile(zipFile, settings.releaseValue), closeableRegistry) + } } - protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath - - private def createUsingCache(zipFile: AbstractFile, settings: Settings): ClassPath = { - cache.getOrCreate(List(zipFile.file.toPath), () => createForZipFile(zipFile, settings.releaseValue)) - } + protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath with Closeable } /** @@ -75,7 +78,7 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory { * with a particularly prepared scala-library.jar. It should have all classes listed in the manifest like e.g. this entry: * Name: scala/Function2$mcFJD$sp.class */ - private case class ManifestResourcesClassPath(file: ManifestResources) extends ClassPath with NoSourcePaths { + private case class ManifestResourcesClassPath(file: ManifestResources) extends ClassPath with NoSourcePaths with Closeable { override def findClassFile(className: String): Option[AbstractFile] = { val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className) classes(pkg).find(_.name == simpleClassName).map(_.file) @@ -84,6 +87,7 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory { override def asClassPathStrings: Seq[String] = Seq(file.path) override def asURLs: Seq[URL] = file.toURLs() + override def close(): Unit = file.close() import ManifestResourcesClassPath.PackageFileInfo import ManifestResourcesClassPath.PackageInfo @@ -152,7 +156,7 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory { case class PackageInfo(packageName: String, subpackages: List[AbstractFile]) } - override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath = + override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath with Closeable = if (zipFile.file == null) createWithoutUnderlyingFile(zipFile) else ZipArchiveClassPath(zipFile.file, release) @@ -183,15 +187,33 @@ object ZipAndJarSourcePathFactory extends ZipAndJarFileLookupFactory { override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isScalaOrJavaSource } - override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath = ZipArchiveSourcePath(zipFile.file) + override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath with Closeable = ZipArchiveSourcePath(zipFile.file) } final class FileBasedCache[T] { import java.nio.file.Path private case class Stamp(lastModified: FileTime, fileKey: Object) - private val cache = collection.mutable.Map.empty[Seq[Path], (Seq[Stamp], T)] + private case class Entry(stamps: Seq[Stamp], t: T) { + val referenceCount: AtomicInteger = new AtomicInteger(1) + def referenceCountDecrementer: Closeable = new Closeable { + var closed = false + override def close(): Unit = { + if (!closed) { + closed = true + val count = referenceCount.decrementAndGet() + if (count == 0) { + t match { + case cl: Closeable => FileBasedCache.deferredClose(referenceCount, cl) + case _ => + } + } + } + } + } + } + private val cache = collection.mutable.Map.empty[Seq[Path], Entry] - def getOrCreate(paths: Seq[Path], create: () => T): T = cache.synchronized { + def getOrCreate(paths: Seq[Path], create: () => T, closeableRegistry: CloseableRegistry): T = cache.synchronized { val stamps = paths.map { path => val attrs = Files.readAttributes(path, classOf[BasicFileAttributes]) val lastModified = attrs.lastModifiedTime() @@ -201,10 +223,34 @@ final class FileBasedCache[T] { } cache.get(paths) match { - case Some((cachedStamps, cached)) if cachedStamps == stamps => cached + case Some(e@Entry(cachedStamps, cached)) => + if (cachedStamps == stamps) { + // Cache hit + e.referenceCount.incrementAndGet() + closeableRegistry.registerClosable(e.referenceCountDecrementer) + cached + } else { + // Cache miss: we found an entry but the underlying files have been modified + cached match { + case c: Closeable => + if (e.referenceCount.get() == 0) { + c.close() + } else { + // TODO: What do do here? Maybe add to a list of closeables polled by a cleanup thread? + } + } + val value = create() + val entry = Entry(stamps, value) + cache.put(paths, entry) + closeableRegistry.registerClosable(entry.referenceCountDecrementer) + value + } case _ => + // Cache miss val value = create() - cache.put(paths, (stamps, value)) + val entry = Entry(stamps, value) + cache.put(paths, entry) + closeableRegistry.registerClosable(entry.referenceCountDecrementer) value } } @@ -215,3 +261,26 @@ final class FileBasedCache[T] { cache.clear() } } + +object FileBasedCache { + private val deferCloseMs = Integer.getInteger("scalac.filebasedcache.defer.close.ms", 1000) + private val timer: Option[Timer] = { + if (deferCloseMs > 0) + Some(new java.util.Timer(true)) + else None + } + private def deferredClose(referenceCount: AtomicInteger, closable: Closeable): Unit = { + timer match { + case Some(timer) => + val task = new TimerTask { + override def run(): Unit = { + if (referenceCount.get == 0) + closable.close() + } + } + timer.schedule(task, FileBasedCache.deferCloseMs.toLong) + case None => + closable.close() + } + } +} diff --git a/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala b/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala index 32ec4cde4485..c658d4c01664 100644 --- a/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala +++ b/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala @@ -12,7 +12,7 @@ package scala.tools.nsc.classpath -import java.io.File +import java.io.{Closeable, File} import java.net.URL import scala.collection.Seq import scala.reflect.io.AbstractFile @@ -25,7 +25,7 @@ import scala.tools.nsc.util.{ClassPath, ClassRepresentation} * It provides common logic for classes handling class and source files. * It's aware of things like e.g. META-INF directory which is correctly skipped. */ -trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends ClassPath { +trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends ClassPath with Closeable { val zipFile: File def release: Option[String] @@ -33,8 +33,8 @@ trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends ClassPa override def asURLs: Seq[URL] = Seq(zipFile.toURI.toURL) override def asClassPathStrings: Seq[String] = Seq(zipFile.getPath) - private val archive = new FileZipArchive(zipFile, release) + override def close(): Unit = archive.close() override private[nsc] def packages(inPackage: String): Seq[PackageEntry] = { val prefix = PackageNameUtils.packagePrefix(inPackage) diff --git a/src/compiler/scala/tools/nsc/plugins/Plugin.scala b/src/compiler/scala/tools/nsc/plugins/Plugin.scala index b76f67ccf6aa..910b2b258d64 100644 --- a/src/compiler/scala/tools/nsc/plugins/Plugin.scala +++ b/src/compiler/scala/tools/nsc/plugins/Plugin.scala @@ -17,7 +17,9 @@ import scala.tools.nsc.io.Jar import scala.reflect.internal.util.ScalaClassLoader import scala.reflect.io.{Directory, File, Path} import java.io.InputStream +import java.net.URL +import scala.collection.JavaConverters._ import scala.collection.mutable import scala.tools.nsc.classpath.FileBasedCache import scala.util.{Failure, Success, Try} @@ -93,26 +95,7 @@ object Plugin { private val PluginXML = "scalac-plugin.xml" - private val pluginClassLoadersCache = new FileBasedCache[ScalaClassLoader]() - - /** Create a class loader with the specified locations plus - * the loader that loaded the Scala compiler. - * - * If the class loader has already been created before and the - * file stamps are the same, the previous loader is returned to - * mitigate the cost of dynamic classloading as it has been - * measured in https://github.com/scala/scala-dev/issues/458. - */ - private def loaderFor(locations: Seq[Path], disableCache: Boolean): ScalaClassLoader = { - def newLoader = () => { - val compilerLoader = classOf[Plugin].getClassLoader - val urls = locations map (_.toURL) - ScalaClassLoader fromURLs (urls, compilerLoader) - } - - if (disableCache || locations.exists(!Jar.isJarOrZip(_))) newLoader() - else pluginClassLoadersCache.getOrCreate(locations.map(_.jfile.toPath()), newLoader) - } + private[nsc] val pluginClassLoadersCache = new FileBasedCache[ScalaClassLoader.URLClassLoader]() /** Try to load a plugin description from the specified location. */ @@ -155,40 +138,26 @@ object Plugin { paths: List[List[Path]], dirs: List[Path], ignoring: List[String], - disableClassLoaderCache: Boolean): List[Try[AnyClass]] = + findPluginClassloader: (Seq[Path] => ClassLoader)): List[Try[AnyClass]] = { - // List[(jar, Try(descriptor))] in dir - def scan(d: Directory) = - d.files.toList sortBy (_.name) filter (Jar isJarOrZip _) map (j => (j, loadDescriptionFromJar(j))) - type PDResults = List[Try[(PluginDescription, ScalaClassLoader)]] - // scan plugin dirs for jars containing plugins, ignoring dirs with none and other jars - val fromDirs: PDResults = dirs filter (_.isDirectory) flatMap { d => - scan(d.toDirectory) collect { - case (j, Success(pd)) => Success((pd, loaderFor(Seq(j), disableClassLoaderCache))) + val fromLoaders = paths.map {path => + val loader = findPluginClassloader(path) + loader.getResource(PluginXML) match { + case null => Failure(new MissingPluginException(path)) + case url => + val inputStream = url.openStream + try { + Try((PluginDescription.fromXML(inputStream), loader)) + } finally { + inputStream.close() + } } } - // scan jar paths for plugins, taking the first plugin you find. - // a path element can be either a plugin.jar or an exploded dir. - def findDescriptor(ps: List[Path]) = { - def loop(qs: List[Path]): Try[PluginDescription] = qs match { - case Nil => Failure(new MissingPluginException(ps)) - case p :: rest => - if (p.isDirectory) loadDescriptionFromFile(p.toDirectory / PluginXML) orElse loop(rest) - else if (p.isFile) loadDescriptionFromJar(p.toFile) orElse loop(rest) - else loop(rest) - } - loop(ps) - } - val fromPaths: PDResults = paths map (p => (p, findDescriptor(p))) map { - case (p, Success(pd)) => Success((pd, loaderFor(p, disableClassLoaderCache))) - case (_, Failure(e)) => Failure(e) - } - val seen = mutable.HashSet[String]() - val enabled = (fromPaths ::: fromDirs) map { + val enabled = fromLoaders map { case Success((pd, loader)) if seen(pd.classname) => // a nod to scala/bug#7494, take the plugin classes distinctly Failure(new PluginLoadException(pd.name, s"Ignoring duplicate plugin ${pd.name} (${pd.classname})")) diff --git a/src/compiler/scala/tools/nsc/plugins/Plugins.scala b/src/compiler/scala/tools/nsc/plugins/Plugins.scala index bba855ba541a..ac8f301885b2 100644 --- a/src/compiler/scala/tools/nsc/plugins/Plugins.scala +++ b/src/compiler/scala/tools/nsc/plugins/Plugins.scala @@ -13,7 +13,14 @@ package scala.tools.nsc package plugins +import java.net.URL + +import scala.reflect.internal.util.ScalaClassLoader import scala.reflect.io.Path +import scala.tools.nsc +import scala.tools.nsc.io.Jar +import scala.tools.nsc.plugins.Plugin.pluginClassLoadersCache +import scala.tools.nsc.typechecker.Macros import scala.tools.nsc.util.ClassPath import scala.tools.util.PathResolver.Defaults @@ -37,7 +44,7 @@ trait Plugins { global: Global => def injectDefault(s: String) = if (s.isEmpty) Defaults.scalaPluginPath else s asPath(settings.pluginsDir.value) map injectDefault map Path.apply } - val maybes = Plugin.loadAllFrom(paths, dirs, settings.disable.value, settings.YcachePluginClassLoader.value == settings.CachePolicy.None.name) + val maybes = Plugin.loadAllFrom(paths, dirs, settings.disable.value, findPluginClassLoader(_)) val (goods, errors) = maybes partition (_.isSuccess) // Explicit parameterization of recover to avoid -Xlint warning about inferred Any errors foreach (_.recover[Any] { @@ -53,6 +60,37 @@ trait Plugins { global: Global => classes map (Plugin.instantiate(_, this)) } + /** + * Locate or create the classloader to load a compiler plugin with `classpath`. + * + * Subclasses may override to customise the behaviour. + * + * @param classpath + * @return + */ + protected def findPluginClassLoader(classpath: Seq[Path]): ClassLoader = { + val disableCache = settings.YcachePluginClassLoader.value == settings.CachePolicy.None.name + def newLoader = () => { + val compilerLoader = classOf[Plugin].getClassLoader + val urls = classpath map (_.toURL) + ScalaClassLoader fromURLs (urls, compilerLoader) + } + + // Create a class loader with the specified locations plus + // the loader that loaded the Scala compiler. + // + // If the class loader has already been created before and the + // file stamps are the same, the previous loader is returned to + // mitigate the cost of dynamic classloading as it has been + // measured in https://github.com/scala/scala-dev/issues/458. + + if (disableCache || classpath.exists(!Jar.isJarOrZip(_))) { + val loader = newLoader() + closeableRegistry.registerClosable(loader) + loader + } else pluginClassLoadersCache.getOrCreate(classpath.map(_.jfile.toPath()), newLoader, closeableRegistry) + } + protected lazy val roughPluginsList: List[Plugin] = loadRoughPluginsList() /** Load all available plugins. Skips plugins that @@ -123,4 +161,49 @@ trait Plugins { global: Global => (for (plug <- roughPluginsList ; help <- plug.optionsHelp) yield { "\nOptions for plugin '%s':\n%s\n".format(plug.name, help) }).mkString + + /** Obtains a `ClassLoader` instance used for macro expansion. + * + * By default a new `ScalaClassLoader` is created using the classpath + * from global and the classloader of self as parent. + * + * Mirrors with runtime definitions (e.g. Repl) need to adjust this method. + */ + protected[scala] def findMacroClassLoader(): ClassLoader = { + val classpath: Seq[URL] = if (settings.YmacroClasspath.isSetByUser) { + for { + file <- scala.tools.nsc.util.ClassPath.expandPath(settings.YmacroClasspath.value, true) + af <- Option(nsc.io.AbstractFile getDirectory file) + } yield af.file.toURI.toURL + } else global.classPath.asURLs + def newLoader: () => ScalaClassLoader.URLClassLoader = () => { + analyzer.macroLogVerbose("macro classloader: initializing from -cp: %s".format(classpath)) + ScalaClassLoader.fromURLs(classpath, getClass.getClassLoader) + } + + val disableCache = settings.YcacheMacroClassLoader.value == settings.CachePolicy.None.name + if (disableCache) newLoader() + else { + import scala.tools.nsc.io.Jar + import scala.reflect.io.{AbstractFile, Path} + + val urlsAndFiles = classpath.map(u => u -> AbstractFile.getURL(u)) + val hasNullURL = urlsAndFiles.filter(_._2 eq null) + if (hasNullURL.nonEmpty) { + // TODO if the only null is jrt:// we can still cache + // TODO filter out classpath elements pointing to non-existing files before we get here, that's another source of null + analyzer.macroLogVerbose(s"macro classloader: caching is disabled because `AbstractFile.getURL` returned `null` for ${hasNullURL.map(_._1).mkString(", ")}.") + perRunCaches.recordClassloader(newLoader()) + } else { + val locations = urlsAndFiles.map(t => Path(t._2.file)) + val nonJarZips = locations.filterNot(Jar.isJarOrZip(_)) + if (nonJarZips.nonEmpty) { + analyzer.macroLogVerbose(s"macro classloader: caching is disabled because the following paths are not supported: ${nonJarZips.mkString(",")}.") + perRunCaches.recordClassloader(newLoader()) + } else { + Macros.macroClassLoadersCache.getOrCreate(locations.map(_.jfile.toPath()), newLoader, closeableRegistry) + } + } + } + } } diff --git a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala index 2783b74a9d59..42fb7d6d02bb 100644 --- a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala +++ b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala @@ -243,6 +243,8 @@ trait ScalaSettings extends AbsScalaSettings val YpartialUnification = BooleanSetting ("-Ypartial-unification", "Enable partial unification in type constructor inference") val Yvirtpatmat = BooleanSetting ("-Yvirtpatmat", "Enable pattern matcher virtualization") + val YmacroClasspath = PathSetting ("-Ymacro-classpath", "The classpath used to reflectively load macro implementations", "") + val exposeEmptyPackage = BooleanSetting ("-Yexpose-empty-package", "Internal only: expose the empty package.").internalOnly() val Ydelambdafy = ChoiceSetting ("-Ydelambdafy", "strategy", "Strategy used for translating lambdas into JVM code.", List("inline", "method"), "method") diff --git a/src/compiler/scala/tools/nsc/typechecker/Macros.scala b/src/compiler/scala/tools/nsc/typechecker/Macros.scala index 1755042d339e..8120019444ca 100644 --- a/src/compiler/scala/tools/nsc/typechecker/Macros.scala +++ b/src/compiler/scala/tools/nsc/typechecker/Macros.scala @@ -14,6 +14,8 @@ package scala.tools.nsc package typechecker import java.lang.Math.min +import java.net.URL + import symtab.Flags._ import scala.reflect.internal.util.ScalaClassLoader import scala.reflect.runtime.ReflectionUtils @@ -22,6 +24,7 @@ import scala.reflect.internal.TypesStats import scala.reflect.macros.util._ import scala.util.control.ControlThrowable import scala.reflect.internal.util.ListOfNil +import scala.reflect.io.AbstractFile import scala.reflect.macros.runtime.{AbortMacroException, MacroRuntimes} import scala.reflect.macros.compiler.DefaultMacroCompiler import scala.tools.reflect.FastTrack @@ -64,49 +67,6 @@ trait Macros extends MacroRuntimes with Traces with Helpers { def globalSettings = global.settings - private final val macroClassLoadersCache = - new scala.tools.nsc.classpath.FileBasedCache[ScalaClassLoader]() - - /** Obtains a `ClassLoader` instance used for macro expansion. - * - * By default a new `ScalaClassLoader` is created using the classpath - * from global and the classloader of self as parent. - * - * Mirrors with runtime definitions (e.g. Repl) need to adjust this method. - */ - protected def findMacroClassLoader(): ClassLoader = { - val classpath = global.classPath.asURLs - def newLoader = () => { - macroLogVerbose("macro classloader: initializing from -cp: %s".format(classpath)) - ScalaClassLoader.fromURLs(classpath, self.getClass.getClassLoader) - } - - val disableCache = settings.YcacheMacroClassLoader.value == settings.CachePolicy.None.name - if (disableCache) newLoader() - else { - import scala.tools.nsc.io.Jar - import scala.reflect.io.{AbstractFile, Path} - - val urlsAndFiles = classpath.map(u => u -> AbstractFile.getURL(u)) - val hasNullURL = urlsAndFiles.filter(_._2 eq null) - if (hasNullURL.nonEmpty) { - // TODO if the only null is jrt:// we can still cache - // TODO filter out classpath elements pointing to non-existing files before we get here, that's another source of null - macroLogVerbose(s"macro classloader: caching is disabled because `AbstractFile.getURL` returned `null` for ${hasNullURL.map(_._1).mkString(", ")}.") - newLoader() - } else { - val locations = urlsAndFiles.map(t => Path(t._2.file)) - val nonJarZips = locations.filterNot(Jar.isJarOrZip(_)) - if (nonJarZips.nonEmpty) { - macroLogVerbose(s"macro classloader: caching is disabled because the following paths are not supported: ${nonJarZips.mkString(",")}.") - newLoader() - } else { - macroClassLoadersCache.getOrCreate(locations.map(_.jfile.toPath()), newLoader) - } - } - } - } - /** `MacroImplBinding` and its companion module are responsible for * serialization/deserialization of macro def -> impl bindings. * @@ -963,6 +923,11 @@ trait Macros extends MacroRuntimes with Traces with Helpers { }.transform(expandee) } +object Macros { + final val macroClassLoadersCache = + new scala.tools.nsc.classpath.FileBasedCache[ScalaClassLoader.URLClassLoader]() +} + trait MacrosStats { self: TypesStats with Statistics => val macroExpandCount = newCounter ("#macro expansions", "typer") diff --git a/src/compiler/scala/tools/nsc/util/ClassPath.scala b/src/compiler/scala/tools/nsc/util/ClassPath.scala index 827c7ce5dbd7..b2ad49c5a38c 100644 --- a/src/compiler/scala/tools/nsc/util/ClassPath.scala +++ b/src/compiler/scala/tools/nsc/util/ClassPath.scala @@ -13,6 +13,8 @@ package scala.tools.nsc package util +import java.io.Closeable + import io.{AbstractFile, Directory, File, Jar} import java.net.MalformedURLException import java.net.URL diff --git a/src/compiler/scala/tools/reflect/ReflectGlobal.scala b/src/compiler/scala/tools/reflect/ReflectGlobal.scala index e1cf834c6fb6..2efd699e9f44 100644 --- a/src/compiler/scala/tools/reflect/ReflectGlobal.scala +++ b/src/compiler/scala/tools/reflect/ReflectGlobal.scala @@ -25,18 +25,14 @@ import scala.tools.nsc.typechecker.Analyzer class ReflectGlobal(currentSettings: Settings, reporter: Reporter, override val rootClassLoader: ClassLoader) extends Global(currentSettings, reporter) with scala.tools.reflect.ReflectSetup with scala.reflect.runtime.SymbolTable { - override lazy val analyzer = new { - val global: ReflectGlobal.this.type = ReflectGlobal.this - } with Analyzer { - /** Obtains the classLoader used for runtime macro expansion. - * - * Macro expansion can use everything available in [[global.classPath]] or [[rootClassLoader]]. - * The [[rootClassLoader]] is used to obtain runtime defined macros. - */ - override protected def findMacroClassLoader(): ClassLoader = { - val classpath = global.classPath.asURLs - ScalaClassLoader.fromURLs(classpath, rootClassLoader) - } + /** Obtains the classLoader used for runtime macro expansion. + * + * Macro expansion can use everything available in `global.classPath` or `rootClassLoader`. + * The `rootClassLoader` is used to obtain runtime defined macros. + */ + override protected[scala] def findMacroClassLoader(): ClassLoader = { + val classpath = classPath.asURLs + perRunCaches.recordClassloader(ScalaClassLoader.fromURLs(classpath, rootClassLoader)) } override def transformedType(sym: Symbol) = diff --git a/src/compiler/scala/tools/reflect/ReflectMain.scala b/src/compiler/scala/tools/reflect/ReflectMain.scala index 3abd5f390761..a290c6bfafc8 100644 --- a/src/compiler/scala/tools/reflect/ReflectMain.scala +++ b/src/compiler/scala/tools/reflect/ReflectMain.scala @@ -14,15 +14,13 @@ package scala.tools package reflect import scala.reflect.internal.util.ScalaClassLoader -import scala.tools.nsc.Driver -import scala.tools.nsc.Global -import scala.tools.nsc.Settings +import scala.tools.nsc.{Driver, Global, CloseableRegistry, Settings} import scala.tools.util.PathResolver object ReflectMain extends Driver { private def classloaderFromSettings(settings: Settings) = { - val classPathURLs = new PathResolver(settings).resultAsURLs + val classPathURLs = new PathResolver(settings, new CloseableRegistry).resultAsURLs ScalaClassLoader.fromURLs(classPathURLs, getClass.getClassLoader) } diff --git a/src/compiler/scala/tools/util/PathResolver.scala b/src/compiler/scala/tools/util/PathResolver.scala index 1ad471e40f8b..cf454d5854f8 100644 --- a/src/compiler/scala/tools/util/PathResolver.scala +++ b/src/compiler/scala/tools/util/PathResolver.scala @@ -15,8 +15,9 @@ package tools package util import java.net.URL + import scala.tools.reflect.WrappedProperties.AccessControl -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import scala.tools.nsc.util.ClassPath import scala.reflect.io.{Directory, File, Path} import PartialFunction.condOpt @@ -189,19 +190,24 @@ object PathResolver { } else { val settings = new Settings() val rest = settings.processArguments(args.toList, processAll = false)._2 - val pr = new PathResolver(settings) - println("COMMAND: 'scala %s'".format(args.mkString(" "))) - println("RESIDUAL: 'scala %s'\n".format(rest.mkString(" "))) - - pr.result match { - case cp: AggregateClassPath => - println(s"ClassPath has ${cp.aggregates.size} entries and results in:\n${cp.asClassPathStrings}") + val registry = new CloseableRegistry + try { + val pr = new PathResolver(settings, registry) + println("COMMAND: 'scala %s'".format(args.mkString(" "))) + println("RESIDUAL: 'scala %s'\n".format(rest.mkString(" "))) + + pr.result match { + case cp: AggregateClassPath => + println(s"ClassPath has ${cp.aggregates.size} entries and results in:\n${cp.asClassPathStrings}") + } + } finally { + registry.close() } } } -final class PathResolver(settings: Settings) { - private val classPathFactory = new ClassPathFactory(settings) +final class PathResolver(settings: Settings, closeableRegistry: CloseableRegistry) { + private val classPathFactory = new ClassPathFactory(settings, closeableRegistry) import PathResolver.{ AsLines, Defaults, ppcp } @@ -250,7 +256,7 @@ final class PathResolver(settings: Settings) { // Assemble the elements! def basis = List[Traversable[ClassPath]]( - JrtClassPath.apply(settings.releaseValue), // 0. The Java 9 classpath (backed by the jrt:/ virtual system, if available) + jrt, // 0. The Java 9+ classpath (backed by the ct.sym or jrt:/ virtual system, if available) classesInPath(javaBootClassPath), // 1. The Java bootstrap class path. contentsOfDirsInPath(javaExtDirs), // 2. The Java extension class path. classesInExpandedPath(javaUserClassPath), // 3. The Java application class path. @@ -261,6 +267,8 @@ final class PathResolver(settings: Settings) { sourcesInPath(sourcePath) // 7. The Scala source path. ) + private def jrt: Option[ClassPath] = JrtClassPath.apply(settings.releaseValue, closeableRegistry) + lazy val containers = basis.flatten.distinct override def toString = s""" diff --git a/src/partest-extras/scala/tools/partest/BytecodeTest.scala b/src/partest-extras/scala/tools/partest/BytecodeTest.scala index b016778bf428..309a6d49c482 100644 --- a/src/partest-extras/scala/tools/partest/BytecodeTest.scala +++ b/src/partest-extras/scala/tools/partest/BytecodeTest.scala @@ -18,6 +18,7 @@ import scala.tools.asm.tree._ import java.io.{InputStream, File => JFile} import AsmNode._ +import scala.tools.nsc.CloseableRegistry /** * Provides utilities for inspecting bytecode using ASM library. @@ -144,7 +145,7 @@ abstract class BytecodeTest { import scala.tools.nsc.Settings // logic inspired by scala.tools.util.PathResolver implementation // `Settings` is used to check YdisableFlatCpCaching in ZipArchiveFlatClassPath - val factory = new ClassPathFactory(new Settings()) + val factory = new ClassPathFactory(new Settings(), new CloseableRegistry) val containers = factory.classesInExpandedPath(sys.props("partest.output") + java.io.File.pathSeparator + Defaults.javaUserClassPath) new AggregateClassPath(containers) } diff --git a/src/reflect/scala/reflect/internal/SymbolTable.scala b/src/reflect/scala/reflect/internal/SymbolTable.scala index 99fd5edd7ac7..0645e60659c2 100644 --- a/src/reflect/scala/reflect/internal/SymbolTable.scala +++ b/src/reflect/scala/reflect/internal/SymbolTable.scala @@ -14,6 +14,8 @@ package scala package reflect package internal +import java.net.URLClassLoader + import scala.annotation.elidable import scala.collection.mutable import util._ @@ -416,6 +418,22 @@ abstract class SymbolTable extends macros.Universe cache } + /** Closes the provided classloader at the conclusion of this Run */ + final def recordClassloader(loader: ClassLoader): ClassLoader = { + def attemptClose(loader: ClassLoader): Unit = { + loader match { + case u: URLClassLoader => debuglog("Closing classloader " + u); u.close() + case _ => + } + } + caches ::= new WeakReference((new Clearable { + def clear(): Unit = { + attemptClose(loader) + } + })) + loader + } + /** * Removes a cache from the per-run caches. This is useful for testing: it allows running the * compiler and then inspect the state of a cache. diff --git a/src/reflect/scala/reflect/io/ZipArchive.scala b/src/reflect/scala/reflect/io/ZipArchive.scala index ee109799f3d5..05c591b9d535 100644 --- a/src/reflect/scala/reflect/io/ZipArchive.scala +++ b/src/reflect/scala/reflect/io/ZipArchive.scala @@ -149,6 +149,7 @@ abstract class ZipArchive(override val file: JFile, release: Option[String]) ext if (entry.isDirectory) ensureDir(dirs, entry.getName, entry) else ensureDir(dirs, dirName(entry.getName), null) } + def close(): Unit } /** ''Note: This library is considered experimental and should not be used unless you know what you are doing.'' */ final class FileZipArchive(file: JFile, release: Option[String]) extends ZipArchive(file, release) { @@ -232,6 +233,7 @@ final class FileZipArchive(file: JFile, release: Option[String]) extends ZipArch } } finally { if (ZipArchive.closeZipFile) zipFile.close() + else closeables ::= zipFile } root } @@ -259,6 +261,10 @@ final class FileZipArchive(file: JFile, release: Option[String]) extends ZipArch case x: FileZipArchive => file.getAbsoluteFile == x.file.getAbsoluteFile case _ => false } + private[this] var closeables: List[java.io.Closeable] = Nil + override def close(): Unit = { + closeables.foreach(_.close) + } } /** ''Note: This library is considered experimental and should not be used unless you know what you are doing.'' */ final class URLZipArchive(val url: URL) extends ZipArchive(null) { @@ -266,6 +272,7 @@ final class URLZipArchive(val url: URL) extends ZipArchive(null) { val root = new DirEntry("/") val dirs = mutable.HashMap[String, DirEntry]("" -> root) val in = new ZipInputStream(new ByteArrayInputStream(Streamable.bytes(input))) + closeables ::= in @tailrec def loop() { val zipEntry = in.getNextEntry() @@ -327,6 +334,10 @@ final class URLZipArchive(val url: URL) extends ZipArchive(null) { case x: URLZipArchive => url == x.url case _ => false } + private[this] var closeables: List[java.io.Closeable] = Nil + def close(): Unit = { + closeables.foreach(_.close()) + } } final class ManifestResources(val url: URL) extends ZipArchive(null) { @@ -334,6 +345,8 @@ final class ManifestResources(val url: URL) extends ZipArchive(null) { val root = new DirEntry("/") val dirs = mutable.HashMap[String, DirEntry]("" -> root) val manifest = new Manifest(input) + closeables ::= input + val iter = manifest.getEntries().keySet().iterator().asScala.filter(_.endsWith(".class")).map(new ZipEntry(_)) for (zipEntry <- iter) { @@ -385,4 +398,8 @@ final class ManifestResources(val url: URL) extends ZipArchive(null) { } } } + private[this] var closeables: List[java.io.Closeable] = Nil + override def close(): Unit = { + closeables.foreach(_.close()) + } } diff --git a/src/repl/scala/tools/nsc/interpreter/IMain.scala b/src/repl/scala/tools/nsc/interpreter/IMain.scala index 764bb4d48543..73cbc828eea4 100644 --- a/src/repl/scala/tools/nsc/interpreter/IMain.scala +++ b/src/repl/scala/tools/nsc/interpreter/IMain.scala @@ -28,6 +28,7 @@ import scala.tools.nsc.util._ import ScalaClassLoader.URLClassLoader import scala.tools.nsc.util.Exceptional.unwrap import java.net.URL +import java.io.Closeable import scala.tools.util.PathResolver import scala.util.{Try => Trying} @@ -63,7 +64,7 @@ import scala.util.{Try => Trying} * @author Moez A. Abdel-Gawad * @author Lex Spoon */ -class IMain(initialSettings: Settings, protected val out: JPrintWriter) extends Imports with PresentationCompilation { +class IMain(initialSettings: Settings, protected val out: JPrintWriter) extends Imports with PresentationCompilation with Closeable { imain => def this(initialSettings: Settings) = this(initialSettings, IMain.defaultOut) @@ -100,7 +101,7 @@ class IMain(initialSettings: Settings, protected val out: JPrintWriter) extends def compilerClasspath: Seq[java.net.URL] = ( if (isInitializeComplete) global.classPath.asURLs - else new PathResolver(settings).resultAsURLs // the compiler's classpath + else new PathResolver(settings, global.closeableRegistry).resultAsURLs // the compiler's classpath ) def settings = initialSettings // Run the code body with the given boolean settings flipped to true. @@ -683,6 +684,9 @@ class IMain(initialSettings: Settings, protected val out: JPrintWriter) extends */ def close() { reporter.flush() + if (isInitializeComplete) { + global.close() + } } /** Here is where we: diff --git a/src/repl/scala/tools/nsc/interpreter/PresentationCompilation.scala b/src/repl/scala/tools/nsc/interpreter/PresentationCompilation.scala index 4c7f05318c53..296cc5a4b1df 100644 --- a/src/repl/scala/tools/nsc/interpreter/PresentationCompilation.scala +++ b/src/repl/scala/tools/nsc/interpreter/PresentationCompilation.scala @@ -16,7 +16,7 @@ import scala.reflect.internal.util.RangePosition import scala.reflect.io.AbstractFile import scala.tools.nsc.backend.JavaPlatform import scala.tools.nsc.util.ClassPath -import scala.tools.nsc.{interactive, Settings} +import scala.tools.nsc.{interactive, CloseableRegistry, Settings} import scala.tools.nsc.reporters.StoreReporter import scala.tools.nsc.classpath._ @@ -63,10 +63,6 @@ trait PresentationCompilation { * You may downcast the `reporter` to `StoreReporter` to access type errors. */ def newPresentationCompiler(): interactive.Global = { - def mergedFlatClasspath = { - val replOutClasspath = ClassPathFactory.newClassPath(replOutput.dir, settings) - AggregateClassPath(replOutClasspath :: global.platform.classPath :: Nil) - } def copySettings: Settings = { val s = new Settings(_ => () /* ignores "bad option -nc" errors, etc */) s.processArguments(global.settings.recreateArgs, processAll = false) @@ -75,6 +71,11 @@ trait PresentationCompilation { } val storeReporter: StoreReporter = new StoreReporter val interactiveGlobal = new interactive.Global(copySettings, storeReporter) { self => + def mergedFlatClasspath = { + val replOutClasspath = ClassPathFactory.newClassPath(replOutput.dir, settings, closeableRegistry) + AggregateClassPath(replOutClasspath :: global.platform.classPath :: Nil) + } + override lazy val platform: ThisPlatform = { new JavaPlatform { lazy val global: self.type = self diff --git a/src/repl/scala/tools/nsc/interpreter/ReplGlobal.scala b/src/repl/scala/tools/nsc/interpreter/ReplGlobal.scala index 1273d6ac92fb..72b5a7424ceb 100644 --- a/src/repl/scala/tools/nsc/interpreter/ReplGlobal.scala +++ b/src/repl/scala/tools/nsc/interpreter/ReplGlobal.scala @@ -30,16 +30,11 @@ trait ReplGlobal extends Global { super.abort(msg) } - override lazy val analyzer = new { - val global: ReplGlobal.this.type = ReplGlobal.this - } with Analyzer { - - override protected def findMacroClassLoader(): ClassLoader = { - val loader = super.findMacroClassLoader - macroLogVerbose("macro classloader: initializing from a REPL classloader: %s".format(global.classPath.asURLs)) - val virtualDirectory = globalSettings.outputDirs.getSingleOutput.get - new util.AbstractFileClassLoader(virtualDirectory, loader) {} - } + override protected[scala] def findMacroClassLoader(): ClassLoader = { + val loader = super.findMacroClassLoader + analyzer.macroLogVerbose("macro classloader: initializing from a REPL classloader: %s".format(classPath.asURLs)) + val virtualDirectory = analyzer.globalSettings.outputDirs.getSingleOutput.get + new util.AbstractFileClassLoader(virtualDirectory, loader) {} } override def optimizerClassPath(base: ClassPath): ClassPath = { @@ -47,7 +42,7 @@ trait ReplGlobal extends Global { case None => base case Some(out) => // Make bytecode of previous lines available to the inliner - val replOutClasspath = ClassPathFactory.newClassPath(settings.outputDirs.getSingleOutput.get, settings) + val replOutClasspath = ClassPathFactory.newClassPath(settings.outputDirs.getSingleOutput.get, settings, closeableRegistry) AggregateClassPath.createAggregate(platform.classPath, replOutClasspath) } } diff --git a/src/scalap/scala/tools/scalap/Main.scala b/src/scalap/scala/tools/scalap/Main.scala index 42b0fdfb236a..5e3d633d429f 100644 --- a/src/scalap/scala/tools/scalap/Main.scala +++ b/src/scalap/scala/tools/scalap/Main.scala @@ -14,8 +14,9 @@ package scala package tools.scalap import java.io.{ByteArrayOutputStream, OutputStreamWriter, PrintStream} + import scala.reflect.NameTransformer -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import scala.tools.nsc.classpath.{AggregateClassPath, ClassPathFactory} import scala.tools.nsc.util.ClassPath import scala.tools.util.PathResolver @@ -185,14 +186,18 @@ object Main extends Main { settings.YdisableFlatCpCaching.value = arguments contains opts.disableFlatClassPathCaching settings.Ylogcp.value = arguments contains opts.logClassPath - val path = createClassPath(cpArg, settings) - - // print the classpath if output is verbose - if (verbose) - Console.println(Console.BOLD + "CLASSPATH" + Console.RESET + " = " + path.asClassPathString) - - // process all given classes - arguments.getOthers foreach process(arguments, path) + val registry = new CloseableRegistry + try { + val path = createClassPath(cpArg, settings, registry) + // print the classpath if output is verbose + if (verbose) + Console.println(Console.BOLD + "CLASSPATH" + Console.RESET + " = " + path.asClassPathString) + + // process all given classes + arguments.getOthers foreach process(arguments, path) + } finally { + registry.close() + } } private def parseArguments(args: Array[String]) = @@ -208,11 +213,11 @@ object Main extends Main { .withOption(opts.logClassPath) .parse(args) - private def createClassPath(cpArg: Option[String], settings: Settings) = cpArg match { + private def createClassPath(cpArg: Option[String], settings: Settings, closeableRegistry: CloseableRegistry) = cpArg match { case Some(cp) => - AggregateClassPath(new ClassPathFactory(settings).classesInExpandedPath(cp)) + AggregateClassPath(new ClassPathFactory(settings, closeableRegistry).classesInExpandedPath(cp)) case _ => settings.classpath.value = "." // include '.' in the default classpath scala/bug#6669 - new PathResolver(settings).result + new PathResolver(settings, closeableRegistry).result } } diff --git a/test/junit/scala/tools/nsc/GlobalCustomizeClassloaderTest.scala b/test/junit/scala/tools/nsc/GlobalCustomizeClassloaderTest.scala new file mode 100644 index 000000000000..500379706090 --- /dev/null +++ b/test/junit/scala/tools/nsc/GlobalCustomizeClassloaderTest.scala @@ -0,0 +1,72 @@ +package scala.tools.nsc + +import org.junit.{Assert, Test} +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +import scala.reflect.internal.util.{AbstractFileClassLoader, NoSourceFile} +import scala.reflect.io.{Path, VirtualDirectory} +import scala.tools.nsc.plugins.{Plugin, PluginComponent} + +@RunWith(classOf[JUnit4]) +class GlobalCustomizeClassloaderTest { + // Demonstrate extension points to customise creation of the classloaders used to load compiler + // plugins and macro implementations. + // + // A use case could be for a build tool to take control of caching of these classloaders in a way + // that properly closes them before one of the elements needs to be overwritten. + @Test def test(): Unit = { + val g = new Global(new Settings) { + override protected[scala] def findMacroClassLoader(): ClassLoader = getClass.getClassLoader + override protected def findPluginClassLoader(classpath: Seq[Path]): ClassLoader = { + val d = new VirtualDirectory("", None) + val xml = d.fileNamed("scalac-plugin.xml") + val out = xml.bufferedOutput + out.write( + s""" + |sample-plugin + |${classOf[SamplePlugin].getName} + | + |""".stripMargin.getBytes()) + out.close() + new AbstractFileClassLoader(d, getClass.getClassLoader) + } + } + g.settings.usejavacp.value = true + g.settings.plugin.value = List("sample") + new g.Run + assert(g.settings.log.value == List("typer")) + + val unit = new g.CompilationUnit(NoSourceFile) + val context = g.analyzer.rootContext(unit) + val typer = g.analyzer.newTyper(context) + import g._ + SampleMacro.data = "in this classloader" + val typed = typer.typed(q"scala.tools.nsc.SampleMacro.m") + assert(!reporter.hasErrors) + typed match { + case Typed(Literal(Constant(s: String)), _) => Assert.assertEquals(SampleMacro.data, s) + case _ => Assert.fail() + } + g.close() + } +} + +object SampleMacro { + var data: String = _ + import language.experimental.macros + import scala.reflect.macros.blackbox.Context + def m: String = macro impl + def impl(c: Context): c.Tree = c.universe.Literal(c.universe.Constant(data)) +} + +class SamplePlugin(val global: Global) extends Plugin { + override val name: String = "sample" + override val description: String = "sample" + override val components: List[PluginComponent] = Nil + override def init(options: List[String], error: String => Unit): Boolean = { + val result = super.init(options, error) + global.settings.log.value = List("typer") + result + } +} diff --git a/test/junit/scala/tools/nsc/classpath/JrtClassPathTest.scala b/test/junit/scala/tools/nsc/classpath/JrtClassPathTest.scala index b46677d6d477..fdc2b9caae69 100644 --- a/test/junit/scala/tools/nsc/classpath/JrtClassPathTest.scala +++ b/test/junit/scala/tools/nsc/classpath/JrtClassPathTest.scala @@ -8,7 +8,7 @@ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import scala.tools.nsc.backend.jvm.AsmUtils import scala.tools.nsc.util.ClassPath import scala.tools.util.PathResolver @@ -19,14 +19,15 @@ class JrtClassPathTest { @Test def lookupJavaClasses(): Unit = { val specVersion = scala.util.Properties.javaSpecVersion // Run the test using the JDK8 or 9 provider for rt.jar depending on the platform the test is running on. + val closeableRegistry = new CloseableRegistry val cp: ClassPath = if (specVersion == "" || specVersion == "1.8") { val settings = new Settings() - val resolver = new PathResolver(settings) - val elements = new ClassPathFactory(settings).classesInPath(resolver.Calculated.javaBootClassPath) + val resolver = new PathResolver(settings, closeableRegistry) + val elements = new ClassPathFactory(settings, closeableRegistry).classesInPath(resolver.Calculated.javaBootClassPath) AggregateClassPath(elements) } - else JrtClassPath(None).get + else JrtClassPath(None, closeableRegistry).get assertEquals(Nil, cp.classes("")) assertTrue(cp.packages("java").toString, cp.packages("java").exists(_.name == "java.lang")) @@ -37,5 +38,7 @@ class JrtClassPathTest { assertTrue(cp.list("java.lang").classesAndSources.exists(_.name == "Object")) assertTrue(cp.findClass("java.lang.Object").isDefined) assertTrue(cp.findClassFile("java.lang.Object").isDefined) + + closeableRegistry.close() } } diff --git a/test/junit/scala/tools/nsc/classpath/PathResolverBaseTest.scala b/test/junit/scala/tools/nsc/classpath/PathResolverBaseTest.scala index d3d4289d8b94..e8025ec69ec5 100644 --- a/test/junit/scala/tools/nsc/classpath/PathResolverBaseTest.scala +++ b/test/junit/scala/tools/nsc/classpath/PathResolverBaseTest.scala @@ -4,13 +4,15 @@ package scala.tools.nsc.classpath import java.io.File + import org.junit.Assert._ import org.junit._ import org.junit.rules.TemporaryFolder import org.junit.runner.RunWith import org.junit.runners.JUnit4 + import scala.tools.nsc.util.ClassPath -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import scala.tools.util.PathResolver @RunWith(classOf[JUnit4]) @@ -57,7 +59,7 @@ class PathResolverBaseTest { def deleteTempDir: Unit = tempDir.delete() private def createFlatClassPath(settings: Settings) = - new PathResolver(settings).result + new PathResolver(settings, new CloseableRegistry).result @Test def testEntriesFromListOperationAgainstSeparateMethods: Unit = { diff --git a/test/junit/scala/tools/nsc/classpath/ZipAndJarFileLookupFactoryTest.scala b/test/junit/scala/tools/nsc/classpath/ZipAndJarFileLookupFactoryTest.scala index f49f04d2c56d..b58effbcfa36 100644 --- a/test/junit/scala/tools/nsc/classpath/ZipAndJarFileLookupFactoryTest.scala +++ b/test/junit/scala/tools/nsc/classpath/ZipAndJarFileLookupFactoryTest.scala @@ -14,7 +14,8 @@ class ZipAndJarFileLookupFactoryTest { Files.delete(f) val g = new scala.tools.nsc.Global(new scala.tools.nsc.Settings()) assert(!g.settings.YdisableFlatCpCaching.value) // we're testing with our JAR metadata caching enabled. - def createCp = ZipAndJarClassPathFactory.create(AbstractFile.getFile(f.toFile), g.settings) + val closeableRegistry = new CloseableRegistry + def createCp = ZipAndJarClassPathFactory.create(AbstractFile.getFile(f.toFile), g.settings, closeableRegistry) try { createZip(f, Array(), "p1/C.class") createZip(f, Array(), "p2/X.class") @@ -41,7 +42,10 @@ class ZipAndJarFileLookupFactoryTest { // And that instance should see D, not C, in package p1. assert(cp3.findClass("p1.C").isEmpty) assert(cp3.findClass("p1.D").isDefined) - } finally Files.delete(f) + } finally { + Files.delete(f) + closeableRegistry.close() + } } def createZip(zipLocation: Path, content: Array[Byte], internalPath: String): Unit = { diff --git a/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala b/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala index cbd5634f292f..e2b11cfecd29 100644 --- a/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala +++ b/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala @@ -36,7 +36,7 @@ class SymbolTableForUnitTesting extends SymbolTable { def platformPhases: List[SubComponent] = Nil - private[nsc] lazy val classPath: ClassPath = new PathResolver(settings).result + private[nsc] lazy val classPath: ClassPath = new PathResolver(settings, new CloseableRegistry).result def isMaybeBoxed(sym: Symbol): Boolean = ??? def needCompile(bin: AbstractFile, src: AbstractFile): Boolean = ???