From 75eb460da12552a825001665b9f3dd1e38e12583 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 5 Feb 2018 19:17:25 +0100 Subject: [PATCH 1/4] Restart presentation compilers if memory is low I have noted during long editing sessions (lasting several days, typically) that memory can get full because the Dotty compiler has some space leaks. The leaks looks really hard to fix, and we don't know yet whether it's at all possible. To mitigate the leaks, this commit makes the language server watch available memory, and, if it is low (i.e. free memory after a GC < 10% of maximal available) restart all interactive drivers. This will free all memory of the compiler(s) except the shared nametable. There's a stressTest option in `Memory.scala`, which, when turned on, causes a restart every 10 editing actions. I verified that the compiler stays functional and reasonably responsive in that mode. --- compiler/src/dotty/tools/dotc/Run.scala | 4 +- .../dotc/interactive/InteractiveDriver.scala | 19 +++++--- .../languageserver/DottyLanguageServer.scala | 38 +++++++++++++-- .../dotty/tools/languageserver/Memory.scala | 47 +++++++++++++++++++ 4 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 language-server/src/dotty/tools/languageserver/Memory.scala diff --git a/compiler/src/dotty/tools/dotc/Run.scala b/compiler/src/dotty/tools/dotc/Run.scala index 925890d4ff4c..b6c401f55f22 100644 --- a/compiler/src/dotty/tools/dotc/Run.scala +++ b/compiler/src/dotty/tools/dotc/Run.scala @@ -147,7 +147,9 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint } protected def compileUnits()(implicit ctx: Context) = Stats.maybeMonitored { - ctx.checkSingleThreaded() + if (!ctx.mode.is(Mode.Interactive)) // IDEs might have multi-threaded access, accesses are synchronized + ctx.checkSingleThreaded() + compiling = true // If testing pickler, make sure to stop after pickling phase: diff --git a/compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala b/compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala index 8d2b0f434c64..58c464de97f5 100644 --- a/compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala +++ b/compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala @@ -24,7 +24,7 @@ import reporting._, reporting.diagnostic.MessageContainer import util._ /** A Driver subclass designed to be used from IDEs */ -class InteractiveDriver(settings: List[String]) extends Driver { +class InteractiveDriver(val settings: List[String]) extends Driver { import tpd._ import InteractiveDriver._ @@ -216,7 +216,17 @@ class InteractiveDriver(settings: List[String]) extends Driver { cleanupTree(tree) } - def run(uri: URI, sourceCode: String): List[MessageContainer] = { + private def toSource(uri: URI, sourceCode: String): SourceFile = { + val virtualFile = new VirtualFile(uri.toString, Paths.get(uri).toString) + val writer = new BufferedWriter(new OutputStreamWriter(virtualFile.output, "UTF-8")) + writer.write(sourceCode) + writer.close() + new SourceFile(virtualFile, Codec.UTF8) + } + + def run(uri: URI, sourceCode: String): List[MessageContainer] = run(uri, toSource(uri, sourceCode)) + + def run(uri: URI, source: SourceFile): List[MessageContainer] = { val previousCtx = myCtx try { val reporter = @@ -227,11 +237,6 @@ class InteractiveDriver(settings: List[String]) extends Driver { implicit val ctx = myCtx - val virtualFile = new VirtualFile(uri.toString, Paths.get(uri).toString) - val writer = new BufferedWriter(new OutputStreamWriter(virtualFile.output, "UTF-8")) - writer.write(sourceCode) - writer.close() - val source = new SourceFile(virtualFile, Codec.UTF8) myOpenedFiles(uri) = source run.compileSources(List(source)) diff --git a/language-server/src/dotty/tools/languageserver/DottyLanguageServer.scala b/language-server/src/dotty/tools/languageserver/DottyLanguageServer.scala index bf648c863356..4c11277a294a 100644 --- a/language-server/src/dotty/tools/languageserver/DottyLanguageServer.scala +++ b/language-server/src/dotty/tools/languageserver/DottyLanguageServer.scala @@ -25,6 +25,7 @@ import reporting._, reporting.diagnostic.MessageContainer import util._ import interactive._, interactive.InteractiveDriver._ import Interactive.Include +import config.Printers.interactiv import languageserver.config.ProjectConfig @@ -78,12 +79,40 @@ class DottyLanguageServer extends LanguageServer .update("-classpath", (config.classDirectory +: config.dependencyClasspath).mkString(File.pathSeparator)) .update("-sourcepath", config.sourceDirectories.mkString(File.pathSeparator)) :+ "-scansource" - myDrivers.put(config, new InteractiveDriver(settings)) + myDrivers(config) = new InteractiveDriver(settings) } } myDrivers } + /** Restart all presentation compiler drivers, copying open files over */ + private def restart() = thisServer.synchronized { + interactiv.println("restarting presentation compiler") + val driverConfigs = for ((config, driver) <- myDrivers.toList) yield + (config, new InteractiveDriver(driver.settings), driver.openedFiles) + for ((config, driver, _) <- driverConfigs) + myDrivers(config) = driver + System.gc() + for ((_, driver, opened) <- driverConfigs; (uri, source) <- opened) + driver.run(uri, source) + if (Memory.isCritical()) + println(s"WARNING: Insufficient memory to run Scala language server on these projects.") + } + + private def checkMemory() = + if (Memory.isCritical()) + CompletableFutures.computeAsync { _ => restart(); new Object() } + // new Object() necessary or we get a BootstrapMethodError: + // + // Caused by: java.lang.invoke.LambdaConversionException: Type mismatch for lambda expected return: void is not convertible to class java.lang.Object + // at java.lang.invoke.AbstractValidatingLambdaMetafactory.validateMetafactoryArgs(AbstractValidatingLambdaMetafactory.java:286) + // at java.lang.invoke.LambdaMetafactory.metafactory(LambdaMetafactory.java:303) + // at java.lang.invoke.CallSite.makeSite(CallSite.java:302) + // ... 11 more + // + // This looks like a problem with Dottys code generation for void-returning closures passed + // to Java methods. (or SAM functions in general?) + /** The driver instance responsible for compiling `uri` */ def driverFor(uri: URI): InteractiveDriver = { val matchingConfig = @@ -112,10 +141,11 @@ class DottyLanguageServer extends LanguageServer } private[this] def computeAsync[R](fun: CancelChecker => R): CompletableFuture[R] = - CompletableFutures.computeAsync({(cancelToken: CancelChecker) => + CompletableFutures.computeAsync { cancelToken => // We do not support any concurrent use of the compiler currently. thisServer.synchronized { cancelToken.checkCanceled() + checkMemory() try { fun(cancelToken) } catch { @@ -124,7 +154,7 @@ class DottyLanguageServer extends LanguageServer throw ex } } - }) + } override def initialize(params: InitializeParams) = computeAsync { cancelToken => rootUri = params.getRootUri @@ -160,6 +190,7 @@ class DottyLanguageServer extends LanguageServer } override def didOpen(params: DidOpenTextDocumentParams): Unit = thisServer.synchronized { + checkMemory() val document = params.getTextDocument val uri = new URI(document.getUri) val driver = driverFor(uri) @@ -173,6 +204,7 @@ class DottyLanguageServer extends LanguageServer } override def didChange(params: DidChangeTextDocumentParams): Unit = thisServer.synchronized { + checkMemory() val document = params.getTextDocument val uri = new URI(document.getUri) val driver = driverFor(uri) diff --git a/language-server/src/dotty/tools/languageserver/Memory.scala b/language-server/src/dotty/tools/languageserver/Memory.scala new file mode 100644 index 000000000000..7193df0732ea --- /dev/null +++ b/language-server/src/dotty/tools/languageserver/Memory.scala @@ -0,0 +1,47 @@ +package dotty.tools +package languageserver + +object Memory { + + /** Memory is judged to be critical if after a GC the amount of used memory + * divided by total available memory exceeds this threshold. + */ + val UsedThreshold = 0.9 + + /** If total available memory is unknown, memory is judged to be critical if + * after a GC free memory divided by used memory is under this threshold. + */ + val FreeThreshold = 0.1 + + /** Turn this flag on to stress test restart capability in compiler. + * It will restart the presentation compiler after every 10 editing actions + */ + private final val stressTest = false + private var stressTestCounter = 0 + + /** Is memory critically low? */ + def isCritical(): Boolean = { + if (stressTest) { + stressTestCounter += 1 + if (stressTestCounter % 10 == 0) return true + } + val runtime = Runtime.getRuntime + def total = runtime.totalMemory + def maximal = runtime.maxMemory + def free = runtime.freeMemory + def used = total - free + def usedIsCloseToMax = + if (maximal == Long.MaxValue) free.toDouble / used < FreeThreshold + else used.toDouble / maximal > UsedThreshold + usedIsCloseToMax && { runtime.gc(); usedIsCloseToMax } + } + + def stats(): String = { + final val M = 2 << 20 + val runtime = Runtime.getRuntime + def total = runtime.totalMemory / M + def maximal = runtime.maxMemory / M + def free = runtime.freeMemory / M + s"total used memory: $total MB, free: $free MB, maximal available = $maximal MB" + } +} \ No newline at end of file From 1f5a2f8921268c5e2ae14ef749401c2e5640d999 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 21 Feb 2018 10:33:52 +0100 Subject: [PATCH 2/4] Address reviewers comments --- .../src/dotty/tools/dotc/interactive/Interactive.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/interactive/Interactive.scala b/compiler/src/dotty/tools/dotc/interactive/Interactive.scala index 0e1456764088..b2ef621e4441 100644 --- a/compiler/src/dotty/tools/dotc/interactive/Interactive.scala +++ b/compiler/src/dotty/tools/dotc/interactive/Interactive.scala @@ -341,11 +341,10 @@ object Interactive { } def pathTo(tree: Tree, pos: Position)(implicit ctx: Context): List[Tree] = - if (tree.pos.contains(pos)) { - // FIXME: We shouldn't need a cast. Change NavigateAST.pathTo to return a List of Tree? - val path = NavigateAST.pathTo(pos, tree, skipZeroExtent = true).asInstanceOf[List[untpd.Tree]] - path.dropWhile(!_.hasType) collect { case t: tpd.Tree @unchecked => t } - } + if (tree.pos.contains(pos)) + NavigateAST.pathTo(pos, tree, skipZeroExtent = true) + .collect { case t: untpd.Tree => t } + .dropWhile(!_.hasType).asInstanceOf[List[tpd.Tree]] else Nil def contextOfStat(stats: List[Tree], stat: Tree, exprOwner: Symbol, ctx: Context): Context = stats match { From 0af475c9f3b01c29646c4d2db1ffd73c1a6c907e Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 27 Feb 2018 18:23:16 +0100 Subject: [PATCH 3/4] Drop workaround for #3984 --- .../tools/languageserver/DottyLanguageServer.scala | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/language-server/src/dotty/tools/languageserver/DottyLanguageServer.scala b/language-server/src/dotty/tools/languageserver/DottyLanguageServer.scala index 4c11277a294a..f70310f21d30 100644 --- a/language-server/src/dotty/tools/languageserver/DottyLanguageServer.scala +++ b/language-server/src/dotty/tools/languageserver/DottyLanguageServer.scala @@ -100,18 +100,7 @@ class DottyLanguageServer extends LanguageServer } private def checkMemory() = - if (Memory.isCritical()) - CompletableFutures.computeAsync { _ => restart(); new Object() } - // new Object() necessary or we get a BootstrapMethodError: - // - // Caused by: java.lang.invoke.LambdaConversionException: Type mismatch for lambda expected return: void is not convertible to class java.lang.Object - // at java.lang.invoke.AbstractValidatingLambdaMetafactory.validateMetafactoryArgs(AbstractValidatingLambdaMetafactory.java:286) - // at java.lang.invoke.LambdaMetafactory.metafactory(LambdaMetafactory.java:303) - // at java.lang.invoke.CallSite.makeSite(CallSite.java:302) - // ... 11 more - // - // This looks like a problem with Dottys code generation for void-returning closures passed - // to Java methods. (or SAM functions in general?) + if (Memory.isCritical()) CompletableFutures.computeAsync { _ => restart() } /** The driver instance responsible for compiling `uri` */ def driverFor(uri: URI): InteractiveDriver = { From 659ab0becaa1637fd37b0962d204c4d4f03f9a0c Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 27 Feb 2018 18:24:52 +0100 Subject: [PATCH 4/4] Avoid NoMatchError in IDE I observed a NoType when editing some code which caused ThisType#underlying to crash. --- compiler/src/dotty/tools/dotc/core/Types.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/core/Types.scala b/compiler/src/dotty/tools/dotc/core/Types.scala index c85cbcdf2622..30a1c4b1aa3b 100644 --- a/compiler/src/dotty/tools/dotc/core/Types.scala +++ b/compiler/src/dotty/tools/dotc/core/Types.scala @@ -2184,7 +2184,7 @@ object Types { if (ctx.erasedTypes) tref else cls.info match { case cinfo: ClassInfo => cinfo.selfType - case cinfo: ErrorType if ctx.mode.is(Mode.Interactive) => cinfo + case _: ErrorType | NoType if ctx.mode.is(Mode.Interactive) => cls.info // can happen in IDE if `cls` is stale }