From 1093c41eba5b6fd6148244ab8ee7f7205dc815fd Mon Sep 17 00:00:00 2001 From: Allan Renucci Date: Tue, 11 Dec 2018 15:47:27 +0100 Subject: [PATCH 1/3] Cleanup tests - Remove std lib blacklist - Factor out some code --- .../test/dotc/scala-collections.blacklist | 5 -- compiler/test/dotty/tools/TestSources.scala | 65 ++++++++++++------- .../dotty/tools/dotc/CompilationTests.scala | 43 ++++-------- .../dotty/tools/dotc/IdempotencyTests.scala | 5 +- .../tools/dottydoc/WhitelistedStdLib.scala | 2 +- 5 files changed, 59 insertions(+), 61 deletions(-) delete mode 100644 compiler/test/dotc/scala-collections.blacklist diff --git a/compiler/test/dotc/scala-collections.blacklist b/compiler/test/dotc/scala-collections.blacklist deleted file mode 100644 index dead17647e64..000000000000 --- a/compiler/test/dotc/scala-collections.blacklist +++ /dev/null @@ -1,5 +0,0 @@ -## Compiling the scala.reflect package object together with other files sometimes leads to cycles, -## but only when compiling with a non-bootstrapped dotty-library, because unpickling DottyPredef -## requires scala.reflect.ScalaSignature which forces the scala.reflect package object. -## Since we'll eventually be fully bootstrapped, it's not worth fixing by adding workarounds. -scala/reflect/package.scala diff --git a/compiler/test/dotty/tools/TestSources.scala b/compiler/test/dotty/tools/TestSources.scala index 034d67466135..1a90e47d2a4d 100644 --- a/compiler/test/dotty/tools/TestSources.scala +++ b/compiler/test/dotty/tools/TestSources.scala @@ -1,27 +1,18 @@ package dotty.tools import java.io.File +import java.nio.file._ -import scala.io.Source +import scala.collection.JavaConverters._ object TestSources { // Std Lib - - private final val stdLibPath = "tests/scala2-library/src/library/" - - private def blacklistFile: String = "compiler/test/dotc/scala-collections.blacklist" - - def stdLibWhitelisted: List[String] = all.diff(stdLibBlacklisted) - def stdLibBlacklisted: List[String] = loadList(blacklistFile).map(stdLibPath + _) - - private def all: List[String] = { - def collectAllFilesInDir(dir: File, acc: List[String]): List[String] = { - val files = dir.listFiles() - val acc2 = files.foldLeft(acc)((acc1, file) => if (file.isFile && file.getPath.endsWith(".scala")) file.getPath :: acc1 else acc1) - files.foldLeft(acc2)((acc3, file) => if (file.isDirectory) collectAllFilesInDir(file, acc3) else acc3) - } - collectAllFilesInDir(new File(stdLibPath), Nil).sorted + def stdLibSources: List[String] = { + val blacklisted = List( + "StructuralCallSite.java" // See #4739 + ) + sources(Paths.get("tests/scala2-library/src/library/"), excludedFiles = blacklisted) } // pos tests lists @@ -50,11 +41,41 @@ object TestSources { // load lists - private def loadList(path: String): List[String] = Source.fromFile(path, "UTF8").getLines() - .map(_.trim) // allow identation - .filter(!_.startsWith("#")) // allow comment lines prefixed by # - .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line - .filter(_.nonEmpty) - .toList + private def loadList(path: String): List[String] = { + val list = Files.readAllLines(Paths.get(path)) + .iterator() + .asScala + .map(_.trim) // allow identation + .filterNot(_.startsWith("#")) // allow comment lines prefixed by # + .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line + .filter(_.nonEmpty) + .toList + + assert(list.nonEmpty) + list + } + + /** Retrieve sources from a directory */ + def sources(path: Path, excludedFiles: List[String] = Nil, shallow: Boolean = false): List[String] = { + def fileFilter(path: Path) = { + val fileName = path.getFileName.toString + (fileName.endsWith(".scala") || fileName.endsWith(".java")) && !excludedFiles.contains(fileName) + } + assert(Files.isDirectory(path)) + val files = if (shallow) Files.list(path) else Files.walk(path) + try { + val sources = files + .filter(fileFilter) + .sorted // make compilation order deterministic + .iterator() + .asScala + .map(_.toString) + .toList + + assert(sources.nonEmpty) + sources + } + finally files.close() + } } diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 9c86c8cdc818..523c3cef3a24 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -12,8 +12,8 @@ import java.util.stream.{ Stream => JStream } import scala.collection.JavaConverters._ import scala.util.matching.Regex import scala.concurrent.duration._ +import TestSources.sources import vulpix._ -import dotty.tools.io.JFile class CompilationTests extends ParallelTesting { import ParallelTesting._ @@ -34,12 +34,12 @@ class CompilationTests extends ParallelTesting { // @Test // enable to test compileStdLib separately with detailed stats def compileStdLibOnly: Unit = { implicit val testGroup: TestGroup = TestGroup("compileStdLibOnly") - compileList("compileStdLib", TestSources.stdLibWhitelisted, scala2Mode.and("-migration", "-Yno-inline", "-Ydetailed-stats")) + compileList("compileStdLib", TestSources.stdLibSources, scala2Mode.and("-migration", "-Yno-inline", "-Ydetailed-stats")) }.checkCompile() @Test def pos: Unit = { implicit val testGroup: TestGroup = TestGroup("compilePos") - compileList("compileStdLib", TestSources.stdLibWhitelisted, scala2Mode.and("-migration", "-Yno-inline")) + + compileList("compileStdLib", TestSources.stdLibSources, scala2Mode.and("-migration", "-Yno-inline")) + compileFile("tests/pos/nullarify.scala", defaultOptions.and("-Ycheck:nullarify")) + compileFile("tests/pos-scala2/rewrites.scala", scala2Mode.and("-rewrite")).copyToTarget() + compileFile("tests/pos-special/utf8encoded.scala", explicitUTF8) + @@ -232,17 +232,16 @@ class CompilationTests extends ParallelTesting { ) val libraryDirs = List(Paths.get("library/src"), Paths.get("library/src-bootstrapped")) - val librarySources = libraryDirs.flatMap(d => sources(Files.walk(d))) + val librarySources = libraryDirs.flatMap(sources(_)) val lib = compileList("src", librarySources, defaultOptions.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))(libGroup) - val compilerDir = Paths.get("compiler/src") - val compilerSources = sources(Files.walk(compilerDir)) + val compilerSources = sources(Paths.get("compiler/src")) val scalaJSIRDir = Paths.get("compiler/target/scala-2.12/src_managed/main/scalajs-ir-src/org/scalajs/ir") - val scalaJSIRSources = sources(Files.list(scalaJSIRDir)) + val scalaJSIRSources = sources(scalaJSIRDir, shallow = true) val dotty1 = compileList("dotty", compilerSources ++ scalaJSIRSources, opt)(dotty1Group) val dotty2 = compileList("dotty", compilerSources ++ scalaJSIRSources, opt)(dotty2Group) @@ -267,9 +266,10 @@ class CompilationTests extends ParallelTesting { }.keepOutput :: Nil }.map(_.checkCompile()) - assert(new java.io.File(s"out/$dotty1Group/dotty/").exists) - assert(new java.io.File(s"out/$dotty2Group/dotty/").exists) - assert(new java.io.File(s"out/$libGroup/src/").exists) + def assertExists(path: String) = assertTrue(Files.exists(Paths.get(path))) + assertExists(s"out/$dotty1Group/dotty/") + assertExists(s"out/$dotty2Group/dotty/") + assertExists(s"out/$libGroup/src/") compileList("idempotency", List("tests/idempotency/BootstrapChecker.scala", "tests/idempotency/IdempotencyCheck.scala"), defaultOptions).checkRuns() tests.foreach(_.delete()) @@ -282,17 +282,13 @@ class CompilationTests extends ParallelTesting { // 2. copy `pluginFile` to destination def compileFilesInDir(dir: String): CompilationTest = { val outDir = defaultOutputDir + "testPlugins/" - val sourceDir = new JFile(dir) - - val dirs = sourceDir.listFiles.foldLeft(List.empty[JFile]) { case (dirs, f) => - if (f.isDirectory) f :: dirs else dirs - } + val sourceDir = new java.io.File(dir) + val dirs = sourceDir.listFiles.toList.filter(_.isDirectory) val targets = dirs.map { dir => val compileDir = createOutputDirsForDir(dir, sourceDir, outDir) - import java.nio.file.StandardCopyOption.REPLACE_EXISTING - Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), REPLACE_EXISTING) - val flags = TestFlags(withCompilerClasspath, noCheckOptions) and ("-Xplugin:" + compileDir.getAbsolutePath) + Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), StandardCopyOption.REPLACE_EXISTING) + val flags = TestFlags(withCompilerClasspath, noCheckOptions).and("-Xplugin:" + compileDir.getAbsolutePath) SeparateCompilationSource("testPlugins", dir, flags, compileDir) } @@ -306,15 +302,4 @@ class CompilationTests extends ParallelTesting { object CompilationTests { implicit val summaryReport: SummaryReporting = new SummaryReport @AfterClass def cleanup(): Unit = summaryReport.echoSummary() - - def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] = { - val sources = paths.iterator().asScala - .filter(path => - (path.toString.endsWith(".scala") || path.toString.endsWith(".java")) - && !excludedFiles.contains(path.getFileName.toString)) - .map(_.toString).toList - - paths.close() - sources - } } diff --git a/compiler/test/dotty/tools/dotc/IdempotencyTests.scala b/compiler/test/dotty/tools/dotc/IdempotencyTests.scala index e0e1d5ee879a..1a565ca97f2f 100644 --- a/compiler/test/dotty/tools/dotc/IdempotencyTests.scala +++ b/compiler/test/dotty/tools/dotc/IdempotencyTests.scala @@ -31,9 +31,6 @@ class IdempotencyTests extends ParallelTesting { implicit val testGroup: TestGroup = TestGroup("idempotency") val opt = defaultOptions - def sourcesFrom(dir: Path) = CompilationTests.sources(Files.walk(dir)) - - val posIdempotency = { compileFilesInDir("tests/pos", opt)(TestGroup("idempotency/posIdempotency1")) + compileFilesInDir("tests/pos", opt)(TestGroup("idempotency/posIdempotency2")) @@ -43,7 +40,7 @@ class IdempotencyTests extends ParallelTesting { (for { testDir <- new JFile("tests/order-idempotency").listFiles() if testDir.isDirectory } yield { - val sources = sourcesFrom(testDir.toPath) + val sources = TestSources.sources(testDir.toPath) compileList(testDir.getName, sources, opt)(TestGroup("idempotency/orderIdempotency1")) + compileList(testDir.getName, sources.reverse, opt)(TestGroup("idempotency/orderIdempotency2")) }).reduce(_ + _) diff --git a/doc-tool/test/dotty/tools/dottydoc/WhitelistedStdLib.scala b/doc-tool/test/dotty/tools/dottydoc/WhitelistedStdLib.scala index dc94b404c94e..ba15e46a9ee9 100644 --- a/doc-tool/test/dotty/tools/dottydoc/WhitelistedStdLib.scala +++ b/doc-tool/test/dotty/tools/dottydoc/WhitelistedStdLib.scala @@ -32,6 +32,6 @@ class TestWhitelistedCollections extends DottyDocTest with CheckFromSource { object TestWhitelistedCollections { val files: List[String] = - TestSources.stdLibWhitelisted + TestSources.stdLibSources .filterNot(_.endsWith("package.scala")) } From 0a5e1a6a666c50652d050a7e4d92c88ae5449033 Mon Sep 17 00:00:00 2001 From: Allan Renucci Date: Fri, 14 Dec 2018 11:13:20 +0100 Subject: [PATCH 2/3] Blacklist volatile in std lib test --- compiler/test/dotty/tools/TestSources.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/test/dotty/tools/TestSources.scala b/compiler/test/dotty/tools/TestSources.scala index 1a90e47d2a4d..57628e3c5106 100644 --- a/compiler/test/dotty/tools/TestSources.scala +++ b/compiler/test/dotty/tools/TestSources.scala @@ -10,7 +10,8 @@ object TestSources { // Std Lib def stdLibSources: List[String] = { val blacklisted = List( - "StructuralCallSite.java" // See #4739 + "StructuralCallSite.java", // see #4739 + "volatile.scala", // see #5610 ) sources(Paths.get("tests/scala2-library/src/library/"), excludedFiles = blacklisted) } From affdaf2b2232e08047004773bb2f9b1ef2c6f3c3 Mon Sep 17 00:00:00 2001 From: Allan Renucci Date: Tue, 5 Mar 2019 16:50:45 +0100 Subject: [PATCH 3/3] Remove StructuralCallSite.java from stdlib blacklist Following fix to #4739. --- compiler/test/dotty/tools/TestSources.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/test/dotty/tools/TestSources.scala b/compiler/test/dotty/tools/TestSources.scala index 57628e3c5106..32712763f61c 100644 --- a/compiler/test/dotty/tools/TestSources.scala +++ b/compiler/test/dotty/tools/TestSources.scala @@ -10,7 +10,6 @@ object TestSources { // Std Lib def stdLibSources: List[String] = { val blacklisted = List( - "StructuralCallSite.java", // see #4739 "volatile.scala", // see #5610 ) sources(Paths.get("tests/scala2-library/src/library/"), excludedFiles = blacklisted) @@ -46,7 +45,7 @@ object TestSources { val list = Files.readAllLines(Paths.get(path)) .iterator() .asScala - .map(_.trim) // allow identation + .map(_.trim) // allow indentation .filterNot(_.startsWith("#")) // allow comment lines prefixed by # .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line .filter(_.nonEmpty)