Merge pull request #5600 from dotty-staging/test-cleanup

Cleanup tests

Author: allanrenucci

compiler/test/dotc/scala-collections.blacklist

@@ -1,27 +1,18 @@
 package dotty.tools
 
 import java.io.File
+import java.nio.file._
 
-import scala.io.Source
+import scala.collection.JavaConverters._
 
 object TestSources {
 
   // Std Lib
-
-  private final val stdLibPath = "tests/scala2-library/src/library/"
-
-  private def blacklistFile: String = "compiler/test/dotc/scala-collections.blacklist"
-
-  def stdLibWhitelisted: List[String] = all.diff(stdLibBlacklisted)
-  def stdLibBlacklisted: List[String] = loadList(blacklistFile).map(stdLibPath + _)
-
-  private def all: List[String] = {
-    def collectAllFilesInDir(dir: File, acc: List[String]): List[String] = {
-      val files = dir.listFiles()
-      val acc2 = files.foldLeft(acc)((acc1, file) => if (file.isFile && file.getPath.endsWith(".scala")) file.getPath :: acc1 else acc1)
-      files.foldLeft(acc2)((acc3, file) => if (file.isDirectory) collectAllFilesInDir(file, acc3) else acc3)
-    }
-    collectAllFilesInDir(new File(stdLibPath), Nil).sorted
+  def stdLibSources: List[String] = {
+    val blacklisted = List(
+      "volatile.scala",          // see #5610
+    )
+    sources(Paths.get("tests/scala2-library/src/library/"), excludedFiles = blacklisted)
   }
 
   // pos tests lists
@@ -50,11 +41,41 @@ object TestSources {
 
   // load lists
 
-  private def loadList(path: String): List[String] = Source.fromFile(path, "UTF8").getLines()
-    .map(_.trim) // allow identation
-    .filter(!_.startsWith("#")) // allow comment lines prefixed by #
-    .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line
-    .filter(_.nonEmpty)
-    .toList
+  private def loadList(path: String): List[String] = {
+    val list = Files.readAllLines(Paths.get(path))
+      .iterator()
+      .asScala
+      .map(_.trim)                     // allow indentation
+      .filterNot(_.startsWith("#"))    // allow comment lines prefixed by #
+      .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line
+      .filter(_.nonEmpty)
+      .toList
+
+    assert(list.nonEmpty)
+    list
+  }
+
+  /** Retrieve sources from a directory */
+  def sources(path: Path, excludedFiles: List[String] = Nil, shallow: Boolean = false): List[String] = {
+    def fileFilter(path: Path) = {
+      val fileName = path.getFileName.toString
+      (fileName.endsWith(".scala") || fileName.endsWith(".java")) && !excludedFiles.contains(fileName)
+    }
 
+    assert(Files.isDirectory(path))
+    val files = if (shallow) Files.list(path) else Files.walk(path)
+    try {
+      val sources = files
+        .filter(fileFilter)
+        .sorted // make compilation order deterministic
+        .iterator()
+        .asScala
+        .map(_.toString)
+        .toList
+
+      assert(sources.nonEmpty)
+      sources
+    }
+    finally files.close()
+  }
 }

compiler/test/dotty/tools/TestSources.scala

@@ -12,8 +12,8 @@ import java.util.stream.{ Stream => JStream }
 import scala.collection.JavaConverters._
 import scala.util.matching.Regex
 import scala.concurrent.duration._
+import TestSources.sources
 import vulpix._
-import dotty.tools.io.JFile
 
 class CompilationTests extends ParallelTesting {
   import ParallelTesting._
@@ -34,12 +34,12 @@ class CompilationTests extends ParallelTesting {
   // @Test  // enable to test compileStdLib separately with detailed stats
   def compileStdLibOnly: Unit = {
     implicit val testGroup: TestGroup = TestGroup("compileStdLibOnly")
-    compileList("compileStdLib", TestSources.stdLibWhitelisted, scala2Mode.and("-migration", "-Yno-inline", "-Ydetailed-stats"))
+    compileList("compileStdLib", TestSources.stdLibSources, scala2Mode.and("-migration", "-Yno-inline", "-Ydetailed-stats"))
   }.checkCompile()
 
   @Test def pos: Unit = {
     implicit val testGroup: TestGroup = TestGroup("compilePos")
-    compileList("compileStdLib", TestSources.stdLibWhitelisted, scala2Mode.and("-migration", "-Yno-inline")) +
+    compileList("compileStdLib", TestSources.stdLibSources, scala2Mode.and("-migration", "-Yno-inline")) +
     compileFile("tests/pos/nullarify.scala", defaultOptions.and("-Ycheck:nullarify")) +
     compileFile("tests/pos-scala2/rewrites.scala", scala2Mode.and("-rewrite")).copyToTarget() +
     compileFile("tests/pos-special/utf8encoded.scala", explicitUTF8) +
@@ -232,17 +232,16 @@ class CompilationTests extends ParallelTesting {
     )
 
     val libraryDirs = List(Paths.get("library/src"), Paths.get("library/src-bootstrapped"))
-    val librarySources = libraryDirs.flatMap(d => sources(Files.walk(d)))
+    val librarySources = libraryDirs.flatMap(sources(_))
 
     val lib =
       compileList("src", librarySources,
         defaultOptions.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))(libGroup)
 
-    val compilerDir = Paths.get("compiler/src")
-    val compilerSources = sources(Files.walk(compilerDir))
+    val compilerSources = sources(Paths.get("compiler/src"))
 
     val scalaJSIRDir = Paths.get("compiler/target/scala-2.12/src_managed/main/scalajs-ir-src/org/scalajs/ir")
-    val scalaJSIRSources = sources(Files.list(scalaJSIRDir))
+    val scalaJSIRSources = sources(scalaJSIRDir, shallow = true)
 
     val dotty1 = compileList("dotty", compilerSources ++ scalaJSIRSources, opt)(dotty1Group)
     val dotty2 = compileList("dotty", compilerSources ++ scalaJSIRSources, opt)(dotty2Group)
@@ -267,9 +266,10 @@ class CompilationTests extends ParallelTesting {
       }.keepOutput :: Nil
     }.map(_.checkCompile())
 
-    assert(new java.io.File(s"out/$dotty1Group/dotty/").exists)
-    assert(new java.io.File(s"out/$dotty2Group/dotty/").exists)
-    assert(new java.io.File(s"out/$libGroup/src/").exists)
+    def assertExists(path: String) = assertTrue(Files.exists(Paths.get(path)))
+    assertExists(s"out/$dotty1Group/dotty/")
+    assertExists(s"out/$dotty2Group/dotty/")
+    assertExists(s"out/$libGroup/src/")
     compileList("idempotency", List("tests/idempotency/BootstrapChecker.scala", "tests/idempotency/IdempotencyCheck.scala"), defaultOptions).checkRuns()
 
     tests.foreach(_.delete())
@@ -282,17 +282,13 @@ class CompilationTests extends ParallelTesting {
     // 2. copy `pluginFile` to destination
     def compileFilesInDir(dir: String): CompilationTest = {
       val outDir = defaultOutputDir + "testPlugins/"
-      val sourceDir = new JFile(dir)
-
-      val dirs = sourceDir.listFiles.foldLeft(List.empty[JFile]) { case (dirs, f) =>
-        if (f.isDirectory) f :: dirs else dirs
-      }
+      val sourceDir = new java.io.File(dir)
 
+      val dirs = sourceDir.listFiles.toList.filter(_.isDirectory)
       val targets = dirs.map { dir =>
         val compileDir = createOutputDirsForDir(dir, sourceDir, outDir)
-        import java.nio.file.StandardCopyOption.REPLACE_EXISTING
-        Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), REPLACE_EXISTING)
-        val flags = TestFlags(withCompilerClasspath, noCheckOptions) and ("-Xplugin:" + compileDir.getAbsolutePath)
+        Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), StandardCopyOption.REPLACE_EXISTING)
+        val flags = TestFlags(withCompilerClasspath, noCheckOptions).and("-Xplugin:" + compileDir.getAbsolutePath)
         SeparateCompilationSource("testPlugins", dir, flags, compileDir)
       }
 
@@ -306,15 +302,4 @@ class CompilationTests extends ParallelTesting {
 object CompilationTests {
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def cleanup(): Unit = summaryReport.echoSummary()
-
-  def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] = {
-    val sources = paths.iterator().asScala
-      .filter(path =>
-        (path.toString.endsWith(".scala") || path.toString.endsWith(".java"))
-          && !excludedFiles.contains(path.getFileName.toString))
-      .map(_.toString).toList
-
-    paths.close()
-    sources
-  }
 }

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -31,9 +31,6 @@ class IdempotencyTests extends ParallelTesting {
     implicit val testGroup: TestGroup = TestGroup("idempotency")
     val opt = defaultOptions
 
-    def sourcesFrom(dir: Path) = CompilationTests.sources(Files.walk(dir))
-
-
     val posIdempotency = {
       compileFilesInDir("tests/pos", opt)(TestGroup("idempotency/posIdempotency1")) +
       compileFilesInDir("tests/pos", opt)(TestGroup("idempotency/posIdempotency2"))
@@ -43,7 +40,7 @@ class IdempotencyTests extends ParallelTesting {
       (for {
         testDir <- new JFile("tests/order-idempotency").listFiles() if testDir.isDirectory
       } yield {
-        val sources = sourcesFrom(testDir.toPath)
+        val sources = TestSources.sources(testDir.toPath)
         compileList(testDir.getName, sources, opt)(TestGroup("idempotency/orderIdempotency1")) +
         compileList(testDir.getName, sources.reverse, opt)(TestGroup("idempotency/orderIdempotency2"))
       }).reduce(_ + _)

compiler/test/dotty/tools/dotc/IdempotencyTests.scala

@@ -32,6 +32,6 @@ class TestWhitelistedCollections extends DottyDocTest with CheckFromSource {
 
 object TestWhitelistedCollections {
   val files: List[String] =
-    TestSources.stdLibWhitelisted
+    TestSources.stdLibSources
     .filterNot(_.endsWith("package.scala"))
 }