Cleanup tests

- Remove std lib blacklist
- Factor out some code

Author: allanrenucci

compiler/test/dotc/scala-collections.blacklist

@@ -1,27 +1,18 @@
 package dotty.tools
 
 import java.io.File
+import java.nio.file._
 
-import scala.io.Source
+import scala.collection.JavaConverters._
 
 object TestSources {
 
   // Std Lib
-
-  private final val stdLibPath = "scala2-library/src/library/"
-
-  private def blacklistFile: String = "compiler/test/dotc/scala-collections.blacklist"
-
-  def stdLibWhitelisted: List[String] = all.diff(stdLibBlacklisted)
-  def stdLibBlacklisted: List[String] = loadList(blacklistFile).map(stdLibPath + _)
-
-  private def all: List[String] = {
-    def collectAllFilesInDir(dir: File, acc: List[String]): List[String] = {
-      val files = dir.listFiles()
-      val acc2 = files.foldLeft(acc)((acc1, file) => if (file.isFile && file.getPath.endsWith(".scala")) file.getPath :: acc1 else acc1)
-      files.foldLeft(acc2)((acc3, file) => if (file.isDirectory) collectAllFilesInDir(file, acc3) else acc3)
-    }
-    collectAllFilesInDir(new File(stdLibPath), Nil)
+  def stdLibSources: List[String] = {
+    val blacklisted = List(
+      "StructuralCallSite.java" // See #4739
+    )
+    sources(Paths.get("scala2-library/src/library/"), excludedFiles = blacklisted)
   }
 
   // pos tests lists
@@ -50,11 +41,41 @@ object TestSources {
 
   // load lists
 
-  private def loadList(path: String): List[String] = Source.fromFile(path, "UTF8").getLines()
-    .map(_.trim) // allow identation
-    .filter(!_.startsWith("#")) // allow comment lines prefixed by #
-    .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line
-    .filter(_.nonEmpty)
-    .toList
+  private def loadList(path: String): List[String] = {
+    val list = Files.readAllLines(Paths.get(path))
+      .iterator()
+      .asScala
+      .map(_.trim)                     // allow identation
+      .filterNot(_.startsWith("#"))    // allow comment lines prefixed by #
+      .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line
+      .filter(_.nonEmpty)
+      .toList
+
+    assert(list.nonEmpty)
+    list
+  }
+
+  /** Retrieve sources from a directory */
+  def sources(path: Path, excludedFiles: List[String] = Nil, shallow: Boolean = false): List[String] = {
+    def fileFilter(path: Path) = {
+      val fileName = path.getFileName.toString
+      (fileName.endsWith(".scala") || fileName.endsWith(".java")) && !excludedFiles.contains(fileName)
+    }
 
+    assert(Files.isDirectory(path))
+    val files = Files.walk(path, if (shallow) 1 else Int.MaxValue)
+    try {
+      val sources = files
+        .filter(fileFilter)
+        .sorted // make compilation order deterministic
+        .iterator()
+        .asScala
+        .map(_.toString)
+        .toList
+
+      assert(sources.nonEmpty)
+      sources
+    }
+    finally files.close()
+  }
 }

compiler/test/dotty/tools/TestSources.scala

@@ -12,8 +12,8 @@ import java.util.stream.{ Stream => JStream }
 import scala.collection.JavaConverters._
 import scala.util.matching.Regex
 import scala.concurrent.duration._
+import TestSources.sources
 import vulpix._
-import dotty.tools.io.JFile
 
 
 class CompilationTests extends ParallelTesting {
@@ -34,12 +34,12 @@ class CompilationTests extends ParallelTesting {
   // @Test  // enable to test compileStdLib separately with detailed stats
   def compileStdLibOnly: Unit = {
     implicit val testGroup: TestGroup = TestGroup("compileStdLibOnly")
-    compileList("compileStdLib", TestSources.stdLibWhitelisted, scala2Mode.and("-migration", "-Yno-inline", "-Ydetailed-stats"))
+    compileList("compileStdLib", TestSources.stdLibSources, scala2Mode.and("-migration", "-Yno-inline", "-Ydetailed-stats"))
   }.checkCompile()
 
   @Test def pos: Unit = {
     implicit val testGroup: TestGroup = TestGroup("compilePos")
-    compileList("compileStdLib", TestSources.stdLibWhitelisted, scala2Mode.and("-migration", "-Yno-inline")) +
+    compileList("compileStdLib", TestSources.stdLibSources, scala2Mode.and("-migration", "-Yno-inline")) +
     compileFile("tests/pos/nullarify.scala", defaultOptions.and("-Ycheck:nullarify")) +
     compileFile("tests/pos-scala2/rewrites.scala", scala2Mode.and("-rewrite")).copyToTarget() +
     compileFile("tests/pos-special/utf8encoded.scala", explicitUTF8) +
@@ -218,14 +218,14 @@ class CompilationTests extends ParallelTesting {
     )
 
     val libraryDirs = List(Paths.get("library/src"), Paths.get("library/src-bootstrapped"))
-    val librarySources = libraryDirs.flatMap(d => sources(Files.walk(d)))
+    val librarySources = libraryDirs.flatMap(sources(_))
 
     val lib =
       compileList("src", librarySources,
         defaultOptions.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))(libGroup)
 
     val compilerDir = Paths.get("compiler/src")
-    val compilerSources = sources(Files.walk(compilerDir))
+    val compilerSources = sources(compilerDir)
 
     val backendDir = Paths.get("scala-backend/src/compiler/scala/tools/nsc/backend")
     val backendJvmDir = Paths.get("scala-backend/src/compiler/scala/tools/nsc/backend/jvm")
@@ -238,11 +238,11 @@ class CompilationTests extends ParallelTesting {
       List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala", "BackendStats.scala", "BCodeAsmEncode.scala")
 
     val backendSources =
-      sources(Files.list(backendDir), excludedFiles = backendExcluded)
+      sources(backendDir, excludedFiles = backendExcluded, shallow = true)
     val backendJvmSources =
-      sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded)
+      sources(backendJvmDir, excludedFiles = backendJvmExcluded, shallow = true)
     val scalaJSIRSources =
-      sources(Files.list(scalaJSIRDir))
+      sources(scalaJSIRDir, shallow = true)
 
     val dotty1 = compileList("dotty", compilerSources ++ backendSources ++ backendJvmSources ++ scalaJSIRSources, opt)(dotty1Group)
     val dotty2 = compileList("dotty", compilerSources ++ backendSources ++ backendJvmSources ++ scalaJSIRSources, opt)(dotty2Group)
@@ -267,9 +267,10 @@ class CompilationTests extends ParallelTesting {
       }.keepOutput :: Nil
     }.map(_.checkCompile())
 
-    assert(new java.io.File(s"out/$dotty1Group/dotty/").exists)
-    assert(new java.io.File(s"out/$dotty2Group/dotty/").exists)
-    assert(new java.io.File(s"out/$libGroup/src/").exists)
+    def assertExists(path: String) = assertTrue(Files.exists(Paths.get(path)))
+    assertExists(s"out/$dotty1Group/dotty/")
+    assertExists(s"out/$dotty2Group/dotty/")
+    assertExists(s"out/$libGroup/src/")
     compileList("idempotency", List("tests/idempotency/BootstrapChecker.scala", "tests/idempotency/IdempotencyCheck.scala"), defaultOptions).checkRuns()
 
     tests.foreach(_.delete())
@@ -282,17 +283,13 @@ class CompilationTests extends ParallelTesting {
     // 2. copy `pluginFile` to destination
     def compileFilesInDir(dir: String): CompilationTest = {
       val outDir = defaultOutputDir + "testPlugins/"
-      val sourceDir = new JFile(dir)
-
-      val dirs = sourceDir.listFiles.foldLeft(List.empty[JFile]) { case (dirs, f) =>
-        if (f.isDirectory) f :: dirs else dirs
-      }
+      val sourceDir = new java.io.File(dir)
 
+      val dirs = sourceDir.listFiles.toList.filter(_.isDirectory)
       val targets = dirs.map { dir =>
         val compileDir = createOutputDirsForDir(dir, sourceDir, outDir)
-        import java.nio.file.StandardCopyOption.REPLACE_EXISTING
-        Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), REPLACE_EXISTING)
-        val flags = TestFlags(withCompilerClasspath, noCheckOptions) and ("-Xplugin:" + compileDir.getAbsolutePath)
+        Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), StandardCopyOption.REPLACE_EXISTING)
+        val flags = TestFlags(withCompilerClasspath, noCheckOptions).and("-Xplugin:" + compileDir.getAbsolutePath)
         SeparateCompilationSource("testPlugins", dir, flags, compileDir)
       }
 
@@ -306,15 +303,4 @@ class CompilationTests extends ParallelTesting {
 object CompilationTests {
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def cleanup(): Unit = summaryReport.echoSummary()
-
-  def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] = {
-    val sources = paths.iterator().asScala
-      .filter(path =>
-        (path.toString.endsWith(".scala") || path.toString.endsWith(".java"))
-          && !excludedFiles.contains(path.getFileName.toString))
-      .map(_.toString).toList
-
-    paths.close()
-    sources
-  }
 }

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -30,9 +30,7 @@ class IdempotencyTests extends ParallelTesting {
     implicit val testGroup: TestGroup = TestGroup("idempotency")
     val opt = defaultOptions
 
-    def sourcesFrom(dir: Path) = CompilationTests.sources(Files.walk(dir))
-
-    val strawmanSources = sourcesFrom(Paths.get("collection-strawman/collections/src/main"))
+    val strawmanSources = TestSources.sources(Paths.get("collection-strawman/collections/src/main"))
     val strawmanSourcesSorted = strawmanSources.sorted
     val strawmanSourcesRevSorted = strawmanSourcesSorted.reverse
 
@@ -45,7 +43,7 @@ class IdempotencyTests extends ParallelTesting {
       (for {
         testDir <- new JFile("tests/order-idempotency").listFiles() if testDir.isDirectory
       } yield {
-        val sources = sourcesFrom(testDir.toPath)
+        val sources = TestSources.sources(testDir.toPath)
         compileList(testDir.getName, sources, opt)(TestGroup("idempotency/orderIdempotency1")) +
         compileList(testDir.getName, sources.reverse, opt)(TestGroup("idempotency/orderIdempotency2"))
       }).reduce(_ + _)

compiler/test/dotty/tools/dotc/IdempotencyTests.scala

@@ -32,6 +32,6 @@ class TestWhitelistedCollections extends DottyDocTest with CheckFromSource {
 
 object TestWhitelistedCollections {
   val files: List[String] =
-    TestSources.stdLibWhitelisted
+    TestSources.stdLibSources
     .filterNot(_.endsWith("package.scala"))
 }