Implement :jar (deprecate :require) (#22343)

#21658

Author: aherlihy

compiler/src/dotty/tools/dotc/config/JavaPlatform.scala

@@ -27,6 +27,13 @@ class JavaPlatform extends Platform {
       case _ => false
     })
 
+  def addToClassPath(cPath: ClassPath)(using Context): Unit = classPath match {
+    case AggregateClassPath(entries) =>
+      currentClassPath = Some(AggregateClassPath(entries :+ cPath))
+    case cp: ClassPath =>
+      currentClassPath = Some(AggregateClassPath(cp :: cPath :: Nil))
+  }
+
   /** Update classpath with a substituted subentry */
   def updateClassPath(subst: Map[ClassPath, ClassPath]): Unit = currentClassPath.get match {
     case AggregateClassPath(entries) =>

compiler/src/dotty/tools/dotc/config/Platform.scala

@@ -21,6 +21,9 @@ abstract class Platform {
   /** Update classpath with a substitution that maps entries to entries */
   def updateClassPath(subst: Map[ClassPath, ClassPath]): Unit
 
+  /** Add new entry to classpath */
+  def addToClassPath(cPath: ClassPath)(using Context): Unit
+
   /** Any platform-specific phases. */
   //def platformPhases: List[SubComponent]
 

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -7,6 +7,7 @@ import java.nio.channels.ClosedByInterruptException
 
 import scala.util.control.NonFatal
 
+import dotty.tools.dotc.classpath.{ ClassPathFactory, PackageNameUtils }
 import dotty.tools.dotc.classpath.FileUtils.{hasTastyExtension, hasBetastyExtension}
 import dotty.tools.io.{ ClassPath, ClassRepresentation, AbstractFile, NoAbstractFile }
 import dotty.tools.backend.jvm.DottyBackendInterface.symExtensions
@@ -272,7 +273,7 @@ object SymbolLoaders {
     def maybeModuleClass(classRep: ClassRepresentation): Boolean =
       classRep.name.nonEmpty && classRep.name.last == '$'
 
-    private def enterClasses(root: SymDenotation, packageName: String, flat: Boolean)(using Context) = {
+    def enterClasses(root: SymDenotation, packageName: String, flat: Boolean)(using Context) = {
       def isAbsent(classRep: ClassRepresentation) =
         !root.unforcedDecls.lookup(classRep.name.toTypeName).exists
 
@@ -316,6 +317,32 @@ object SymbolLoaders {
         }
     }
   }
+
+  def mergeNewEntries(
+    packageClass: ClassSymbol, fullPackageName: String,
+    jarClasspath: ClassPath, fullClasspath: ClassPath,
+  )(using Context): Unit =
+    if jarClasspath.classes(fullPackageName).nonEmpty then
+      // if the package contains classes in jarClasspath, the package is invalidated (or removed if there are no more classes in it)
+      val packageVal = packageClass.sourceModule.asInstanceOf[TermSymbol]
+      if packageClass.isRoot then
+        val loader = new PackageLoader(packageVal, fullClasspath)
+        loader.enterClasses(defn.EmptyPackageClass, fullPackageName, flat = false)
+        loader.enterClasses(defn.EmptyPackageClass, fullPackageName, flat = true)
+      else if packageClass.ownersIterator.contains(defn.ScalaPackageClass) then
+        () // skip
+      else if fullClasspath.hasPackage(fullPackageName) then
+        packageClass.info = new PackageLoader(packageVal, fullClasspath)
+      else
+        packageClass.owner.info.decls.openForMutations.unlink(packageVal)
+    else
+      for p <- jarClasspath.packages(fullPackageName) do
+        val subPackageName = PackageNameUtils.separatePkgAndClassNames(p.name)._2.toTermName
+        val subPackage = packageClass.info.decl(subPackageName).orElse:
+          // package does not exist in symbol table, create a new symbol
+          enterPackage(packageClass, subPackageName, (module, modcls) => new PackageLoader(module, fullClasspath))
+        mergeNewEntries(subPackage.asSymDenotation.moduleClass.asClass, p.name, jarClasspath, fullClasspath)
+  end mergeNewEntries
 }
 
 /** A lazy type that completes itself by calling parameter doComplete.

compiler/src/dotty/tools/repl/ParseResult.scala

@@ -52,6 +52,20 @@ object Load {
   val command: String = ":load"
 }
 
+/** `:require` is a deprecated alias for :jar`
+ */
+case class Require(path: String) extends Command
+object Require {
+  val command: String = ":require"
+}
+
+/** `:jar <path>` adds a jar to the classpath
+ */
+case class JarCmd(path: String) extends Command
+object JarCmd {
+  val command: String = ":jar"
+}
+
 /** `:kind <type>` display the kind of a type. see also :help kind
  */
 case class KindOf(expr: String) extends Command
@@ -151,8 +165,10 @@ object ParseResult {
     Help.command -> (_  => Help),
     Reset.command -> (arg  => Reset(arg)),
     Imports.command -> (_  => Imports),
+    JarCmd.command -> (arg => JarCmd(arg)),
     KindOf.command -> (arg => KindOf(arg)),
     Load.command -> (arg => Load(arg)),
+    Require.command -> (arg => Require(arg)),
     TypeOf.command -> (arg => TypeOf(arg)),
     DocOf.command -> (arg => DocOf(arg)),
     Settings.command -> (arg => Settings(arg)),

compiler/src/dotty/tools/repl/ReplDriver.scala

@@ -7,6 +7,7 @@ import java.nio.charset.StandardCharsets
 
 import dotty.tools.dotc.ast.Trees.*
 import dotty.tools.dotc.ast.{tpd, untpd}
+import dotty.tools.dotc.classpath.ClassPathFactory
 import dotty.tools.dotc.config.CommandLineParser.tokenize
 import dotty.tools.dotc.config.Properties.{javaVersion, javaVmName, simpleVersionString}
 import dotty.tools.dotc.core.Contexts.*
@@ -21,6 +22,7 @@ import dotty.tools.dotc.core.NameOps.*
 import dotty.tools.dotc.core.Names.Name
 import dotty.tools.dotc.core.StdNames.*
 import dotty.tools.dotc.core.Symbols.{Symbol, defn}
+import dotty.tools.dotc.core.SymbolLoaders
 import dotty.tools.dotc.interfaces
 import dotty.tools.dotc.interactive.Completion
 import dotty.tools.dotc.printing.SyntaxHighlighting
@@ -39,6 +41,7 @@ import scala.annotation.tailrec
 import scala.collection.mutable
 import scala.compiletime.uninitialized
 import scala.jdk.CollectionConverters.*
+import scala.tools.asm.ClassReader
 import scala.util.control.NonFatal
 import scala.util.Using
 
@@ -512,10 +515,65 @@ class ReplDriver(settings: Array[String],
         state
       }
 
+    case Require(path) =>
+      out.println(":require is no longer supported, but has been replaced with :jar. Please use :jar")
+      state
+
+    case JarCmd(path) =>
+      val jarFile = AbstractFile.getDirectory(path)
+      if (jarFile == null)
+        out.println(s"""Cannot add "$path" to classpath.""")
+        state
+      else
+        def flatten(f: AbstractFile): Iterator[AbstractFile] =
+          if (f.isClassContainer) f.iterator.flatMap(flatten)
+          else Iterator(f)
+
+        def tryClassLoad(classFile: AbstractFile): Option[String] = {
+          val input = classFile.input
+          try {
+            val reader = new ClassReader(input)
+            val clsName = reader.getClassName.replace('/', '.')
+            rendering.myClassLoader.loadClass(clsName)
+            Some(clsName)
+          } catch
+            case _: ClassNotFoundException => None
+          finally {
+            input.close()
+          }
+        }
+
+        try {
+          val entries = flatten(jarFile)
+
+          val existingClass = entries.filter(_.ext.isClass).find(tryClassLoad(_).isDefined)
+          if (existingClass.nonEmpty)
+            out.println(s"The path '$path' cannot be loaded, it contains a classfile that already exists on the classpath: ${existingClass.get}")
+          else inContext(state.context):
+            val jarClassPath = ClassPathFactory.newClassPath(jarFile)
+            val prevOutputDir = ctx.settings.outputDir.value
+
+            // add to compiler class path
+            ctx.platform.addToClassPath(jarClassPath)
+            SymbolLoaders.mergeNewEntries(defn.RootClass, ClassPath.RootPackage, jarClassPath, ctx.platform.classPath)
+
+            // new class loader with previous output dir and specified jar
+            val prevClassLoader = rendering.classLoader()
+            val jarClassLoader = fromURLsParallelCapable(
+              jarClassPath.asURLs, prevClassLoader)
+            rendering.myClassLoader = new AbstractFileClassLoader(
+              prevOutputDir, jarClassLoader)
+
+            out.println(s"Added '$path' to classpath.")
+        } catch {
+          case e: Throwable =>
+            out.println(s"Failed to load '$path' to classpath: ${e.getMessage}")
+        }
+        state
+
     case KindOf(expr) =>
       out.println(s"""The :kind command is not currently supported.""")
       state
-
     case TypeOf(expr) =>
       expr match {
         case "" => out.println(s":type <expression>")

compiler/test-resources/jars/MyLibrary.scala

@@ -0,0 +1,8 @@
+/**
+ * JAR used for testing repl :jar
+ * Generated using: mkdir out; scalac -d out MyLibrary.scala; jar cf mylibrary.jar -C out .
+ */
+package mylibrary
+
+object Utils:
+  def greet(name: String): String = s"Hello, $name!"

compiler/test-resources/jars/MyLibrary2.scala

@@ -0,0 +1,9 @@
+/**
+ * JAR used for testing repl :jar
+ * Generated using: mkdir out2; scalac -d out MyLibrary2.scala; jar cf mylibrary2.jar -C out2 .
+ */
+package mylibrary2
+
+object Utils2:
+  def greet(name: String): String = s"Greetings, $name!"
+

compiler/test-resources/jars/mylibrary.jar

@@ -0,0 +1,13 @@
+scala> val z = 1
+val z: Int = 1
+
+scala>:jar sbt-test/source-dependencies/canon/actual/a.jar
+Added 'sbt-test/source-dependencies/canon/actual/a.jar' to classpath.
+
+scala> import A.x
+
+scala> x
+val res0: Int = 3
+
+scala> z
+val res1: Int = 1

compiler/test-resources/jars/mylibrary2.jar

@@ -0,0 +1,11 @@
+scala>:jar path/does/not/exist
+Cannot add "path/does/not/exist" to classpath.
+
+scala>:jar sbt-test/source-dependencies/canon/actual/a.jar
+Added 'sbt-test/source-dependencies/canon/actual/a.jar' to classpath.
+
+scala>:jar sbt-test/source-dependencies/canon/actual/a.jar
+The path 'sbt-test/source-dependencies/canon/actual/a.jar' cannot be loaded, it contains a classfile that already exists on the classpath: sbt-test/source-dependencies/canon/actual/a.jar(A.class)
+
+scala>:require sbt-test/source-dependencies/canon/actual/a.jar
+:require is no longer supported, but has been replaced with :jar. Please use :jar

compiler/test-resources/repl/jar-command

@@ -0,0 +1,32 @@
+scala> val z = 1
+val z: Int = 1
+
+scala>:jar compiler/test-resources/jars/mylibrary.jar
+Added 'compiler/test-resources/jars/mylibrary.jar' to classpath.
+
+scala> import mylibrary.Utils
+
+scala> Utils.greet("Alice")
+val res0: String = Hello, Alice!
+
+scala>:jar compiler/test-resources/jars/mylibrary2.jar
+Added 'compiler/test-resources/jars/mylibrary2.jar' to classpath.
+
+scala> import mylibrary2.Utils2
+
+scala> Utils2.greet("Alice")
+val res1: String = Greetings, Alice!
+
+scala> Utils.greet("Alice")
+val res2: String = Hello, Alice!
+
+scala> import mylibrary.Utils.greet
+
+scala> greet("Tom")
+val res3: String = Hello, Tom!
+
+scala> Utils.greet("Alice")
+val res4: String = Hello, Alice!
+
+scala> z
+val res5: Int = 1

compiler/test-resources/repl/jar-errors

@@ -98,10 +98,7 @@ extends ReplDriver(options, new PrintStream(out, true, StandardCharsets.UTF_8.na
         FileDiff.dump(checkFile.toPath.toString, actualOutput)
         println(s"Wrote updated script file to $checkFile")
       else
-        println("expected =========>")
-        println(expectedOutput.mkString(EOL))
-        println("actual ===========>")
-        println(actualOutput.mkString(EOL))
+        println(dotc.util.DiffUtil.mkColoredHorizontalLineDiff(actualOutput.mkString(EOL), expectedOutput.mkString(EOL)))
 
         fail(s"Error in script $name, expected output did not match actual")
     end if

compiler/test-resources/repl/jar-multiple

@@ -213,9 +213,11 @@ class TabcompleteTests extends ReplTest {
         ":exit",
         ":help",
         ":imports",
+        ":jar",
         ":kind",
         ":load",
         ":quit",
+        ":require",
         ":reset",
         ":settings",
         ":sh",