Support links in using file directive (implements #1328)

modules/build/src/main/scala/scala/build/Build.scala

@@ -267,7 +267,8 @@ object Build {
       ),
       logger,
       options.suppressWarningOptions,
-      options.internal.exclude
+      options.internal.exclude,
+      download = options.downloader
     )
 
   private def build(

modules/build/src/main/scala/scala/build/CrossSources.scala

@@ -7,7 +7,8 @@ import scala.build.errors.{
   CompositeBuildException,
   ExcludeDefinitionError,
   MalformedDirectiveError,
-  Severity
+  Severity,
+  UsingFileFromUriError
 }
 import scala.build.input.*
 import scala.build.input.ElementsUtils.*
@@ -156,7 +157,8 @@ object CrossSources {
     logger: Logger,
     suppressWarningOptions: SuppressWarningOptions,
     exclude: Seq[Positioned[String]] = Nil,
-    maybeRecoverOnError: BuildException => Option[BuildException] = e => Some(e)
+    maybeRecoverOnError: BuildException => Option[BuildException] = e => Some(e),
+    download: BuildOptions.Download = BuildOptions.Download.notSupported
   )(using ScalaCliInvokeData): Either[BuildException, (CrossSources, Inputs)] = either {
 
     def preprocessSources(elems: Seq[SingleElement])
@@ -204,8 +206,9 @@ object CrossSources {
         .flatMap(_.options)
         .flatMap(_.internal.extraSourceFiles)
         .distinct
-    val inputsElemFromDirectives: Seq[SingleFile] =
-      value(resolveInputsFromSources(sourcesFromDirectives, inputs.enableMarkdown))
+
+    val inputsElemFromDirectives: Seq[SingleElement] =
+      value(resolveInputsFromSources(sourcesFromDirectives, inputs.enableMarkdown, download))
     val preprocessedSourcesFromDirectives: Seq[PreprocessedSource] =
       value(preprocessSources(inputsElemFromDirectives.pipe(elements =>
         value(excludeSources(elements, inputs.workspace, allExclude))
@@ -398,7 +401,32 @@ object CrossSources {
     fromInputs ++ fromSources ++ fromSourcesWithRequirements
   }
 
-  private def resolveInputsFromSources(sources: Seq[Positioned[os.Path]], enableMarkdown: Boolean) =
+  private def downloadFile(download: BuildOptions.Download)(pUri: Positioned[java.net.URI]) =
+    download(pUri.value.toString).left.map(
+      new UsingFileFromUriError(pUri.value, pUri.positions, _)
+    ).map(content =>
+      Seq(Virtual(pUri.value.toString, content))
+    )
+
+  type CodeFile = os.Path | java.net.URI
+
+  private def resolveInputsFromSources(
+    sources: Seq[Positioned[CodeFile]],
+    enableMarkdown: Boolean,
+    download: BuildOptions.Download
+  ) =
+    val links = sources.collect {
+      case Positioned(pos, value: java.net.URI) => Positioned(pos, value)
+    }
+    val paths = sources.collect {
+      case Positioned(pos, value: os.Path) => Positioned(pos, value)
+    }
+
+    (resolveInputsFromPath(paths, enableMarkdown) ++ links.map(downloadFile(download))).sequence
+      .left.map(CompositeBuildException(_))
+      .map(_.flatten)
+
+  private def resolveInputsFromPath(sources: Seq[Positioned[os.Path]], enableMarkdown: Boolean) =
     sources.map { source =>
       val sourcePath   = source.value
       lazy val dir     = sourcePath / os.up
@@ -419,9 +447,7 @@ object CrossSources {
           else s"$sourcePath: not found path defined in using directive."
         Left(new MalformedDirectiveError(msg, source.positions))
       }
-    }.sequence
-      .left.map(CompositeBuildException(_))
-      .map(_.flatten)
+    }
 
   /** Filters out the sources from the input sequence based on the provided 'exclude' patterns. The
     * exclude patterns can be absolute paths, relative paths, or glob patterns.

modules/build/src/main/scala/scala/build/bsp/BspImpl.scala

@@ -113,7 +113,8 @@ final class BspImpl(
         logger = persistentLogger,
         suppressWarningOptions = buildOptions.suppressWarningOptions,
         exclude = buildOptions.internal.exclude,
-        maybeRecoverOnError = maybeRecoverOnError(Scope.Main)
+        maybeRecoverOnError = maybeRecoverOnError(Scope.Main),
+        download = buildOptions.downloader
       ).left.map((_, Scope.Main))
     }
 

modules/build/src/main/scala/scala/build/input/Inputs.scala

@@ -11,7 +11,7 @@ import scala.build.errors.{BuildException, InputsException, WorkspaceError}
 import scala.build.input.ElementsUtils.*
 import scala.build.internal.Constants
 import scala.build.internal.zip.WrappedZipInputStream
-import scala.build.options.Scope
+import scala.build.options.{BuildOptions, Scope}
 import scala.build.preprocessing.SheBang.isShebangScript
 import scala.util.matching.Regex
 import scala.util.{Properties, Try}
@@ -225,7 +225,7 @@ object Inputs {
   def validateArgs(
     args: Seq[String],
     cwd: os.Path,
-    download: String => Either[String, Array[Byte]],
+    download: BuildOptions.Download,
     stdinOpt: => Option[Array[Byte]],
     acceptFds: Boolean,
     enableMarkdown: Boolean
@@ -423,7 +423,7 @@ object Inputs {
     args: Seq[String],
     cwd: os.Path,
     defaultInputs: () => Option[Inputs] = () => None,
-    download: String => Either[String, Array[Byte]] = _ => Left("URL not supported"),
+    download: BuildOptions.Download = BuildOptions.Download.notSupported,
     stdinOpt: => Option[Array[Byte]] = None,
     scriptSnippetList: List[String] = List.empty,
     scalaSnippetList: List[String] = List.empty,

modules/cli/src/main/scala/scala/cli/commands/bsp/Bsp.scala

@@ -108,7 +108,8 @@ object Bsp extends ScalaCommand[BspOptions] {
             ),
             persistentLogger,
             baseOptions.suppressWarningOptions,
-            baseOptions.internal.exclude
+            baseOptions.internal.exclude,
+            download = baseOptions.downloader
           )
 
           val (allInputs, finalBuildOptions) = {

modules/cli/src/main/scala/scala/cli/commands/dependencyupdate/DependencyUpdate.scala

@@ -44,7 +44,8 @@ object DependencyUpdate extends ScalaCommand[DependencyUpdateOptions] {
         ),
         logger,
         buildOptions.suppressWarningOptions,
-        buildOptions.internal.exclude
+        buildOptions.internal.exclude,
+        download = buildOptions.downloader
       ).orExit(logger)
 
     val sharedOptions = crossSources.sharedOptions(buildOptions)

modules/cli/src/main/scala/scala/cli/commands/export0/Export.scala

@@ -43,7 +43,8 @@ object Export extends ScalaCommand[ExportOptions] {
         ),
         logger,
         buildOptions.suppressWarningOptions,
-        buildOptions.internal.exclude
+        buildOptions.internal.exclude,
+        download = buildOptions.downloader
       )
     }
 

modules/cli/src/main/scala/scala/cli/commands/fix/BuiltInRules.scala

@@ -169,7 +169,8 @@ object BuiltInRules extends CommandHelpers {
       ),
       logger = logger,
       suppressWarningOptions = SuppressWarningOptions.suppressAll,
-      exclude = buildOptions.internal.exclude
+      exclude = buildOptions.internal.exclude,
+      download = buildOptions.downloader
     ).orExit(logger)
 
     val sharedOptions = crossSources.sharedOptions(buildOptions)

modules/cli/src/main/scala/scala/cli/commands/publish/PublishSetup.scala

@@ -89,7 +89,8 @@ object PublishSetup extends ScalaCommand[PublishSetupOptions] {
         ),
         logger,
         cliBuildOptions.suppressWarningOptions,
-        cliBuildOptions.internal.exclude
+        cliBuildOptions.internal.exclude,
+        download = cliBuildOptions.downloader
       ).orExit(logger)
 
       val crossSourcesSharedOptions = crossSources.sharedOptions(cliBuildOptions)

modules/cli/src/main/scala/scala/cli/commands/setupide/SetupIde.scala

@@ -43,7 +43,8 @@ object SetupIde extends ScalaCommand[SetupIdeOptions] {
           ),
           logger,
           options.suppressWarningOptions,
-          options.internal.exclude
+          options.internal.exclude,
+          download = options.downloader
         )
       }
 

modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala

@@ -8,7 +8,7 @@ import com.github.plokhotnyuk.jsoniter_scala.core.*
 import com.github.plokhotnyuk.jsoniter_scala.macros.*
 import coursier.cache.FileCache
 import coursier.core.Version
-import coursier.util.{Artifact, Task}
+import coursier.util.Task
 import dependency.AnyDependency
 import dependency.parser.DependencyParser
 
@@ -26,8 +26,7 @@ import scala.build.interactive.Interactive.{InteractiveAsk, InteractiveNop}
 import scala.build.internal.util.WarningMessages
 import scala.build.internal.{Constants, FetchExternalBinary, OsLibc}
 import scala.build.internals.ConsoleUtils.ScalaCliConsole
-import scala.build.options.ScalaVersionUtil.fileWithTtl0
-import scala.build.options.{Platform, ShadowingSeq}
+import scala.build.options.{BuildOptions, Platform, ShadowingSeq}
 import scala.build.preprocessing.directives.ClasspathUtils.*
 import scala.build.preprocessing.directives.Toolkit.maxScalaNativeWarningMsg
 import scala.build.preprocessing.directives.{Python, Toolkit}
@@ -648,7 +647,7 @@ final case class SharedOptions(
     Inputs.validateArgs(
       args,
       Os.pwd,
-      SharedOptions.downloadInputs(coursierCache),
+      BuildOptions.Download.changing(coursierCache),
       SharedOptions.readStdin(logger = logger),
       !Properties.isWin,
       enableMarkdown = true
@@ -664,15 +663,6 @@ object SharedOptions {
   implicit lazy val help: Help[SharedOptions]                = Help.derive
   implicit lazy val jsonCodec: JsonValueCodec[SharedOptions] = JsonCodecMaker.make
 
-  private def downloadInputs(cache: FileCache[Task]): String => Either[String, Array[Byte]] = {
-    url =>
-      val artifact = Artifact(url).withChanging(true)
-      cache.fileWithTtl0(artifact)
-        .left
-        .map(_.describe)
-        .map(f => os.read.bytes(os.Path(f, Os.pwd)))
-  }
-
   /** [[Inputs]] builder, handy when you don't have a [[SharedOptions]] instance at hand */
   def inputs(
     args: Seq[String],
@@ -704,7 +694,7 @@ object SharedOptions {
       args,
       Os.pwd,
       defaultInputs = defaultInputs,
-      download = downloadInputs(cache),
+      download = BuildOptions.Download.changing(cache),
       stdinOpt = readStdin(logger = logger),
       scriptSnippetList = scriptSnippetList,
       scalaSnippetList = scalaSnippetList,

modules/directives/src/main/scala/scala/build/errors/UsingFileFromUriError.scala

@@ -0,0 +1,11 @@
+package scala.build.errors
+
+import java.net.URI
+
+import scala.build.Position
+
+final class UsingFileFromUriError(uri: URI, positions: Seq[Position], description: String)
+    extends BuildException(
+      message = s"Error using file from $uri - $description",
+      positions = positions
+    )

modules/directives/src/main/scala/scala/build/preprocessing/directives/Sources.scala

@@ -10,6 +10,9 @@ import scala.cli.commands.SpecificationLevel
 
 @DirectiveGroupName("Custom sources")
 @DirectiveExamples("//> using file utils.scala")
+@DirectiveExamples(
+  "//> using file https://raw.githubusercontent.com/softwaremill/sttp/refs/heads/master/examples/src/main/scala/sttp/client4/examples/json/GetAndParseJsonCatsEffectCirce.scala"
+)
 @DirectiveUsage(
   "`//> using file `_path_ | `//> using files `_path1_ _path2_ …",
   """`//> using file` _path_
@@ -26,6 +29,17 @@ final case class Sources(
   files: DirectiveValueParser.WithScopePath[List[Positioned[String]]] =
     DirectiveValueParser.WithScopePath.empty(Nil)
 ) extends HasBuildOptions {
+
+  private def codeFile(codeFile: String, root: os.Path): Sources.CodeFile =
+    scala.util.Try {
+      val uri = java.net.URI.create(codeFile)
+      uri.getScheme match {
+        case "file" | "http" | "https" => uri
+      }
+    }.getOrElse {
+      os.Path(codeFile, root)
+    }
+
   def buildOptions: Either[BuildException, BuildOptions] = either {
 
     val paths = files
@@ -34,7 +48,7 @@ final case class Sources(
         for {
           root <- Directive.osRoot(files.scopePath, positioned.positions.headOption)
           path <- {
-            try Right(positioned.map(os.Path(_, root)))
+            try Right(positioned.map(codeFile(_, root)))
             catch {
               case e: IllegalArgumentException =>
                 Left(new WrongSourcePathError(positioned.value, e, positioned.positions))
@@ -54,5 +68,8 @@ final case class Sources(
 }
 
 object Sources {
+
+  type CodeFile = os.Path | java.net.URI
+
   val handler: DirectiveHandler[Sources] = DirectiveHandler.derive
 }

modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala

@@ -9,15 +9,16 @@ trait RunGistTestDefinitions { _: RunTestDefinitions =>
     if (Properties.isWin) "\"" + url + "\""
     else url
 
+  protected val scalaScriptUrl =
+    "https://gist.github.com/alexarchambault/f972d941bc4a502d70267cfbbc4d6343/raw/b0285fa0305f76856897517b06251970578565af/test.sc"
+  protected val scalaScriptMessage = "Hello from GitHub Gist"
+
   test("Script URL") {
-    val url =
-      "https://gist.github.com/alexarchambault/f972d941bc4a502d70267cfbbc4d6343/raw/b0285fa0305f76856897517b06251970578565af/test.sc"
-    val message = "Hello from GitHub Gist"
     emptyInputs.fromRoot { root =>
-      val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(url))
+      val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(scalaScriptUrl))
         .call(cwd = root)
         .out.trim()
-      expect(output == message)
+      expect(output == scalaScriptMessage)
     }
   }
 

modules/integration/src/test/scala/scala/cli/integration/RunTestsDefault.scala

@@ -111,6 +111,17 @@ class RunTestsDefault extends RunTestDefinitions
     }
   }
 
+  test("using file + http[s] link directive") {
+    val inputPath = os.rel / "usingFileLinkExample.scala"
+    TestInputs(inputPath -> s"//> using file $scalaScriptUrl\n").fromRoot {
+      root =>
+        val res = os.proc(TestUtil.cli, "run", extraOptions, inputPath)
+          .call(cwd = root)
+        val out = res.out.trim()
+        expect(out == scalaScriptMessage)
+    }
+  }
+
   for {
     suppressDeprecatedWarnings <- Seq(true, false)
     suppressByConfig           <- if (suppressDeprecatedWarnings) Seq(true, false) else Seq(false)

modules/options/src/main/scala/scala/build/options/BuildOptions.scala

@@ -2,7 +2,7 @@ package scala.build.options
 import coursier.cache.{ArchiveCache, FileCache}
 import coursier.core.{Repository, Version}
 import coursier.parse.RepositoryParser
-import coursier.util.Task
+import coursier.util.{Artifact, Task}
 import dependency.*
 
 import java.io.File
@@ -20,7 +20,7 @@ import scala.build.internal.Regexes.scala3NightlyNicknameRegex
 import scala.build.internal.{Constants, OsLibc, Util}
 import scala.build.internals.EnvVar
 import scala.build.options.validation.BuildOptionsRule
-import scala.build.{Artifacts, Logger, Position, Positioned}
+import scala.build.{Artifacts, Logger, Os, Position, Positioned}
 import scala.collection.immutable.Seq
 import scala.concurrent.Await
 import scala.concurrent.duration.*
@@ -579,6 +579,8 @@ final case class BuildOptions(
     }
   }
 
+  lazy val downloader: BuildOptions.Download = BuildOptions.Download(finalCache)
+
   lazy val interactive: Either[BuildException, Interactive] =
     internal.interactive.map(_()).getOrElse(Right(InteractiveNop))
 }
@@ -628,6 +630,23 @@ object BuildOptions {
     }
   }
 
+  type Download = String => Either[String, Array[Byte]]
+  object Download {
+    def apply(
+      cache: FileCache[Task],
+      toArtifact: String => Artifact = Artifact.fromUrl
+    ): Download = {
+      import scala.build.options.ScalaVersionUtil.fileWithTtl0
+      url =>
+        cache.fileWithTtl0(toArtifact(url))
+          .left
+          .map(_.describe)
+          .map(f => os.read.bytes(os.Path(f, Os.pwd)))
+    }
+    def changing(cache: FileCache[Task]): Download = apply(cache, Artifact(_).withChanging(true))
+    val notSupported: Download                     = _ => Left("URL not supported")
+  }
+
   implicit val hasHashData: HasHashData[BuildOptions] = HasHashData.derive
   implicit val monoid: ConfigMonoid[BuildOptions]     = ConfigMonoid.derive
 }