Support links in using file directive (implements #1328)

modules/build/src/main/scala/scala/build/CrossSources.scala

@@ -1,6 +1,7 @@
 package scala.build
 
-import java.io.File
+import coursier.cache.FileCache
+import coursier.util.{Artifact, Task}
 
 import scala.build.CollectionOps.*
 import scala.build.EitherCps.{either, value}
@@ -209,7 +210,8 @@ object CrossSources {
         .flatMap(_.options)
         .flatMap(_.internal.extraSourceFiles)
         .distinct
-    val inputsElemFromDirectives: Seq[SingleFile] =
+
+    val inputsElemFromDirectives: Seq[SingleElement] =
       value(resolveInputsFromSources(sourcesFromDirectives, inputs.enableMarkdown))
     val preprocessedSourcesFromDirectives: Seq[PreprocessedSource] =
       value(preprocessSources(inputsElemFromDirectives.pipe(elements =>
@@ -403,7 +405,39 @@ object CrossSources {
     fromInputs ++ fromSources ++ fromSourcesWithRequirements
   }
 
-  private def resolveInputsFromSources(sources: Seq[Positioned[os.Path]], enableMarkdown: Boolean) =
+  // TODO: reuse existing one? e.g. scala.cli.commands.shared.SharedOptions.coursierCache
+  lazy val fileCache: FileCache[coursier.util.Task] = FileCache()
+
+  private def downloadFile(pUri: Positioned[java.net.URI]) =
+    import scala.build.options.ScalaVersionUtil.fileWithTtl0
+    val artifact = Artifact(pUri.value.toString).withChanging(true)
+    fileCache.fileWithTtl0(artifact)
+      .left
+      .map(err =>
+        new MalformedDirectiveError(err.describe, pUri.positions)
+      ) // TODO: better error type
+      .map(f => os.read.bytes(os.Path(f, Os.pwd))).map(content =>
+        Seq(Virtual(pUri.value.toString, content))
+      )
+
+  type CodeFile = os.Path | java.net.URI
+
+  private def resolveInputsFromSources(
+    sources: Seq[Positioned[CodeFile]],
+    enableMarkdown: Boolean
+  ) =
+    val links = sources.collect {
+      case Positioned(pos, value: java.net.URI) => Positioned(pos, value)
+    }
+    val paths = sources.collect {
+      case Positioned(pos, value: os.Path) => Positioned(pos, value)
+    }
+
+    (resolveInputsFromPath(paths, enableMarkdown) ++ links.map(downloadFile)).sequence
+      .left.map(CompositeBuildException(_))
+      .map(_.flatten)
+
+  private def resolveInputsFromPath(sources: Seq[Positioned[os.Path]], enableMarkdown: Boolean) =
     sources.map { source =>
       val sourcePath   = source.value
       lazy val dir     = sourcePath / os.up
@@ -424,9 +458,7 @@ object CrossSources {
           else s"$sourcePath: not found path defined in using directive."
         Left(new MalformedDirectiveError(msg, source.positions))
       }
-    }.sequence
-      .left.map(CompositeBuildException(_))
-      .map(_.flatten)
+    }
 
   /** Filters out the sources from the input sequence based on the provided 'exclude' patterns. The
     * exclude patterns can be absolute paths, relative paths, or glob patterns.

modules/directives/src/main/scala/scala/build/preprocessing/directives/Sources.scala

@@ -11,6 +11,9 @@ import scala.util.Try
 
 @DirectiveGroupName("Custom sources")
 @DirectiveExamples("//> using file utils.scala")
+@DirectiveExamples(
+  "//> using file https://raw.githubusercontent.com/softwaremill/sttp/refs/heads/master/examples/src/main/scala/sttp/client4/examples/json/GetAndParseJsonCatsEffectCirce.scala"
+)
 @DirectiveUsage(
   "`//> using file `_path_ | `//> using files `_path1_ _path2_ …",
   """`//> using file` _path_
@@ -27,6 +30,17 @@ final case class Sources(
   files: DirectiveValueParser.WithScopePath[List[Positioned[String]]] =
     DirectiveValueParser.WithScopePath.empty(Nil)
 ) extends HasBuildOptions {
+
+  private def codeFile(codeFile: String, root: os.Path): Sources.CodeFile =
+    scala.util.Try {
+      val uri = java.net.URI.create(codeFile)
+      uri.getScheme match {
+        case "file" | "http" | "https" => uri
+      }
+    }.getOrElse {
+      os.Path(codeFile, root)
+    }
+
   def buildOptions: Either[BuildException, BuildOptions] = either {
 
     val paths = files
@@ -35,7 +49,7 @@ final case class Sources(
         for {
           root <- Directive.osRoot(files.scopePath, positioned.positions.headOption)
           path <- {
-            try Right(positioned.map(os.Path(_, root)))
+            try Right(positioned.map(codeFile(_, root)))
             catch {
               case e: IllegalArgumentException =>
                 Left(new WrongSourcePathError(positioned.value, e, positioned.positions))
@@ -55,5 +69,8 @@ final case class Sources(
 }
 
 object Sources {
+
+  type CodeFile = os.Path | java.net.URI
+
   val handler: DirectiveHandler[Sources] = DirectiveHandler.derive
 }

modules/options/src/main/scala/scala/build/options/InternalOptions.scala

@@ -8,6 +8,8 @@ import scala.build.errors.BuildException
 import scala.build.interactive.Interactive
 import scala.build.interactive.Interactive.InteractiveNop
 
+type CodeFile = os.Path | java.net.URI
+
 final case class InternalOptions(
   keepDiagnostics: Boolean = false,
   cache: Option[FileCache[Task]] = None,
@@ -24,7 +26,7 @@ final case class InternalOptions(
     * really needed.
     */
   keepResolution: Boolean = false,
-  extraSourceFiles: Seq[Positioned[os.Path]] = Nil,
+  extraSourceFiles: Seq[Positioned[CodeFile]] = Nil,
   exclude: Seq[Positioned[String]] = Nil,
   offline: Option[Boolean] = None
 ) {

website/docs/reference/directives.md

@@ -126,6 +126,8 @@ Manually add sources to the project. Does not support chaining, sources are adde
 #### Examples
 `//> using file utils.scala`
 
+`//> using file https://raw.githubusercontent.com/softwaremill/sttp/refs/heads/master/examples/src/main/scala/sttp/client4/examples/json/GetAndParseJsonCatsEffectCirce.scala`
+
 ### Dependency
 
 Add dependencies

website/docs/reference/scala-command/directives.md

@@ -197,6 +197,8 @@ Manually add sources to the project. Does not support chaining, sources are adde
 #### Examples
 `//> using file utils.scala`
 
+`//> using file https://raw.githubusercontent.com/softwaremill/sttp/refs/heads/master/examples/src/main/scala/sttp/client4/examples/json/GetAndParseJsonCatsEffectCirce.scala`
+
 ### Exclude sources
 
 Exclude sources from the project