From 17e82e6c49ad0366d2d5e1553c463cc1d1558086 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Fri, 3 Oct 2025 08:19:35 -0600 Subject: [PATCH 1/5] initial demo works --- .../src/dotty/tools/repl/ReplDriver.scala | 31 +++++++++++++++++-- project/Build.scala | 1 + 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/repl/ReplDriver.scala b/compiler/src/dotty/tools/repl/ReplDriver.scala index befb3de9a941..39b933b1b9ad 100644 --- a/compiler/src/dotty/tools/repl/ReplDriver.scala +++ b/compiler/src/dotty/tools/repl/ReplDriver.scala @@ -301,8 +301,35 @@ class ReplDriver(settings: Array[String], protected def interpret(res: ParseResult)(using state: State): State = { res match { - case parsed: Parsed if parsed.trees.nonEmpty => - compile(parsed, state) + case parsed: Parsed => + // Check for magic comments specifying dependencies + val sourceCode = parsed.source.content().mkString + val depStrings = DependencyResolver.extractDependencies(sourceCode) + + if depStrings.nonEmpty then + val deps = depStrings.flatMap(DependencyResolver.parseDependency) + if deps.nonEmpty then + DependencyResolver.resolveDependencies(deps) match + case Right(files) => + if files.nonEmpty then + inContext(state.context): + // Update both compiler classpath and classloader + val prevOutputDir = ctx.settings.outputDir.value + val prevClassLoader = rendering.classLoader() + rendering.myClassLoader = DependencyResolver.addToCompilerClasspath( + files, + prevClassLoader, + prevOutputDir + ) + out.println(s"Resolved ${deps.size} dependencies (${files.size} JARs)") + case Left(error) => + out.println(s"Error resolving dependencies: $error") + + // Only compile if there are actual trees to compile + if parsed.trees.nonEmpty then + compile(parsed, state) + else + state case SyntaxErrors(_, errs, _) => displayErrors(errs) diff --git a/project/Build.scala b/project/Build.scala index 8c233af8b2b3..65e034de1785 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -717,6 +717,7 @@ object Build { "org.jline" % "jline-terminal" % "3.29.0", "org.jline" % "jline-terminal-jni" % "3.29.0", // needed for Windows ("io.get-coursier" %% "coursier" % "2.0.16" % Test).cross(CrossVersion.for3Use2_13), + "io.get-coursier" % "interface" % "1.0.19", // used by the REPL for dependency resolution ), (Compile / sourceGenerators) += ShadedSourceGenerator.task.taskValue, From 44abbbfe1ddfcba037b927d5a62fd3cadbc4aa9d Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Fri, 3 Oct 2025 08:37:25 -0600 Subject: [PATCH 2/5] . --- .../dotty/tools/repl/DependencyResolver.scala | 121 ++++++++++++++++++ project/Build.scala | 1 + 2 files changed, 122 insertions(+) create mode 100644 compiler/src/dotty/tools/repl/DependencyResolver.scala diff --git a/compiler/src/dotty/tools/repl/DependencyResolver.scala b/compiler/src/dotty/tools/repl/DependencyResolver.scala new file mode 100644 index 000000000000..e01590435d00 --- /dev/null +++ b/compiler/src/dotty/tools/repl/DependencyResolver.scala @@ -0,0 +1,121 @@ +package dotty.tools.repl + +import scala.language.unsafeNulls + +import java.io.File +import java.net.{URL, URLClassLoader} +import scala.jdk.CollectionConverters.* +import scala.util.control.NonFatal + +import coursierapi.{Repository, Dependency, MavenRepository} +import com.virtuslab.using_directives.UsingDirectivesProcessor +import com.virtuslab.using_directives.custom.model.{Path, StringValue, Value} + +/** Handles dependency resolution using Coursier for the REPL */ +object DependencyResolver: + + /** Parse a dependency string of the form `org::artifact:version` or `org:artifact:version` + * and return the (organization, artifact, version) triple if successful. + * + * Supports both Maven-style (single colon) and Scala-style (double colon) notation: + * - Maven: `com.lihaoyi:scalatags_3:0.13.1` + * - Scala: `com.lihaoyi::scalatags:0.13.1` (automatically appends _3) + */ + def parseDependency(dep: String): Option[(String, String, String)] = + // Match either org:artifact:version or org::artifact:version + val pattern = """([^:]+)::?([^:]+):([^:]+)""".r + + dep match + case pattern(org, artifact, version) => + val isScalaStyle = dep.contains("::") + val fullArtifact = if isScalaStyle then s"${artifact}_3" else artifact + Some((org, fullArtifact, version)) + case _ => None + + /** Extract all dependencies from using directives in source code */ + def extractDependencies(sourceCode: String): List[String] = + try + val processor = new UsingDirectivesProcessor() + val directives = processor.extract(sourceCode.toCharArray) + + val deps = scala.collection.mutable.ListBuffer[String]() + + directives.asScala.foreach { directive => + val flatMap = directive.getFlattenedMap + flatMap.asScala.foreach { case (path, values) => + // Check if this is a "dep" directive (path segments: ["dep"]) + if path.getPath.asScala.toList == List("dep") then + values.asScala.foreach { value => + value match + case strValue: StringValue => + deps += strValue.get() + case _ => + } + } + } + + deps.toList + catch + case NonFatal(e) => + // If parsing fails, fall back to empty list + Nil + + /** Resolve dependencies using Coursier Interface and return the classpath as a list of File objects */ + def resolveDependencies(dependencies: List[(String, String, String)]): Either[String, List[File]] = + if dependencies.isEmpty then Right(Nil) + else + try + // Add Maven Central and Sonatype repositories + val repos = Array( + MavenRepository.of("https://repo1.maven.org/maven2"), + MavenRepository.of("https://oss.sonatype.org/content/repositories/releases") + ) + + // Create dependency objects + val deps = dependencies.map { case (org, artifact, version) => + Dependency.of(org, artifact, version) + }.toArray + + val fetch = coursierapi.Fetch.create() + .withRepositories(repos*) + .withDependencies(deps*) + + val files = fetch.fetch().asScala.toList + Right(files) + + catch + case NonFatal(e) => + Left(s"Failed to resolve dependencies: ${e.getMessage}") + + /** Add resolved dependencies to the compiler classpath and classloader. + * Returns the new classloader. + * + * This follows the same pattern as the `:jar` command. + */ + def addToCompilerClasspath( + files: List[File], + prevClassLoader: ClassLoader, + prevOutputDir: dotty.tools.io.AbstractFile + )(using ctx: dotty.tools.dotc.core.Contexts.Context): AbstractFileClassLoader = + import dotty.tools.dotc.classpath.ClassPathFactory + import dotty.tools.dotc.core.SymbolLoaders + import dotty.tools.dotc.core.Symbols.defn + import dotty.tools.io.* + import dotty.tools.runner.ScalaClassLoader.fromURLsParallelCapable + + // Create a classloader with all the resolved JAR files + val urls = files.map(_.toURI.toURL).toArray + val depsClassLoader = new URLClassLoader(urls, prevClassLoader) + + // Add each JAR to the compiler's classpath + for file <- files do + val jarFile = AbstractFile.getDirectory(file.getAbsolutePath) + if jarFile != null then + val jarClassPath = ClassPathFactory.newClassPath(jarFile) + ctx.platform.addToClassPath(jarClassPath) + SymbolLoaders.mergeNewEntries(defn.RootClass, ClassPath.RootPackage, jarClassPath, ctx.platform.classPath) + + // Create new classloader with previous output dir and resolved dependencies + new AbstractFileClassLoader(prevOutputDir, depsClassLoader) + +end DependencyResolver diff --git a/project/Build.scala b/project/Build.scala index 65e034de1785..6c43815cf587 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -718,6 +718,7 @@ object Build { "org.jline" % "jline-terminal-jni" % "3.29.0", // needed for Windows ("io.get-coursier" %% "coursier" % "2.0.16" % Test).cross(CrossVersion.for3Use2_13), "io.get-coursier" % "interface" % "1.0.19", // used by the REPL for dependency resolution + "org.virtuslab" % "using_directives" % "1.1.4", // used by the REPL for parsing magic comments ), (Compile / sourceGenerators) += ShadedSourceGenerator.task.taskValue, From 34a8fb7c77b0440a67552e925946e05d955109ae Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Fri, 3 Oct 2025 08:51:03 -0600 Subject: [PATCH 3/5] . --- .../dotty/tools/repl/DependencyResolver.scala | 58 ++++++++----------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/compiler/src/dotty/tools/repl/DependencyResolver.scala b/compiler/src/dotty/tools/repl/DependencyResolver.scala index e01590435d00..96eb4d11cac0 100644 --- a/compiler/src/dotty/tools/repl/DependencyResolver.scala +++ b/compiler/src/dotty/tools/repl/DependencyResolver.scala @@ -22,43 +22,34 @@ object DependencyResolver: * - Scala: `com.lihaoyi::scalatags:0.13.1` (automatically appends _3) */ def parseDependency(dep: String): Option[(String, String, String)] = - // Match either org:artifact:version or org::artifact:version - val pattern = """([^:]+)::?([^:]+):([^:]+)""".r - dep match - case pattern(org, artifact, version) => - val isScalaStyle = dep.contains("::") - val fullArtifact = if isScalaStyle then s"${artifact}_3" else artifact - Some((org, fullArtifact, version)) - case _ => None + case s"$org::$artifact:$version" => Some((org, s"${artifact}_3", version)) + case s"$org:$artifact:$version" => Some((org, artifact, version)) + case _ => + System.err.println("Unable to parse dependency \"" + dep + "\"") + None /** Extract all dependencies from using directives in source code */ def extractDependencies(sourceCode: String): List[String] = try - val processor = new UsingDirectivesProcessor() - val directives = processor.extract(sourceCode.toCharArray) - - val deps = scala.collection.mutable.ListBuffer[String]() - - directives.asScala.foreach { directive => - val flatMap = directive.getFlattenedMap - flatMap.asScala.foreach { case (path, values) => - // Check if this is a "dep" directive (path segments: ["dep"]) - if path.getPath.asScala.toList == List("dep") then - values.asScala.foreach { value => - value match - case strValue: StringValue => - deps += strValue.get() - case _ => - } - } - } + val directives = new UsingDirectivesProcessor().extract(sourceCode.toCharArray) + val deps = scala.collection.mutable.Buffer[String]() + + for + directive <- directives.asScala + (path, values) <- directive.getFlattenedMap.asScala + do + if path.getPath.asScala.toList == List("dep") then + values.asScala.foreach { + case strValue: StringValue => deps += strValue.get() + case value => System.err.println("Unrecognized directive value " + value) + } + else + System.err.println("Unrecognized directive " + path.getPath) deps.toList catch - case NonFatal(e) => - // If parsing fails, fall back to empty list - Nil + case NonFatal(e) => Nil // If parsing fails, fall back to empty list /** Resolve dependencies using Coursier Interface and return the classpath as a list of File objects */ def resolveDependencies(dependencies: List[(String, String, String)]): Either[String, List[File]] = @@ -72,16 +63,15 @@ object DependencyResolver: ) // Create dependency objects - val deps = dependencies.map { case (org, artifact, version) => - Dependency.of(org, artifact, version) - }.toArray + val deps = dependencies + .map { case (org, artifact, version) => Dependency.of(org, artifact, version) } + .toArray val fetch = coursierapi.Fetch.create() .withRepositories(repos*) .withDependencies(deps*) - val files = fetch.fetch().asScala.toList - Right(files) + Right(fetch.fetch().asScala.toList) catch case NonFatal(e) => From 976e7c1575bbcdc6eb21cd3d9cac96066f9ba765 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Fri, 3 Oct 2025 09:12:20 -0600 Subject: [PATCH 4/5] . --- project/Build.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project/Build.scala b/project/Build.scala index 6c43815cf587..1f82f362fd5c 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -2139,6 +2139,8 @@ object Build { "org.jline" % "jline-terminal" % "3.29.0", "org.jline" % "jline-terminal-jni" % "3.29.0", ("io.get-coursier" %% "coursier" % "2.0.16" % Test).cross(CrossVersion.for3Use2_13), + "io.get-coursier" % "interface" % "1.0.19", // used by the REPL for dependency resolution + "org.virtuslab" % "using_directives" % "1.1.4", // used by the REPL for parsing magic comments ), // NOTE: The only difference here is that we drop `-Werror` and semanticDB for now Compile / scalacOptions := Seq("-deprecation", "-feature", "-unchecked", "-encoding", "UTF8", "-language:implicitConversions"), From cd2a13b6bdea1518768d5a0e4c4592d70a929888 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Fri, 3 Oct 2025 09:50:53 -0600 Subject: [PATCH 5/5] . --- project/Build.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project/Build.scala b/project/Build.scala index 1f82f362fd5c..1e043b2b5143 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -2313,6 +2313,8 @@ object Build { "org.jline" % "jline-terminal-jni" % "3.29.0", "com.github.sbt" % "junit-interface" % "0.13.3" % Test, ("io.get-coursier" %% "coursier" % "2.0.16" % Test).cross(CrossVersion.for3Use2_13), + "io.get-coursier" % "interface" % "1.0.19", // used by the REPL for dependency resolution + "org.virtuslab" % "using_directives" % "1.1.4", // used by the REPL for parsing magic comments ), // NOTE: The only difference here is that we drop `-Werror` and semanticDB for now Compile / scalacOptions := Seq("-deprecation", "-feature", "-unchecked", "-encoding", "UTF8", "-language:implicitConversions"),