General purpose web IDE for MLscript

hkmc2/js/src/main/scala/hkmc2/Compiler.scala

@@ -64,7 +64,7 @@ class Compiler(paths: MLsCompiler.Paths)(using cctx: CompilerCtx):
     perFileDiagnostics
 
 @JSExportTopLevel("Paths")
-final class Paths(prelude: Str, runtime: Str, term: Str) extends MLsCompiler.Paths:
+final class Paths(prelude: Str, runtime: Str, term: Str, std: Str) extends MLsCompiler.Paths:
   val preludeFile = Path(prelude)
   val runtimeFile = Path(runtime)
   val termFile = Path(term)

hkmc2/js/src/test/scala/hkmc2/CompilerTest.scala

@@ -8,34 +8,44 @@ import scala.scalajs.js.annotation._
 import scala.scalajs.js.Dynamic.global
 
 class CompilerTest extends AnyFunSuite:
+  val projectRoot = node.process.cwd()
+  val compilePath = node.path.join(projectRoot, "hkmc2", "shared", "src", "test", "mlscript-compile")
+  val runtimePath = node.path.join(projectRoot, "hkmc2", "shared", "src", "test", "mlscript-compile", "RuntimeJS.mjs")
+  val preludePath = node.path.join(projectRoot, "hkmc2", "shared", "src", "test", "mlscript", "decls", "Prelude.mls")
+
   private def loadStandardLibrary(): Map[String, String] =
-    val projectRoot = node.process.cwd()
-    val compilePath = node.path.join(projectRoot, "hkmc2", "shared", "src", "test", "mlscript-compile")
-    val preludePath = node.path.join(projectRoot, "hkmc2", "shared", "src", "test", "mlscript", "decls", "Prelude.mls")
 
-    node.fs.readdirSync(compilePath).filter(_.endsWith(".mls")).toSeq.flatMap: fileName =>
+    // Actually, there's no need to load `.mjs` files. But we did it since we
+    // were importing them to reduce duplicated parsing and elaboration. The
+    // imports can be reverted back to `.mls` files, but we should check that
+    // doing so does not cause any significant slowdown first.
+    node.fs.readdirSync(compilePath).filter:
+      fileName => fileName.endsWith(".mls") || fileName.endsWith(".mjs")
+    .toSeq.flatMap: fileName =>
       val filePath = node.path.join(compilePath, fileName)
       if node.fs.existsSync(filePath) then
         Some(s"/std/$fileName" -> node.fs.readFileSync(filePath, "utf-8"))
       else
         None
-    .toMap + ("/std/Prelude.mls" -> node.fs.readFileSync(preludePath, "utf-8"))
+    .toMap
+      + ("/std/RuntimeJS.mjs" -> node.fs.readFileSync(runtimePath, "utf-8"))
+      + ("/std/Prelude.mls" -> node.fs.readFileSync(preludePath, "utf-8"))
 
-  private val paths = new Paths("/std/Prelude.mls", "/std/Runtime.mjs", "/std/Term.mjs")
+  private val paths = new Paths("/std/Prelude.mls", "/std/Runtime.mjs", "/std/Term.mjs", "/std")
 
   private def createCompiler(): (InMemoryFileSystem, Compiler) =
     val stdLib = loadStandardLibrary()
     val fs = new InMemoryFileSystem(stdLib)
-    given CompilerCtx = CompilerCtx.fresh(fs)
+    given CompilerCtx = CompilerCtx.fresh(fs, LocalTestModuleResolver(io.Path("/std")))
     (fs, new Compiler(paths))
 
   test("compiler can compile a simple program"):
     val (fs, compiler) = createCompiler()
 
     // Write test program to the file system
-    val code = """|import "./std/Option.mls"
-                  |import "./std/Stack.mls"
-                  |import "./std/Predef.mls"
+    val code = """|import "std/Option.mls"
+                  |import "std/Stack.mls"
+                  |import "std/Predef.mls"
                   |
                   |open Stack
                   |open Option

hkmc2/jvm/src/test/scala/hkmc2/AppTestRunner.scala

@@ -0,0 +1,73 @@
+package hkmc2
+
+import org.scalatest.{funsuite, funspec, ParallelTestExecution}
+import org.scalatest.time._
+import org.scalatest.concurrent.{TimeLimitedTests, Signaler}
+
+import mlscript.utils.*, shorthands.*
+import io.PlatformPath.given
+
+import AppTestRunner.given
+import hkmc2.codegen.Local
+
+/**
+  * A simple test runner that compiles apps written in MLscript.
+  */
+class AppTestRunner
+  extends funspec.AnyFunSpec
+  // with ParallelTestExecution // Can `MLsCompiler` handle parallel compilation?
+  // with TimeLimitedTests // TODO
+:
+  import AppTestRunner.*
+
+  private val inParallel = isInstanceOf[ParallelTestExecution]
+
+  // val timeLimit = TimeLimit
+
+  for app <- os.list(appsDir).filter(os.isDir) do
+    val allFiles = os.walk(app).filter(os.isFile).filter(_.ext == "mls").toSeq
+    val appName = app.baseName
+
+    given Config = Config.default
+    val wrap: (=> Unit) => Unit = body => AppTestRunner.synchronized(body)
+    val report = ReportFormatter(System.out.println, colorize = true, wrap = Some(wrap))
+    val compiler = MLsCompiler(paths, mkRaise = report.mkRaise)
+
+    describe(s"$appName (${"file" countBy allFiles.size})"):
+
+      allFiles.foreach: file =>
+        val relativeName = file.relativeTo(app).toString()
+
+        it(relativeName):
+
+          AppTestRunner.synchronized:
+            println(s"Compiling: [${fansi.Bold.On(appName)}] ${fansi.Color.Green(relativeName)}")
+
+          assert(true, s"Placeholder test for app: $relativeName")
+
+          compiler.compileModule(file)
+
+          if report.badLines.nonEmpty then
+            fail(s"Unexpected diagnostic at: " +
+              report.badLines.distinct.sorted
+                .map("\n\t"+relativeName+"."+file.ext+":"+_).mkString(", "))
+end AppTestRunner
+
+object AppTestRunner:
+
+  val mainTestDir = os.pwd / "hkmc2" / "shared" / "src" / "test"
+  val appsDir = mainTestDir / "mlscript-apps"
+  val stdlibDir = mainTestDir / "mlscript-compile"
+
+  val paths = new MLsCompiler.Paths:
+    val preludeFile = mainTestDir / "mlscript" / "decls" / "Prelude.mls"
+    val runtimeFile = stdlibDir / "Runtime.mjs"
+    val termFile = stdlibDir / "Term.mjs"
+
+  val nodeModulesPath = os.pwd / "node_modules"
+
+  // We may use a different module resolver for URL modules in browsers. For
+  // example, `import "https://esm.sh/nanoid"` should be accepted.
+  given cctx: CompilerCtx = CompilerCtx.fresh(io.FileSystem.default, LocalTestModuleResolver(stdlibDir, S(nodeModulesPath)))
+
+end AppTestRunner

hkmc2/jvm/src/test/scala/hkmc2/CompileTestRunner.scala

@@ -8,7 +8,7 @@ import os.up
 import mlscript.utils._, shorthands._
 import io.PlatformPath.given
 
-import CompileTestRunner.given
+import CompileTestRunner.{*, given}
 
 
 class CompileTestRunner
@@ -21,11 +21,6 @@ class CompileTestRunner
 
   // val timeLimit = TimeLimit
 
-  val pwd = os.pwd
-  val workingDir = pwd
-
-  val mainTestDir = workingDir/"hkmc2"/"shared"/"src"/"test"  
-
   // The compilation tests currently include compiling the benchmark instrumentation code.
   val dirs = mainTestDir :: workingDir/"hkmc2Benchmarks"/"src"/"test" :: Nil
 
@@ -58,10 +53,7 @@ class CompileTestRunner
         val wrap: (=> Unit) => Unit = body => CompileTestRunner.synchronized(body)
         val report = ReportFormatter(System.out.println, colorize = true, wrap = Some(wrap))
         val compiler = MLsCompiler(
-          paths = new MLsCompiler.Paths:
-            val preludeFile = mainTestDir / "mlscript" / "decls" / "Prelude.mls"
-            val runtimeFile = mainTestDir / "mlscript-compile" / "Runtime.mjs"
-            val termFile = mainTestDir / "mlscript-compile" / "Term.mjs",
+          paths = compilerPaths,
           mkRaise = report.mkRaise
         )
         compiler.compileModule(file)
@@ -77,7 +69,20 @@ end CompileTestRunner
 
 object CompileTestRunner:
 
-  given cctx: CompilerCtx = CompilerCtx.fresh(io.FileSystem.default)
+  val pwd = os.pwd
+  val workingDir = pwd
+
+  val mainTestDir = workingDir/"hkmc2"/"shared"/"src"/"test"
+  val stdPath = mainTestDir / "mlscript-compile"
+
+  val compilerPaths = new MLsCompiler.Paths:
+    val preludeFile = mainTestDir / "mlscript" / "decls" / "Prelude.mls"
+    val runtimeFile = mainTestDir / "mlscript-compile" / "Runtime.mjs"
+    val termFile = mainTestDir / "mlscript-compile" / "Term.mjs"
+
+  val nodeModulesPath = workingDir / "node_modules"
+
+  given cctx: CompilerCtx = CompilerCtx.fresh(io.FileSystem.default, LocalTestModuleResolver(stdPath, S(nodeModulesPath)))
 
 end CompileTestRunner
 

hkmc2/shared/src/main/scala/hkmc2/CompilerCtx.scala

@@ -22,6 +22,7 @@ class CompilerCtx(
     val importing: Opt[(io.Path, CompilerCtx)],
     val beingCompiled: Set[io.Path],
     val fs: io.FileSystem,
+    val moduleResolver: ModuleResolver,
     cache: CompilerCache,
 ):
 
@@ -31,7 +32,7 @@ class CompilerCtx(
     case N => Nil
 
   def derive(newFile: io.Path): CompilerCtx =
-    CompilerCtx(S(newFile, this), beingCompiled + newFile, fs, cache)
+    CompilerCtx(S(newFile, this), beingCompiled + newFile, fs, moduleResolver, cache)
 
   def getElaboratedBlock
         (file: io.Path, prelude: Ctx)
@@ -65,7 +66,8 @@ object CompilerCtx:
 
   inline def get(using cctx: CompilerCtx) = cctx
 
-  def fresh(fs: io.FileSystem): CompilerCtx = CompilerCtx(N, Set.empty, fs, new PlatformCompilerCache)
+  def fresh(fs: io.FileSystem, moduleResolver: io.FileSystem ?=> ModuleResolver): CompilerCtx =
+    CompilerCtx(N, Set.empty, fs, moduleResolver(using fs), new PlatformCompilerCache)
 
 end CompilerCtx
 

hkmc2/shared/src/main/scala/hkmc2/MLsCompiler.scala

@@ -9,6 +9,7 @@ import utils.*
 import hkmc2.semantics.*
 import hkmc2.syntax.Keyword.`override`
 import semantics.Elaborator.{Ctx, State}
+import hkmc2.io.Path
 
 
 class ParserSetup(file: io.Path, dbgParsing: Bool)(using state: Elaborator.State, raise: Raise, cctx: CompilerCtx):

hkmc2/shared/src/main/scala/hkmc2/ModuleResolver.scala

@@ -0,0 +1,96 @@
+package hkmc2
+
+import mlscript.utils.*, shorthands.*
+
+trait ModuleResolver:
+  /**
+    * Try to resolve path if it refers to a module or a source file in a module.
+    *
+    * @param path the import path
+    * @return the resolved path and module identifier, if any.
+    */
+  def tryResolveModulePath(path: Str): Opt[(Str \/ io.Path, Opt[Str])]
+
+/**
+  * For local tests, including `CompileTestRunner`, `DiffTestRunner`, etc.
+  *
+  * @param stdPath the path to the standard library folder
+  * @param nodeModulesPath the optional path to the `node_modules` folder.
+  *                        If provided, we will do a simple check to see if
+  *                        the requested Node.js built-in module exists.
+  */
+class LocalTestModuleResolver(stdPath: io.Path, nodeModulesPath: Opt[io.Path])(using fs: io.FileSystem) extends ModuleResolver:
+  import LocalTestModuleResolver.*
+
+  private def existsNodeModule(orgNameOpt: Opt[Str], moduleName: Str): Bool =
+    nodeModulesPath match
+    case S(basePath) =>
+      val packagePath = orgNameOpt match
+        case S(orgName) => basePath / s"@$orgName" / moduleName
+        case N => basePath / moduleName
+      fs.exists(packagePath)
+    case N => false
+
+  /**
+    * Resolve an imported module name to a directory (or file) path. This method
+    * should be overridden by subclasses to provide module resolution logic.
+    *
+    * @param orgName the optional organization name of the module
+    * @param moduleName the name of the module being imported
+    * @param noSubPath if the import does not have a sub-path (i.e., only the module name)
+    * @return the pure module name or the resolved path
+    */
+  private def resolveModule(orgName: Opt[Str], moduleName: Str, noSubPath: Bool): Opt[(Str \/ io.Path, Opt[Str])] =
+    // Currently, there is only one std module, so the implementation here is
+    // hard-coded. If one day we implement the mechanism of a module manifest
+    // (for example, through `.mlson` file or `.witton` file), this part will
+    // need to be updated accordingly.
+    if orgName.isEmpty then
+      if noSubPath then
+        val realName = if moduleName.startsWith("node:") then moduleName.drop(5) else moduleName
+        if allowedNodeJsModules contains realName then S(L(moduleName) -> S(realName))
+        else if existsNodeModule(orgName, moduleName) then S(L(moduleName) -> N)
+        else N
+      else if moduleName == "std" then S(R(stdPath) -> N)
+      else N
+    else N
+
+  def tryResolveModulePath(path: Str): Opt[(Str \/ io.Path, Opt[Str])] = path match
+    case r(orgName, modName, subPath) =>
+      val orgNameOpt = Opt(orgName)
+      if subPath is null then
+        resolveModule(orgNameOpt, modName, true)
+      else
+        val res = resolveModule(orgNameOpt, modName, false).map:
+          case (R(p), id) => (R(p / io.RelPath(subPath)), id)
+          case other => other
+        res
+    case _ => N
+
+object LocalTestModuleResolver:
+  def apply(stdPath: io.Path, nodeModulesPath: Opt[io.Path] = N)(using fs: io.FileSystem): LocalTestModuleResolver =
+    new LocalTestModuleResolver(stdPath, nodeModulesPath)
+
+  /**
+    * The pattern of valid Node.js module specifiers.
+    * 
+    * 1. **Group 1:** Scope (no `@`)
+    *      + Example: `@my-org/foo/bar` → `"my-org"`
+    * 2. **Group 2:** Package name
+    *      + Example: `@my-org/foo/bar` → `"foo"`
+    *      + Example: `mypkg/test` → `"mypkg"`
+    * 3. **Group 3:** The remaining text after the first slash
+    *      + Example: `@my-org/foo/bar/baz` → `"bar/baz"`
+    *      + Example: `mypkg/sub/path` → `"sub/path"`
+    *      + Example: `mypkg` → `null`
+    */
+  private val r = """^(?:@([a-z0-9-~][a-z0-9-._~]*)\/)?((?:node:)?[a-z0-9-~][a-z0-9-._~]*)(?:\/(.*))?$""".r
+
+  /**
+    * An incomplete list of allowed Node.js built-in modules. Add more as needed.
+    */
+  private val allowedNodeJsModules = Set(
+    "fs", "path", "http", "https", "url", "util", "events", "stream", "buffer",
+    "os", "child_process", "vm", "assert", "tty", "process")
+
+// TODO: WebModuleResolver that can resolve URL imports.

hkmc2/shared/src/main/scala/hkmc2/semantics/Elaborator.scala

@@ -1000,7 +1000,7 @@ extends Importer with ucs.SplitElaborator:
       case (m @ PrefixApp(Keywrd(Keyword.`import`), arg)) :: sts =>
         reportUnusedAnnotations
         val (newCtx, newAcc) = arg match
-          case StrLit(path) =>
+          case path: StrLit =>
             val stmt = importPath(path).withLocOf(m)
             (ctx + (stmt.sym.nme -> stmt.sym),
             stmt :: acc)