Add support for npm packages in PackageHub

Previously, PackageHub only supported Maven and JDK packages. Now,
PackageHub can additionally serve sources of npm packages. This makes it
possible to LSIF index the sources of npm packages similar to how VS
Code and IntelliJ index the `node_modules/` directory.

Author: olafurpg

Dockerfile

@@ -6,6 +6,6 @@ RUN git config --global user.name "Your Name"
 RUN git config --global http.postBuffer 1048576000
 RUN curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src
 RUN chmod +x /src
-RUN /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:0.5.1-26-2d4609cc-SNAPSHOT -o /packagehub
+RUN /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:0.5.1-43-da43fdbb-SNAPSHOT -o /packagehub
 ENV COURSIER_REPOSITORIES=central|https://maven.google.com/|jitpack
 CMD ["/packagehub.sh"]

auto-indexing/Dockerfile

@@ -1,6 +1,7 @@
 FROM gradle:7.0.0-jdk8@sha256:25ecb23e3516b697219f5a780bf68598318cf13b35ca616f4521c7e9e115a11e
-RUN apt-get update && \
-      apt-get install --yes maven && \
-      curl -fLo /coursier https://git.io/coursier-cli && \
-      chmod +x /coursier && \
-      /coursier launch --contrib lsif-java -- --help
+RUN apt-get update
+RUN apt-get install --yes maven npm
+RUN curl -fLo /coursier https://git.io/coursier-cli
+RUN chmod +x /coursier
+RUN apt-get install --yes yarn
+RUN /coursier launch --contrib lsif-java -- --help

build.sbt

@@ -229,6 +229,7 @@ lazy val packagehub = project
         "com.zaxxer" % "HikariCP" % "4.0.3",
         "org.flywaydb" % "flyway-core" % "7.7.1",
         "org.postgresql" % "postgresql" % "42.2.14",
+        "org.rauschig" % "jarchivelib" % "1.1.0",
         "org.scalameta" %% "scalameta" % V.scalameta,
         "com.lihaoyi" %% "cask" % "0.7.8"
       )

packagehub/src/main/scala/com/sourcegraph/packagehub/Package.scala

@@ -31,6 +31,9 @@ sealed abstract class Package(
   def relativePath: Path = Paths.get(path)
 }
 object Package {
+  def npm(name: String, version: String): NpmPackage = {
+    NpmPackage(name, version)
+  }
   def jdk(version: String): JdkPackage = {
     JdkPackage(version)
   }
@@ -49,6 +52,8 @@ object Package {
       case s"maven:$library" =>
         val Right(dep) = Dependencies.parseDependencyEither(library)
         MavenPackage(dep)
+      case s"npm:$name:$version" =>
+        NpmPackage(name, version)
     }
   }
   def fromPath(path: List[String]): Option[(Package, List[String])] =
@@ -57,19 +62,37 @@ object Package {
         Some(Package.maven(org, name, version) -> requestPath)
       case "jdk" :: version :: requestPath =>
         Some(Package.jdk(version) -> requestPath)
+      case "npm" :: GitRequestPrefix(parts, requestPath) =>
+        val name = parts.init.mkString("/")
+        val version = parts.last
+        val actualName =
+          if (name.startsWith("-"))
+            "@" + name.stripPrefix("-")
+          else
+            name
+        Some(Package.npm(actualName, version) -> requestPath)
       case _ =>
         None
     }
+
+  object GitRequestPrefix {
+    private val suffixes = List(
+      ".git" :: "info" :: "refs" :: Nil,
+      "info" :: "refs" :: Nil,
+      ".git" :: "git-upload-pack" :: Nil,
+      "git-upload-pack" :: Nil
+    )
+    def unapply(path: List[String]): Option[(List[String], List[String])] = {
+      suffixes.find(path.endsWith) match {
+        case Some(suffix) =>
+          Some(path.dropRight(suffix.length) -> suffix)
+        case None =>
+          None
+      }
+    }
+  }
   def fromString(value: String, coursier: String): Either[String, Package] = {
     value match {
-      case s"jdk:$version" =>
-        val exit = os
-          .proc(coursier, "java-home", "--jvm", version)
-          .call(check = false)
-        if (exit.exitCode == 0)
-          Right(JdkPackage(version))
-        else
-          Left(exit.out.trim())
       case s"maven:$library" =>
         try {
           val deps = Dependencies
@@ -85,6 +108,29 @@ object Package {
           case NonFatal(e) =>
             Left(e.getMessage())
         }
+      case s"jdk:$version" =>
+        val exit = os
+          .proc(coursier, "java-home", "--jvm", version)
+          .call(check = false)
+        if (exit.exitCode == 0)
+          Right(JdkPackage(version))
+        else
+          Left(exit.out.trim())
+      case s"npm:$name:$version" =>
+        try {
+          val out = os
+            .proc("npm", "info", s"$name@$version")
+            .call(check = false)
+            .out
+            .trim()
+          if (out.nonEmpty)
+            Right(NpmPackage(name, version))
+          else
+            Left(s"no such npm package: $name@$version")
+        } catch {
+          case NonFatal(e) =>
+            Left(e.getMessage)
+        }
       case other =>
         Left(
           s"unsupported package '$other'. To fix this problem, use a valid syntax " +
@@ -120,3 +166,13 @@ case class JdkPackage(override val version: String)
     extends Package(s"jdk:${version}", s"jdk/${version}", version) {
   def repr = id.stripPrefix("jdk:")
 }
+
+case class NpmPackage(packageName: String, override val version: String)
+    extends Package(
+      s"npm:$packageName:$version",
+      s"npm/$packageName/$version".replace("@", "-"),
+      version
+    ) {
+  def npmName = s"$packageName@$version"
+  def tarballFilename = s"$packageName-$version.tgz".replace('/', '-')
+}