Fix pkg.pr.new redirects (#1204)

Author: ije

README.md

@@ -147,6 +147,15 @@ You can also add the `?standalone` flag to bundle the module along with all its
 import { Button } from "https://esm.sh/antd?standalone";
 ```
 
+You can disable the default transforming/bundling behavior by adding `?raw` query to the import URL.
+
+```js
+import { render } from "https://esm.sh/preact?raw";
+```
+
+> [!TIP]
+> The `?raw` query is useful when you want to import the raw JavaScript source code of a package, as-is, without transformation into ES modules.
+
 ### Tree Shaking
 
 By default, esm.sh exports a module with all its exported members. However, if you want to import only a specific set of
@@ -329,8 +338,9 @@ In development mode (open the page on localhost), `esm.sh/tsx` uses [@esm.sh/tsx
 
 ## Escape Hatch: Raw Source Files
 
-In rare cases, you may want to request JS source files from packages, as-is, without transformation into ES modules. To
-do so, you need to add a `?raw` query to the request URL.
+By default, esm.sh transforms (and bundles if necessary) the JavaScript source code. However, in rare cases, you may want to request JS source files from packages, as-is, without transformation into ES modules. To do so, you need to add a `?raw` query to the request URL.
+
+The `raw` mode works just like other CDN services, unpkg.com(https://unpkg.com/), jsdelivr.net(https://www.jsdelivr.net/), etc.
 
 ```html
 <script src="https://esm.sh/p5/lib/p5.min.js?raw"></script>

internal/fetch/fetch.go

@@ -36,8 +36,8 @@ func NewClient(userAgent string, timeout int, reserveRedirect bool, allowedHosts
 				return http.ErrUseLastResponse
 			}
 		}
-		if len(via) >= 3 {
-			return errors.New("stopped after 3 redirects")
+		if len(via) >= 6 {
+			return errors.New("too many redirects")
 		}
 		return nil
 	}

server/npmrc.go

@@ -508,23 +508,25 @@ RETRY:
 			time.Sleep(time.Duration(retryTimes) * 100 * time.Millisecond)
 			goto RETRY
 		}
+		err = fmt.Errorf("failed to download tarball of package '%s': %v", pkgName, err)
 		return
 	}
 	defer res.Body.Close()
 
 	if res.StatusCode == 404 || res.StatusCode == 401 {
-		err = fmt.Errorf("tarball of package '%s' not found", path.Base(installDir))
+		err = fmt.Errorf("tarball of package '%s' not found", pkgName)
 		return
 	}
 
 	if res.StatusCode != 200 {
 		msg, _ := io.ReadAll(res.Body)
-		err = fmt.Errorf("could not download tarball of package '%s' (%s: %s)", path.Base(installDir), res.Status, string(msg))
+		err = fmt.Errorf("could not download tarball of package '%s' (%s: %s)", pkgName, res.Status, string(msg))
 		return
 	}
 
 	err = extractPackageTarball(installDir, pkgName, io.LimitReader(res.Body, maxPackageTarballSize))
 	if err != nil {
+		err = fmt.Errorf("failed to extract tarball of package '%s': %v", pkgName, err)
 		// clear installDir if failed to extract tarball
 		os.RemoveAll(installDir)
 	}

server/path.go

@@ -3,11 +3,15 @@ package server
 import (
 	"errors"
 	"fmt"
+	"net/http"
 	"net/url"
+	"regexp"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/Masterminds/semver/v3"
+	"github.com/esm-dev/esm.sh/internal/fetch"
 	"github.com/esm-dev/esm.sh/internal/npm"
 	"github.com/ije/gox/set"
 	"github.com/ije/gox/utils"
@@ -70,7 +74,6 @@ func parseEsmPath(npmrc *NpmRC, pathname string) (esm EsmPath, extraQuery string
 			err = errors.New("invalid path")
 			return
 		}
-		exactVersion = true
 		hasTargetSegment = validateTargetSegment(strings.Split(subPath, "/"))
 		esm = EsmPath{
 			PkgName:       pkgName,
@@ -79,6 +82,17 @@ func parseEsmPath(npmrc *NpmRC, pathname string) (esm EsmPath, extraQuery string
 			SubModuleName: stripEntryModuleExt(subPath),
 			PrPrefix:      true,
 		}
+		if isCommitish(esm.PkgVersion) {
+			exactVersion = true
+			return
+		}
+		esm.PkgVersion, err = resolvePrPackageVersion(esm)
+		if err != nil {
+			return
+		}
+		if !isCommitish(esm.PkgVersion) {
+			err = errors.New("pkg.pr.new: tag or branch not found")
+		}
 		return
 	}
 
@@ -151,69 +165,28 @@ func parseEsmPath(npmrc *NpmRC, pathname string) (esm EsmPath, extraQuery string
 	}
 
 	if ghPrefix {
-		if npm.IsExactVersion(strings.TrimPrefix(esm.PkgVersion, "v")) {
+		if npm.IsExactVersion(strings.TrimPrefix(esm.PkgVersion, "v")) || isCommitish(esm.PkgVersion) {
 			exactVersion = true
 			return
 		}
-		var refs []GitRef
-		refs, err = listGhRepoRefs(fmt.Sprintf("https://github.com/%s", esm.PkgName))
+
+		esm.PkgVersion, err = resolveGhPackageVersion(esm)
 		if err != nil {
 			return
 		}
-		if esm.PkgVersion == "" {
-			for _, ref := range refs {
-				if ref.Ref == "HEAD" {
-					esm.PkgVersion = ref.Sha[:7]
-					return
-				}
-			}
-		} else {
-			// try to find the exact tag or branch
-			for _, ref := range refs {
-				if ref.Ref == "refs/tags/"+esm.PkgVersion || ref.Ref == "refs/heads/"+esm.PkgVersion {
-					esm.PkgVersion = ref.Sha[:7]
-					return
-				}
-			}
-			// try to find the 'semver' tag
-			if semv, erro := semver.NewConstraint(strings.TrimPrefix(esm.PkgVersion, "semver:")); erro == nil {
-				semtags := make([]*semver.Version, len(refs))
-				i := 0
-				for _, ref := range refs {
-					if strings.HasPrefix(ref.Ref, "refs/tags/") {
-						v, e := semver.NewVersion(strings.TrimPrefix(ref.Ref, "refs/tags/"))
-						if e == nil && semv.Check(v) {
-							semtags[i] = v
-							i++
-						}
-					}
-				}
-				if i > 0 {
-					semtags = semtags[:i]
-					if i > 1 {
-						sort.Sort(semver.Collection(semtags))
-					}
-					esm.PkgVersion = semtags[i-1].String()
-					return
-				}
-			}
-		}
 
 		if !isCommitish(esm.PkgVersion) {
-			err = errors.New("git: tag or branch not found")
-			return
+			err = errors.New("github: tag or branch not found")
 		}
-
-		exactVersion = true
 		return
 	}
 
 	originalExactVersion := len(esm.PkgVersion) > 0 && npm.IsExactVersion(esm.PkgVersion)
 	exactVersion = originalExactVersion
-	
+
 	// Check if version is a date format (yyyy-mm-dd)
 	isDateVersion := npm.IsDateVersion(esm.PkgVersion)
-	
+
 	if !originalExactVersion {
 		var p *npm.PackageJSON
 		if isDateVersion {
@@ -286,3 +259,79 @@ func isPackageInExternalNamespace(pkgName string, external set.ReadOnlySet[strin
 	}
 	return false
 }
+
+func resolveGhPackageVersion(esm EsmPath) (version string, err error) {
+	return withCache("gh/"+esm.PkgName+"@"+esm.PkgVersion, time.Duration(config.NpmQueryCacheTTL)*time.Second, func() (version string, aliasKey string, err error) {
+		var refs []GitRef
+		refs, err = listGhRepoRefs(fmt.Sprintf("https://github.com/%s", esm.PkgName))
+		if err != nil {
+			return
+		}
+		if esm.PkgVersion == "" {
+			for _, ref := range refs {
+				if ref.Ref == "HEAD" {
+					version = ref.Sha[:7]
+					return
+				}
+			}
+		} else {
+			// try to find the exact tag or branch
+			for _, ref := range refs {
+				if ref.Ref == "refs/tags/"+esm.PkgVersion || ref.Ref == "refs/heads/"+esm.PkgVersion {
+					version = ref.Sha[:7]
+					return
+				}
+			}
+			// try to find the 'semver' tag
+			if semv, erro := semver.NewConstraint(strings.TrimPrefix(esm.PkgVersion, "semver:")); erro == nil {
+				semtags := make([]*semver.Version, len(refs))
+				i := 0
+				for _, ref := range refs {
+					if after, ok := strings.CutPrefix(ref.Ref, "refs/tags/"); ok {
+						v, e := semver.NewVersion(after)
+						if e == nil && semv.Check(v) {
+							semtags[i] = v
+							i++
+						}
+					}
+				}
+				if i > 0 {
+					semtags = semtags[:i]
+					if i > 1 {
+						sort.Sort(semver.Collection(semtags))
+					}
+					version = semtags[i-1].String()
+					return
+				}
+			}
+		}
+		version = esm.PkgVersion
+		return
+	})
+}
+
+func resolvePrPackageVersion(esm EsmPath) (version string, err error) {
+	return withCache("pr/"+esm.PkgName+"@"+esm.PkgVersion, time.Duration(config.NpmQueryCacheTTL)*time.Second, func() (version string, aliasKey string, err error) {
+		u, err := url.Parse(fmt.Sprintf("https://pkg.pr.new/%s@%s", esm.PkgName, esm.PkgVersion))
+		if err != nil {
+			return
+		}
+		versionRegex := regexp.MustCompile(`[^/]@([\da-f]{7,})$`)
+		client, recycle := fetch.NewClient("esmd/"+VERSION, 30, false, nil)
+		version = esm.PkgVersion
+		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			match := versionRegex.FindStringSubmatch(req.URL.Path)
+			if len(match) > 1 {
+				version = match[1][:7]
+			}
+			if len(via) >= 6 {
+				return errors.New("too many redirects")
+			}
+			return nil
+		}
+		defer recycle()
+
+		_, err = client.Fetch(u, nil)
+		return
+	})
+}

server/router.go

@@ -1026,7 +1026,53 @@ func esmRouter(db Database, esmStorage storage.Storage, logger *log.Logger) rex.
 				ctx.SetHeader("Cache-Control", fmt.Sprintf("public, max-age=%d", config.NpmQueryCacheTTL))
 				return redirect(ctx, fmt.Sprintf("%s%s/%s@%s%s%s", origin, registryPrefix, pkgName, pkgVersion, subPath, query), false)
 			}
-		} else {
+		}
+
+		// fix url that is related to `import.meta.url`
+		if hasTargetSegment && isExactVersion && pathKind == RawFile && !rawFlag {
+			extname := path.Ext(esm.SubPath)
+			dir := path.Join(npmrc.StoreDir(), esm.Name())
+			if !existsDir(dir) {
+				_, err := npmrc.installPackage(esm.Package())
+				if err != nil {
+					return rex.Status(500, err.Error())
+				}
+			}
+			pkgRoot := path.Join(dir, "node_modules", esm.PkgName)
+			files, err := findFiles(pkgRoot, "", func(fp string) bool {
+				return strings.HasSuffix(fp, extname)
+			})
+			if err != nil {
+				return rex.Status(500, err.Error())
+			}
+			var file string
+			if l := len(files); l == 1 {
+				file = files[0]
+			} else if l > 1 {
+				for _, f := range files {
+					if strings.HasSuffix(esm.SubPath, f) {
+						file = f
+						break
+					}
+				}
+				if file == "" {
+					for _, f := range files {
+						if path.Base(esm.SubPath) == path.Base(f) {
+							file = f
+							break
+						}
+					}
+				}
+			}
+			if file == "" {
+				return rex.Status(404, "File not found")
+			}
+			url := fmt.Sprintf("%s%s/%s@%s/%s", origin, registryPrefix, esm.PkgName, esm.PkgVersion, file)
+			return redirect(ctx, url, true)
+		}
+
+		// try to serve package static files if the version is exact
+		if isExactVersion {
 			// return wasm file as an es6 module when `?module` query is present (requires `top-level-await` support)
 			if pathKind == RawFile && strings.HasSuffix(esm.SubPath, ".wasm") && query.Has("module") {
 				wasmUrl := origin + pathname
@@ -1066,50 +1112,7 @@ func esmRouter(db Database, esmStorage storage.Storage, logger *log.Logger) rex.
 				return buf
 			}
 
-			// fix url that is related to `import.meta.url`
-			if hasTargetSegment && pathKind == RawFile && !rawFlag {
-				extname := path.Ext(esm.SubPath)
-				dir := path.Join(npmrc.StoreDir(), esm.Name())
-				if !existsDir(dir) {
-					_, err := npmrc.installPackage(esm.Package())
-					if err != nil {
-						return rex.Status(500, err.Error())
-					}
-				}
-				pkgRoot := path.Join(dir, "node_modules", esm.PkgName)
-				files, err := findFiles(pkgRoot, "", func(fp string) bool {
-					return strings.HasSuffix(fp, extname)
-				})
-				if err != nil {
-					return rex.Status(500, err.Error())
-				}
-				var file string
-				if l := len(files); l == 1 {
-					file = files[0]
-				} else if l > 1 {
-					for _, f := range files {
-						if strings.HasSuffix(esm.SubPath, f) {
-							file = f
-							break
-						}
-					}
-					if file == "" {
-						for _, f := range files {
-							if path.Base(esm.SubPath) == path.Base(f) {
-								file = f
-								break
-							}
-						}
-					}
-				}
-				if file == "" {
-					return rex.Status(404, "File not found")
-				}
-				url := fmt.Sprintf("%s%s/%s@%s/%s", origin, registryPrefix, esm.PkgName, esm.PkgVersion, file)
-				return redirect(ctx, url, true)
-			}
-
-			// package raw files
+			// serve package raw files
 			if pathKind == RawFile {
 				if esm.SubPath == "" {
 					b := &BuildContext{
@@ -1192,7 +1195,7 @@ func esmRouter(db Database, esmStorage storage.Storage, logger *log.Logger) rex.
 				return content // auto closed
 			}
 
-			// build/dts files
+			// serve build/dts files
 			if pathKind == EsmBuild || pathKind == EsmSourceMap || pathKind == EsmDts {
 				var savePath string
 				if asteriskPrefix {