Add "add" sub-command

Author: code-asher

README.md

@@ -5,8 +5,8 @@ Marketplace for use in editors like
 [code-server](https://github.com/cdr/code-server).
 
 This marketplace reads extensions from file storage and provides an API for
-editors to consume. It does not have a frontend or any mechanisms for adding or
-updating extensions in the marketplace.
+editors to consume. It does not have a frontend or any mechanisms for extension
+authors to add or update extensions in the marketplace.
 
 ## Deployment
 
@@ -26,34 +26,6 @@ reject connecting to the API.
 The `/healthz` endpoint can be used to determine if the marketplace is ready to
 receive requests.
 
-## File Storage
-
-Extensions must be both copied as a vsix and extracted to the following path:
-
-```
-<extensions-dir>/<publisher>/<extension name>/<version>/
-```
-
-For example:
-
-```
-extensions
-|-- ms-python
-|   `-- python
-|       `-- 2022.14.0
-|           |-- [Content_Types].xml
-|           |-- extension
-|           |-- extension.vsixmanifest
-|           `-- ms-python.python-2022.14.0.vsix
-`-- vscodevim
-    `-- vim
-        `-- 1.23.2
-            |-- [Content_Types].xml
-            |-- extension
-            |-- extension.vsixmanifest
-            `-- vscodevim.vim-1.23.2.vsix
-```
-
 ## Usage in code-server
 
 ```
@@ -72,31 +44,38 @@ or both the `X-Forwarded-Host` and `X-Forwarded-Proto` headers.
 The marketplace does not support being hosted behind a base path; it must be
 proxied at the root of your domain.
 
-## Getting extensions
+## Adding extensions
+
+Extensions can be added to the marketplace by file or URL.  The extensions
+directory does not need to be created beforehand.
+
+```
+./code-marketplace add extension.vsix --extensions-dir ./extensions
+./code-marketplace add https://domain.tld/extension.vsix --extensions-dir ./extensions
+```
+
+Extensions listed as dependencies must also be added.
+
+If the extension is part of an extension pack the other extensions in the pack
+can also be added but doing so is optional.
 
 If an extension is open source you can get it from one of three locations:
 
 1. GitHub releases (if the extension publishes releases to GitHub).
 2. Open VSX (if the extension is published to Open VSX).
 3. Building from source.
 
-For example to download the Python extension from Open VSX:
+For example to add the Python extension from Open VSX:
 
 ```
-mkdir -p extensions/ms-python/python/2022.14.0
-wget https://open-vsx.org/api/ms-python/python/2022.14.0/file/ms-python.python-2022.14.0.vsix
-unzip ms-python.python-2022.14.0.vsix -d extensions/ms-python/python/2022.14.0
-mv ms-python.python-2022.14.0.vsix extensions/ms-python/python/2022.14.0
+./code-marketplace add https://open-vsx.org/api/ms-python/python/2022.14.0/file/ms-python.python-2022.14.0.vsix --extensions-dir ./extensions
 ```
 
-Make sure to both extract the contents *and* copy/move the `.vsix` file.
-
-If an extension has dependencies those must be added as well.  An extension's
-dependencies can be found in the extension's `package.json` under
-`extensionDependencies`.
+Or the Vim extension from GitHub:
 
-Extensions under `extensionPack` in the extension's `package.json` can be added
-as well although doing so is not required.
+```
+./code-marketplace add https://github.com/VSCodeVim/Vim/releases/download/v1.24.1/vim-1.24.1.vsix --extensions-dir ./extensions
+```
 
 ## Development
 

api/api_test.go

@@ -26,6 +26,10 @@ import (
 
 type fakeStorage struct{}
 
+func (s *fakeStorage) AddExtension(ctx context.Context, source string) (string, error) {
+	return "", errors.New("not implemented")
+}
+
 func (s *fakeStorage) FileServer() http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/nonexistent" {

cli/add.go

@@ -0,0 +1,63 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/sloghuman"
+
+	"github.com/coder/code-marketplace/storage"
+)
+
+func add() *cobra.Command {
+	var (
+		extdir string
+	)
+
+	cmd := &cobra.Command{
+		Use:   "add <source>",
+		Short: "Add an extension to the marketplace",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx, cancel := context.WithCancel(cmd.Context())
+			defer cancel()
+
+			verbose, err := cmd.Flags().GetBool("verbose")
+			if err != nil {
+				return err
+			}
+			logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr()))
+			if verbose {
+				logger = logger.Leveled(slog.LevelDebug)
+			}
+
+			extdir, err = filepath.Abs(extdir)
+			if err != nil {
+				return err
+			}
+
+			// Always local storage for now.
+			store := &storage.Local{
+				ExtDir: extdir,
+				Logger: logger,
+			}
+
+			dest, err := store.AddExtension(ctx, args[0])
+			if err != nil {
+				return err
+			}
+
+			_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Extension unpacked to %s\n", dest)
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&extdir, "extensions-dir", "", "The path to extensions.")
+	_ = cmd.MarkFlagRequired("extensions-dir")
+
+	return cmd
+}

cli/add_test.go

@@ -0,0 +1,25 @@
+package cli_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/code-marketplace/cli"
+)
+
+func TestAdd(t *testing.T) {
+	t.Parallel()
+
+	cmd := cli.Root()
+	cmd.SetArgs([]string{"add", "--help"})
+	buf := new(bytes.Buffer)
+	cmd.SetOut(buf)
+
+	err := cmd.Execute()
+	require.NoError(t, err)
+
+	output := buf.String()
+	require.Contains(t, output, "Add an extension", "has help")
+}

cli/root.go

@@ -13,7 +13,7 @@ func Root() *cobra.Command {
 		Example:       "  marketplace server --extensions-dir /path/to/extensions",
 	}
 
-	cmd.AddCommand(server(), version())
+	cmd.AddCommand(add(), server(), version())
 
 	cmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose output")
 

database/database_test.go

@@ -2,6 +2,7 @@ package database_test
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -19,6 +20,10 @@ import (
 
 type memoryStorage struct{}
 
+func (s *memoryStorage) AddExtension(ctx context.Context, source string) (string, error) {
+	return "", errors.New("not implemented")
+}
+
 func (s *memoryStorage) FileServer() http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		http.Error(rw, "not implemented", http.StatusNotImplemented)

storage/local.go

@@ -1,7 +1,10 @@
 package storage
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -18,6 +21,57 @@ type Local struct {
 	Logger slog.Logger
 }
 
+func (s *Local) AddExtension(ctx context.Context, source string) (string, error) {
+	vsixBytes, err := readVSIX(ctx, source)
+	if err != nil {
+		return "", err
+	}
+
+	mr, err := GetZipFileReader(vsixBytes, "extension.vsixmanifest")
+	if err != nil {
+		return "", err
+	}
+	defer mr.Close()
+
+	manifest, err := parseVSIXManifest(mr)
+	if err != nil {
+		return "", err
+	}
+
+	err = validateManifest(manifest)
+	if err != nil {
+		return "", err
+	}
+
+	// Extract the zip to the correct path.
+	identity := manifest.Metadata.Identity
+	dir := filepath.Join(s.ExtDir, identity.Publisher, identity.ID, identity.Version)
+	err = ExtractZip(vsixBytes, func(name string) (io.Writer, error) {
+		path := filepath.Join(dir, name)
+		err := os.MkdirAll(filepath.Dir(path), 0o755)
+		if err != nil {
+			return nil, err
+		}
+		return os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
+	})
+	if err != nil {
+		return "", nil
+	}
+
+	// Copy the VSIX itself as well.
+	vsixName := fmt.Sprintf("%s.%s-%s.vsix", identity.Publisher, identity.ID, identity.Version)
+	dst, err := os.OpenFile(filepath.Join(dir, vsixName), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
+	if err != nil {
+		return "", nil
+	}
+	_, err = io.Copy(dst, bytes.NewReader(vsixBytes))
+	if err != nil {
+		return "", nil
+	}
+
+	return dir, nil
+}
+
 func (s *Local) FileServer() http.Handler {
 	return http.FileServer(http.Dir(s.ExtDir))
 }

storage/storage.go

@@ -6,6 +6,10 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"strings"
+
+	"golang.org/x/xerrors"
 )
 
 const VSIXAssetType = "Microsoft.VisualStudio.Services.VSIXPackage"
@@ -76,6 +80,10 @@ type VSIXAsset struct {
 
 // TODO: Add Artifactory implementation of Storage.
 type Storage interface {
+	// AddExtension adds the extension found at the specified source by copying it
+	// into the extension storage directory and returns the location of the new
+	// extension.  The source may be an URI or a local file path.
+	AddExtension(ctx context.Context, source string) (string, error)
 	// FileServer provides a handler for fetching extension repository files from
 	// a client.
 	FileServer() http.Handler
@@ -112,3 +120,44 @@ func parseVSIXManifest(reader io.Reader) (*VSIXManifest, error) {
 
 	return vm, nil
 }
+
+// validateManifest checks a manifest for issues.
+func validateManifest(manifest *VSIXManifest) error {
+	if manifest == nil {
+		return xerrors.Errorf("vsix did not contain a manifest")
+	}
+	identity := manifest.Metadata.Identity
+	if identity.Publisher == "" {
+		return xerrors.Errorf("manifest did not contain a publisher")
+	} else if identity.ID == "" {
+		return xerrors.Errorf("manifest did not contain an ID")
+	} else if identity.Version == "" {
+		return xerrors.Errorf("manifest did not contain a version")
+	}
+
+	return nil
+}
+
+// readVSIX reads the bytes of a VSIX from the specified source.  The source
+// might be a URI or a local file path.
+func readVSIX(ctx context.Context, source string) ([]byte, error) {
+	if !strings.HasPrefix(source, "http://") && !strings.HasPrefix(source, "https://") {
+		// Assume it is a local file path.
+		return os.ReadFile(source)
+	}
+
+	resp, err := http.Get(source)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
+		return nil, xerrors.Errorf("error retrieving vsix: status code %d", resp.StatusCode)
+	}
+
+	return io.ReadAll(&io.LimitedReader{
+		R: resp.Body,
+		N: 100 * 1000 * 1000, // 100 MB
+	})
+}