diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4b2a6be..5f7f4f0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: "~1.19" + go-version: "~1.22" - name: Get Go cache paths id: go-cache-paths diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index beeb080..62dc0f6 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -29,8 +29,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: "~1.19" + go-version: "~1.22" - name: golangci-lint uses: golangci/golangci-lint-action@v6.1.1 with: - version: v1.48.0 + version: v1.58.0 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fe7a8e6..e8bca42 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: "~1.19" + go-version: "~1.22" - name: Echo Go Cache Paths id: go-cache-paths diff --git a/.gitignore b/.gitignore index 504cb83..92996d1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ bin coverage extensions -.idea \ No newline at end of file +.idea diff --git a/api/api.go b/api/api.go index 50f07c5..91a92df 100644 --- a/api/api.go +++ b/api/api.go @@ -112,7 +112,7 @@ func New(options *Options) *API { r.Post("/api/extensionquery", api.extensionQuery) // Endpoint for getting an extension's files or the extension zip. - r.Mount("/files", http.StripPrefix("/files", options.Storage.FileServer())) + r.Mount("/files", http.StripPrefix("/files", storage.HTTPFileServer(options.Storage))) // VS Code can use the files in the response to get file paths but it will // sometimes ignore that and use requests to /assets with hardcoded types to diff --git a/api/api_test.go b/api/api_test.go index a144482..ab7ff40 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -171,7 +171,7 @@ func TestAPI(t *testing.T) { Response: "foobar", }, { - Name: "FileAPI", + Name: "FileAPINotExists", Path: "/files/nonexistent", Status: http.StatusNotFound, }, diff --git a/cli/add.go b/cli/add.go index 05d61f8..1bf17d4 100644 --- a/cli/add.go +++ b/cli/add.go @@ -10,20 +10,12 @@ import ( "github.com/spf13/cobra" "golang.org/x/xerrors" - "cdr.dev/slog" - "cdr.dev/slog/sloggers/sloghuman" - "github.com/coder/code-marketplace/storage" "github.com/coder/code-marketplace/util" ) func add() *cobra.Command { - var ( - artifactory string - extdir string - repo string - ) - + addFlags, opts := serverFlags() cmd := &cobra.Command{ Use: "add ", Short: "Add an extension to the marketplace", @@ -37,21 +29,7 @@ func add() *cobra.Command { ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() - verbose, err := cmd.Flags().GetBool("verbose") - if err != nil { - return err - } - logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) - if verbose { - logger = logger.Leveled(slog.LevelDebug) - } - - store, err := storage.NewStorage(ctx, &storage.Options{ - Artifactory: artifactory, - ExtDir: extdir, - Logger: logger, - Repo: repo, - }) + store, err := storage.NewStorage(ctx, opts) if err != nil { return err } @@ -98,10 +76,7 @@ func add() *cobra.Command { return nil }, } - - cmd.Flags().StringVar(&extdir, "extensions-dir", "", "The path to extensions.") - cmd.Flags().StringVar(&artifactory, "artifactory", "", "Artifactory server URL.") - cmd.Flags().StringVar(&repo, "repo", "", "Artifactory repository.") + addFlags(cmd) return cmd } diff --git a/cli/remove.go b/cli/remove.go index 5c3add7..45aa686 100644 --- a/cli/remove.go +++ b/cli/remove.go @@ -10,20 +10,15 @@ import ( "github.com/spf13/cobra" "golang.org/x/xerrors" - "cdr.dev/slog" - "cdr.dev/slog/sloggers/sloghuman" - "github.com/coder/code-marketplace/storage" "github.com/coder/code-marketplace/util" ) func remove() *cobra.Command { var ( - all bool - artifactory string - extdir string - repo string + all bool ) + addFlags, opts := serverFlags() cmd := &cobra.Command{ Use: "remove ", @@ -37,21 +32,7 @@ func remove() *cobra.Command { ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() - verbose, err := cmd.Flags().GetBool("verbose") - if err != nil { - return err - } - logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) - if verbose { - logger = logger.Leveled(slog.LevelDebug) - } - - store, err := storage.NewStorage(ctx, &storage.Options{ - Artifactory: artifactory, - ExtDir: extdir, - Logger: logger, - Repo: repo, - }) + store, err := storage.NewStorage(ctx, opts) if err != nil { return err } @@ -120,9 +101,7 @@ func remove() *cobra.Command { } cmd.Flags().BoolVar(&all, "all", false, "Whether to delete all versions of the extension.") - cmd.Flags().StringVar(&extdir, "extensions-dir", "", "The path to extensions.") - cmd.Flags().StringVar(&artifactory, "artifactory", "", "Artifactory server URL.") - cmd.Flags().StringVar(&repo, "repo", "", "Artifactory repository.") + addFlags(cmd) return cmd } diff --git a/cli/root.go b/cli/root.go index 435e28f..b6638c3 100644 --- a/cli/root.go +++ b/cli/root.go @@ -1,8 +1,9 @@ package cli import ( - "github.com/spf13/cobra" "strings" + + "github.com/spf13/cobra" ) func Root() *cobra.Command { @@ -16,7 +17,7 @@ func Root() *cobra.Command { }, "\n"), } - cmd.AddCommand(add(), remove(), server(), version()) + cmd.AddCommand(add(), remove(), server(), version(), signature()) cmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose output") diff --git a/cli/server.go b/cli/server.go index befc20b..8cfb7ea 100644 --- a/cli/server.go +++ b/cli/server.go @@ -15,21 +15,68 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/sloghuman" + "github.com/coder/code-marketplace/extensionsign" "github.com/coder/code-marketplace/api" "github.com/coder/code-marketplace/database" "github.com/coder/code-marketplace/storage" ) +func serverFlags() (addFlags func(cmd *cobra.Command), opts *storage.Options) { + opts = &storage.Options{} + var sign bool + return func(cmd *cobra.Command) { + cmd.Flags().StringVar(&opts.ExtDir, "extensions-dir", "", "The path to extensions.") + cmd.Flags().StringVar(&opts.Artifactory, "artifactory", "", "Artifactory server URL.") + cmd.Flags().StringVar(&opts.Repo, "repo", "", "Artifactory repository.") + cmd.Flags().BoolVar(&sign, "sign", false, "Sign extensions.") + _ = cmd.Flags().MarkHidden("sign") // This flag needs to import a key, not just be a bool + + if cmd.Use == "server" { + // Server only flags + cmd.Flags().DurationVar(&opts.ListCacheDuration, "list-cache-duration", time.Minute, "The duration of the extension cache.") + } + + var before func(cmd *cobra.Command, args []string) error + if cmd.PreRunE != nil { + before = cmd.PreRunE + } + if cmd.PreRun != nil { + beforeNoE := cmd.PreRun + before = func(cmd *cobra.Command, args []string) error { + beforeNoE(cmd, args) + return nil + } + } + + cmd.PreRunE = func(cmd *cobra.Command, args []string) error { + opts.Logger = cmdLogger(cmd) + if before != nil { + return before(cmd, args) + } + if sign { // TODO: Remove this for an actual key import + opts.Signer, _ = extensionsign.GenerateKey() + } + return nil + } + }, opts +} + +func cmdLogger(cmd *cobra.Command) slog.Logger { + verbose, _ := cmd.Flags().GetBool("verbose") + logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) + if verbose { + logger = logger.Leveled(slog.LevelDebug) + } + return logger +} + func server() *cobra.Command { var ( - address string - artifactory string - extdir string - repo string - listcacheduration time.Duration - maxpagesize int + address string + maxpagesize int ) + addFlags, opts := serverFlags() cmd := &cobra.Command{ Use: "server", @@ -41,26 +88,12 @@ func server() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() + logger := opts.Logger notifyCtx, notifyStop := signal.NotifyContext(ctx, interruptSignals...) defer notifyStop() - verbose, err := cmd.Flags().GetBool("verbose") - if err != nil { - return err - } - logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) - if verbose { - logger = logger.Leveled(slog.LevelDebug) - } - - store, err := storage.NewStorage(ctx, &storage.Options{ - Artifactory: artifactory, - ExtDir: extdir, - Logger: logger, - Repo: repo, - ListCacheDuration: listcacheduration, - }) + store, err := storage.NewStorage(ctx, opts) if err != nil { return err } @@ -137,12 +170,9 @@ func server() *cobra.Command { }, } - cmd.Flags().StringVar(&extdir, "extensions-dir", "", "The path to extensions.") cmd.Flags().IntVar(&maxpagesize, "max-page-size", api.MaxPageSizeDefault, "The maximum number of pages to request") - cmd.Flags().StringVar(&artifactory, "artifactory", "", "Artifactory server URL.") - cmd.Flags().StringVar(&repo, "repo", "", "Artifactory repository.") cmd.Flags().StringVar(&address, "address", "127.0.0.1:3001", "The address on which to serve the marketplace API.") - cmd.Flags().DurationVar(&listcacheduration, "list-cache-duration", time.Minute, "The duration of the extension cache.") + addFlags(cmd) return cmd } diff --git a/cli/signature.go b/cli/signature.go new file mode 100644 index 0000000..432a14a --- /dev/null +++ b/cli/signature.go @@ -0,0 +1,63 @@ +package cli + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + "golang.org/x/xerrors" + + "github.com/coder/code-marketplace/extensionsign" +) + +func signature() *cobra.Command { + cmd := &cobra.Command{ + Use: "signature", + Short: "Commands for debugging and working with signatures.", + Hidden: true, // Debugging tools + Aliases: []string{"sig", "sigs", "signatures"}, + } + cmd.AddCommand(compareSignatureSigZips()) + return cmd +} + +func compareSignatureSigZips() *cobra.Command { + cmd := &cobra.Command{ + Use: "compare", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + decode := func(path string) (extensionsign.SignatureManifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return extensionsign.SignatureManifest{}, xerrors.Errorf("read %q: %w", args[0], err) + } + + sig, err := extensionsign.ExtractSignatureManifest(data) + if err != nil { + return extensionsign.SignatureManifest{}, xerrors.Errorf("unmarshal %q: %w", path, err) + } + return sig, nil + } + + a, err := decode(args[0]) + if err != nil { + return err + } + b, err := decode(args[1]) + if err != nil { + return err + } + + _, _ = fmt.Fprintf(os.Stdout, "Signature A:%s\n", a) + _, _ = fmt.Fprintf(os.Stdout, "Signature B:%s\n", b) + err = a.Equal(b) + if err != nil { + return err + } + + _, _ = fmt.Fprintf(os.Stdout, "Signatures are equal\n") + return nil + }, + } + return cmd +} diff --git a/extensionsign/doc.go b/extensionsign/doc.go new file mode 100644 index 0000000..b51e216 --- /dev/null +++ b/extensionsign/doc.go @@ -0,0 +1,2 @@ +// Package extensionsign is a Go implementation of https://github.com/filiptronicek/node-ovsx-sign +package extensionsign diff --git a/extensionsign/key.go b/extensionsign/key.go new file mode 100644 index 0000000..9af9778 --- /dev/null +++ b/extensionsign/key.go @@ -0,0 +1,14 @@ +package extensionsign + +import ( + "crypto/ed25519" + "crypto/rand" +) + +func GenerateKey() (ed25519.PrivateKey, error) { + _, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + return private, nil +} diff --git a/extensionsign/sigmanifest.go b/extensionsign/sigmanifest.go new file mode 100644 index 0000000..3ff67e6 --- /dev/null +++ b/extensionsign/sigmanifest.go @@ -0,0 +1,114 @@ +package extensionsign + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + "io" + + "golang.org/x/xerrors" + + "github.com/coder/code-marketplace/storage/easyzip" +) + +// SignatureManifest should be serialized to JSON before being signed. +type SignatureManifest struct { + Package File + // Entries is base64(filepath) -> File + Entries map[string]File +} + +func (a SignatureManifest) String() string { + return fmt.Sprintf("Package %q with Entries: %d", a.Package.Digests.SHA256, len(a.Entries)) +} + +// Equal is helpful for debugging to know if two manifests are equal. +// They can change if any file is removed/added/edited to an extension. +func (a SignatureManifest) Equal(b SignatureManifest) error { + var errs []error + if err := a.Package.Equal(b.Package); err != nil { + errs = append(errs, xerrors.Errorf("package: %w", err)) + } + + if len(a.Entries) != len(b.Entries) { + errs = append(errs, xerrors.Errorf("entry count mismatch: %d != %d", len(a.Entries), len(b.Entries))) + } + + for k, v := range a.Entries { + if _, ok := b.Entries[k]; !ok { + errs = append(errs, xerrors.Errorf("entry %q not found in second set", k)) + continue + } + if err := v.Equal(b.Entries[k]); err != nil { + errs = append(errs, xerrors.Errorf("entry %q: %w", k, err)) + } + } + return errors.Join(errs...) +} + +type File struct { + Size int64 `json:"size"` + Digests Digests `json:"digests"` +} + +func (f File) Equal(b File) error { + if f.Size != b.Size { + return xerrors.Errorf("size mismatch: %d != %d", f.Size, b.Size) + } + if f.Digests.SHA256 != b.Digests.SHA256 { + return xerrors.Errorf("sha256 mismatch: %s != %s", f.Digests.SHA256, b.Digests.SHA256) + } + return nil +} + +func FileManifest(file io.Reader) (File, error) { + hash := sha256.New() + + n, err := io.Copy(hash, file) + if err != nil { + return File{}, xerrors.Errorf("hash file: %w", err) + } + + return File{ + Size: n, + Digests: Digests{ + SHA256: base64.StdEncoding.EncodeToString(hash.Sum(nil)), + }, + }, nil +} + +type Digests struct { + SHA256 string `json:"sha256"` +} + +// GenerateSignatureManifest generates a signature manifest for a VSIX file. +// It does not sign the manifest. The manifest is the base64 encoded file path +// followed by the sha256 hash of the file, and it's size. +func GenerateSignatureManifest(vsixFile []byte) (SignatureManifest, error) { + pkgManifest, err := FileManifest(bytes.NewReader(vsixFile)) + if err != nil { + return SignatureManifest{}, xerrors.Errorf("package manifest: %w", err) + } + + manifest := SignatureManifest{ + Package: pkgManifest, + Entries: make(map[string]File), + } + + err = easyzip.ExtractZip(vsixFile, func(name string, reader io.Reader) error { + fm, err := FileManifest(reader) + if err != nil { + return xerrors.Errorf("file %q: %w", name, err) + } + manifest.Entries[base64.StdEncoding.EncodeToString([]byte(name))] = fm + return nil + }) + + if err != nil { + return SignatureManifest{}, err + } + + return manifest, nil +} diff --git a/extensionsign/sigzip.go b/extensionsign/sigzip.go new file mode 100644 index 0000000..c6948d1 --- /dev/null +++ b/extensionsign/sigzip.go @@ -0,0 +1,73 @@ +package extensionsign + +import ( + "archive/zip" + "bytes" + "crypto" + "crypto/rand" + "encoding/json" + + "golang.org/x/xerrors" + + "github.com/coder/code-marketplace/storage/easyzip" +) + +func ExtractSignatureManifest(zip []byte) (SignatureManifest, error) { + r, err := easyzip.GetZipFileReader(zip, ".signature.manifest") + if err != nil { + return SignatureManifest{}, xerrors.Errorf("get manifest: %w", err) + } + + defer r.Close() + var manifest SignatureManifest + err = json.NewDecoder(r).Decode(&manifest) + if err != nil { + return SignatureManifest{}, xerrors.Errorf("decode manifest: %w", err) + } + return manifest, nil +} + +// SignAndZipManifest signs a manifest and zips it up +func SignAndZipManifest(secret crypto.Signer, manifest json.RawMessage) ([]byte, error) { + var buf bytes.Buffer + w := zip.NewWriter(&buf) + + manFile, err := w.Create(".signature.manifest") + if err != nil { + return nil, xerrors.Errorf("create manifest: %w", err) + } + + _, err = manFile.Write(manifest) + if err != nil { + return nil, xerrors.Errorf("write manifest: %w", err) + } + + // Empty file + _, err = w.Create(".signature.p7s") + if err != nil { + return nil, xerrors.Errorf("create empty p7s signature: %w", err) + } + + // Actual sig + sigFile, err := w.Create(".signature.sig") + if err != nil { + return nil, xerrors.Errorf("create signature: %w", err) + } + + signature, err := secret.Sign(rand.Reader, manifest, crypto.Hash(0)) + if err != nil { + return nil, xerrors.Errorf("sign: %w", err) + } + + _, err = sigFile.Write(signature) + if err != nil { + return nil, xerrors.Errorf("write signature: %w", err) + } + + err = w.Close() + if err != nil { + return nil, xerrors.Errorf("close zip: %w", err) + } + + return buf.Bytes(), nil +} diff --git a/go.mod b/go.mod index 10bbe7d..52220e5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/coder/code-marketplace -go 1.19 +go 1.22.8 require ( cdr.dev/slog v1.6.1 @@ -9,6 +9,7 @@ require ( github.com/go-chi/httprate v0.14.1 github.com/google/uuid v1.6.0 github.com/lithammer/fuzzysearch v1.1.8 + github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 golang.org/x/mod v0.19.0 @@ -32,9 +33,11 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/crypto v0.19.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f49f711..f19c00c 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,14 @@ cdr.dev/slog v1.6.1 h1:IQjWZD0x6//sfv5n+qEhbu3wBkmtBQY5DILXNvMaIv4= cdr.dev/slog v1.6.1/go.mod h1:eHEYQLaZvxnIAXC+XdTSNLb/kgA/X2RVSF72v5wsxEI= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -20,9 +25,13 @@ github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vz github.com/go-chi/httprate v0.14.1 h1:EKZHYEZ58Cg6hWcYzoZILsv7ppb46Wt4uQ738IRtpZs= github.com/go-chi/httprate v0.14.1/go.mod h1:TUepLXaz/pCjmCtf/obgOQJ2Sz6rC8fSf5cAt5cnTt0= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -47,6 +56,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -57,13 +68,15 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= @@ -72,7 +85,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -85,13 +99,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -106,11 +120,16 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg= -google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 h1:XVeBY8d/FaK4848myy41HBqnDwvxeV3zMZhwN1TvAMU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/storage/artifactory.go b/storage/artifactory.go index a20c331..ae4ea88 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "os" "path" @@ -15,10 +16,12 @@ import ( "sync" "time" + "github.com/spf13/afero/mem" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" "cdr.dev/slog" + "github.com/coder/code-marketplace/storage/easyzip" "github.com/coder/code-marketplace/util" ) @@ -41,6 +44,8 @@ type ArtifactoryList struct { Files []ArtifactoryFile `json:"files"` } +var _ Storage = (*Artifactory)(nil) + // Artifactory implements Storage. It stores extensions remotely through // Artifactory by both copying the VSIX and extracting said VSIX to a tree // structure in the form of publisher/extension/version to easily serve @@ -213,7 +218,7 @@ func (s *Artifactory) upload(ctx context.Context, endpoint string, r io.Reader) return code, nil } -func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte) (string, error) { +func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) { // Extract the zip to the correct path. identity := manifest.Metadata.Identity dir := path.Join(identity.Publisher, identity.ID, Version{ @@ -244,7 +249,7 @@ func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, } } - err := ExtractZip(vsix, func(name string, r io.Reader) error { + err := easyzip.ExtractZip(vsix, func(name string, r io.Reader) error { if util.Contains(assets, name) || (browser != "" && strings.HasPrefix(name, browser)) { _, err := s.upload(ctx, path.Join(dir, name), r) return err @@ -262,28 +267,47 @@ func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, return "", err } + for _, file := range extra { + _, err := s.upload(ctx, path.Join(dir, file.RelativePath), bytes.NewReader(file.Content)) + if err != nil { + return "", err + } + } + return s.uri + dir, nil } -func (s *Artifactory) FileServer() http.Handler { - // TODO: Since we only extract a subset of files perhaps if the file does not - // exist we should download the vsix and extract the requested file as a - // fallback. Obviously this seems like quite a bit of overhead so we would - // then emit a warning so we can notice that VS Code has added new asset types - // that we should be extracting to avoid that overhead. Other solutions could - // be implemented though like extracting the VSIX to disk locally and only - // going to Artifactory for the VSIX when it is missing on disk (basically - // using the disk as a cache). - return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - reader, code, err := s.read(r.Context(), r.URL.Path) - if err != nil { - http.Error(rw, err.Error(), code) - return +// Open returns a file from Artifactory. +// TODO: Since we only extract a subset of files perhaps if the file does not +// exist we should download the vsix and extract the requested file as a +// fallback. Obviously this seems like quite a bit of overhead so we would +// then emit a warning so we can notice that VS Code has added new asset types +// that we should be extracting to avoid that overhead. Other solutions could +// be implemented though like extracting the VSIX to disk locally and only +// going to Artifactory for the VSIX when it is missing on disk (basically +// using the disk as a cache). +func (s *Artifactory) Open(ctx context.Context, fp string) (fs.File, error) { + resp, code, err := s.read(ctx, fp) + if code != http.StatusOK || err != nil { + switch code { + case http.StatusNotFound: + return nil, fs.ErrNotExist + case http.StatusForbidden: + return nil, fs.ErrPermission + default: + return nil, err } - defer reader.Close() - rw.WriteHeader(http.StatusOK) - _, _ = io.Copy(rw, reader) - }) + } + + // TODO: Do no copy the bytes into memory, stream them rather than + // storing the entire file into memory. + f := mem.NewFileHandle(mem.CreateFile(fp)) + _, err = io.Copy(f, resp) + if err != nil { + return nil, err + } + + return f, nil } func (s *Artifactory) Manifest(ctx context.Context, publisher, name string, version Version) (*VSIXManifest, error) { diff --git a/storage/zip.go b/storage/easyzip/zip.go similarity index 99% rename from storage/zip.go rename to storage/easyzip/zip.go index da2526f..9a76e89 100644 --- a/storage/zip.go +++ b/storage/easyzip/zip.go @@ -1,4 +1,4 @@ -package storage +package easyzip import ( "archive/zip" diff --git a/storage/zip_test.go b/storage/easyzip/zip_test.go similarity index 99% rename from storage/zip_test.go rename to storage/easyzip/zip_test.go index 3ae2ceb..febe877 100644 --- a/storage/zip_test.go +++ b/storage/easyzip/zip_test.go @@ -1,4 +1,4 @@ -package storage +package easyzip import ( "archive/zip" diff --git a/storage/local.go b/storage/local.go index 363eae2..5293e39 100644 --- a/storage/local.go +++ b/storage/local.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "io/fs" "net/http" "os" "path/filepath" @@ -11,9 +12,14 @@ import ( "sync" "time" + "golang.org/x/xerrors" + "cdr.dev/slog" + "github.com/coder/code-marketplace/storage/easyzip" ) +var _ Storage = (*Local)(nil) + // Local implements Storage. It stores extensions locally on disk by both // copying the VSIX and extracting said VSIX to a tree structure in the form of // publisher/extension/version to easily serve individual assets via HTTP. @@ -89,14 +95,14 @@ func (s *Local) list(ctx context.Context) []extension { return list } -func (s *Local) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte) (string, error) { +func (s *Local) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) { // Extract the zip to the correct path. identity := manifest.Metadata.Identity dir := filepath.Join(s.extdir, identity.Publisher, identity.ID, Version{ Version: identity.Version, TargetPlatform: identity.TargetPlatform, }.String()) - err := ExtractZip(vsix, func(name string, r io.Reader) error { + err := easyzip.ExtractZip(vsix, func(name string, r io.Reader) error { path := filepath.Join(dir, name) err := os.MkdirAll(filepath.Dir(path), 0o755) if err != nil { @@ -121,11 +127,23 @@ func (s *Local) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix [ return "", err } + for _, file := range extra { + path := filepath.Join(dir, file.RelativePath) + err := os.MkdirAll(filepath.Dir(path), 0o755) + if err != nil { + return "", err + } + err = os.WriteFile(path, file.Content, 0o644) + if err != nil { + return dir, xerrors.Errorf("write extra file %q: %w", path, err) + } + } + return dir, nil } -func (s *Local) FileServer() http.Handler { - return http.FileServer(http.Dir(s.extdir)) +func (s *Local) Open(_ context.Context, fp string) (fs.File, error) { + return http.Dir(s.extdir).Open(fp) } func (s *Local) Manifest(ctx context.Context, publisher, name string, version Version) (*VSIXManifest, error) { diff --git a/storage/signature.go b/storage/signature.go new file mode 100644 index 0000000..4bc3afd --- /dev/null +++ b/storage/signature.go @@ -0,0 +1,135 @@ +package storage + +import ( + "context" + "crypto" + "encoding/json" + "io" + "io/fs" + "path/filepath" + + "github.com/spf13/afero/mem" + "golang.org/x/xerrors" + + "github.com/coder/code-marketplace/extensionsign" +) + +var _ Storage = (*Signature)(nil) + +const ( + SigzipFilename = "extension.sigzip" + sigManifestName = ".signature.manifest" +) + +// Signature is a storage wrapper that can sign extensions on demand. +type Signature struct { + // Signer if provided, will be used to sign extensions. If not provided, + // no extensions will be signed. + Signer crypto.Signer + Storage +} + +func NewSignatureStorage(signer crypto.Signer, s Storage) *Signature { + return &Signature{ + Signer: signer, + Storage: s, + } +} + +func (s *Signature) SigningEnabled() bool { + return s.Signer != nil +} + +// AddExtension includes the signature manifest of the vsix. Signing happens on +// demand, so leave the manifest unsigned. This is safe to do even if +// 'signExtensions' is disabled, as these files lay dormant until signed. +func (s *Signature) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) { + sigManifest, err := extensionsign.GenerateSignatureManifest(vsix) + if err != nil { + return "", xerrors.Errorf("generate signature manifest: %w", err) + } + + data, err := json.Marshal(sigManifest) + if err != nil { + return "", xerrors.Errorf("encode signature manifest: %w", err) + } + + return s.Storage.AddExtension(ctx, manifest, vsix, append(extra, File{ + RelativePath: sigManifestName, + Content: data, + })...) +} + +func (s *Signature) Manifest(ctx context.Context, publisher, name string, version Version) (*VSIXManifest, error) { + manifest, err := s.Storage.Manifest(ctx, publisher, name, version) + if err != nil { + return nil, err + } + + if s.SigningEnabled() { + for _, asset := range manifest.Assets.Asset { + if asset.Path == SigzipFilename { + // Already signed + return manifest, nil + } + } + manifest.Assets.Asset = append(manifest.Assets.Asset, VSIXAsset{ + Type: VSIXSignatureType, + Path: SigzipFilename, + Addressable: "true", + }) + return manifest, nil + } + return manifest, nil +} + +// Open will intercept requests for signed extensions payload. +// It does this by looking for 'SigzipFilename' or p7s.sig. +// +// The signed payload and signing process is taken from: +// https://github.com/filiptronicek/node-ovsx-sign +// +// Some notes: +// +// - VSCodium requires a signature to exist, but it does appear to actually read +// the signature. Meaning the signature could be empty, incorrect, or a +// picture of cat and it would work. There is no signature verification. +// +// - VSCode requires a signature payload to exist, but the context appear +// to be somewhat optional. +// Following another open source implementation, it appears the '.signature.p7s' +// file must exist, but it can be empty. +// The signature is stored in a '.signature.sig' file, although it is unclear +// is VSCode ever reads this file. +// TODO: Properly implement the p7s file, and diverge from the other open +// source implementation. Ideally this marketplace would match Microsoft's +// marketplace API. +func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { + if s.SigningEnabled() && filepath.Base(fp) == SigzipFilename { + // hijack this request, sign the sig manifest + manifest, err := s.Storage.Open(ctx, filepath.Join(filepath.Dir(fp), sigManifestName)) + if err != nil { + // If this file is missing, it means the extension was added before + // signatures were handled by the marketplace. + // TODO: Generate the sig manifest payload and insert it? + return nil, xerrors.Errorf("open signature manifest: %w", err) + } + defer manifest.Close() + + manifestData, err := io.ReadAll(manifest) + if err != nil { + return nil, xerrors.Errorf("read signature manifest: %w", err) + } + + signed, err := extensionsign.SignAndZipManifest(s.Signer, manifestData) + if err != nil { + return nil, xerrors.Errorf("sign and zip manifest: %w", err) + } + + f := mem.NewFileHandle(mem.CreateFile(SigzipFilename)) + _, err = f.Write(signed) + return f, err + } + + return s.Storage.Open(ctx, fp) +} diff --git a/storage/signature_test.go b/storage/signature_test.go new file mode 100644 index 0000000..bfcef6d --- /dev/null +++ b/storage/signature_test.go @@ -0,0 +1,37 @@ +package storage_test + +import ( + "crypto" + "testing" + + "github.com/coder/code-marketplace/extensionsign" + "github.com/coder/code-marketplace/storage" +) + +func expectSignature(manifest *storage.VSIXManifest) { + manifest.Assets.Asset = append(manifest.Assets.Asset, storage.VSIXAsset{ + Type: storage.VSIXSignatureType, + Path: storage.SigzipFilename, + Addressable: "true", + }) +} + +//nolint:revive // test control flag +func signed(signer bool, factory func(t *testing.T) testStorage) func(t *testing.T) testStorage { + return func(t *testing.T) testStorage { + st := factory(t) + var key crypto.Signer + var exp func(*storage.VSIXManifest) + if signer { + key, _ = extensionsign.GenerateKey() + exp = expectSignature + } + + return testStorage{ + storage: storage.NewSignatureStorage(key, st.storage), + write: st.write, + exists: st.exists, + expectedManifest: exp, + } + } +} diff --git a/storage/storage.go b/storage/storage.go index ecdaa50..94e5505 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -2,10 +2,12 @@ package storage import ( "context" + "crypto" "encoding/json" "encoding/xml" "fmt" "io" + "io/fs" "net/http" "os" "regexp" @@ -13,9 +15,11 @@ import ( "time" "golang.org/x/mod/semver" + "golang.org/x/xerrors" "cdr.dev/slog" + "github.com/coder/code-marketplace/storage/easyzip" ) // VSIXManifest implement XMLManifest.PackageManifest. @@ -112,6 +116,7 @@ type AssetType string const ( ManifestAssetType AssetType = "Microsoft.VisualStudio.Code.Manifest" // This is the package.json. VSIXAssetType AssetType = "Microsoft.VisualStudio.Services.VSIXPackage" + VSIXSignatureType AssetType = "Microsoft.VisualStudio.Services.VsixSignature" ) // VSIXAsset implements XMLManifest.PackageManifest.Assets.Asset. @@ -123,6 +128,7 @@ type VSIXAsset struct { } type Options struct { + Signer crypto.Signer Artifactory string ExtDir string Repo string @@ -203,11 +209,13 @@ func (vs ByVersion) Less(i, j int) bool { type Storage interface { // AddExtension adds the provided VSIX into storage and returns the location - // for verification purposes. - AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte) (string, error) - // FileServer provides a handler for fetching extension repository files from - // a client. - FileServer() http.Handler + // for verification purposes. Extra files can be included, but not required. + // All extra files will be placed relative to the manifest outside the vsix. + AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) + // Open mirrors the fs.FS interface of Open, except with a context. + // The Open should return files from the extension storage, and used for + // serving extensions. + Open(ctx context.Context, name string) (fs.File, error) // Manifest returns the manifest bytes for the provided extension. The // extension asset itself (the VSIX) will be included on the manifest even if // it does not exist on the manifest on disk. @@ -230,6 +238,22 @@ type Storage interface { WalkExtensions(ctx context.Context, fn func(manifest *VSIXManifest, versions []Version) error) error } +// HTTPFileServer creates an http.Handler that serves files from the provided +// storage. +func HTTPFileServer(s Storage) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + http.FileServerFS(&contextFs{ + ctx: r.Context(), + open: s.Open, + }).ServeHTTP(rw, r) + }) +} + +type File struct { + RelativePath string + Content []byte +} + const ArtifactoryTokenEnvKey = "ARTIFACTORY_TOKEN" // NewStorage returns a storage instance based on the provided extension @@ -240,31 +264,42 @@ func NewStorage(ctx context.Context, options *Options) (Storage, error) { return nil, xerrors.Errorf("cannot use both Artifactory and extension directory") } else if options.Artifactory != "" && options.Repo == "" { return nil, xerrors.Errorf("must provide repository") - } else if options.Artifactory != "" { + } + + var store Storage + var err error + switch { + case options.Artifactory != "": token := os.Getenv(ArtifactoryTokenEnvKey) if token == "" { return nil, xerrors.Errorf("the %s environment variable must be set", ArtifactoryTokenEnvKey) } - return NewArtifactoryStorage(ctx, &ArtifactoryOptions{ + store, err = NewArtifactoryStorage(ctx, &ArtifactoryOptions{ ListCacheDuration: options.ListCacheDuration, Logger: options.Logger, Repo: options.Repo, Token: token, URI: options.Artifactory, }) - } else if options.ExtDir != "" { - return NewLocalStorage(&LocalOptions{ + case options.ExtDir != "": + store, err = NewLocalStorage(&LocalOptions{ ListCacheDuration: options.ListCacheDuration, ExtDir: options.ExtDir, }, options.Logger) + default: + return nil, xerrors.Errorf("must provide an Artifactory repository or local directory") } - return nil, xerrors.Errorf("must provide an Artifactory repository or local directory") + if err != nil { + return nil, err + } + + return NewSignatureStorage(options.Signer, store), nil } // ReadVSIXManifest reads and parses an extension manifest from a vsix file. If // the manifest is invalid it will be returned along with the validation error. func ReadVSIXManifest(vsix []byte) (*VSIXManifest, error) { - vmr, err := GetZipFileReader(vsix, "extension.vsixmanifest") + vmr, err := easyzip.GetZipFileReader(vsix, "extension.vsixmanifest") if err != nil { return nil, err } @@ -322,7 +357,7 @@ type VSIXPackageJSON struct { // ReadVSIXPackageJSON reads and parses an extension's package.json from a vsix // file. func ReadVSIXPackageJSON(vsix []byte, packageJsonPath string) (*VSIXPackageJSON, error) { - vpjr, err := GetZipFileReader(vsix, packageJsonPath) + vpjr, err := easyzip.GetZipFileReader(vsix, packageJsonPath) if err != nil { return nil, err } @@ -406,3 +441,12 @@ func ParseExtensionID(id string) (string, string, string, error) { } return match[0][1], match[0][2], match[0][3], nil } + +type contextFs struct { + ctx context.Context + open func(ctx context.Context, name string) (fs.File, error) +} + +func (c *contextFs) Open(name string) (fs.File, error) { + return c.open(c.ctx, name) +} diff --git a/storage/storage_test.go b/storage/storage_test.go index 4bbd433..8bb649c 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -25,6 +25,8 @@ type testStorage struct { storage storage.Storage write func(content []byte, elem ...string) exists func(elem ...string) bool + + expectedManifest func(man *storage.VSIXManifest) } type storageFactory = func(t *testing.T) testStorage @@ -104,11 +106,13 @@ func TestNewStorage(t *testing.T) { require.Error(t, err) require.Regexp(t, test.error, err.Error()) } else if test.local { - _, ok := s.(*storage.Local) + under := s.(*storage.Signature) + _, ok := under.Storage.(*storage.Local) require.True(t, ok) require.NoError(t, err) } else { - _, ok := s.(*storage.Artifactory) + under := s.(*storage.Signature) + _, ok := under.Storage.(*storage.Artifactory) require.True(t, ok) require.NoError(t, err) } @@ -130,6 +134,14 @@ func TestStorage(t *testing.T) { name: "Artifactory", factory: artifactoryFactory, }, + { + name: "SignedLocal", + factory: signed(true, localFactory), + }, + { + name: "SignedArtifactory", + factory: signed(true, artifactoryFactory), + }, } for _, sf := range factories { t.Run(sf.name, func(t *testing.T) { @@ -189,7 +201,7 @@ func testFileServer(t *testing.T, factory storageFactory) { req := httptest.NewRequest("GET", test.path, nil) rec := httptest.NewRecorder() - server := f.storage.FileServer() + server := storage.HTTPFileServer(f.storage) server.ServeHTTP(rec, req) resp := rec.Result() @@ -322,6 +334,9 @@ func testManifest(t *testing.T, factory storageFactory) { Path: fmt.Sprintf("%s.%s-%s.vsix", test.extension.Publisher, test.extension.Name, test.version), Addressable: "true", }) + if f.expectedManifest != nil { + f.expectedManifest(test.expected) + } require.Equal(t, test.expected, manifest) } }) diff --git a/testutil/extensions.go b/testutil/extensions.go index 62fb385..a32967e 100644 --- a/testutil/extensions.go +++ b/testutil/extensions.go @@ -26,6 +26,13 @@ type Extension struct { Pack []string } +func (e Extension) Copy() Extension { + var n Extension + data, _ := json.Marshal(e) + _ = json.Unmarshal(data, &n) + return n +} + var Extensions = []Extension{ { Publisher: "foo", @@ -113,6 +120,7 @@ var Extensions = []Extension{ } func ConvertExtensionToManifest(ext Extension, version storage.Version) *storage.VSIXManifest { + ext = ext.Copy() return &storage.VSIXManifest{ Metadata: storage.VSIXMetadata{ Identity: storage.VSIXIdentity{ diff --git a/testutil/mockstorage.go b/testutil/mockstorage.go index db49fa2..0066c40 100644 --- a/testutil/mockstorage.go +++ b/testutil/mockstorage.go @@ -3,13 +3,19 @@ package testutil import ( "context" "errors" + "io/fs" "net/http" "os" + "path/filepath" "sort" + "github.com/spf13/afero/mem" + "github.com/coder/code-marketplace/storage" ) +var _ storage.Storage = (*MockStorage)(nil) + // MockStorage implements storage.Storage for tests. type MockStorage struct{} @@ -17,9 +23,18 @@ func NewMockStorage() *MockStorage { return &MockStorage{} } -func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXManifest, vsix []byte) (string, error) { +func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXManifest, vsix []byte, extra ...storage.File) (string, error) { return "", errors.New("not implemented") } +func (s *MockStorage) Open(ctx context.Context, path string) (fs.File, error) { + if filepath.Base(path) == "nonexistent" { + return nil, fs.ErrNotExist + } + + f := mem.NewFileHandle(mem.CreateFile(path)) + _, _ = f.Write([]byte("foobar")) + return f, nil +} func (s *MockStorage) FileServer() http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {