From 644c355e69f7c8b70bbe23c23646ba7e39b766ca Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Wed, 11 Dec 2024 16:49:20 -0600 Subject: [PATCH 01/22] chore: begin work implementing vsix signatures, manifest matching --- cli/add.go | 36 ++++++++++++++++++++++++++++++++---- cli/root.go | 5 +++-- storage/artifactory.go | 2 +- storage/local.go | 16 +++++++++++++++- storage/storage.go | 10 ++++++++-- testutil/mockstorage.go | 2 +- 6 files changed, 60 insertions(+), 11 deletions(-) diff --git a/cli/add.go b/cli/add.go index 05d61f8..929c53e 100644 --- a/cli/add.go +++ b/cli/add.go @@ -2,6 +2,7 @@ package cli import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -12,6 +13,7 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/sloghuman" + "github.com/coder/code-marketplace/internal/extensionsign" "github.com/coder/code-marketplace/storage" "github.com/coder/code-marketplace/util" @@ -22,6 +24,7 @@ func add() *cobra.Command { artifactory string extdir string repo string + signature bool ) cmd := &cobra.Command{ @@ -73,7 +76,7 @@ func add() *cobra.Command { return err } for _, file := range files { - s, err := doAdd(ctx, filepath.Join(args[0], file.Name()), store) + s, err := doAdd(ctx, filepath.Join(args[0], file.Name()), signature, store) if err != nil { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Failed to unpack %s: %s\n", file.Name(), err.Error()) failed = append(failed, file.Name()) @@ -82,7 +85,7 @@ func add() *cobra.Command { } } } else { - s, err := doAdd(ctx, args[0], store) + s, err := doAdd(ctx, args[0], signature, store) if err != nil { return err } @@ -102,11 +105,12 @@ func add() *cobra.Command { cmd.Flags().StringVar(&extdir, "extensions-dir", "", "The path to extensions.") cmd.Flags().StringVar(&artifactory, "artifactory", "", "Artifactory server URL.") cmd.Flags().StringVar(&repo, "repo", "", "Artifactory repository.") + cmd.Flags().BoolVar(&signature, "signature", true, "Include signature") return cmd } -func doAdd(ctx context.Context, source string, store storage.Storage) ([]string, error) { +func doAdd(ctx context.Context, source string, includeSignature bool, store storage.Storage) ([]string, error) { // Read in the extension. In the future we might support stdin as well. vsix, err := storage.ReadVSIX(ctx, source) if err != nil { @@ -120,7 +124,31 @@ func doAdd(ctx context.Context, source string, store storage.Storage) ([]string, return nil, err } - location, err := store.AddExtension(ctx, manifest, vsix) + var extra []storage.File + if includeSignature { + sigManifest, err := extensionsign.GenerateSignatureManifest(vsix) + if err != nil { + return nil, xerrors.Errorf("generate signature manifest: %w", err) + } + + data, err := json.Marshal(sigManifest) + if err != nil { + return nil, xerrors.Errorf("marshal signature manifest: %w", err) + } + + extra = append(extra, storage.File{ + RelativePath: "extension.sigzip", + Content: data, + }) + + manifest.Assets.Asset = append(manifest.Assets.Asset, storage.VSIXAsset{ + Type: storage.VSIXSignatureType, + Path: "extension.sigzip", + Addressable: "true", // TODO: Idk if this is right + }) + } + + location, err := store.AddExtension(ctx, manifest, vsix, extra...) if err != nil { return nil, err } diff --git a/cli/root.go b/cli/root.go index 435e28f..b6638c3 100644 --- a/cli/root.go +++ b/cli/root.go @@ -1,8 +1,9 @@ package cli import ( - "github.com/spf13/cobra" "strings" + + "github.com/spf13/cobra" ) func Root() *cobra.Command { @@ -16,7 +17,7 @@ func Root() *cobra.Command { }, "\n"), } - cmd.AddCommand(add(), remove(), server(), version()) + cmd.AddCommand(add(), remove(), server(), version(), signature()) cmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose output") diff --git a/storage/artifactory.go b/storage/artifactory.go index a20c331..f1bbf71 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -213,7 +213,7 @@ func (s *Artifactory) upload(ctx context.Context, endpoint string, r io.Reader) return code, nil } -func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte) (string, error) { +func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) { // Extract the zip to the correct path. identity := manifest.Metadata.Identity dir := path.Join(identity.Publisher, identity.ID, Version{ diff --git a/storage/local.go b/storage/local.go index 363eae2..65f74f9 100644 --- a/storage/local.go +++ b/storage/local.go @@ -11,6 +11,8 @@ import ( "sync" "time" + "golang.org/x/xerrors" + "cdr.dev/slog" ) @@ -89,7 +91,7 @@ func (s *Local) list(ctx context.Context) []extension { return list } -func (s *Local) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte) (string, error) { +func (s *Local) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) { // Extract the zip to the correct path. identity := manifest.Metadata.Identity dir := filepath.Join(s.extdir, identity.Publisher, identity.ID, Version{ @@ -121,6 +123,18 @@ func (s *Local) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix [ return "", err } + for _, file := range extra { + path := filepath.Join(dir, file.RelativePath) + err := os.MkdirAll(filepath.Dir(path), 0o644) + if err != nil { + return "", err + } + err = os.WriteFile(path, file.Content, 0o644) + if err != nil { + return dir, xerrors.Errorf("write extra file %q: %w", path, err) + } + } + return dir, nil } diff --git a/storage/storage.go b/storage/storage.go index ecdaa50..af710bc 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -112,6 +112,7 @@ type AssetType string const ( ManifestAssetType AssetType = "Microsoft.VisualStudio.Code.Manifest" // This is the package.json. VSIXAssetType AssetType = "Microsoft.VisualStudio.Services.VSIXPackage" + VSIXSignatureType AssetType = "Microsoft.VisualStudio.Services.VsixSignature" ) // VSIXAsset implements XMLManifest.PackageManifest.Assets.Asset. @@ -203,8 +204,8 @@ func (vs ByVersion) Less(i, j int) bool { type Storage interface { // AddExtension adds the provided VSIX into storage and returns the location - // for verification purposes. - AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte) (string, error) + // for verification purposes. Extra files can be included, but not required. + AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) // FileServer provides a handler for fetching extension repository files from // a client. FileServer() http.Handler @@ -230,6 +231,11 @@ type Storage interface { WalkExtensions(ctx context.Context, fn func(manifest *VSIXManifest, versions []Version) error) error } +type File struct { + RelativePath string + Content []byte +} + const ArtifactoryTokenEnvKey = "ARTIFACTORY_TOKEN" // NewStorage returns a storage instance based on the provided extension diff --git a/testutil/mockstorage.go b/testutil/mockstorage.go index db49fa2..a7dc12a 100644 --- a/testutil/mockstorage.go +++ b/testutil/mockstorage.go @@ -17,7 +17,7 @@ func NewMockStorage() *MockStorage { return &MockStorage{} } -func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXManifest, vsix []byte) (string, error) { +func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXManifest, vsix []byte, extra ...storage.File) (string, error) { return "", errors.New("not implemented") } From fe510694a5a29f46e6fcc2c88f4708c66aaa0197 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Wed, 11 Dec 2024 17:03:05 -0600 Subject: [PATCH 02/22] properly zip sigzips --- cli/add.go | 7 ++--- internal/extensionsign/sigzip.go | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 internal/extensionsign/sigzip.go diff --git a/cli/add.go b/cli/add.go index 929c53e..c8aa295 100644 --- a/cli/add.go +++ b/cli/add.go @@ -2,7 +2,6 @@ package cli import ( "context" - "encoding/json" "fmt" "os" "path/filepath" @@ -131,14 +130,14 @@ func doAdd(ctx context.Context, source string, includeSignature bool, store stor return nil, xerrors.Errorf("generate signature manifest: %w", err) } - data, err := json.Marshal(sigManifest) + zipped, err := extensionsign.Zip(sigManifest) if err != nil { - return nil, xerrors.Errorf("marshal signature manifest: %w", err) + return nil, xerrors.Errorf("zip signature manifest: %w", err) } extra = append(extra, storage.File{ RelativePath: "extension.sigzip", - Content: data, + Content: zipped, }) manifest.Assets.Asset = append(manifest.Assets.Asset, storage.VSIXAsset{ diff --git a/internal/extensionsign/sigzip.go b/internal/extensionsign/sigzip.go new file mode 100644 index 0000000..672d69d --- /dev/null +++ b/internal/extensionsign/sigzip.go @@ -0,0 +1,48 @@ +package extensionsign + +import ( + "archive/zip" + "bytes" + "encoding/json" + + "golang.org/x/xerrors" + + "github.com/coder/code-marketplace/storage" +) + +func ExtractSignatureManifest(zip []byte) (SignatureManifest, error) { + r, err := storage.GetZipFileReader(zip, ".signature.manifest") + if err != nil { + return SignatureManifest{}, xerrors.Errorf("get manifest: %w", err) + } + + defer r.Close() + var manifest SignatureManifest + err = json.NewDecoder(r).Decode(&manifest) + if err != nil { + return SignatureManifest{}, xerrors.Errorf("decode manifest: %w", err) + } + return manifest, nil +} + +func Zip(manifest SignatureManifest) ([]byte, error) { + var buf bytes.Buffer + w := zip.NewWriter(&buf) + + manFile, err := w.Create(".signature.manifest") + if err != nil { + return nil, xerrors.Errorf("create manifest: %w", err) + } + + err = json.NewEncoder(manFile).Encode(manifest) + if err != nil { + return nil, xerrors.Errorf("encode manifest: %w", err) + } + + err = w.Close() + if err != nil { + return nil, xerrors.Errorf("close zip: %w", err) + } + + return buf.Bytes(), nil +} From 3b91874dbf0b29e1654fbae032793a443263254c Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Wed, 11 Dec 2024 17:23:12 -0600 Subject: [PATCH 03/22] have some sort of signing going --- cli/signature.go | 61 +++++++++++ internal/extensionsign/doc.go | 2 + internal/extensionsign/key.go | 14 +++ internal/extensionsign/sigmanifest.go | 111 +++++++++++++++++++++ internal/extensionsign/sigmanifest_test.go | 7 ++ internal/extensionsign/sigzip.go | 37 ++++++- 6 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 cli/signature.go create mode 100644 internal/extensionsign/doc.go create mode 100644 internal/extensionsign/key.go create mode 100644 internal/extensionsign/sigmanifest.go create mode 100644 internal/extensionsign/sigmanifest_test.go diff --git a/cli/signature.go b/cli/signature.go new file mode 100644 index 0000000..e1b130a --- /dev/null +++ b/cli/signature.go @@ -0,0 +1,61 @@ +package cli + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + "golang.org/x/xerrors" + + "github.com/coder/code-marketplace/internal/extensionsign" +) + +func signature() *cobra.Command { + cmd := &cobra.Command{ + Use: "signature", + Aliases: []string{"sig", "sigs", "signatures"}, + } + cmd.AddCommand(compareSignatureSigZips()) + return cmd +} + +func compareSignatureSigZips() *cobra.Command { + cmd := &cobra.Command{ + Use: "compare", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + decode := func(path string) (extensionsign.SignatureManifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return extensionsign.SignatureManifest{}, xerrors.Errorf("read %q: %w", args[0], err) + } + + sig, err := extensionsign.ExtractSignatureManifest(data) + if err != nil { + return extensionsign.SignatureManifest{}, xerrors.Errorf("unmarshal %q: %w", path, err) + } + return sig, nil + } + + a, err := decode(args[0]) + if err != nil { + return err + } + b, err := decode(args[1]) + if err != nil { + return err + } + + _, _ = fmt.Fprintf(os.Stdout, "Signature A:%s\n", a) + _, _ = fmt.Fprintf(os.Stdout, "Signature B:%s\n", b) + err = a.Equal(b) + if err != nil { + return err + } + + _, _ = fmt.Fprintf(os.Stdout, "Signatures are equal\n") + return nil + }, + } + return cmd +} diff --git a/internal/extensionsign/doc.go b/internal/extensionsign/doc.go new file mode 100644 index 0000000..b51e216 --- /dev/null +++ b/internal/extensionsign/doc.go @@ -0,0 +1,2 @@ +// Package extensionsign is a Go implementation of https://github.com/filiptronicek/node-ovsx-sign +package extensionsign diff --git a/internal/extensionsign/key.go b/internal/extensionsign/key.go new file mode 100644 index 0000000..9af9778 --- /dev/null +++ b/internal/extensionsign/key.go @@ -0,0 +1,14 @@ +package extensionsign + +import ( + "crypto/ed25519" + "crypto/rand" +) + +func GenerateKey() (ed25519.PrivateKey, error) { + _, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + return private, nil +} diff --git a/internal/extensionsign/sigmanifest.go b/internal/extensionsign/sigmanifest.go new file mode 100644 index 0000000..60deddc --- /dev/null +++ b/internal/extensionsign/sigmanifest.go @@ -0,0 +1,111 @@ +package extensionsign + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "io" + + "github.com/cloudflare/cfssl/scan/crypto/sha256" + "golang.org/x/xerrors" + + "github.com/coder/code-marketplace/storage" +) + +type SignatureManifest struct { + Package File + // Entries is base64(filepath) -> File + Entries map[string]File +} + +func (a SignatureManifest) String() string { + return fmt.Sprintf("Package %q with Entries: %d", a.Package.Digests.SHA256, len(a.Entries)) +} + +// Equal is helpful for debugging +func (a SignatureManifest) Equal(b SignatureManifest) error { + var err error + if err := a.Package.Equal(b.Package); err != nil { + err = errors.Join(err, xerrors.Errorf("package: %w", err)) + } + + if len(a.Entries) != len(b.Entries) { + err = errors.Join(err, xerrors.Errorf("entry count mismatch: %d != %d", len(a.Entries), len(b.Entries))) + } + + for k, v := range a.Entries { + if _, ok := b.Entries[k]; !ok { + err = errors.Join(err, xerrors.Errorf("entry %q not found in second set", k)) + continue + } + if err := v.Equal(b.Entries[k]); err != nil { + err = errors.Join(err, xerrors.Errorf("entry %q: %w", k, err)) + } + } + return err +} + +type File struct { + Size int64 `json:"size"` + Digests Digests `json:"digests"` +} + +func (f File) Equal(b File) error { + if f.Size != b.Size { + return xerrors.Errorf("size mismatch: %d != %d", f.Size, b.Size) + } + if f.Digests.SHA256 != b.Digests.SHA256 { + return xerrors.Errorf("sha256 mismatch: %s != %s", f.Digests.SHA256, b.Digests.SHA256) + } + return nil +} + +func FileManifest(file io.Reader) (File, error) { + hash := sha256.New() + + n, err := io.Copy(hash, file) + if err != nil { + return File{}, xerrors.Errorf("hash file: %w", err) + } + + return File{ + Size: n, + Digests: Digests{ + SHA256: base64.StdEncoding.EncodeToString(hash.Sum(nil)), + }, + }, nil +} + +type Digests struct { + SHA256 string `json:"sha256"` +} + +// GenerateSignatureManifest generates a signature manifest for a VSIX file. +// It does not sign the manifest. +func GenerateSignatureManifest(vsixFile []byte) (SignatureManifest, error) { + pkgManifest, err := FileManifest(bytes.NewReader(vsixFile)) + if err != nil { + return SignatureManifest{}, xerrors.Errorf("package manifest: %w", err) + } + + manifest := SignatureManifest{ + Package: pkgManifest, + Entries: make(map[string]File), + } + + err = storage.ExtractZip(vsixFile, func(name string, reader io.Reader) error { + fm, err := FileManifest(reader) + if err != nil { + return xerrors.Errorf("file %q: %w", name, err) + } + manifest.Entries[base64.StdEncoding.EncodeToString([]byte(name))] = fm + return nil + }) + + if err != nil { + return SignatureManifest{}, err + } + + return manifest, nil +} diff --git a/internal/extensionsign/sigmanifest_test.go b/internal/extensionsign/sigmanifest_test.go new file mode 100644 index 0000000..d9efb6a --- /dev/null +++ b/internal/extensionsign/sigmanifest_test.go @@ -0,0 +1,7 @@ +package extensionsign_test + +import "testing" + +func TestManifestEqual(t *testing.T) { + +} diff --git a/internal/extensionsign/sigzip.go b/internal/extensionsign/sigzip.go index 672d69d..fe292d3 100644 --- a/internal/extensionsign/sigzip.go +++ b/internal/extensionsign/sigzip.go @@ -3,6 +3,7 @@ package extensionsign import ( "archive/zip" "bytes" + "crypto/ed25519" "encoding/json" "golang.org/x/xerrors" @@ -25,7 +26,10 @@ func ExtractSignatureManifest(zip []byte) (SignatureManifest, error) { return manifest, nil } -func Zip(manifest SignatureManifest) ([]byte, error) { +// SignAndZip signs a manifest and zips it up +// Should be a PCKS8 key +// TODO: Support other key types +func SignAndZip(key ed25519.PrivateKey, manifest SignatureManifest) ([]byte, error) { var buf bytes.Buffer w := zip.NewWriter(&buf) @@ -34,11 +38,40 @@ func Zip(manifest SignatureManifest) ([]byte, error) { return nil, xerrors.Errorf("create manifest: %w", err) } - err = json.NewEncoder(manFile).Encode(manifest) + manifestData, err := json.Marshal(manifest) if err != nil { return nil, xerrors.Errorf("encode manifest: %w", err) } + _, err = manFile.Write(manifestData) + if err != nil { + return nil, xerrors.Errorf("write manifest: %w", err) + } + + // Empty file + _, err = w.Create(".signature.p7s") + if err != nil { + return nil, xerrors.Errorf("create empty p7s signature: %w", err) + } + + // Actual sig + sigFile, err := w.Create(".signature.sig") + if err != nil { + return nil, xerrors.Errorf("create signature: %w", err) + } + + signature := ed25519.Sign(key, manifestData) + + //signature, err := key.Sign(rand.Reader, manifestData, crypto.SHA512) + //if err != nil { + // return nil, xerrors.Errorf("sign: %w", err) + //} + + _, err = sigFile.Write(signature) + if err != nil { + return nil, xerrors.Errorf("write signature: %w", err) + } + err = w.Close() if err != nil { return nil, xerrors.Errorf("close zip: %w", err) From 75208fe97c3fc0c4e35956381ede066dd3e30e1d Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Wed, 11 Dec 2024 18:02:08 -0600 Subject: [PATCH 04/22] chore: signatures working, time to refactor --- cli/add.go | 48 +++++++++++++++----------------- internal/extensionsign/sigzip.go | 10 ++++++- storage/local.go | 6 ++++ testutil/mockstorage.go | 2 +- 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/cli/add.go b/cli/add.go index c8aa295..40e20cc 100644 --- a/cli/add.go +++ b/cli/add.go @@ -2,6 +2,7 @@ package cli import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -23,7 +24,6 @@ func add() *cobra.Command { artifactory string extdir string repo string - signature bool ) cmd := &cobra.Command{ @@ -75,7 +75,7 @@ func add() *cobra.Command { return err } for _, file := range files { - s, err := doAdd(ctx, filepath.Join(args[0], file.Name()), signature, store) + s, err := doAdd(ctx, filepath.Join(args[0], file.Name()), store) if err != nil { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Failed to unpack %s: %s\n", file.Name(), err.Error()) failed = append(failed, file.Name()) @@ -84,7 +84,7 @@ func add() *cobra.Command { } } } else { - s, err := doAdd(ctx, args[0], signature, store) + s, err := doAdd(ctx, args[0], store) if err != nil { return err } @@ -104,12 +104,11 @@ func add() *cobra.Command { cmd.Flags().StringVar(&extdir, "extensions-dir", "", "The path to extensions.") cmd.Flags().StringVar(&artifactory, "artifactory", "", "Artifactory server URL.") cmd.Flags().StringVar(&repo, "repo", "", "Artifactory repository.") - cmd.Flags().BoolVar(&signature, "signature", true, "Include signature") return cmd } -func doAdd(ctx context.Context, source string, includeSignature bool, store storage.Storage) ([]string, error) { +func doAdd(ctx context.Context, source string, store storage.Storage) ([]string, error) { // Read in the extension. In the future we might support stdin as well. vsix, err := storage.ReadVSIX(ctx, source) if err != nil { @@ -123,31 +122,28 @@ func doAdd(ctx context.Context, source string, includeSignature bool, store stor return nil, err } - var extra []storage.File - if includeSignature { - sigManifest, err := extensionsign.GenerateSignatureManifest(vsix) - if err != nil { - return nil, xerrors.Errorf("generate signature manifest: %w", err) - } + sigManifest, err := extensionsign.GenerateSignatureManifest(vsix) + if err != nil { + return nil, xerrors.Errorf("zip signature manifest: %w", err) + } - zipped, err := extensionsign.Zip(sigManifest) - if err != nil { - return nil, xerrors.Errorf("zip signature manifest: %w", err) - } + data, err := json.Marshal(sigManifest) + if err != nil { + return nil, xerrors.Errorf("encode manifest: %w", err) + } - extra = append(extra, storage.File{ - RelativePath: "extension.sigzip", - Content: zipped, - }) + key, _ := extensionsign.GenerateKey() + sigZip, _ := extensionsign.SignAndZipVSIX(key, vsix) - manifest.Assets.Asset = append(manifest.Assets.Asset, storage.VSIXAsset{ - Type: storage.VSIXSignatureType, - Path: "extension.sigzip", - Addressable: "true", // TODO: Idk if this is right + location, err := store.AddExtension(ctx, manifest, vsix, + storage.File{ + RelativePath: "extension.sigzip", + Content: sigZip, + }, + storage.File{ + RelativePath: ".signature.manifest", + Content: data, }) - } - - location, err := store.AddExtension(ctx, manifest, vsix, extra...) if err != nil { return nil, err } diff --git a/internal/extensionsign/sigzip.go b/internal/extensionsign/sigzip.go index fe292d3..4f49b7f 100644 --- a/internal/extensionsign/sigzip.go +++ b/internal/extensionsign/sigzip.go @@ -26,10 +26,18 @@ func ExtractSignatureManifest(zip []byte) (SignatureManifest, error) { return manifest, nil } +func SignAndZipVSIX(key ed25519.PrivateKey, vsix []byte) ([]byte, error) { + manifest, err := GenerateSignatureManifest(vsix) + if err != nil { + return nil, xerrors.Errorf("generate manifest: %w", err) + } + return SignAndZipManifest(key, manifest) +} + // SignAndZip signs a manifest and zips it up // Should be a PCKS8 key // TODO: Support other key types -func SignAndZip(key ed25519.PrivateKey, manifest SignatureManifest) ([]byte, error) { +func SignAndZipManifest(key ed25519.PrivateKey, manifest SignatureManifest) ([]byte, error) { var buf bytes.Buffer w := zip.NewWriter(&buf) diff --git a/storage/local.go b/storage/local.go index 65f74f9..1d0837f 100644 --- a/storage/local.go +++ b/storage/local.go @@ -164,6 +164,12 @@ func (s *Local) Manifest(ctx context.Context, publisher, name string, version Ve Addressable: "true", }) + manifest.Assets.Asset = append(manifest.Assets.Asset, VSIXAsset{ + Type: VSIXSignatureType, + Path: "extension.sigzip", + Addressable: "true", + }) + return manifest, nil } diff --git a/testutil/mockstorage.go b/testutil/mockstorage.go index a7dc12a..db49fa2 100644 --- a/testutil/mockstorage.go +++ b/testutil/mockstorage.go @@ -17,7 +17,7 @@ func NewMockStorage() *MockStorage { return &MockStorage{} } -func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXManifest, vsix []byte, extra ...storage.File) (string, error) { +func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXManifest, vsix []byte) (string, error) { return "", errors.New("not implemented") } From f4091e744b6c5ea8020e200fb981f7739abe637e Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 09:53:14 -0600 Subject: [PATCH 05/22] chore: refactor cli to support resuing the server flags --- .gitignore | 2 +- cli/add.go | 56 ++------------------- cli/remove.go | 29 ++--------- cli/server.go | 72 +++++++++++++++++---------- cli/signature.go | 1 + internal/extensionsign/sigmanifest.go | 4 +- internal/extensionsign/sigzip.go | 4 +- storage/artifactory.go | 13 ++++- storage/{ => easyzip}/zip.go | 2 +- storage/{ => easyzip}/zip_test.go | 2 +- storage/local.go | 11 ++-- storage/signature.go | 63 +++++++++++++++++++++++ storage/storage.go | 29 ++++++++--- testutil/mockstorage.go | 4 +- 14 files changed, 166 insertions(+), 126 deletions(-) rename storage/{ => easyzip}/zip.go (99%) rename storage/{ => easyzip}/zip_test.go (99%) create mode 100644 storage/signature.go diff --git a/.gitignore b/.gitignore index 504cb83..92996d1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ bin coverage extensions -.idea \ No newline at end of file +.idea diff --git a/cli/add.go b/cli/add.go index 40e20cc..1bf17d4 100644 --- a/cli/add.go +++ b/cli/add.go @@ -2,7 +2,6 @@ package cli import ( "context" - "encoding/json" "fmt" "os" "path/filepath" @@ -11,21 +10,12 @@ import ( "github.com/spf13/cobra" "golang.org/x/xerrors" - "cdr.dev/slog" - "cdr.dev/slog/sloggers/sloghuman" - "github.com/coder/code-marketplace/internal/extensionsign" - "github.com/coder/code-marketplace/storage" "github.com/coder/code-marketplace/util" ) func add() *cobra.Command { - var ( - artifactory string - extdir string - repo string - ) - + addFlags, opts := serverFlags() cmd := &cobra.Command{ Use: "add ", Short: "Add an extension to the marketplace", @@ -39,21 +29,7 @@ func add() *cobra.Command { ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() - verbose, err := cmd.Flags().GetBool("verbose") - if err != nil { - return err - } - logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) - if verbose { - logger = logger.Leveled(slog.LevelDebug) - } - - store, err := storage.NewStorage(ctx, &storage.Options{ - Artifactory: artifactory, - ExtDir: extdir, - Logger: logger, - Repo: repo, - }) + store, err := storage.NewStorage(ctx, opts) if err != nil { return err } @@ -100,10 +76,7 @@ func add() *cobra.Command { return nil }, } - - cmd.Flags().StringVar(&extdir, "extensions-dir", "", "The path to extensions.") - cmd.Flags().StringVar(&artifactory, "artifactory", "", "Artifactory server URL.") - cmd.Flags().StringVar(&repo, "repo", "", "Artifactory repository.") + addFlags(cmd) return cmd } @@ -122,28 +95,7 @@ func doAdd(ctx context.Context, source string, store storage.Storage) ([]string, return nil, err } - sigManifest, err := extensionsign.GenerateSignatureManifest(vsix) - if err != nil { - return nil, xerrors.Errorf("zip signature manifest: %w", err) - } - - data, err := json.Marshal(sigManifest) - if err != nil { - return nil, xerrors.Errorf("encode manifest: %w", err) - } - - key, _ := extensionsign.GenerateKey() - sigZip, _ := extensionsign.SignAndZipVSIX(key, vsix) - - location, err := store.AddExtension(ctx, manifest, vsix, - storage.File{ - RelativePath: "extension.sigzip", - Content: sigZip, - }, - storage.File{ - RelativePath: ".signature.manifest", - Content: data, - }) + location, err := store.AddExtension(ctx, manifest, vsix) if err != nil { return nil, err } diff --git a/cli/remove.go b/cli/remove.go index 5c3add7..45aa686 100644 --- a/cli/remove.go +++ b/cli/remove.go @@ -10,20 +10,15 @@ import ( "github.com/spf13/cobra" "golang.org/x/xerrors" - "cdr.dev/slog" - "cdr.dev/slog/sloggers/sloghuman" - "github.com/coder/code-marketplace/storage" "github.com/coder/code-marketplace/util" ) func remove() *cobra.Command { var ( - all bool - artifactory string - extdir string - repo string + all bool ) + addFlags, opts := serverFlags() cmd := &cobra.Command{ Use: "remove ", @@ -37,21 +32,7 @@ func remove() *cobra.Command { ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() - verbose, err := cmd.Flags().GetBool("verbose") - if err != nil { - return err - } - logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) - if verbose { - logger = logger.Leveled(slog.LevelDebug) - } - - store, err := storage.NewStorage(ctx, &storage.Options{ - Artifactory: artifactory, - ExtDir: extdir, - Logger: logger, - Repo: repo, - }) + store, err := storage.NewStorage(ctx, opts) if err != nil { return err } @@ -120,9 +101,7 @@ func remove() *cobra.Command { } cmd.Flags().BoolVar(&all, "all", false, "Whether to delete all versions of the extension.") - cmd.Flags().StringVar(&extdir, "extensions-dir", "", "The path to extensions.") - cmd.Flags().StringVar(&artifactory, "artifactory", "", "Artifactory server URL.") - cmd.Flags().StringVar(&repo, "repo", "", "Artifactory repository.") + addFlags(cmd) return cmd } diff --git a/cli/server.go b/cli/server.go index befc20b..6c3cf85 100644 --- a/cli/server.go +++ b/cli/server.go @@ -21,15 +21,52 @@ import ( "github.com/coder/code-marketplace/storage" ) +func serverFlags() (addFlags func(cmd *cobra.Command), opts *storage.Options) { + opts = &storage.Options{} + return func(cmd *cobra.Command) { + cmd.Flags().StringVar(&opts.ExtDir, "extensions-dir", "", "The path to extensions.") + cmd.Flags().StringVar(&opts.Artifactory, "artifactory", "", "Artifactory server URL.") + cmd.Flags().StringVar(&opts.Repo, "repo", "", "Artifactory repository.") + cmd.Flags().DurationVar(&opts.ListCacheDuration, "list-cache-duration", time.Minute, "The duration of the extension cache.") + cmd.Flags().BoolVar(&opts.SignExtensions, "sign", false, "Sign extensions.") + + var before func(cmd *cobra.Command, args []string) error + if cmd.PreRunE != nil { + before = cmd.PreRunE + } + if cmd.PreRun != nil { + beforeNoE := cmd.PreRun + before = func(cmd *cobra.Command, args []string) error { + beforeNoE(cmd, args) + return nil + } + } + + cmd.PreRunE = func(cmd *cobra.Command, args []string) error { + opts.Logger = cmdLogger(cmd) + if before != nil { + return before(cmd, args) + } + return nil + } + }, opts +} + +func cmdLogger(cmd *cobra.Command) slog.Logger { + verbose, _ := cmd.Flags().GetBool("verbose") + logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) + if verbose { + logger = logger.Leveled(slog.LevelDebug) + } + return logger +} + func server() *cobra.Command { var ( - address string - artifactory string - extdir string - repo string - listcacheduration time.Duration - maxpagesize int + address string + maxpagesize int ) + addFlags, opts := serverFlags() cmd := &cobra.Command{ Use: "server", @@ -41,26 +78,12 @@ func server() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() + logger := opts.Logger notifyCtx, notifyStop := signal.NotifyContext(ctx, interruptSignals...) defer notifyStop() - verbose, err := cmd.Flags().GetBool("verbose") - if err != nil { - return err - } - logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) - if verbose { - logger = logger.Leveled(slog.LevelDebug) - } - - store, err := storage.NewStorage(ctx, &storage.Options{ - Artifactory: artifactory, - ExtDir: extdir, - Logger: logger, - Repo: repo, - ListCacheDuration: listcacheduration, - }) + store, err := storage.NewStorage(ctx, opts) if err != nil { return err } @@ -137,12 +160,9 @@ func server() *cobra.Command { }, } - cmd.Flags().StringVar(&extdir, "extensions-dir", "", "The path to extensions.") cmd.Flags().IntVar(&maxpagesize, "max-page-size", api.MaxPageSizeDefault, "The maximum number of pages to request") - cmd.Flags().StringVar(&artifactory, "artifactory", "", "Artifactory server URL.") - cmd.Flags().StringVar(&repo, "repo", "", "Artifactory repository.") cmd.Flags().StringVar(&address, "address", "127.0.0.1:3001", "The address on which to serve the marketplace API.") - cmd.Flags().DurationVar(&listcacheduration, "list-cache-duration", time.Minute, "The duration of the extension cache.") + addFlags(cmd) return cmd } diff --git a/cli/signature.go b/cli/signature.go index e1b130a..5b6a683 100644 --- a/cli/signature.go +++ b/cli/signature.go @@ -13,6 +13,7 @@ import ( func signature() *cobra.Command { cmd := &cobra.Command{ Use: "signature", + Hidden: true, // Debugging tools Aliases: []string{"sig", "sigs", "signatures"}, } cmd.AddCommand(compareSignatureSigZips()) diff --git a/internal/extensionsign/sigmanifest.go b/internal/extensionsign/sigmanifest.go index 60deddc..58b8ec6 100644 --- a/internal/extensionsign/sigmanifest.go +++ b/internal/extensionsign/sigmanifest.go @@ -10,7 +10,7 @@ import ( "github.com/cloudflare/cfssl/scan/crypto/sha256" "golang.org/x/xerrors" - "github.com/coder/code-marketplace/storage" + "github.com/coder/code-marketplace/storage/easyzip" ) type SignatureManifest struct { @@ -94,7 +94,7 @@ func GenerateSignatureManifest(vsixFile []byte) (SignatureManifest, error) { Entries: make(map[string]File), } - err = storage.ExtractZip(vsixFile, func(name string, reader io.Reader) error { + err = easyzip.ExtractZip(vsixFile, func(name string, reader io.Reader) error { fm, err := FileManifest(reader) if err != nil { return xerrors.Errorf("file %q: %w", name, err) diff --git a/internal/extensionsign/sigzip.go b/internal/extensionsign/sigzip.go index 4f49b7f..4245e80 100644 --- a/internal/extensionsign/sigzip.go +++ b/internal/extensionsign/sigzip.go @@ -8,11 +8,11 @@ import ( "golang.org/x/xerrors" - "github.com/coder/code-marketplace/storage" + "github.com/coder/code-marketplace/storage/easyzip" ) func ExtractSignatureManifest(zip []byte) (SignatureManifest, error) { - r, err := storage.GetZipFileReader(zip, ".signature.manifest") + r, err := easyzip.GetZipFileReader(zip, ".signature.manifest") if err != nil { return SignatureManifest{}, xerrors.Errorf("get manifest: %w", err) } diff --git a/storage/artifactory.go b/storage/artifactory.go index f1bbf71..c71e5ca 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -19,6 +19,7 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog" + "github.com/coder/code-marketplace/storage/easyzip" "github.com/coder/code-marketplace/util" ) @@ -41,6 +42,8 @@ type ArtifactoryList struct { Files []ArtifactoryFile `json:"files"` } +var _ Storage = (*Artifactory)(nil) + // Artifactory implements Storage. It stores extensions remotely through // Artifactory by both copying the VSIX and extracting said VSIX to a tree // structure in the form of publisher/extension/version to easily serve @@ -244,7 +247,7 @@ func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, } } - err := ExtractZip(vsix, func(name string, r io.Reader) error { + err := easyzip.ExtractZip(vsix, func(name string, r io.Reader) error { if util.Contains(assets, name) || (browser != "" && strings.HasPrefix(name, browser)) { _, err := s.upload(ctx, path.Join(dir, name), r) return err @@ -262,6 +265,14 @@ func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, return "", err } + for _, file := range extra { + // TODO: I think this is correct? + _, err := s.upload(ctx, path.Join(dir, file.RelativePath), bytes.NewReader(file.Content)) + if err != nil { + return "", err + } + } + return s.uri + dir, nil } diff --git a/storage/zip.go b/storage/easyzip/zip.go similarity index 99% rename from storage/zip.go rename to storage/easyzip/zip.go index da2526f..9a76e89 100644 --- a/storage/zip.go +++ b/storage/easyzip/zip.go @@ -1,4 +1,4 @@ -package storage +package easyzip import ( "archive/zip" diff --git a/storage/zip_test.go b/storage/easyzip/zip_test.go similarity index 99% rename from storage/zip_test.go rename to storage/easyzip/zip_test.go index 3ae2ceb..febe877 100644 --- a/storage/zip_test.go +++ b/storage/easyzip/zip_test.go @@ -1,4 +1,4 @@ -package storage +package easyzip import ( "archive/zip" diff --git a/storage/local.go b/storage/local.go index 1d0837f..6fa908a 100644 --- a/storage/local.go +++ b/storage/local.go @@ -14,8 +14,11 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog" + "github.com/coder/code-marketplace/storage/easyzip" ) +var _ Storage = (*Local)(nil) + // Local implements Storage. It stores extensions locally on disk by both // copying the VSIX and extracting said VSIX to a tree structure in the form of // publisher/extension/version to easily serve individual assets via HTTP. @@ -98,7 +101,7 @@ func (s *Local) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix [ Version: identity.Version, TargetPlatform: identity.TargetPlatform, }.String()) - err := ExtractZip(vsix, func(name string, r io.Reader) error { + err := easyzip.ExtractZip(vsix, func(name string, r io.Reader) error { path := filepath.Join(dir, name) err := os.MkdirAll(filepath.Dir(path), 0o755) if err != nil { @@ -164,12 +167,6 @@ func (s *Local) Manifest(ctx context.Context, publisher, name string, version Ve Addressable: "true", }) - manifest.Assets.Asset = append(manifest.Assets.Asset, VSIXAsset{ - Type: VSIXSignatureType, - Path: "extension.sigzip", - Addressable: "true", - }) - return manifest, nil } diff --git a/storage/signature.go b/storage/signature.go new file mode 100644 index 0000000..6623f62 --- /dev/null +++ b/storage/signature.go @@ -0,0 +1,63 @@ +package storage + +import ( + "context" + "encoding/json" + + "golang.org/x/xerrors" + + "github.com/coder/code-marketplace/internal/extensionsign" +) + +var _ Storage = (*Signature)(nil) + +type Signature struct { + // SignDesignExtensions is a flag that determines if the signature should + // include the extension payloads. + signExtensions bool + Storage +} + +func NewSignatureStorage(signExtensions bool, s Storage) *Signature { + return &Signature{ + signExtensions: signExtensions, + Storage: s, + } +} + +// AddExtension includes the signature manifest of the vsix. Signing happens on +// demand, so leave the manifest unsigned. This is safe to do even if +// 'signExtensions' is disabled, as these files lay dormant until signed. +func (s *Signature) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) { + sigManifest, err := extensionsign.GenerateSignatureManifest(vsix) + if err != nil { + return "", xerrors.Errorf("generate signature manifest: %w", err) + } + + data, err := json.Marshal(sigManifest) + if err != nil { + return "", xerrors.Errorf("encode signature manifest: %w", err) + } + + return s.Storage.AddExtension(ctx, manifest, vsix, append(extra, File{ + RelativePath: ".signature.manifest", + Content: data, + })...) +} + +func (s *Signature) Manifest(ctx context.Context, publisher, name string, version Version) (*VSIXManifest, error) { + manifest, err := s.Storage.Manifest(ctx, publisher, name, version) + if err != nil { + return nil, err + } + + if s.signExtensions { + manifest.Assets.Asset = append(manifest.Assets.Asset, VSIXAsset{ + Type: VSIXSignatureType, + Path: "extension.sigzip", + Addressable: "true", + }) + } + + return manifest, nil +} diff --git a/storage/storage.go b/storage/storage.go index af710bc..0d552d9 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -13,9 +13,11 @@ import ( "time" "golang.org/x/mod/semver" + "golang.org/x/xerrors" "cdr.dev/slog" + "github.com/coder/code-marketplace/storage/easyzip" ) // VSIXManifest implement XMLManifest.PackageManifest. @@ -124,6 +126,7 @@ type VSIXAsset struct { } type Options struct { + SignExtensions bool Artifactory string ExtDir string Repo string @@ -205,6 +208,7 @@ func (vs ByVersion) Less(i, j int) bool { type Storage interface { // AddExtension adds the provided VSIX into storage and returns the location // for verification purposes. Extra files can be included, but not required. + // All extra files will be placed relative to the manifest outside the vsix. AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) // FileServer provides a handler for fetching extension repository files from // a client. @@ -246,31 +250,42 @@ func NewStorage(ctx context.Context, options *Options) (Storage, error) { return nil, xerrors.Errorf("cannot use both Artifactory and extension directory") } else if options.Artifactory != "" && options.Repo == "" { return nil, xerrors.Errorf("must provide repository") - } else if options.Artifactory != "" { + } + + var store Storage + var err error + switch { + case options.Artifactory != "": token := os.Getenv(ArtifactoryTokenEnvKey) if token == "" { return nil, xerrors.Errorf("the %s environment variable must be set", ArtifactoryTokenEnvKey) } - return NewArtifactoryStorage(ctx, &ArtifactoryOptions{ + store, err = NewArtifactoryStorage(ctx, &ArtifactoryOptions{ ListCacheDuration: options.ListCacheDuration, Logger: options.Logger, Repo: options.Repo, Token: token, URI: options.Artifactory, }) - } else if options.ExtDir != "" { - return NewLocalStorage(&LocalOptions{ + case options.ExtDir != "": + store, err = NewLocalStorage(&LocalOptions{ ListCacheDuration: options.ListCacheDuration, ExtDir: options.ExtDir, }, options.Logger) + default: + return nil, xerrors.Errorf("must provide an Artifactory repository or local directory") } - return nil, xerrors.Errorf("must provide an Artifactory repository or local directory") + if err != nil { + return nil, err + } + + return NewSignatureStorage(options.SignExtensions, store), nil } // ReadVSIXManifest reads and parses an extension manifest from a vsix file. If // the manifest is invalid it will be returned along with the validation error. func ReadVSIXManifest(vsix []byte) (*VSIXManifest, error) { - vmr, err := GetZipFileReader(vsix, "extension.vsixmanifest") + vmr, err := easyzip.GetZipFileReader(vsix, "extension.vsixmanifest") if err != nil { return nil, err } @@ -328,7 +343,7 @@ type VSIXPackageJSON struct { // ReadVSIXPackageJSON reads and parses an extension's package.json from a vsix // file. func ReadVSIXPackageJSON(vsix []byte, packageJsonPath string) (*VSIXPackageJSON, error) { - vpjr, err := GetZipFileReader(vsix, packageJsonPath) + vpjr, err := easyzip.GetZipFileReader(vsix, packageJsonPath) if err != nil { return nil, err } diff --git a/testutil/mockstorage.go b/testutil/mockstorage.go index db49fa2..c1f6249 100644 --- a/testutil/mockstorage.go +++ b/testutil/mockstorage.go @@ -10,6 +10,8 @@ import ( "github.com/coder/code-marketplace/storage" ) +var _ storage.Storage = (*MockStorage)(nil) + // MockStorage implements storage.Storage for tests. type MockStorage struct{} @@ -17,7 +19,7 @@ func NewMockStorage() *MockStorage { return &MockStorage{} } -func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXManifest, vsix []byte) (string, error) { +func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXManifest, vsix []byte, extra ...storage.File) (string, error) { return "", errors.New("not implemented") } From 2e4c55e6a3294adbbc42c9b3665fbf1a38541a30 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 09:56:36 -0600 Subject: [PATCH 06/22] move extensionsign pkg --- cli/signature.go | 2 +- {internal/extensionsign => extensionsign}/doc.go | 0 {internal/extensionsign => extensionsign}/key.go | 0 {internal/extensionsign => extensionsign}/sigmanifest.go | 0 {internal/extensionsign => extensionsign}/sigmanifest_test.go | 0 {internal/extensionsign => extensionsign}/sigzip.go | 0 storage/signature.go | 2 +- 7 files changed, 2 insertions(+), 2 deletions(-) rename {internal/extensionsign => extensionsign}/doc.go (100%) rename {internal/extensionsign => extensionsign}/key.go (100%) rename {internal/extensionsign => extensionsign}/sigmanifest.go (100%) rename {internal/extensionsign => extensionsign}/sigmanifest_test.go (100%) rename {internal/extensionsign => extensionsign}/sigzip.go (100%) diff --git a/cli/signature.go b/cli/signature.go index 5b6a683..5adf4b2 100644 --- a/cli/signature.go +++ b/cli/signature.go @@ -7,7 +7,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/xerrors" - "github.com/coder/code-marketplace/internal/extensionsign" + "github.com/coder/code-marketplace/extensionsign" ) func signature() *cobra.Command { diff --git a/internal/extensionsign/doc.go b/extensionsign/doc.go similarity index 100% rename from internal/extensionsign/doc.go rename to extensionsign/doc.go diff --git a/internal/extensionsign/key.go b/extensionsign/key.go similarity index 100% rename from internal/extensionsign/key.go rename to extensionsign/key.go diff --git a/internal/extensionsign/sigmanifest.go b/extensionsign/sigmanifest.go similarity index 100% rename from internal/extensionsign/sigmanifest.go rename to extensionsign/sigmanifest.go diff --git a/internal/extensionsign/sigmanifest_test.go b/extensionsign/sigmanifest_test.go similarity index 100% rename from internal/extensionsign/sigmanifest_test.go rename to extensionsign/sigmanifest_test.go diff --git a/internal/extensionsign/sigzip.go b/extensionsign/sigzip.go similarity index 100% rename from internal/extensionsign/sigzip.go rename to extensionsign/sigzip.go diff --git a/storage/signature.go b/storage/signature.go index 6623f62..9c1f43b 100644 --- a/storage/signature.go +++ b/storage/signature.go @@ -6,7 +6,7 @@ import ( "golang.org/x/xerrors" - "github.com/coder/code-marketplace/internal/extensionsign" + "github.com/coder/code-marketplace/extensionsign" ) var _ Storage = (*Signature)(nil) From a72ead1a2670507e82155899df66318e8fd7585a Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 10:41:53 -0600 Subject: [PATCH 07/22] chore: implement signature storage, still need to load a key --- api/api.go | 19 +++++++++- extensionsign/sigmanifest.go | 2 +- extensionsign/sigzip.go | 35 +++++------------- storage/artifactory.go | 72 +++++++++++++++++++++++++++--------- storage/local.go | 5 ++- storage/signature.go | 54 ++++++++++++++++++++++++++- storage/storage.go | 4 +- 7 files changed, 141 insertions(+), 50 deletions(-) diff --git a/api/api.go b/api/api.go index 50f07c5..526838f 100644 --- a/api/api.go +++ b/api/api.go @@ -1,7 +1,9 @@ package api import ( + "context" "encoding/json" + "io/fs" "net/http" "os" "strconv" @@ -112,7 +114,13 @@ func New(options *Options) *API { r.Post("/api/extensionquery", api.extensionQuery) // Endpoint for getting an extension's files or the extension zip. - r.Mount("/files", http.StripPrefix("/files", options.Storage.FileServer())) + r.Mount("/files", http.StripPrefix("/files", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + // Convert the storage into a fs.FS, including the request context + http.FileServerFS(&contextFs{ + ctx: r.Context(), + open: options.Storage.Open, + }).ServeHTTP(rw, r) + }))) // VS Code can use the files in the response to get file paths but it will // sometimes ignore that and use requests to /assets with hardcoded types to @@ -256,3 +264,12 @@ func (api *API) assetRedirect(rw http.ResponseWriter, r *http.Request) { http.Redirect(rw, r, url, http.StatusMovedPermanently) } + +type contextFs struct { + ctx context.Context + open func(ctx context.Context, name string) (fs.File, error) +} + +func (c *contextFs) Open(name string) (fs.File, error) { + return c.open(c.ctx, name) +} diff --git a/extensionsign/sigmanifest.go b/extensionsign/sigmanifest.go index 58b8ec6..55ac4c6 100644 --- a/extensionsign/sigmanifest.go +++ b/extensionsign/sigmanifest.go @@ -2,12 +2,12 @@ package extensionsign import ( "bytes" + "crypto/sha256" "encoding/base64" "errors" "fmt" "io" - "github.com/cloudflare/cfssl/scan/crypto/sha256" "golang.org/x/xerrors" "github.com/coder/code-marketplace/storage/easyzip" diff --git a/extensionsign/sigzip.go b/extensionsign/sigzip.go index 4245e80..9e4ab15 100644 --- a/extensionsign/sigzip.go +++ b/extensionsign/sigzip.go @@ -3,7 +3,8 @@ package extensionsign import ( "archive/zip" "bytes" - "crypto/ed25519" + "crypto" + "crypto/rand" "encoding/json" "golang.org/x/xerrors" @@ -26,18 +27,9 @@ func ExtractSignatureManifest(zip []byte) (SignatureManifest, error) { return manifest, nil } -func SignAndZipVSIX(key ed25519.PrivateKey, vsix []byte) ([]byte, error) { - manifest, err := GenerateSignatureManifest(vsix) - if err != nil { - return nil, xerrors.Errorf("generate manifest: %w", err) - } - return SignAndZipManifest(key, manifest) -} - -// SignAndZip signs a manifest and zips it up -// Should be a PCKS8 key -// TODO: Support other key types -func SignAndZipManifest(key ed25519.PrivateKey, manifest SignatureManifest) ([]byte, error) { +// SignAndZipManifest signs a manifest and zips it up +// Sign +func SignAndZipManifest(secret crypto.Signer, manifest json.RawMessage) ([]byte, error) { var buf bytes.Buffer w := zip.NewWriter(&buf) @@ -46,12 +38,7 @@ func SignAndZipManifest(key ed25519.PrivateKey, manifest SignatureManifest) ([]b return nil, xerrors.Errorf("create manifest: %w", err) } - manifestData, err := json.Marshal(manifest) - if err != nil { - return nil, xerrors.Errorf("encode manifest: %w", err) - } - - _, err = manFile.Write(manifestData) + _, err = manFile.Write(manifest) if err != nil { return nil, xerrors.Errorf("write manifest: %w", err) } @@ -68,12 +55,10 @@ func SignAndZipManifest(key ed25519.PrivateKey, manifest SignatureManifest) ([]b return nil, xerrors.Errorf("create signature: %w", err) } - signature := ed25519.Sign(key, manifestData) - - //signature, err := key.Sign(rand.Reader, manifestData, crypto.SHA512) - //if err != nil { - // return nil, xerrors.Errorf("sign: %w", err) - //} + signature, err := secret.Sign(rand.Reader, manifest, nil) + if err != nil { + return nil, xerrors.Errorf("sign: %w", err) + } _, err = sigFile.Write(signature) if err != nil { diff --git a/storage/artifactory.go b/storage/artifactory.go index c71e5ca..8fa3984 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -7,9 +7,11 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "os" "path" + "path/filepath" "sort" "strings" "sync" @@ -276,25 +278,32 @@ func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, return s.uri + dir, nil } -func (s *Artifactory) FileServer() http.Handler { - // TODO: Since we only extract a subset of files perhaps if the file does not - // exist we should download the vsix and extract the requested file as a - // fallback. Obviously this seems like quite a bit of overhead so we would - // then emit a warning so we can notice that VS Code has added new asset types - // that we should be extracting to avoid that overhead. Other solutions could - // be implemented though like extracting the VSIX to disk locally and only - // going to Artifactory for the VSIX when it is missing on disk (basically - // using the disk as a cache). - return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - reader, code, err := s.read(r.Context(), r.URL.Path) - if err != nil { - http.Error(rw, err.Error(), code) - return +// Open returns a file from Artifactory. +// TODO: Since we only extract a subset of files perhaps if the file does not +// exist we should download the vsix and extract the requested file as a +// fallback. Obviously this seems like quite a bit of overhead so we would +// then emit a warning so we can notice that VS Code has added new asset types +// that we should be extracting to avoid that overhead. Other solutions could +// be implemented though like extracting the VSIX to disk locally and only +// going to Artifactory for the VSIX when it is missing on disk (basically +// using the disk as a cache). +func (s *Artifactory) Open(ctx context.Context, fp string) (fs.File, error) { + resp, code, err := s.request(ctx, http.MethodGet, path.Join(s.repo, fp), nil) + if err != nil { + switch code { + case http.StatusNotFound: + return nil, fs.ErrNotExist + case http.StatusForbidden: + return nil, fs.ErrPermission + default: + return nil, err } - defer reader.Close() - rw.WriteHeader(http.StatusOK) - _, _ = io.Copy(rw, reader) - }) + } + + return artifactoryFile{ + Response: resp, + path: fp, + }, nil } func (s *Artifactory) Manifest(ctx context.Context, publisher, name string, version Version) (*VSIXManifest, error) { @@ -443,3 +452,30 @@ func (s *Artifactory) Versions(ctx context.Context, publisher, name string) ([]V sort.Sort(ByVersion(versions)) return versions, nil } + +type contextFs struct { + ctx context.Context + open func(ctx context.Context, name string) (fs.File, error) +} + +func (c *contextFs) Open(name string) (fs.File, error) { + return c.open(c.ctx, name) +} + +var _ fs.File = (*artifactoryFile)(nil) +var _ fs.FileInfo = (*artifactoryFile)(nil) + +type artifactoryFile struct { + *http.Response + path string +} + +func (a artifactoryFile) Name() string { return filepath.Base(a.path) } +func (a artifactoryFile) Size() int64 { return a.Response.ContentLength } +func (a artifactoryFile) Mode() fs.FileMode { return fs.FileMode(0) } // ? +func (a artifactoryFile) ModTime() time.Time { return time.Now() } +func (a artifactoryFile) IsDir() bool { return false } +func (a artifactoryFile) Sys() any { return nil } +func (a artifactoryFile) Stat() (fs.FileInfo, error) { return a, nil } +func (a artifactoryFile) Read(i []byte) (int, error) { return a.Response.Body.Read(i) } +func (a artifactoryFile) Close() error { return a.Response.Body.Close() } diff --git a/storage/local.go b/storage/local.go index 6fa908a..4d57c47 100644 --- a/storage/local.go +++ b/storage/local.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "io/fs" "net/http" "os" "path/filepath" @@ -141,8 +142,8 @@ func (s *Local) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix [ return dir, nil } -func (s *Local) FileServer() http.Handler { - return http.FileServer(http.Dir(s.extdir)) +func (s *Local) Open(_ context.Context, fp string) (fs.File, error) { + return http.Dir(s.extdir).Open(fp) } func (s *Local) Manifest(ctx context.Context, publisher, name string, version Version) (*VSIXManifest, error) { diff --git a/storage/signature.go b/storage/signature.go index 9c1f43b..6c5a870 100644 --- a/storage/signature.go +++ b/storage/signature.go @@ -1,8 +1,13 @@ package storage import ( + "bytes" "context" "encoding/json" + "io" + "io/fs" + "path/filepath" + "time" "golang.org/x/xerrors" @@ -11,6 +16,11 @@ import ( var _ Storage = (*Signature)(nil) +const ( + sigzipFilename = "extension.sigzip" + sigManifestName = ".signature.manifest" +) + type Signature struct { // SignDesignExtensions is a flag that determines if the signature should // include the extension payloads. @@ -40,7 +50,7 @@ func (s *Signature) AddExtension(ctx context.Context, manifest *VSIXManifest, vs } return s.Storage.AddExtension(ctx, manifest, vsix, append(extra, File{ - RelativePath: ".signature.manifest", + RelativePath: sigManifestName, Content: data, })...) } @@ -54,10 +64,50 @@ func (s *Signature) Manifest(ctx context.Context, publisher, name string, versio if s.signExtensions { manifest.Assets.Asset = append(manifest.Assets.Asset, VSIXAsset{ Type: VSIXSignatureType, - Path: "extension.sigzip", + Path: sigzipFilename, Addressable: "true", }) } return manifest, nil } + +func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { + if s.signExtensions && filepath.Base(fp) == sigzipFilename { + // hijack this request, sign the sig manifest + manifest, err := s.Storage.Open(ctx, sigManifestName) + if err != nil { + return nil, err + } + defer manifest.Close() + + key, _ := extensionsign.GenerateKey() + manifestData, err := io.ReadAll(manifest) + if err != nil { + return nil, xerrors.Errorf("read signature manifest: %w", err) + } + + signed, err := extensionsign.SignAndZipManifest(key, manifestData) + if err != nil { + return nil, xerrors.Errorf("sign and zip manifest: %w", err) + } + return memFile{data: bytes.NewBuffer(signed), name: sigzipFilename}, nil + } + + return s.Storage.Open(ctx, fp) +} + +type memFile struct { + data *bytes.Buffer + name string +} + +func (a memFile) Name() string { return a.name } +func (a memFile) Size() int64 { return int64(a.data.Len()) } +func (a memFile) Mode() fs.FileMode { return fs.FileMode(0) } // ? +func (a memFile) ModTime() time.Time { return time.Now() } +func (a memFile) IsDir() bool { return false } +func (a memFile) Sys() any { return nil } +func (a memFile) Stat() (fs.FileInfo, error) { return a, nil } +func (a memFile) Read(i []byte) (int, error) { return a.data.Read(i) } +func (a memFile) Close() error { return nil } diff --git a/storage/storage.go b/storage/storage.go index 0d552d9..5a1cfee 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -6,6 +6,7 @@ import ( "encoding/xml" "fmt" "io" + "io/fs" "net/http" "os" "regexp" @@ -212,7 +213,8 @@ type Storage interface { AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) // FileServer provides a handler for fetching extension repository files from // a client. - FileServer() http.Handler + //FileServer() fs.FS + Open(ctx context.Context, name string) (fs.File, error) // Manifest returns the manifest bytes for the provided extension. The // extension asset itself (the VSIX) will be included on the manifest even if // it does not exist on the manifest on disk. From 21661f060c4d79fbfbcb6a7067817a8af6968bdc Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 13:10:26 -0600 Subject: [PATCH 08/22] user afero mem files --- extensionsign/sigzip.go | 2 +- go.mod | 8 +++++--- go.sum | 31 ++++++++++++++++++------------- storage/artifactory.go | 9 --------- storage/signature.go | 34 +++++++++++++--------------------- 5 files changed, 37 insertions(+), 47 deletions(-) diff --git a/extensionsign/sigzip.go b/extensionsign/sigzip.go index 9e4ab15..37a655f 100644 --- a/extensionsign/sigzip.go +++ b/extensionsign/sigzip.go @@ -55,7 +55,7 @@ func SignAndZipManifest(secret crypto.Signer, manifest json.RawMessage) ([]byte, return nil, xerrors.Errorf("create signature: %w", err) } - signature, err := secret.Sign(rand.Reader, manifest, nil) + signature, err := secret.Sign(rand.Reader, manifest, crypto.Hash(0)) if err != nil { return nil, xerrors.Errorf("sign: %w", err) } diff --git a/go.mod b/go.mod index 10bbe7d..2654974 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,13 @@ go 1.19 require ( cdr.dev/slog v1.6.1 + github.com/cloudflare/cfssl v1.6.5 github.com/go-chi/chi/v5 v5.1.0 github.com/go-chi/cors v1.2.1 github.com/go-chi/httprate v0.14.1 github.com/google/uuid v1.6.0 github.com/lithammer/fuzzysearch v1.1.8 + github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 golang.org/x/mod v0.19.0 @@ -32,9 +34,9 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/crypto v0.19.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f49f711..646e554 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ cdr.dev/slog v1.6.1 h1:IQjWZD0x6//sfv5n+qEhbu3wBkmtBQY5DILXNvMaIv4= cdr.dev/slog v1.6.1/go.mod h1:eHEYQLaZvxnIAXC+XdTSNLb/kgA/X2RVSF72v5wsxEI= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= @@ -10,6 +11,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E= github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= +github.com/cloudflare/cfssl v1.6.5 h1:46zpNkm6dlNkMZH/wMW22ejih6gIaJbzL2du6vD7ZeI= +github.com/cloudflare/cfssl v1.6.5/go.mod h1:Bk1si7sq8h2+yVEDrFJiz3d7Aw+pfjjJSZVaD+Taky4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -47,6 +50,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -62,8 +67,8 @@ go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZE go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= @@ -72,7 +77,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -85,13 +90,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -106,11 +111,11 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg= -google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 h1:XVeBY8d/FaK4848myy41HBqnDwvxeV3zMZhwN1TvAMU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/storage/artifactory.go b/storage/artifactory.go index 8fa3984..1db0cc7 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -453,15 +453,6 @@ func (s *Artifactory) Versions(ctx context.Context, publisher, name string) ([]V return versions, nil } -type contextFs struct { - ctx context.Context - open func(ctx context.Context, name string) (fs.File, error) -} - -func (c *contextFs) Open(name string) (fs.File, error) { - return c.open(c.ctx, name) -} - var _ fs.File = (*artifactoryFile)(nil) var _ fs.FileInfo = (*artifactoryFile)(nil) diff --git a/storage/signature.go b/storage/signature.go index 6c5a870..d26b599 100644 --- a/storage/signature.go +++ b/storage/signature.go @@ -1,14 +1,14 @@ package storage import ( - "bytes" "context" "encoding/json" + "fmt" "io" "io/fs" "path/filepath" - "time" + "github.com/spf13/afero/mem" "golang.org/x/xerrors" "github.com/coder/code-marketplace/extensionsign" @@ -68,16 +68,20 @@ func (s *Signature) Manifest(ctx context.Context, publisher, name string, versio Addressable: "true", }) } - return manifest, nil } func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { + if s.signExtensions && filepath.Base(fp) == "p7s.sig" { + // This file must exist, and it is always empty + return mem.NewFileHandle(mem.CreateFile("p7s.sig")), nil + } if s.signExtensions && filepath.Base(fp) == sigzipFilename { // hijack this request, sign the sig manifest - manifest, err := s.Storage.Open(ctx, sigManifestName) + manifest, err := s.Storage.Open(ctx, filepath.Join(filepath.Dir(fp), sigManifestName)) if err != nil { - return nil, err + fmt.Println(err) + return nil, xerrors.Errorf("open signature manifest: %w", err) } defer manifest.Close() @@ -91,23 +95,11 @@ func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { if err != nil { return nil, xerrors.Errorf("sign and zip manifest: %w", err) } - return memFile{data: bytes.NewBuffer(signed), name: sigzipFilename}, nil + + f := mem.NewFileHandle(mem.CreateFile(sigzipFilename)) + _, err = f.Write(signed) + return f, err } return s.Storage.Open(ctx, fp) } - -type memFile struct { - data *bytes.Buffer - name string -} - -func (a memFile) Name() string { return a.name } -func (a memFile) Size() int64 { return int64(a.data.Len()) } -func (a memFile) Mode() fs.FileMode { return fs.FileMode(0) } // ? -func (a memFile) ModTime() time.Time { return time.Now() } -func (a memFile) IsDir() bool { return false } -func (a memFile) Sys() any { return nil } -func (a memFile) Stat() (fs.FileInfo, error) { return a, nil } -func (a memFile) Read(i []byte) (int, error) { return a.data.Read(i) } -func (a memFile) Close() error { return nil } From 62d0a170d1a77de8dfcbea76d02578736f7e474c Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 13:15:34 -0600 Subject: [PATCH 09/22] bump go version --- .github/workflows/build.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yaml | 2 +- go.mod | 5 +++-- go.sum | 18 ++++++++++++++++-- storage/storage.go | 6 +++--- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4b2a6be..5f7f4f0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: "~1.19" + go-version: "~1.22" - name: Get Go cache paths id: go-cache-paths diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index beeb080..08d2d49 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: "~1.19" + go-version: "~1.22" - name: golangci-lint uses: golangci/golangci-lint-action@v6.1.1 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fe7a8e6..e8bca42 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: "~1.19" + go-version: "~1.22" - name: Echo Go Cache Paths id: go-cache-paths diff --git a/go.mod b/go.mod index 2654974..52220e5 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ module github.com/coder/code-marketplace -go 1.19 +go 1.22.8 require ( cdr.dev/slog v1.6.1 - github.com/cloudflare/cfssl v1.6.5 github.com/go-chi/chi/v5 v5.1.0 github.com/go-chi/cors v1.2.1 github.com/go-chi/httprate v0.14.1 @@ -35,8 +34,10 @@ require ( go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect golang.org/x/crypto v0.19.0 // indirect + golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 646e554..f19c00c 100644 --- a/go.sum +++ b/go.sum @@ -2,17 +2,19 @@ cdr.dev/slog v1.6.1 h1:IQjWZD0x6//sfv5n+qEhbu3wBkmtBQY5DILXNvMaIv4= cdr.dev/slog v1.6.1/go.mod h1:eHEYQLaZvxnIAXC+XdTSNLb/kgA/X2RVSF72v5wsxEI= cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E= github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= -github.com/cloudflare/cfssl v1.6.5 h1:46zpNkm6dlNkMZH/wMW22ejih6gIaJbzL2du6vD7ZeI= -github.com/cloudflare/cfssl v1.6.5/go.mod h1:Bk1si7sq8h2+yVEDrFJiz3d7Aw+pfjjJSZVaD+Taky4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -23,9 +25,13 @@ github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vz github.com/go-chi/httprate v0.14.1 h1:EKZHYEZ58Cg6hWcYzoZILsv7ppb46Wt4uQ738IRtpZs= github.com/go-chi/httprate v0.14.1/go.mod h1:TUepLXaz/pCjmCtf/obgOQJ2Sz6rC8fSf5cAt5cnTt0= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -62,7 +68,9 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -78,6 +86,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -112,10 +121,15 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/storage/storage.go b/storage/storage.go index 5a1cfee..ef1596a 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -211,9 +211,9 @@ type Storage interface { // for verification purposes. Extra files can be included, but not required. // All extra files will be placed relative to the manifest outside the vsix. AddExtension(ctx context.Context, manifest *VSIXManifest, vsix []byte, extra ...File) (string, error) - // FileServer provides a handler for fetching extension repository files from - // a client. - //FileServer() fs.FS + // Open mirrors the fs.FS interface of Open, except with a context. + // The Open should return files from the extension storage, and used for + // serving extensions. Open(ctx context.Context, name string) (fs.File, error) // Manifest returns the manifest bytes for the provided extension. The // extension asset itself (the VSIX) will be included on the manifest even if From 404d3a7e96dd51fbe1d4f2796f1ec22baa88c698 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 13:25:23 -0600 Subject: [PATCH 10/22] fix artifactory mem files --- api/api.go | 19 +------------------ storage/artifactory.go | 31 ++++++++----------------------- storage/storage.go | 20 ++++++++++++++++++++ storage/storage_test.go | 2 +- testutil/mockstorage.go | 13 +++++++++++++ 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/api/api.go b/api/api.go index 526838f..91a92df 100644 --- a/api/api.go +++ b/api/api.go @@ -1,9 +1,7 @@ package api import ( - "context" "encoding/json" - "io/fs" "net/http" "os" "strconv" @@ -114,13 +112,7 @@ func New(options *Options) *API { r.Post("/api/extensionquery", api.extensionQuery) // Endpoint for getting an extension's files or the extension zip. - r.Mount("/files", http.StripPrefix("/files", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - // Convert the storage into a fs.FS, including the request context - http.FileServerFS(&contextFs{ - ctx: r.Context(), - open: options.Storage.Open, - }).ServeHTTP(rw, r) - }))) + r.Mount("/files", http.StripPrefix("/files", storage.HTTPFileServer(options.Storage))) // VS Code can use the files in the response to get file paths but it will // sometimes ignore that and use requests to /assets with hardcoded types to @@ -264,12 +256,3 @@ func (api *API) assetRedirect(rw http.ResponseWriter, r *http.Request) { http.Redirect(rw, r, url, http.StatusMovedPermanently) } - -type contextFs struct { - ctx context.Context - open func(ctx context.Context, name string) (fs.File, error) -} - -func (c *contextFs) Open(name string) (fs.File, error) { - return c.open(c.ctx, name) -} diff --git a/storage/artifactory.go b/storage/artifactory.go index 1db0cc7..d385d9c 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -11,12 +11,12 @@ import ( "net/http" "os" "path" - "path/filepath" "sort" "strings" "sync" "time" + "github.com/spf13/afero/mem" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" @@ -300,10 +300,13 @@ func (s *Artifactory) Open(ctx context.Context, fp string) (fs.File, error) { } } - return artifactoryFile{ - Response: resp, - path: fp, - }, nil + f := mem.NewFileHandle(mem.CreateFile(fp)) + _, err = io.Copy(f, resp.Body) + if err != nil { + return nil, err + } + + return f, nil } func (s *Artifactory) Manifest(ctx context.Context, publisher, name string, version Version) (*VSIXManifest, error) { @@ -452,21 +455,3 @@ func (s *Artifactory) Versions(ctx context.Context, publisher, name string) ([]V sort.Sort(ByVersion(versions)) return versions, nil } - -var _ fs.File = (*artifactoryFile)(nil) -var _ fs.FileInfo = (*artifactoryFile)(nil) - -type artifactoryFile struct { - *http.Response - path string -} - -func (a artifactoryFile) Name() string { return filepath.Base(a.path) } -func (a artifactoryFile) Size() int64 { return a.Response.ContentLength } -func (a artifactoryFile) Mode() fs.FileMode { return fs.FileMode(0) } // ? -func (a artifactoryFile) ModTime() time.Time { return time.Now() } -func (a artifactoryFile) IsDir() bool { return false } -func (a artifactoryFile) Sys() any { return nil } -func (a artifactoryFile) Stat() (fs.FileInfo, error) { return a, nil } -func (a artifactoryFile) Read(i []byte) (int, error) { return a.Response.Body.Read(i) } -func (a artifactoryFile) Close() error { return a.Response.Body.Close() } diff --git a/storage/storage.go b/storage/storage.go index ef1596a..3fdd22d 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -237,6 +237,17 @@ type Storage interface { WalkExtensions(ctx context.Context, fn func(manifest *VSIXManifest, versions []Version) error) error } +// HTTPFileServer creates an http.Handler that serves files from the provided +// storage. +func HTTPFileServer(s Storage) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + http.FileServerFS(&contextFs{ + ctx: r.Context(), + open: s.Open, + }).ServeHTTP(rw, r) + }) +} + type File struct { RelativePath string Content []byte @@ -429,3 +440,12 @@ func ParseExtensionID(id string) (string, string, string, error) { } return match[0][1], match[0][2], match[0][3], nil } + +type contextFs struct { + ctx context.Context + open func(ctx context.Context, name string) (fs.File, error) +} + +func (c *contextFs) Open(name string) (fs.File, error) { + return c.open(c.ctx, name) +} diff --git a/storage/storage_test.go b/storage/storage_test.go index 4bbd433..cb98ede 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -189,7 +189,7 @@ func testFileServer(t *testing.T, factory storageFactory) { req := httptest.NewRequest("GET", test.path, nil) rec := httptest.NewRecorder() - server := f.storage.FileServer() + server := storage.HTTPFileServer(f.storage) server.ServeHTTP(rec, req) resp := rec.Result() diff --git a/testutil/mockstorage.go b/testutil/mockstorage.go index c1f6249..e9adbad 100644 --- a/testutil/mockstorage.go +++ b/testutil/mockstorage.go @@ -3,10 +3,14 @@ package testutil import ( "context" "errors" + "io/fs" "net/http" "os" + "path/filepath" "sort" + "github.com/spf13/afero/mem" + "github.com/coder/code-marketplace/storage" ) @@ -22,6 +26,15 @@ func NewMockStorage() *MockStorage { func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXManifest, vsix []byte, extra ...storage.File) (string, error) { return "", errors.New("not implemented") } +func (s *MockStorage) Open(ctx context.Context, path string) (fs.File, error) { + if filepath.Base(path) == "/nonexistent" { + return nil, fs.ErrNotExist + } + + f := mem.NewFileHandle(mem.CreateFile(path)) + _, _ = f.Write([]byte("foobar")) + return f, nil +} func (s *MockStorage) FileServer() http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { From 0cdd5a3aa1a5370ec076f129aa1467efcb3c93bb Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 13:30:02 -0600 Subject: [PATCH 11/22] setup signer opts pass through --- cli/server.go | 8 +++++++- storage/signature.go | 26 +++++++++++++++----------- storage/storage.go | 5 +++-- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/cli/server.go b/cli/server.go index 6c3cf85..702b6a0 100644 --- a/cli/server.go +++ b/cli/server.go @@ -15,6 +15,7 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/sloghuman" + "github.com/coder/code-marketplace/extensionsign" "github.com/coder/code-marketplace/api" "github.com/coder/code-marketplace/database" @@ -23,12 +24,14 @@ import ( func serverFlags() (addFlags func(cmd *cobra.Command), opts *storage.Options) { opts = &storage.Options{} + var sign bool return func(cmd *cobra.Command) { cmd.Flags().StringVar(&opts.ExtDir, "extensions-dir", "", "The path to extensions.") cmd.Flags().StringVar(&opts.Artifactory, "artifactory", "", "Artifactory server URL.") cmd.Flags().StringVar(&opts.Repo, "repo", "", "Artifactory repository.") cmd.Flags().DurationVar(&opts.ListCacheDuration, "list-cache-duration", time.Minute, "The duration of the extension cache.") - cmd.Flags().BoolVar(&opts.SignExtensions, "sign", false, "Sign extensions.") + cmd.Flags().BoolVar(&sign, "sign", false, "Sign extensions.") + _ = cmd.Flags().MarkHidden("sign") // This flag needs to import a key, not just be a bool var before func(cmd *cobra.Command, args []string) error if cmd.PreRunE != nil { @@ -47,6 +50,9 @@ func serverFlags() (addFlags func(cmd *cobra.Command), opts *storage.Options) { if before != nil { return before(cmd, args) } + if sign { // TODO: Remove this for an actual key import + opts.Signer, _ = extensionsign.GenerateKey() + } return nil } }, opts diff --git a/storage/signature.go b/storage/signature.go index d26b599..ac8c201 100644 --- a/storage/signature.go +++ b/storage/signature.go @@ -2,6 +2,7 @@ package storage import ( "context" + "crypto" "encoding/json" "fmt" "io" @@ -22,19 +23,23 @@ const ( ) type Signature struct { - // SignDesignExtensions is a flag that determines if the signature should - // include the extension payloads. - signExtensions bool + // Signer if provided, will be used to sign extensions. If not provided, + // no extensions will be signed. + Signer crypto.Signer Storage } -func NewSignatureStorage(signExtensions bool, s Storage) *Signature { +func NewSignatureStorage(signer crypto.Signer, s Storage) *Signature { return &Signature{ - signExtensions: signExtensions, - Storage: s, + Signer: signer, + Storage: s, } } +func (s *Signature) SigningEnabled() bool { + return s.Signer != nil +} + // AddExtension includes the signature manifest of the vsix. Signing happens on // demand, so leave the manifest unsigned. This is safe to do even if // 'signExtensions' is disabled, as these files lay dormant until signed. @@ -61,7 +66,7 @@ func (s *Signature) Manifest(ctx context.Context, publisher, name string, versio return nil, err } - if s.signExtensions { + if s.SigningEnabled() { manifest.Assets.Asset = append(manifest.Assets.Asset, VSIXAsset{ Type: VSIXSignatureType, Path: sigzipFilename, @@ -72,11 +77,11 @@ func (s *Signature) Manifest(ctx context.Context, publisher, name string, versio } func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { - if s.signExtensions && filepath.Base(fp) == "p7s.sig" { + if s.SigningEnabled() && filepath.Base(fp) == "p7s.sig" { // This file must exist, and it is always empty return mem.NewFileHandle(mem.CreateFile("p7s.sig")), nil } - if s.signExtensions && filepath.Base(fp) == sigzipFilename { + if s.SigningEnabled() && filepath.Base(fp) == sigzipFilename { // hijack this request, sign the sig manifest manifest, err := s.Storage.Open(ctx, filepath.Join(filepath.Dir(fp), sigManifestName)) if err != nil { @@ -85,13 +90,12 @@ func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { } defer manifest.Close() - key, _ := extensionsign.GenerateKey() manifestData, err := io.ReadAll(manifest) if err != nil { return nil, xerrors.Errorf("read signature manifest: %w", err) } - signed, err := extensionsign.SignAndZipManifest(key, manifestData) + signed, err := extensionsign.SignAndZipManifest(s.Signer, manifestData) if err != nil { return nil, xerrors.Errorf("sign and zip manifest: %w", err) } diff --git a/storage/storage.go b/storage/storage.go index 3fdd22d..94e5505 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -2,6 +2,7 @@ package storage import ( "context" + "crypto" "encoding/json" "encoding/xml" "fmt" @@ -127,7 +128,7 @@ type VSIXAsset struct { } type Options struct { - SignExtensions bool + Signer crypto.Signer Artifactory string ExtDir string Repo string @@ -292,7 +293,7 @@ func NewStorage(ctx context.Context, options *Options) (Storage, error) { return nil, err } - return NewSignatureStorage(options.SignExtensions, store), nil + return NewSignatureStorage(options.Signer, store), nil } // ReadVSIXManifest reads and parses an extension manifest from a vsix file. If From cc33da1105ce38452bfbd5a70d170739571d211a Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 13:37:53 -0600 Subject: [PATCH 12/22] fixup tests --- api/api_test.go | 2 +- storage/storage_test.go | 6 ++++-- testutil/mockstorage.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index a144482..ab7ff40 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -171,7 +171,7 @@ func TestAPI(t *testing.T) { Response: "foobar", }, { - Name: "FileAPI", + Name: "FileAPINotExists", Path: "/files/nonexistent", Status: http.StatusNotFound, }, diff --git a/storage/storage_test.go b/storage/storage_test.go index cb98ede..b42abfc 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -104,11 +104,13 @@ func TestNewStorage(t *testing.T) { require.Error(t, err) require.Regexp(t, test.error, err.Error()) } else if test.local { - _, ok := s.(*storage.Local) + under := s.(*storage.Signature) + _, ok := under.Storage.(*storage.Local) require.True(t, ok) require.NoError(t, err) } else { - _, ok := s.(*storage.Artifactory) + under := s.(*storage.Signature) + _, ok := under.Storage.(*storage.Artifactory) require.True(t, ok) require.NoError(t, err) } diff --git a/testutil/mockstorage.go b/testutil/mockstorage.go index e9adbad..0066c40 100644 --- a/testutil/mockstorage.go +++ b/testutil/mockstorage.go @@ -27,7 +27,7 @@ func (s *MockStorage) AddExtension(ctx context.Context, manifest *storage.VSIXMa return "", errors.New("not implemented") } func (s *MockStorage) Open(ctx context.Context, path string) (fs.File, error) { - if filepath.Base(path) == "/nonexistent" { + if filepath.Base(path) == "nonexistent" { return nil, fs.ErrNotExist } From 43d8b5055f4938a2550bc33393fd6abc100e25f4 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 13:44:09 -0600 Subject: [PATCH 13/22] Add more comments --- extensionsign/sigmanifest.go | 6 ++++-- extensionsign/sigzip.go | 1 - storage/signature.go | 11 +++++++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/extensionsign/sigmanifest.go b/extensionsign/sigmanifest.go index 55ac4c6..2a42a1d 100644 --- a/extensionsign/sigmanifest.go +++ b/extensionsign/sigmanifest.go @@ -23,7 +23,8 @@ func (a SignatureManifest) String() string { return fmt.Sprintf("Package %q with Entries: %d", a.Package.Digests.SHA256, len(a.Entries)) } -// Equal is helpful for debugging +// Equal is helpful for debugging to know if two manifests are equal. +// They can change if any file is removed/added/edited to an extension. func (a SignatureManifest) Equal(b SignatureManifest) error { var err error if err := a.Package.Equal(b.Package); err != nil { @@ -82,7 +83,8 @@ type Digests struct { } // GenerateSignatureManifest generates a signature manifest for a VSIX file. -// It does not sign the manifest. +// It does not sign the manifest. The manifest is the base64 encoded file path +// followed by the sha256 hash of the file, and it's size. func GenerateSignatureManifest(vsixFile []byte) (SignatureManifest, error) { pkgManifest, err := FileManifest(bytes.NewReader(vsixFile)) if err != nil { diff --git a/extensionsign/sigzip.go b/extensionsign/sigzip.go index 37a655f..c6948d1 100644 --- a/extensionsign/sigzip.go +++ b/extensionsign/sigzip.go @@ -28,7 +28,6 @@ func ExtractSignatureManifest(zip []byte) (SignatureManifest, error) { } // SignAndZipManifest signs a manifest and zips it up -// Sign func SignAndZipManifest(secret crypto.Signer, manifest json.RawMessage) ([]byte, error) { var buf bytes.Buffer w := zip.NewWriter(&buf) diff --git a/storage/signature.go b/storage/signature.go index ac8c201..b5c7e58 100644 --- a/storage/signature.go +++ b/storage/signature.go @@ -4,7 +4,6 @@ import ( "context" "crypto" "encoding/json" - "fmt" "io" "io/fs" "path/filepath" @@ -76,16 +75,24 @@ func (s *Signature) Manifest(ctx context.Context, publisher, name string, versio return manifest, nil } +// Open will intercept requests for signed extensions payload. +// It does this by looking for 'sigzipFilename' or p7s.sig. +// +// The signed payload and signing process is taken from: +// https://github.com/filiptronicek/node-ovsx-sign func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { if s.SigningEnabled() && filepath.Base(fp) == "p7s.sig" { // This file must exist, and it is always empty return mem.NewFileHandle(mem.CreateFile("p7s.sig")), nil } + if s.SigningEnabled() && filepath.Base(fp) == sigzipFilename { // hijack this request, sign the sig manifest manifest, err := s.Storage.Open(ctx, filepath.Join(filepath.Dir(fp), sigManifestName)) if err != nil { - fmt.Println(err) + // If this file is missing, it means the extension was added before + // signatures were handled by the marketplace. + // TODO: Generate the sig manifest payload and insert it? return nil, xerrors.Errorf("open signature manifest: %w", err) } defer manifest.Close() From 77a862c3434a0e38e2c59c345783713b8c3033c3 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 13:53:57 -0600 Subject: [PATCH 14/22] add comments --- storage/signature.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/storage/signature.go b/storage/signature.go index b5c7e58..0ba2f87 100644 --- a/storage/signature.go +++ b/storage/signature.go @@ -80,12 +80,23 @@ func (s *Signature) Manifest(ctx context.Context, publisher, name string, versio // // The signed payload and signing process is taken from: // https://github.com/filiptronicek/node-ovsx-sign +// +// Some notes: +// +// - VSCodium requires a signature to exist, but it does appear to actually read +// the signature. Meaning the signature could be empty, incorrect, or a +// picture of cat and it would work. There is so signature verification. +// +// - VSCode requires a signature payload to exist, but the context appear +// to be somewhat optional. +// Following another open source implementation, it appears the '.signature.p7s' +// file must exist, but it can be empty. +// The signature is stored in a '.signature.sig' file, although it is unclear +// is VSCode ever reads this file. +// TODO: Properly implement the p7s file, and diverge from the other open +// source implementation. Ideally this marketplace would match Microsoft's +// marketplace API. func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { - if s.SigningEnabled() && filepath.Base(fp) == "p7s.sig" { - // This file must exist, and it is always empty - return mem.NewFileHandle(mem.CreateFile("p7s.sig")), nil - } - if s.SigningEnabled() && filepath.Base(fp) == sigzipFilename { // hijack this request, sign the sig manifest manifest, err := s.Storage.Open(ctx, filepath.Join(filepath.Dir(fp), sigManifestName)) From f8b371492c88161da7ba85276ad02517bc051014 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 14:01:35 -0600 Subject: [PATCH 15/22] update golangci lint in gh actions --- .github/workflows/lint.yaml | 2 +- cli/signature.go | 1 + extensionsign/sigmanifest.go | 1 + extensionsign/sigmanifest_test.go | 7 ------- storage/signature.go | 1 + storage/signature_test.go | 21 +++++++++++++++++++++ storage/storage_test.go | 8 ++++++++ 7 files changed, 33 insertions(+), 8 deletions(-) delete mode 100644 extensionsign/sigmanifest_test.go create mode 100644 storage/signature_test.go diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 08d2d49..f91a6d7 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -33,4 +33,4 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6.1.1 with: - version: v1.48.0 + version: v1.60.0 diff --git a/cli/signature.go b/cli/signature.go index 5adf4b2..432a14a 100644 --- a/cli/signature.go +++ b/cli/signature.go @@ -13,6 +13,7 @@ import ( func signature() *cobra.Command { cmd := &cobra.Command{ Use: "signature", + Short: "Commands for debugging and working with signatures.", Hidden: true, // Debugging tools Aliases: []string{"sig", "sigs", "signatures"}, } diff --git a/extensionsign/sigmanifest.go b/extensionsign/sigmanifest.go index 2a42a1d..5e6ee6f 100644 --- a/extensionsign/sigmanifest.go +++ b/extensionsign/sigmanifest.go @@ -13,6 +13,7 @@ import ( "github.com/coder/code-marketplace/storage/easyzip" ) +// SignatureManifest should be serialized to JSON before being signed. type SignatureManifest struct { Package File // Entries is base64(filepath) -> File diff --git a/extensionsign/sigmanifest_test.go b/extensionsign/sigmanifest_test.go deleted file mode 100644 index d9efb6a..0000000 --- a/extensionsign/sigmanifest_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package extensionsign_test - -import "testing" - -func TestManifestEqual(t *testing.T) { - -} diff --git a/storage/signature.go b/storage/signature.go index 0ba2f87..182e009 100644 --- a/storage/signature.go +++ b/storage/signature.go @@ -21,6 +21,7 @@ const ( sigManifestName = ".signature.manifest" ) +// Signature is a storage wrapper that can sign extensions on demand. type Signature struct { // Signer if provided, will be used to sign extensions. If not provided, // no extensions will be signed. diff --git a/storage/signature_test.go b/storage/signature_test.go new file mode 100644 index 0000000..8612534 --- /dev/null +++ b/storage/signature_test.go @@ -0,0 +1,21 @@ +package storage_test + +import ( + "testing" + + "github.com/coder/code-marketplace/extensionsign" + "github.com/coder/code-marketplace/storage" +) + +func signed(factory func(t *testing.T) testStorage) func(t *testing.T) testStorage { + return func(t *testing.T) testStorage { + st := factory(t) + key, _ := extensionsign.GenerateKey() + + return testStorage{ + storage: storage.NewSignatureStorage(key, st.storage), + write: st.write, + exists: st.exists, + } + } +} diff --git a/storage/storage_test.go b/storage/storage_test.go index b42abfc..26299ea 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -132,6 +132,14 @@ func TestStorage(t *testing.T) { name: "Artifactory", factory: artifactoryFactory, }, + //{ + // name: "SignedLocal", + // factory: signed(localFactory), + //}, + //{ + // name: "SignedArtifactory", + // factory: signed(artifactoryFactory), + //}, } for _, sf := range factories { t.Run(sf.name, func(t *testing.T) { From 35c997b01aa875a15d4709916a926ffffe002151 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 14:31:53 -0600 Subject: [PATCH 16/22] handle cached artifactory manifests --- .github/workflows/lint.yaml | 2 +- storage/signature.go | 17 ++++++++++++----- storage/signature_test.go | 26 +++++++++++++++++++++----- storage/storage_test.go | 26 +++++++++++++++++--------- testutil/extensions.go | 8 ++++++++ 5 files changed, 59 insertions(+), 20 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index f91a6d7..62dc0f6 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -33,4 +33,4 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6.1.1 with: - version: v1.60.0 + version: v1.58.0 diff --git a/storage/signature.go b/storage/signature.go index 182e009..0ab1bdc 100644 --- a/storage/signature.go +++ b/storage/signature.go @@ -17,7 +17,7 @@ import ( var _ Storage = (*Signature)(nil) const ( - sigzipFilename = "extension.sigzip" + SigzipFilename = "extension.sigzip" sigManifestName = ".signature.manifest" ) @@ -67,17 +67,24 @@ func (s *Signature) Manifest(ctx context.Context, publisher, name string, versio } if s.SigningEnabled() { + for _, asset := range manifest.Assets.Asset { + if asset.Path == SigzipFilename { + // Already signed + return manifest, nil + } + } manifest.Assets.Asset = append(manifest.Assets.Asset, VSIXAsset{ Type: VSIXSignatureType, - Path: sigzipFilename, + Path: SigzipFilename, Addressable: "true", }) + return manifest, nil } return manifest, nil } // Open will intercept requests for signed extensions payload. -// It does this by looking for 'sigzipFilename' or p7s.sig. +// It does this by looking for 'SigzipFilename' or p7s.sig. // // The signed payload and signing process is taken from: // https://github.com/filiptronicek/node-ovsx-sign @@ -98,7 +105,7 @@ func (s *Signature) Manifest(ctx context.Context, publisher, name string, versio // source implementation. Ideally this marketplace would match Microsoft's // marketplace API. func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { - if s.SigningEnabled() && filepath.Base(fp) == sigzipFilename { + if s.SigningEnabled() && filepath.Base(fp) == SigzipFilename { // hijack this request, sign the sig manifest manifest, err := s.Storage.Open(ctx, filepath.Join(filepath.Dir(fp), sigManifestName)) if err != nil { @@ -119,7 +126,7 @@ func (s *Signature) Open(ctx context.Context, fp string) (fs.File, error) { return nil, xerrors.Errorf("sign and zip manifest: %w", err) } - f := mem.NewFileHandle(mem.CreateFile(sigzipFilename)) + f := mem.NewFileHandle(mem.CreateFile(SigzipFilename)) _, err = f.Write(signed) return f, err } diff --git a/storage/signature_test.go b/storage/signature_test.go index 8612534..bfcef6d 100644 --- a/storage/signature_test.go +++ b/storage/signature_test.go @@ -1,21 +1,37 @@ package storage_test import ( + "crypto" "testing" "github.com/coder/code-marketplace/extensionsign" "github.com/coder/code-marketplace/storage" ) -func signed(factory func(t *testing.T) testStorage) func(t *testing.T) testStorage { +func expectSignature(manifest *storage.VSIXManifest) { + manifest.Assets.Asset = append(manifest.Assets.Asset, storage.VSIXAsset{ + Type: storage.VSIXSignatureType, + Path: storage.SigzipFilename, + Addressable: "true", + }) +} + +//nolint:revive // test control flag +func signed(signer bool, factory func(t *testing.T) testStorage) func(t *testing.T) testStorage { return func(t *testing.T) testStorage { st := factory(t) - key, _ := extensionsign.GenerateKey() + var key crypto.Signer + var exp func(*storage.VSIXManifest) + if signer { + key, _ = extensionsign.GenerateKey() + exp = expectSignature + } return testStorage{ - storage: storage.NewSignatureStorage(key, st.storage), - write: st.write, - exists: st.exists, + storage: storage.NewSignatureStorage(key, st.storage), + write: st.write, + exists: st.exists, + expectedManifest: exp, } } } diff --git a/storage/storage_test.go b/storage/storage_test.go index 26299ea..394c537 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -15,6 +15,7 @@ import ( "strconv" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/coder/code-marketplace/storage" @@ -25,6 +26,8 @@ type testStorage struct { storage storage.Storage write func(content []byte, elem ...string) exists func(elem ...string) bool + + expectedManifest func(man *storage.VSIXManifest) } type storageFactory = func(t *testing.T) testStorage @@ -132,14 +135,14 @@ func TestStorage(t *testing.T) { name: "Artifactory", factory: artifactoryFactory, }, - //{ - // name: "SignedLocal", - // factory: signed(localFactory), - //}, - //{ - // name: "SignedArtifactory", - // factory: signed(artifactoryFactory), - //}, + { + name: "SignedLocal", + factory: signed(true, localFactory), + }, + { + name: "SignedArtifactory", + factory: signed(true, artifactoryFactory), + }, } for _, sf := range factories { t.Run(sf.name, func(t *testing.T) { @@ -332,7 +335,12 @@ func testManifest(t *testing.T, factory storageFactory) { Path: fmt.Sprintf("%s.%s-%s.vsix", test.extension.Publisher, test.extension.Name, test.version), Addressable: "true", }) - require.Equal(t, test.expected, manifest) + if f.expectedManifest != nil { + f.expectedManifest(test.expected) + } + if !assert.Equal(t, test.expected, manifest) { + fmt.Println("Asd") + } } }) } diff --git a/testutil/extensions.go b/testutil/extensions.go index 62fb385..a32967e 100644 --- a/testutil/extensions.go +++ b/testutil/extensions.go @@ -26,6 +26,13 @@ type Extension struct { Pack []string } +func (e Extension) Copy() Extension { + var n Extension + data, _ := json.Marshal(e) + _ = json.Unmarshal(data, &n) + return n +} + var Extensions = []Extension{ { Publisher: "foo", @@ -113,6 +120,7 @@ var Extensions = []Extension{ } func ConvertExtensionToManifest(ext Extension, version storage.Version) *storage.VSIXManifest { + ext = ext.Copy() return &storage.VSIXManifest{ Metadata: storage.VSIXMetadata{ Identity: storage.VSIXIdentity{ From 4f7cd917795e17d16af816990be779326f195282 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 14:46:32 -0600 Subject: [PATCH 17/22] fix err groups --- extensionsign/sigmanifest.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extensionsign/sigmanifest.go b/extensionsign/sigmanifest.go index 5e6ee6f..9c333b7 100644 --- a/extensionsign/sigmanifest.go +++ b/extensionsign/sigmanifest.go @@ -27,25 +27,25 @@ func (a SignatureManifest) String() string { // Equal is helpful for debugging to know if two manifests are equal. // They can change if any file is removed/added/edited to an extension. func (a SignatureManifest) Equal(b SignatureManifest) error { - var err error + var errs error if err := a.Package.Equal(b.Package); err != nil { - err = errors.Join(err, xerrors.Errorf("package: %w", err)) + errs = errors.Join(errs, xerrors.Errorf("package: %w", err)) } if len(a.Entries) != len(b.Entries) { - err = errors.Join(err, xerrors.Errorf("entry count mismatch: %d != %d", len(a.Entries), len(b.Entries))) + errs = errors.Join(errs, xerrors.Errorf("entry count mismatch: %d != %d", len(a.Entries), len(b.Entries))) } for k, v := range a.Entries { if _, ok := b.Entries[k]; !ok { - err = errors.Join(err, xerrors.Errorf("entry %q not found in second set", k)) + errs = errors.Join(errs, xerrors.Errorf("entry %q not found in second set", k)) continue } if err := v.Equal(b.Entries[k]); err != nil { err = errors.Join(err, xerrors.Errorf("entry %q: %w", k, err)) } } - return err + return errs } type File struct { From 1ab2dcc04aeffbb6d22d989210944aaf14f78b93 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 12 Dec 2024 14:51:47 -0600 Subject: [PATCH 18/22] fix err groups --- extensionsign/sigmanifest.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extensionsign/sigmanifest.go b/extensionsign/sigmanifest.go index 9c333b7..3ff67e6 100644 --- a/extensionsign/sigmanifest.go +++ b/extensionsign/sigmanifest.go @@ -27,25 +27,25 @@ func (a SignatureManifest) String() string { // Equal is helpful for debugging to know if two manifests are equal. // They can change if any file is removed/added/edited to an extension. func (a SignatureManifest) Equal(b SignatureManifest) error { - var errs error + var errs []error if err := a.Package.Equal(b.Package); err != nil { - errs = errors.Join(errs, xerrors.Errorf("package: %w", err)) + errs = append(errs, xerrors.Errorf("package: %w", err)) } if len(a.Entries) != len(b.Entries) { - errs = errors.Join(errs, xerrors.Errorf("entry count mismatch: %d != %d", len(a.Entries), len(b.Entries))) + errs = append(errs, xerrors.Errorf("entry count mismatch: %d != %d", len(a.Entries), len(b.Entries))) } for k, v := range a.Entries { if _, ok := b.Entries[k]; !ok { - errs = errors.Join(errs, xerrors.Errorf("entry %q not found in second set", k)) + errs = append(errs, xerrors.Errorf("entry %q not found in second set", k)) continue } if err := v.Equal(b.Entries[k]); err != nil { - err = errors.Join(err, xerrors.Errorf("entry %q: %w", k, err)) + errs = append(errs, xerrors.Errorf("entry %q: %w", k, err)) } } - return errs + return errors.Join(errs...) } type File struct { From e51ea05c83582df2b09166831ecd73eab67ee454 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Fri, 13 Dec 2024 09:29:54 -0600 Subject: [PATCH 19/22] PR fixes --- storage/artifactory.go | 4 ++-- storage/local.go | 2 +- storage/signature.go | 2 +- storage/storage_test.go | 5 +---- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/storage/artifactory.go b/storage/artifactory.go index d385d9c..db08caa 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -288,7 +288,7 @@ func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, // going to Artifactory for the VSIX when it is missing on disk (basically // using the disk as a cache). func (s *Artifactory) Open(ctx context.Context, fp string) (fs.File, error) { - resp, code, err := s.request(ctx, http.MethodGet, path.Join(s.repo, fp), nil) + resp, code, err := s.read(ctx, path.Join(s.repo, fp)) if err != nil { switch code { case http.StatusNotFound: @@ -301,7 +301,7 @@ func (s *Artifactory) Open(ctx context.Context, fp string) (fs.File, error) { } f := mem.NewFileHandle(mem.CreateFile(fp)) - _, err = io.Copy(f, resp.Body) + _, err = io.Copy(f, resp) if err != nil { return nil, err } diff --git a/storage/local.go b/storage/local.go index 4d57c47..5293e39 100644 --- a/storage/local.go +++ b/storage/local.go @@ -129,7 +129,7 @@ func (s *Local) AddExtension(ctx context.Context, manifest *VSIXManifest, vsix [ for _, file := range extra { path := filepath.Join(dir, file.RelativePath) - err := os.MkdirAll(filepath.Dir(path), 0o644) + err := os.MkdirAll(filepath.Dir(path), 0o755) if err != nil { return "", err } diff --git a/storage/signature.go b/storage/signature.go index 0ab1bdc..4bc3afd 100644 --- a/storage/signature.go +++ b/storage/signature.go @@ -93,7 +93,7 @@ func (s *Signature) Manifest(ctx context.Context, publisher, name string, versio // // - VSCodium requires a signature to exist, but it does appear to actually read // the signature. Meaning the signature could be empty, incorrect, or a -// picture of cat and it would work. There is so signature verification. +// picture of cat and it would work. There is no signature verification. // // - VSCode requires a signature payload to exist, but the context appear // to be somewhat optional. diff --git a/storage/storage_test.go b/storage/storage_test.go index 394c537..8bb649c 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -15,7 +15,6 @@ import ( "strconv" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/coder/code-marketplace/storage" @@ -338,9 +337,7 @@ func testManifest(t *testing.T, factory storageFactory) { if f.expectedManifest != nil { f.expectedManifest(test.expected) } - if !assert.Equal(t, test.expected, manifest) { - fmt.Println("Asd") - } + require.Equal(t, test.expected, manifest) } }) } From dd761773570842ec45ea37d14395878b9b06a51e Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Fri, 13 Dec 2024 09:32:08 -0600 Subject: [PATCH 20/22] server flags only on server --- cli/server.go | 6 +++++- storage/artifactory.go | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cli/server.go b/cli/server.go index 702b6a0..8cfb7ea 100644 --- a/cli/server.go +++ b/cli/server.go @@ -29,10 +29,14 @@ func serverFlags() (addFlags func(cmd *cobra.Command), opts *storage.Options) { cmd.Flags().StringVar(&opts.ExtDir, "extensions-dir", "", "The path to extensions.") cmd.Flags().StringVar(&opts.Artifactory, "artifactory", "", "Artifactory server URL.") cmd.Flags().StringVar(&opts.Repo, "repo", "", "Artifactory repository.") - cmd.Flags().DurationVar(&opts.ListCacheDuration, "list-cache-duration", time.Minute, "The duration of the extension cache.") cmd.Flags().BoolVar(&sign, "sign", false, "Sign extensions.") _ = cmd.Flags().MarkHidden("sign") // This flag needs to import a key, not just be a bool + if cmd.Use == "server" { + // Server only flags + cmd.Flags().DurationVar(&opts.ListCacheDuration, "list-cache-duration", time.Minute, "The duration of the extension cache.") + } + var before func(cmd *cobra.Command, args []string) error if cmd.PreRunE != nil { before = cmd.PreRunE diff --git a/storage/artifactory.go b/storage/artifactory.go index db08caa..ec1eea2 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -268,7 +268,6 @@ func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, } for _, file := range extra { - // TODO: I think this is correct? _, err := s.upload(ctx, path.Join(dir, file.RelativePath), bytes.NewReader(file.Content)) if err != nil { return "", err From adb3c6626e518aa8f09fef64e92d19ca17129608 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Fri, 13 Dec 2024 09:43:54 -0600 Subject: [PATCH 21/22] add a comment to prevent copy into memory --- storage/artifactory.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/artifactory.go b/storage/artifactory.go index ec1eea2..1a92038 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -299,6 +299,8 @@ func (s *Artifactory) Open(ctx context.Context, fp string) (fs.File, error) { } } + // TODO: Do no copy the bytes into memory, stream them rather than + // storing the entire file into memory. f := mem.NewFileHandle(mem.CreateFile(fp)) _, err = io.Copy(f, resp) if err != nil { From 0fa5b4e6bc050adad3e384a847c542aac7fa9edd Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Fri, 13 Dec 2024 09:46:55 -0600 Subject: [PATCH 22/22] fix artifactory read --- storage/artifactory.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/artifactory.go b/storage/artifactory.go index 1a92038..ae4ea88 100644 --- a/storage/artifactory.go +++ b/storage/artifactory.go @@ -287,8 +287,8 @@ func (s *Artifactory) AddExtension(ctx context.Context, manifest *VSIXManifest, // going to Artifactory for the VSIX when it is missing on disk (basically // using the disk as a cache). func (s *Artifactory) Open(ctx context.Context, fp string) (fs.File, error) { - resp, code, err := s.read(ctx, path.Join(s.repo, fp)) - if err != nil { + resp, code, err := s.read(ctx, fp) + if code != http.StatusOK || err != nil { switch code { case http.StatusNotFound: return nil, fs.ErrNotExist