diff --git a/.gitignore b/.gitignore index cd4c5f2..f4c6fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ ### Local development ### # Local build system configuration /local.mk +.idea node_modules diff --git a/go.mod b/go.mod index 62f9ec7..458f9b8 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/google/uuid v1.6.0 github.com/muesli/termenv v0.15.2 github.com/oapi-codegen/runtime v1.1.1 + github.com/puzpuzpuz/xsync v1.5.2 github.com/rs/zerolog v1.34.0 github.com/snyk/code-client-go v1.21.3 github.com/snyk/error-catalog-golang-public v0.0.0-20250625135845-2d6f9a31f318 @@ -81,7 +82,6 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/puzpuzpuz/xsync v1.5.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect diff --git a/internal/fileupload/client.go b/internal/fileupload/client.go new file mode 100644 index 0000000..dd8262b --- /dev/null +++ b/internal/fileupload/client.go @@ -0,0 +1,237 @@ +package fileupload + +import ( + "context" + "fmt" + "github.com/snyk/cli-extension-os-flows/internal/fileupload/filters" + "net/http" + "os" + "path/filepath" + "runtime" + + "github.com/google/uuid" + "github.com/puzpuzpuz/xsync" + + listsources "github.com/snyk/cli-extension-os-flows/internal/files" + "github.com/snyk/cli-extension-os-flows/internal/fileupload/uploadrevision" +) + +// Config contains configuration for the file upload client. +type Config struct { + BaseURL string + OrgID OrgID +} + +// Client provides high-level file upload functionality. +type Client struct { + uploadRevisionSealableClient uploadrevision.SealableClient + filtersClient filters.Client + cfg Config + filters Filters +} + +// NewClient creates a new high-level file upload client. +func NewClient(httpClient *http.Client, cfg Config, opts ...Option) *Client { + client := &Client{ + cfg: cfg, + filters: Filters{ + supportedExtensions: xsync.NewMapOf[bool](), + supportedConfigFiles: xsync.NewMapOf[bool](), + }, + } + + for _, opt := range opts { + opt(client) + } + + if client.uploadRevisionSealableClient == nil { + client.uploadRevisionSealableClient = uploadrevision.NewClient(uploadrevision.Config{ + BaseURL: cfg.BaseURL, + }, uploadrevision.WithHTTPClient(httpClient)) + } + + if client.filtersClient == nil { + client.filtersClient = filters.NewDeeproxyClient(filters.Config{ + BaseURL: cfg.BaseURL, + }, filters.WithHTTPClient(httpClient)) + } + + return client +} + +func (c *Client) loadFilters(ctx context.Context) error { + c.filters.once.Do(func() { + filtersResp, err := c.filtersClient.GetFilters(ctx, c.cfg.OrgID) + if err != nil { + c.filters.initErr = err + return + } + + for _, ext := range filtersResp.Extensions { + c.filters.supportedExtensions.Store(ext, true) + } + for _, configFile := range filtersResp.ConfigFiles { + // .gitignore and .dcignore should not be uploaded + // (https://github.com/snyk/code-client/blob/d6f6a2ce4c14cb4b05aa03fb9f03533d8cf6ca4a/src/files.ts#L138) + if configFile == ".gitignore" || configFile == ".dcignore" { + continue + } + c.filters.supportedConfigFiles.Store(configFile, true) + } + }) + return c.filters.initErr +} + +// createFileFilter creates a filter function based on the current filtering configuration. +// Returns nil if filtering should be skipped, otherwise returns a function that evaluates files. +func (c *Client) createFileFilter(ctx context.Context) (func(string) bool, error) { + if err := c.loadFilters(ctx); err != nil { + return nil, fmt.Errorf("failed to load deeproxy filters: %w", err) + } + + return func(path string) bool { + fileExt := filepath.Ext(path) + fileName := filepath.Base(path) + _, isSupportedExtension := c.filters.supportedExtensions.Load(fileExt) + _, isSupportedConfigFile := c.filters.supportedConfigFiles.Load(fileName) + return isSupportedExtension || isSupportedConfigFile + }, nil +} + +func (c *Client) uploadPaths(ctx context.Context, revID RevisionID, rootPath string, paths []string) error { + files := make([]uploadrevision.UploadFile, 0, c.uploadRevisionSealableClient.GetLimits().FileCountLimit) + defer func() { + for _, file := range files { + file.File.Close() + } + }() + + for _, pth := range paths { + relPth, err := filepath.Rel(rootPath, pth) + if err != nil { + return fmt.Errorf("failed to get relative path for %s: %w", pth, err) + } + + f, err := os.Open(pth) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", pth, err) + } + + files = append(files, uploadrevision.UploadFile{ + Path: relPth, + File: f, + }) + } + + err := c.uploadRevisionSealableClient.UploadFiles(ctx, c.cfg.OrgID, revID, files) + if err != nil { + return fmt.Errorf("failed to upload files: %w", err) + } + + return nil +} + +// addPathsToRevision adds multiple file paths to an existing revision. +func (c *Client) addPathsToRevision(ctx context.Context, revisionID RevisionID, rootPath string, pathsChan <-chan string, opts UploadOptions) error { + var chunks <-chan []string + + if opts.SkipFiltering { + chunks = chunkChan(pathsChan, c.uploadRevisionSealableClient.GetLimits().FileCountLimit) + } else { + filter, err := c.createFileFilter(ctx) + if err != nil { + return err + } + chunks = chunkChanFiltered(pathsChan, c.uploadRevisionSealableClient.GetLimits().FileCountLimit, filter) + } + + for chunk := range chunks { + err := c.uploadPaths(ctx, revisionID, rootPath, chunk) + if err != nil { + return err + } + } + + return nil +} + +// CreateRevision creates a new revision and returns its ID. +func (c *Client) CreateRevision(ctx context.Context) (RevisionID, error) { + revision, err := c.uploadRevisionSealableClient.CreateRevision(ctx, c.cfg.OrgID) + if err != nil { + return uuid.Nil, fmt.Errorf("failed to create revision: %w", err) + } + return revision.Data.ID, nil +} + +// AddFileToRevision adds a single file to an existing revision. +func (c *Client) AddFileToRevision(ctx context.Context, revisionID RevisionID, filePath string, opts UploadOptions) error { + writableChan := make(chan string, 1) + writableChan <- filePath + close(writableChan) + + return c.addPathsToRevision(ctx, revisionID, filepath.Dir(filePath), writableChan, opts) +} + +// AddDirToRevision adds a directory and all its contents to an existing revision. +func (c *Client) AddDirToRevision(ctx context.Context, revisionID RevisionID, dirPath string, opts UploadOptions) error { + sources, err := listsources.ForPath(dirPath, nil, runtime.NumCPU()) + if err != nil { + return fmt.Errorf("failed to list files in directory %s: %w", dirPath, err) + } + + return c.addPathsToRevision(ctx, revisionID, dirPath, sources, opts) +} + +// SealRevision seals a revision, making it immutable. +func (c *Client) SealRevision(ctx context.Context, revisionID RevisionID) error { + _, err := c.uploadRevisionSealableClient.SealRevision(ctx, c.cfg.OrgID, revisionID) + if err != nil { + return fmt.Errorf("failed to seal revision: %w", err) + } + return nil +} + +// CreateRevisionFromPaths uploads multiple paths (files or directories), returning a revision ID. +// This is a convenience method that creates, uploads, and seals a revision. +func (c *Client) CreateRevisionFromPaths(ctx context.Context, paths []string, opts UploadOptions) (RevisionID, error) { + revisionID, err := c.CreateRevision(ctx) + if err != nil { + return uuid.Nil, err + } + + for _, pth := range paths { + info, err := os.Stat(pth) + if err != nil { + return uuid.Nil, fmt.Errorf("failed to stat path %s: %w", pth, err) + } + + if info.IsDir() { + if err := c.AddDirToRevision(ctx, revisionID, pth, opts); err != nil { + return uuid.Nil, fmt.Errorf("failed to add directory %s: %w", pth, err) + } + } else { + if err := c.AddFileToRevision(ctx, revisionID, pth, opts); err != nil { + return uuid.Nil, fmt.Errorf("failed to add file %s: %w", pth, err) + } + } + } + + if err := c.SealRevision(ctx, revisionID); err != nil { + return uuid.Nil, err + } + + return revisionID, nil +} + +// CreateRevisionFromDir uploads a directory and all its contents, returning a revision ID. +// This is a convenience method equivalent to CreateRevisionFromPaths with a single directory. +func (c *Client) CreateRevisionFromDir(ctx context.Context, dirPath string, opts UploadOptions) (RevisionID, error) { + return c.CreateRevisionFromPaths(ctx, []string{dirPath}, opts) +} + +// CreateRevisionFromFile uploads a single file, returning a revision ID. +// This is a convenience method equivalent to CreateRevisionFromPaths with a single file. +func (c *Client) CreateRevisionFromFile(ctx context.Context, filePath string, opts UploadOptions) (RevisionID, error) { + return c.CreateRevisionFromPaths(ctx, []string{filePath}, opts) +} diff --git a/internal/fileupload/client_test.go b/internal/fileupload/client_test.go new file mode 100644 index 0000000..58dbf2d --- /dev/null +++ b/internal/fileupload/client_test.go @@ -0,0 +1,466 @@ +package fileupload_test + +import ( + "context" + "github.com/snyk/cli-extension-os-flows/internal/fileupload/filters" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/snyk/cli-extension-os-flows/internal/fileupload" + "github.com/snyk/cli-extension-os-flows/internal/fileupload/uploadrevision" +) + +func Test_GranularAPI(t *testing.T) { + llcfg := uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 10, + FileSizeLimit: 100, + }, + } + + filters := filters.AllowList{ + ConfigFiles: []string{"go.mod"}, + Extensions: []string{".txt", ".go", ".md"}, + } + + t.Run("manual revision lifecycle with individual files", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + {Path: "main.go", Content: "package main\n\nfunc main() {}"}, + {Path: "utils.go", Content: "package main\n\nfunc utils() {}"}, + {Path: "README.md", Content: "# My Project"}, + } + + ctx, fakeSealableClient, client, dir, cleanup := setupTest(t, llcfg, expectedFiles, filters) + defer cleanup() + + revisionID, err := client.CreateRevision(ctx) + require.NoError(t, err) + + for _, expectedFile := range expectedFiles { + filePath := filepath.Join(dir.Name(), expectedFile.Path) + addErr := client.AddFileToRevision(ctx, revisionID, filePath, fileupload.UploadOptions{}) + require.NoError(t, addErr) + } + + err = client.SealRevision(ctx, revisionID) + require.NoError(t, err) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(revisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("manual revision with directory addition", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + {Path: "src/main.go", Content: "package main"}, + {Path: "src/utils.go", Content: "package utils"}, + {Path: "README.md", Content: "# Project"}, + } + + ctx, fakeSealableClient, client, dir, cleanup := setupTest(t, llcfg, expectedFiles, filters) + defer cleanup() + + revisionID, err := client.CreateRevision(ctx) + require.NoError(t, err) + + err = client.AddDirToRevision(ctx, revisionID, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + err = client.SealRevision(ctx, revisionID) + require.NoError(t, err) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(revisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("mixed file and directory operations", func(t *testing.T) { + dirFiles := []uploadrevision.LoadedFile{ + {Path: "src/main.go", Content: "package main"}, + {Path: "src/utils.go", Content: "package utils"}, + } + + ctx, fakeSealableClient, client, dir, cleanup := setupTest(t, llcfg, dirFiles, filters) + defer cleanup() + + // Create standalone file manually inside temp dir + standaloneFile := filepath.Join(dir.Name(), "README.md") + err := os.WriteFile(standaloneFile, []byte("# Project"), 0o600) + require.NoError(t, err) + + revisionID, err := client.CreateRevision(ctx) + require.NoError(t, err) + + srcDir := filepath.Join(dir.Name(), "src") + err = client.AddDirToRevision(ctx, revisionID, srcDir, fileupload.UploadOptions{}) + require.NoError(t, err) + + err = client.AddFileToRevision(ctx, revisionID, standaloneFile, fileupload.UploadOptions{}) + require.NoError(t, err) + + err = client.SealRevision(ctx, revisionID) + require.NoError(t, err) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(revisionID) + require.NoError(t, err) + require.Len(t, uploadedFiles, 3) + + paths := make([]string, len(uploadedFiles)) + for i, f := range uploadedFiles { + paths[i] = f.Path + } + assert.Contains(t, paths, "main.go") + assert.Contains(t, paths, "utils.go") + assert.Contains(t, paths, "README.md") + }) +} + +func Test_CreateRevisionFromPaths(t *testing.T) { + llcfg := uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 10, + FileSizeLimit: 100, + }, + } + + filters := filters.AllowList{ + ConfigFiles: []string{"go.mod"}, + Extensions: []string{".txt", ".go", ".md"}, + } + + t.Run("mixed files and directories", func(t *testing.T) { + allFiles := []uploadrevision.LoadedFile{ + {Path: "src/main.go", Content: "package main"}, + {Path: "src/utils.go", Content: "package utils"}, + {Path: "config.yaml", Content: "version: 1"}, + {Path: "README.md", Content: "# Project"}, + } + + ctx, fakeSealableClient, client, dir, cleanup := setupTest(t, llcfg, allFiles, filters) + defer cleanup() + + paths := []string{ + filepath.Join(dir.Name(), "src"), // Directory + filepath.Join(dir.Name(), "README.md"), // Individual file + } + + revID, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) + require.NoError(t, err) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(revID) + require.NoError(t, err) + require.Len(t, uploadedFiles, 3) // 2 from src/ + 1 README.md + + uploadedPaths := make([]string, len(uploadedFiles)) + for i, f := range uploadedFiles { + uploadedPaths[i] = f.Path + } + assert.Contains(t, uploadedPaths, "main.go") + assert.Contains(t, uploadedPaths, "utils.go") + assert.Contains(t, uploadedPaths, "README.md") + }) + + t.Run("error handling with better context", func(t *testing.T) { + ctx, _, client, _, cleanup := setupTest(t, llcfg, []uploadrevision.LoadedFile{}, filters) + defer cleanup() + + paths := []string{ + "/nonexistent/file.go", + "/another/missing/path.txt", + } + + _, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to stat path") + assert.Contains(t, err.Error(), "/nonexistent/file.go") // Should include the specific path + }) +} + +func Test_CreateRevisionFromDir(t *testing.T) { + llcfg := uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 2, + FileSizeLimit: 100, + }, + } + + filters := filters.AllowList{ + ConfigFiles: []string{"go.mod"}, + Extensions: []string{".txt", ".go", ".md"}, + } + + t.Run("uploading a shallow directory", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "content1", + }, + { + Path: "file2.txt", + Content: "content2", + }, + } + ctx, fakeSealableClient, client, dir, cleanup := setupTest(t, llcfg, expectedFiles, filters) + defer cleanup() + + revID, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + + require.NoError(t, err) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(revID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory with nested files", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "src/main.go", + Content: "package main\n\nfunc main() {}", + }, + { + Path: "src/utils/helper.go", + Content: "package utils\n\nfunc Helper() {}", + }, + } + ctx, fakeSealableClient, client, dir, cleanup := setupTest(t, llcfg, expectedFiles, filters) + defer cleanup() + + revID, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + + require.NoError(t, err) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(revID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory exceeding the file count limit for a single upload", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "root level file", + }, + { + Path: "src/main.go", + Content: "package main\n\nfunc main() {}", + }, + { + Path: "src/utils/helper.go", + Content: "package utils\n\nfunc Helper() {}", + }, + { + Path: "docs/README.md", + Content: "# Project Documentation", + }, + { + Path: "src/go.mod", + Content: "foo bar", + }, + } + ctx, fakeSealableClient, client, dir, cleanup := setupTest(t, llcfg, expectedFiles, filters) + defer cleanup() + + revID, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(revID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory with file exceeding the file size limit", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "foo bar", + }, + } + ctx, _, client, dir, cleanup := setupTest(t, uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 1, + FileSizeLimit: 6, + }, + }, expectedFiles, filters) + defer cleanup() + + _, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + + require.Error(t, err) + var fileSizeLimitErr *fileupload.FileSizeLimitError + require.ErrorAs(t, err, &fileSizeLimitErr) + assert.Equal(t, fileSizeLimitErr.FilePath, expectedFiles[0].Path) + assert.Equal(t, fileSizeLimitErr.FileSize, int64(len(expectedFiles[0].Content))) + assert.Equal(t, fileSizeLimitErr.Limit, int64(6)) + }) + + t.Run("uploading a directory applies filtering", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "src/main.go", + Content: "package main\n\nfunc main() {}", + }, + { + Path: "src/utils/helper.go", + Content: "package utils\n\nfunc Helper() {}", + }, + { + Path: "src/go.mod", + Content: "foo bar", + }, + } + additionalFiles := []uploadrevision.LoadedFile{ + { + Path: "src/script.js", + Content: "console.log('hi')", + }, + { + Path: "src/package.json", + Content: "{}", + }, + } + //nolint:gocritic // Not an issue for tests. + allFiles := append(expectedFiles, additionalFiles...) + + ctx, fakeSealableClient, client, dir, cleanup := setupTest(t, llcfg, allFiles, filters) + defer cleanup() + + revID, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(revID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory with filtering disabled", func(t *testing.T) { + allFiles := []uploadrevision.LoadedFile{ + { + Path: "src/main.go", + Content: "package main\n\nfunc main() {}", + }, + { + Path: "src/utils/helper.go", + Content: "package utils\n\nfunc Helper() {}", + }, + { + Path: "src/go.mod", + Content: "foo bar", + }, + { + Path: "src/script.js", + Content: "console.log('hi')", + }, + { + Path: "src/package.json", + Content: "{}", + }, + } + + ctx, fakeSealableClient, client, dir, cleanup := setupTest(t, llcfg, allFiles, filters) + defer cleanup() + + revID, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{SkipFiltering: true}) + require.NoError(t, err) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(revID) + require.NoError(t, err) + expectEqualFiles(t, allFiles, uploadedFiles) + }) +} + +func expectEqualFiles(t *testing.T, expectedFiles, uploadedFiles []uploadrevision.LoadedFile) { + t.Helper() + + require.NotEmpty(t, uploadedFiles) + require.Equal(t, len(expectedFiles), len(uploadedFiles)) + + slices.SortFunc(expectedFiles, func(fileA, fileB uploadrevision.LoadedFile) int { + return strings.Compare(fileA.Path, fileB.Path) + }) + + slices.SortFunc(uploadedFiles, func(fileA, fileB uploadrevision.LoadedFile) int { + return strings.Compare(fileA.Path, fileB.Path) + }) + + for i := range uploadedFiles { + assert.Equal(t, expectedFiles[i].Path, uploadedFiles[i].Path) + assert.Equal(t, expectedFiles[i].Content, uploadedFiles[i].Content) + } +} + +func createDirWithFiles(t *testing.T, files []uploadrevision.LoadedFile) (dir *os.File, cleanup func()) { + t.Helper() + + tempDir, err := os.MkdirTemp("", "cliuploadtest*") + if err != nil { + panic(err) + } + + dir, err = os.Open(tempDir) + if err != nil { + panic(err) + } + + for _, file := range files { + fullPath := filepath.Join(tempDir, file.Path) + + parentDir := filepath.Dir(fullPath) + if err := os.MkdirAll(parentDir, 0o755); err != nil { + panic(err) + } + + f, err := os.Create(fullPath) + if err != nil { + panic(err) + } + + if _, err := f.WriteString(file.Content); err != nil { + f.Close() + panic(err) + } + f.Close() + } + + cleanup = func() { + if dir != nil { + dir.Close() + } + if err := os.RemoveAll(tempDir); err != nil { + t.Logf("failed to cleanup temp directory: %s\n", err.Error()) + } + } + + return dir, cleanup +} + +func setupTest( + t *testing.T, + llcfg uploadrevision.FakeClientConfig, + files []uploadrevision.LoadedFile, + allowList filters.AllowList, +) (context.Context, *uploadrevision.FakeSealableClient, *fileupload.Client, *os.File, func()) { + t.Helper() + + ctx := context.Background() + orgID := uuid.New() + + fakeSealeableClient := uploadrevision.NewFakeSealableClient(llcfg) + fakeFiltersClient := filters.NewFakeClient(allowList, nil) + client := fileupload.NewClient(nil, fileupload.Config{ + OrgID: orgID, + }, + fileupload.WithUploadRevisionSealableClient(fakeSealeableClient), + fileupload.WithFiltersClient(fakeFiltersClient)) + + dir, dirCleanup := createDirWithFiles(t, files) + + return ctx, fakeSealeableClient, client, dir, func() { + dirCleanup() + } +} diff --git a/internal/fileupload/errors.go b/internal/fileupload/errors.go new file mode 100644 index 0000000..5db1a98 --- /dev/null +++ b/internal/fileupload/errors.go @@ -0,0 +1,23 @@ +package fileupload + +import "github.com/snyk/cli-extension-os-flows/internal/fileupload/uploadrevision" + +// Aliasing uploadRevisionSealableClient errors so that they're scoped to the fileupload package as well. + +// FileSizeLimitError indicates a file exceeds the maximum allowed size. +type FileSizeLimitError = uploadrevision.FileSizeLimitError + +// FileCountLimitError indicates too many files were provided. +type FileCountLimitError = uploadrevision.FileCountLimitError + +// FileAccessError indicates a file access permission issue. +type FileAccessError = uploadrevision.FileAccessError + +// DirectoryError indicates an issue with directory operations. +type DirectoryError = uploadrevision.DirectoryError + +// HTTPError indicates an HTTP request/response error. +type HTTPError = uploadrevision.HTTPError + +// MultipartError indicates an issue with multipart request handling. +type MultipartError = uploadrevision.MultipartError diff --git a/internal/fileupload/filters/client.go b/internal/fileupload/filters/client.go new file mode 100644 index 0000000..fc61d5e --- /dev/null +++ b/internal/fileupload/filters/client.go @@ -0,0 +1,71 @@ +package filters + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/google/uuid" +) + +// AllowList represents the response structure from the deeproxy filters API. +type AllowList struct { + ConfigFiles []string `json:"configFiles"` + Extensions []string `json:"extensions"` +} + +type Client interface { + GetFilters(ctx context.Context, orgID uuid.UUID) (AllowList, error) +} + +type deeproxyClient struct { + httpClient *http.Client + cfg Config +} + +type Config struct { + BaseURL string + IsFedRamp bool +} + +func NewDeeproxyClient(cfg Config, opts ...Opt) *deeproxyClient { + c := &deeproxyClient{ + cfg: cfg, + httpClient: http.DefaultClient, + } + + for _, opt := range opts { + opt(c) + } + + return c +} + +func (c *deeproxyClient) GetFilters(ctx context.Context, orgID uuid.UUID) (AllowList, error) { + var allowList AllowList + + url := getFilterUrl(c.cfg.BaseURL, orgID, c.cfg.IsFedRamp) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) + if err != nil { + return allowList, fmt.Errorf("failed to create deeproxy filters request: %w", err) + } + + req.Header.Set("snyk-org-name", orgID.String()) + + resp, err := c.httpClient.Do(req) + if err != nil { + return allowList, fmt.Errorf("error making deeproxy filters request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return allowList, fmt.Errorf("unexpected response code: %s", resp.Status) + } + + if err := json.NewDecoder(resp.Body).Decode(&allowList); err != nil { + return allowList, fmt.Errorf("failed to decode deeproxy filters response: %w", err) + } + + return allowList, nil +} diff --git a/internal/fileupload/filters/client_test.go b/internal/fileupload/filters/client_test.go new file mode 100644 index 0000000..571f6b7 --- /dev/null +++ b/internal/fileupload/filters/client_test.go @@ -0,0 +1,77 @@ +package filters + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClients(t *testing.T) { + tests := []struct { + getClient func(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) (Client, func()) + clientName string + }{ + { + clientName: "deeproxyClient", + getClient: func(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) (Client, func()) { + t.Helper() + + s := setupServer(t, orgID, expectedAllow) + cleanup := func() { + s.Close() + } + c := NewDeeproxyClient(Config{BaseURL: s.URL, IsFedRamp: true}, WithHTTPClient(s.Client())) + + return c, cleanup + }, + }, + { + clientName: "fakeClient", + getClient: func(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) (Client, func()) { + t.Helper() + + c := NewFakeClient(expectedAllow, nil) + + return c, func() {} + }, + }, + } + + for _, testData := range tests { + t.Run(testData.clientName+": GetFilters", func(t *testing.T) { + orgID := uuid.New() + expectedAllow := AllowList{ + ConfigFiles: []string{"package.json"}, + Extensions: []string{".ts", ".js"}, + } + client, cleanup := testData.getClient(t, orgID, expectedAllow) + defer cleanup() + + allow, err := client.GetFilters(t.Context(), orgID) + require.NoError(t, err) + + assert.Equal(t, expectedAllow, allow) + }) + } +} + +func setupServer(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) *httptest.Server { + t.Helper() + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expectedURL := getFilterUrl("", orgID, true) + assert.Equal(t, expectedURL, r.URL.Path) + assert.Equal(t, orgID.String(), r.Header.Get("snyk-org-name")) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(expectedAllow); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + return + } + })) + ts.Start() + return ts +} diff --git a/internal/fileupload/filters/fake_client.go b/internal/fileupload/filters/fake_client.go new file mode 100644 index 0000000..657112e --- /dev/null +++ b/internal/fileupload/filters/fake_client.go @@ -0,0 +1,23 @@ +package filters + +import ( + "context" + "github.com/google/uuid" +) + +type fakeClient struct { + getFilters func(ctx context.Context, orgID uuid.UUID) (AllowList, error) +} + +func NewFakeClient(allowList AllowList, err error) Client { + return &fakeClient{ + getFilters: func(ctx context.Context, orgID uuid.UUID) (AllowList, error) { + return allowList, err + }, + } + +} + +func (f *fakeClient) GetFilters(ctx context.Context, orgID uuid.UUID) (AllowList, error) { + return f.getFilters(ctx, orgID) +} diff --git a/internal/fileupload/filters/opts.go b/internal/fileupload/filters/opts.go new file mode 100644 index 0000000..d16cf98 --- /dev/null +++ b/internal/fileupload/filters/opts.go @@ -0,0 +1,13 @@ +package filters + +import "net/http" + +// Opt is a function that configures an deeproxyClient instance. +type Opt func(*deeproxyClient) + +// WithHTTPClient sets a custom HTTP client for the filters client. +func WithHTTPClient(httpClient *http.Client) Opt { + return func(c *deeproxyClient) { + c.httpClient = httpClient + } +} diff --git a/internal/fileupload/filters/utils.go b/internal/fileupload/filters/utils.go new file mode 100644 index 0000000..9f8f74a --- /dev/null +++ b/internal/fileupload/filters/utils.go @@ -0,0 +1,17 @@ +package filters + +import ( + "fmt" + "strings" + + "github.com/google/uuid" +) + +func getFilterUrl(baseUrl string, orgID uuid.UUID, isFedRamp bool) string { + if isFedRamp { + return fmt.Sprintf("%s/hidden/orgs/%s/code/filters", baseUrl, orgID) + } else { + deeproxyUrl := strings.ReplaceAll(baseUrl, "api", "deeproxy") + return fmt.Sprintf("%s/filters", deeproxyUrl) + } +} diff --git a/internal/fileupload/filters/utils_test.go b/internal/fileupload/filters/utils_test.go new file mode 100644 index 0000000..42e08f1 --- /dev/null +++ b/internal/fileupload/filters/utils_test.go @@ -0,0 +1,19 @@ +package filters + +import ( + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "testing" +) + +var orgId = uuid.MustParse("738ef92e-21cc-4a11-8c13-388d89272f4b") + +func Test_getBaseUrl_notFedramp(t *testing.T) { + actualUrl := getFilterUrl("https://api.snyk.io", orgId, false) + assert.Equal(t, "https://deeproxy.snyk.io/filters", actualUrl) +} + +func Test_getBaseUrl_fedramp(t *testing.T) { + actualUrl := getFilterUrl("https://api.snyk.io", orgId, true) + assert.Equal(t, "https://api.snyk.io/hidden/orgs/738ef92e-21cc-4a11-8c13-388d89272f4b/code/filters", actualUrl) +} diff --git a/internal/fileupload/opts.go b/internal/fileupload/opts.go new file mode 100644 index 0000000..5a04e06 --- /dev/null +++ b/internal/fileupload/opts.go @@ -0,0 +1,23 @@ +package fileupload + +import ( + "github.com/snyk/cli-extension-os-flows/internal/fileupload/filters" + "github.com/snyk/cli-extension-os-flows/internal/fileupload/uploadrevision" +) + +// Option allows customizing the Client during construction. +type Option func(*Client) + +// WithUploadRevisionSealableClient allows injecting a custom low-level client (primarily for testing). +func WithUploadRevisionSealableClient(client uploadrevision.SealableClient) Option { + return func(c *Client) { + c.uploadRevisionSealableClient = client + } +} + +// WithFiltersClient allows injecting a custom low-level client (primarily for testing). +func WithFiltersClient(client filters.Client) Option { + return func(c *Client) { + c.filtersClient = client + } +} diff --git a/internal/fileupload/types.go b/internal/fileupload/types.go new file mode 100644 index 0000000..b7bec51 --- /dev/null +++ b/internal/fileupload/types.go @@ -0,0 +1,28 @@ +package fileupload + +import ( + "sync" + + "github.com/puzpuzpuz/xsync" + + "github.com/snyk/cli-extension-os-flows/internal/fileupload/uploadrevision" +) + +// OrgID represents an organization identifier. +type OrgID = uploadrevision.OrgID + +// RevisionID represents a revision identifier. +type RevisionID = uploadrevision.RevisionID + +// Filters holds the filtering configuration for file uploads with thread-safe maps. +type Filters struct { + supportedExtensions *xsync.MapOf[string, bool] + supportedConfigFiles *xsync.MapOf[string, bool] + once sync.Once + initErr error +} + +// UploadOptions configures the behavior of file upload operations. +type UploadOptions struct { + SkipFiltering bool +} diff --git a/internal/fileupload/lowlevel/client.go b/internal/fileupload/uploadrevision/client.go similarity index 90% rename from internal/fileupload/lowlevel/client.go rename to internal/fileupload/uploadrevision/client.go index bbba071..50d6dec 100644 --- a/internal/fileupload/lowlevel/client.go +++ b/internal/fileupload/uploadrevision/client.go @@ -1,4 +1,4 @@ -package lowlevel +package uploadrevision import ( "bytes" @@ -24,21 +24,21 @@ type SealableClient interface { } // This will force go to complain if the type doesn't satisfy the interface. -var _ SealableClient = (*HTTPSealableClient)(nil) +var _ SealableClient = (*httpSealableClient)(nil) // Config contains configuration for the file upload client. type Config struct { BaseURL string } -// HTTPSealableClient implements the SealableClient interface for file upload operations via HTTP API. -type HTTPSealableClient struct { +// httpSealableClient implements the SealableClient interface for file upload operations via HTTP API. +type httpSealableClient struct { cfg Config httpClient *http.Client } -// APIVersion specifies the API version to use for requests. -const APIVersion = "2024-10-15" +// apiVersion specifies the API version to use for requests. +const apiVersion = "2024-10-15" const ( fileSizeLimit = 50_000_000 // arbitrary number, chosen to support max size of SBOMs @@ -46,11 +46,11 @@ const ( ) // NewClient creates a new file upload client with the given configuration and options. -func NewClient(cfg Config, opts ...Opt) *HTTPSealableClient { +func NewClient(cfg Config, opts ...Opt) *httpSealableClient { httpClient := &http.Client{ Transport: http.DefaultTransport, } - c := HTTPSealableClient{cfg, httpClient} + c := httpSealableClient{cfg, httpClient} for _, opt := range opts { opt(&c) @@ -63,7 +63,7 @@ func NewClient(cfg Config, opts ...Opt) *HTTPSealableClient { } // CreateRevision creates a new upload revision for the specified organization. -func (c *HTTPSealableClient) CreateRevision(ctx context.Context, orgID OrgID) (*UploadRevisionResponseBody, error) { +func (c *httpSealableClient) CreateRevision(ctx context.Context, orgID OrgID) (*UploadRevisionResponseBody, error) { if orgID == uuid.Nil { return nil, ErrEmptyOrgID } @@ -81,7 +81,7 @@ func (c *HTTPSealableClient) CreateRevision(ctx context.Context, orgID OrgID) (* return nil, fmt.Errorf("failed to encode request body: %w", err) } - url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions?version=%s", c.cfg.BaseURL, orgID, APIVersion) + url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions?version=%s", c.cfg.BaseURL, orgID, apiVersion) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, buff) if err != nil { return nil, fmt.Errorf("failed to create revision request: %w", err) @@ -107,7 +107,7 @@ func (c *HTTPSealableClient) CreateRevision(ctx context.Context, orgID OrgID) (* } // UploadFiles uploads the provided files to the specified revision. It will not close the file descriptors. -func (c *HTTPSealableClient) UploadFiles(ctx context.Context, orgID OrgID, revisionID RevisionID, files []UploadFile) error { +func (c *httpSealableClient) UploadFiles(ctx context.Context, orgID OrgID, revisionID RevisionID, files []UploadFile) error { if orgID == uuid.Nil { return ErrEmptyOrgID } @@ -128,7 +128,7 @@ func (c *HTTPSealableClient) UploadFiles(ctx context.Context, orgID OrgID, revis go streamFilesToPipe(pipeWriter, mpartWriter, files) - url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions/%s/files?version=%s", c.cfg.BaseURL, orgID, revisionID, APIVersion) + url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions/%s/files?version=%s", c.cfg.BaseURL, orgID, revisionID, apiVersion) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, pipeReader) if err != nil { return fmt.Errorf("failed to create upload files request: %w", err) @@ -202,7 +202,7 @@ func validateFiles(files []UploadFile) error { } // SealRevision seals the specified upload revision, marking it as complete. -func (c *HTTPSealableClient) SealRevision(ctx context.Context, orgID OrgID, revisionID RevisionID) (*SealUploadRevisionResponseBody, error) { +func (c *httpSealableClient) SealRevision(ctx context.Context, orgID OrgID, revisionID RevisionID) (*SealUploadRevisionResponseBody, error) { if orgID == uuid.Nil { return nil, ErrEmptyOrgID } @@ -225,7 +225,7 @@ func (c *HTTPSealableClient) SealRevision(ctx context.Context, orgID OrgID, revi return nil, fmt.Errorf("failed to encode request body: %w", err) } - url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions/%s?version=%s", c.cfg.BaseURL, orgID, revisionID, APIVersion) + url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions/%s?version=%s", c.cfg.BaseURL, orgID, revisionID, apiVersion) req, err := http.NewRequestWithContext(ctx, http.MethodPatch, url, buff) if err != nil { return nil, fmt.Errorf("failed to create seal request: %w", err) @@ -271,7 +271,7 @@ func handleUnexpectedStatusCodes(body io.ReadCloser, statusCode int, status, ope } // GetLimits returns the upload Limits defined in the low level client. -func (c *HTTPSealableClient) GetLimits() Limits { +func (c *httpSealableClient) GetLimits() Limits { return Limits{ FileCountLimit: fileCountLimit, FileSizeLimit: fileSizeLimit, diff --git a/internal/fileupload/lowlevel/client_test.go b/internal/fileupload/uploadrevision/client_test.go similarity index 86% rename from internal/fileupload/lowlevel/client_test.go rename to internal/fileupload/uploadrevision/client_test.go index b1681a7..1290aa0 100644 --- a/internal/fileupload/lowlevel/client_test.go +++ b/internal/fileupload/uploadrevision/client_test.go @@ -1,4 +1,4 @@ -package lowlevel_test +package uploadrevision_test import ( "compress/gzip" @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/snyk/cli-extension-os-flows/internal/fileupload/lowlevel" + "github.com/snyk/cli-extension-os-flows/internal/fileupload/uploadrevision" ) func TestClient_CreateRevision(t *testing.T) { @@ -43,7 +43,7 @@ func TestClient_CreateRevision(t *testing.T) { } }`)) })) - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: srv.URL, }) @@ -55,7 +55,7 @@ func TestClient_CreateRevision(t *testing.T) { } func TestClient_CreateRevision_EmptyOrgID(t *testing.T) { - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: "http://example.com", }) @@ -63,7 +63,7 @@ func TestClient_CreateRevision_EmptyOrgID(t *testing.T) { assert.Error(t, err) assert.Nil(t, resp) - assert.ErrorIs(t, err, lowlevel.ErrEmptyOrgID) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyOrgID) } func TestClient_CreateRevision_ServerError(t *testing.T) { @@ -71,14 +71,14 @@ func TestClient_CreateRevision_ServerError(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: srv.URL, }) resp, err := c.CreateRevision(context.Background(), orgID) assert.Nil(t, resp) - var httpErr *lowlevel.HTTPError + var httpErr *uploadrevision.HTTPError assert.ErrorAs(t, err, &httpErr) assert.Equal(t, http.StatusInternalServerError, httpErr.StatusCode) assert.Equal(t, "create upload revision", httpErr.Operation) @@ -126,7 +126,7 @@ func TestClient_UploadFiles(t *testing.T) { })) defer srv.Close() - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: srv.URL, }) @@ -139,7 +139,7 @@ func TestClient_UploadFiles(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []lowlevel.UploadFile{ + []uploadrevision.UploadFile{ {Path: "foo/bar", File: fd}, }) @@ -196,7 +196,7 @@ func TestClient_UploadFiles_MultipleFiles(t *testing.T) { })) defer srv.Close() - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: srv.URL, }) @@ -213,7 +213,7 @@ func TestClient_UploadFiles_MultipleFiles(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []lowlevel.UploadFile{ + []uploadrevision.UploadFile{ {Path: "file1.txt", File: file1}, {Path: "file2.json", File: file2}, }) @@ -224,7 +224,7 @@ func TestClient_UploadFiles_MultipleFiles(t *testing.T) { func TestClient_UploadFiles_EmptyOrgID(t *testing.T) { revID := uuid.MustParse("ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f") - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: "http://example.com", }) @@ -237,18 +237,18 @@ func TestClient_UploadFiles_EmptyOrgID(t *testing.T) { err = c.UploadFiles(context.Background(), uuid.Nil, // empty orgID revID, - []lowlevel.UploadFile{ + []uploadrevision.UploadFile{ {Path: "test.txt", File: file}, }) assert.Error(t, err) - assert.ErrorIs(t, err, lowlevel.ErrEmptyOrgID) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyOrgID) } func TestClient_UploadFiles_EmptyRevisionID(t *testing.T) { orgID := uuid.MustParse("9102b78b-c28d-4392-a39f-08dd26fd9622") - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: "http://example.com", }) @@ -261,19 +261,19 @@ func TestClient_UploadFiles_EmptyRevisionID(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, uuid.Nil, // empty revisionID - []lowlevel.UploadFile{ + []uploadrevision.UploadFile{ {Path: "test.txt", File: file}, }) assert.Error(t, err) - assert.ErrorIs(t, err, lowlevel.ErrEmptyRevisionID) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyRevisionID) } func TestClient_UploadFiles_FileSizeLimit(t *testing.T) { orgID := uuid.MustParse("9102b78b-c28d-4392-a39f-08dd26fd9622") revID := uuid.MustParse("ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f") - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: "http://example.com", }) @@ -288,12 +288,12 @@ func TestClient_UploadFiles_FileSizeLimit(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []lowlevel.UploadFile{ + []uploadrevision.UploadFile{ {Path: "large_file.txt", File: file}, }) assert.Error(t, err) - var fileSizeErr *lowlevel.FileSizeLimitError + var fileSizeErr *uploadrevision.FileSizeLimitError assert.ErrorAs(t, err, &fileSizeErr) assert.Equal(t, "large_file.txt", fileSizeErr.FilePath) assert.Equal(t, c.GetLimits().FileSizeLimit+1, fileSizeErr.FileSize) @@ -304,11 +304,11 @@ func TestClient_UploadFiles_FileCountLimit(t *testing.T) { orgID := uuid.MustParse("9102b78b-c28d-4392-a39f-08dd26fd9622") revID := uuid.MustParse("ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f") - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: "http://example.com", }) - files := make([]lowlevel.UploadFile, c.GetLimits().FileCountLimit+1) + files := make([]uploadrevision.UploadFile, c.GetLimits().FileCountLimit+1) mockFS := fstest.MapFS{} for i := range c.GetLimits().FileCountLimit + 1 { @@ -318,7 +318,7 @@ func TestClient_UploadFiles_FileCountLimit(t *testing.T) { file, err := mockFS.Open(filename) require.NoError(t, err) - files[i] = lowlevel.UploadFile{ + files[i] = uploadrevision.UploadFile{ Path: filename, File: file, } @@ -327,7 +327,7 @@ func TestClient_UploadFiles_FileCountLimit(t *testing.T) { err := c.UploadFiles(context.Background(), orgID, revID, files) assert.Error(t, err) - var fileCountErr *lowlevel.FileCountLimitError + var fileCountErr *uploadrevision.FileCountLimitError assert.ErrorAs(t, err, &fileCountErr) assert.Equal(t, c.GetLimits().FileCountLimit+1, fileCountErr.Count) assert.Equal(t, c.GetLimits().FileCountLimit, fileCountErr.Limit) @@ -337,7 +337,7 @@ func TestClient_UploadFiles_DirectoryError(t *testing.T) { orgID := uuid.MustParse("9102b78b-c28d-4392-a39f-08dd26fd9622") revID := uuid.MustParse("ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f") - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: "http://example.com", }) @@ -353,12 +353,12 @@ func TestClient_UploadFiles_DirectoryError(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []lowlevel.UploadFile{ + []uploadrevision.UploadFile{ {Path: "test-directory", File: dirFile}, }) assert.Error(t, err) - var dirErr *lowlevel.DirectoryError + var dirErr *uploadrevision.DirectoryError assert.ErrorAs(t, err, &dirErr) assert.Equal(t, "test-directory", dirErr.Path) } @@ -367,14 +367,14 @@ func TestClient_UploadFiles_EmptyFileList(t *testing.T) { orgID := uuid.MustParse("9102b78b-c28d-4392-a39f-08dd26fd9622") revID := uuid.MustParse("ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f") - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: "http://example.com", }) - err := c.UploadFiles(context.Background(), orgID, revID, []lowlevel.UploadFile{}) + err := c.UploadFiles(context.Background(), orgID, revID, []uploadrevision.UploadFile{}) assert.Error(t, err) - assert.ErrorIs(t, err, lowlevel.ErrNoFilesProvided) + assert.ErrorIs(t, err, uploadrevision.ErrNoFilesProvided) } func TestClient_SealRevision(t *testing.T) { @@ -400,7 +400,7 @@ func TestClient_SealRevision(t *testing.T) { } }`)) })) - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: srv.URL, }) @@ -414,7 +414,7 @@ func TestClient_SealRevision(t *testing.T) { func TestClient_SealRevision_EmptyOrgID(t *testing.T) { revID := uuid.MustParse("ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f") - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: "http://example.com", }) @@ -424,14 +424,14 @@ func TestClient_SealRevision_EmptyOrgID(t *testing.T) { ) assert.Error(t, err) - assert.ErrorIs(t, err, lowlevel.ErrEmptyOrgID) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyOrgID) assert.Nil(t, resp) } func TestClient_SealRevision_EmptyRevisionID(t *testing.T) { orgID := uuid.MustParse("9102b78b-c28d-4392-a39f-08dd26fd9622") - c := lowlevel.NewClient(lowlevel.Config{ + c := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: "http://example.com", }) @@ -441,6 +441,6 @@ func TestClient_SealRevision_EmptyRevisionID(t *testing.T) { ) assert.Error(t, err) - assert.ErrorIs(t, err, lowlevel.ErrEmptyRevisionID) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyRevisionID) assert.Nil(t, resp) } diff --git a/internal/fileupload/lowlevel/compression.go b/internal/fileupload/uploadrevision/compression.go similarity index 97% rename from internal/fileupload/lowlevel/compression.go rename to internal/fileupload/uploadrevision/compression.go index 4913c3d..0489988 100644 --- a/internal/fileupload/lowlevel/compression.go +++ b/internal/fileupload/uploadrevision/compression.go @@ -1,4 +1,4 @@ -package lowlevel +package uploadrevision import ( "compress/gzip" @@ -59,7 +59,7 @@ func (crt *CompressionRoundTripper) RoundTrip(r *http.Request) (*http.Response, r.Body = compressedBody r.Header.Set(ContentEncoding, "gzip") - r.Header.Del("Content-Length") + r.Header.Del(ContentLength) r.ContentLength = -1 // Let Go calculate the length //nolint:wrapcheck // No need to wrap the error here. diff --git a/internal/fileupload/lowlevel/compression_test.go b/internal/fileupload/uploadrevision/compression_test.go similarity index 89% rename from internal/fileupload/lowlevel/compression_test.go rename to internal/fileupload/uploadrevision/compression_test.go index 6ab9695..699e5d2 100644 --- a/internal/fileupload/lowlevel/compression_test.go +++ b/internal/fileupload/uploadrevision/compression_test.go @@ -1,4 +1,4 @@ -package lowlevel_test +package uploadrevision_test import ( "compress/gzip" @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/snyk/cli-extension-os-flows/internal/fileupload/lowlevel" + "github.com/snyk/cli-extension-os-flows/internal/fileupload/uploadrevision" ) func TestCompressionRoundTripper_RoundTrip(t *testing.T) { @@ -25,7 +25,7 @@ func TestCompressionRoundTripper_RoundTrip(t *testing.T) { })) defer server.Close() - crt := lowlevel.NewCompressionRoundTripper(http.DefaultTransport) + crt := uploadrevision.NewCompressionRoundTripper(http.DefaultTransport) req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, http.NoBody) require.NoError(t, err) @@ -57,7 +57,7 @@ func TestCompressionRoundTripper_RoundTrip(t *testing.T) { })) defer server.Close() - crt := lowlevel.NewCompressionRoundTripper(http.DefaultTransport) + crt := uploadrevision.NewCompressionRoundTripper(http.DefaultTransport) req, err := http.NewRequestWithContext(ctx, http.MethodPost, server.URL, strings.NewReader(originalBody)) require.NoError(t, err) @@ -81,7 +81,7 @@ func TestCompressionRoundTripper_RoundTrip(t *testing.T) { })) defer server.Close() - crt := lowlevel.NewCompressionRoundTripper(http.DefaultTransport) + crt := uploadrevision.NewCompressionRoundTripper(http.DefaultTransport) req, err := http.NewRequestWithContext(ctx, http.MethodPost, server.URL, strings.NewReader(`{"key":"value"}`)) require.NoError(t, err) @@ -99,7 +99,7 @@ func TestCompressionRoundTripper_RoundTrip(t *testing.T) { t.Run("wraps underlying transport errors", func(t *testing.T) { ctx := context.Background() failingTransport := &failingRoundTripper{err: assert.AnError} - crt := lowlevel.NewCompressionRoundTripper(failingTransport) + crt := uploadrevision.NewCompressionRoundTripper(failingTransport) req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://example.com", strings.NewReader("test")) require.NoError(t, err) diff --git a/internal/fileupload/lowlevel/errors.go b/internal/fileupload/uploadrevision/errors.go similarity index 99% rename from internal/fileupload/lowlevel/errors.go rename to internal/fileupload/uploadrevision/errors.go index 709f2b1..58062d6 100644 --- a/internal/fileupload/lowlevel/errors.go +++ b/internal/fileupload/uploadrevision/errors.go @@ -1,4 +1,4 @@ -package lowlevel +package uploadrevision import ( "errors" diff --git a/internal/fileupload/lowlevel/fake_client.go b/internal/fileupload/uploadrevision/fake_client.go similarity index 99% rename from internal/fileupload/lowlevel/fake_client.go rename to internal/fileupload/uploadrevision/fake_client.go index 9a4d66e..bb32e5d 100644 --- a/internal/fileupload/lowlevel/fake_client.go +++ b/internal/fileupload/uploadrevision/fake_client.go @@ -1,4 +1,4 @@ -package lowlevel +package uploadrevision import ( "context" diff --git a/internal/fileupload/lowlevel/opts.go b/internal/fileupload/uploadrevision/opts.go similarity index 71% rename from internal/fileupload/lowlevel/opts.go rename to internal/fileupload/uploadrevision/opts.go index 29197a3..d6e491b 100644 --- a/internal/fileupload/lowlevel/opts.go +++ b/internal/fileupload/uploadrevision/opts.go @@ -1,13 +1,13 @@ -package lowlevel +package uploadrevision import "net/http" // Opt is a function that configures an HTTPSealableClient instance. -type Opt func(*HTTPSealableClient) +type Opt func(*httpSealableClient) // WithHTTPClient sets a custom HTTP client for the file upload client. func WithHTTPClient(httpClient *http.Client) Opt { - return func(c *HTTPSealableClient) { + return func(c *httpSealableClient) { c.httpClient = httpClient } } diff --git a/internal/fileupload/lowlevel/opts_test.go b/internal/fileupload/uploadrevision/opts_test.go similarity index 76% rename from internal/fileupload/lowlevel/opts_test.go rename to internal/fileupload/uploadrevision/opts_test.go index aea2433..c0751a2 100644 --- a/internal/fileupload/lowlevel/opts_test.go +++ b/internal/fileupload/uploadrevision/opts_test.go @@ -1,4 +1,4 @@ -package lowlevel_test +package uploadrevision_test import ( "context" @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/snyk/cli-extension-os-flows/internal/fileupload/lowlevel" + "github.com/snyk/cli-extension-os-flows/internal/fileupload/uploadrevision" ) type CustomRoundTripper struct{} @@ -26,7 +26,7 @@ func Test_WithHTTPClient(t *testing.T) { fooValue := r.Header.Get("foo") assert.Equal(t, "bar", fooValue) - resp, err := json.Marshal(lowlevel.UploadRevisionResponseBody{}) + resp, err := json.Marshal(uploadrevision.UploadRevisionResponseBody{}) require.NoError(t, err) w.WriteHeader(http.StatusCreated) @@ -37,9 +37,9 @@ func Test_WithHTTPClient(t *testing.T) { customClient := srv.Client() customClient.Transport = &CustomRoundTripper{} - llc := lowlevel.NewClient(lowlevel.Config{ + llc := uploadrevision.NewClient(uploadrevision.Config{ BaseURL: srv.URL, - }, lowlevel.WithHTTPClient(customClient)) + }, uploadrevision.WithHTTPClient(customClient)) _, err := llc.CreateRevision(context.Background(), uuid.New()) diff --git a/internal/fileupload/lowlevel/types.go b/internal/fileupload/uploadrevision/types.go similarity index 97% rename from internal/fileupload/lowlevel/types.go rename to internal/fileupload/uploadrevision/types.go index c6ecfb2..4896925 100644 --- a/internal/fileupload/lowlevel/types.go +++ b/internal/fileupload/uploadrevision/types.go @@ -1,4 +1,4 @@ -package lowlevel +package uploadrevision import ( "io/fs" @@ -121,6 +121,8 @@ const ( ContentType = "Content-Type" // ContentEncoding is the HTTP header name for content encoding. ContentEncoding = "Content-Encoding" + // ContentLength is the HTTP header name for content length. + ContentLength = "Content-Length" ) // Limits contains the limits enforced by the low level client. diff --git a/internal/fileupload/utils.go b/internal/fileupload/utils.go new file mode 100644 index 0000000..ac65f7c --- /dev/null +++ b/internal/fileupload/utils.go @@ -0,0 +1,30 @@ +package fileupload + +func chunkChanFiltered[T any](chn <-chan T, size int, filter func(T) bool) <-chan []T { + out := make(chan []T) + chunk := make([]T, 0, size) + + go func() { + defer close(out) + + for el := range chn { + if filter == nil || filter(el) { + chunk = append(chunk, el) + } + if len(chunk) == size { + out <- chunk + chunk = make([]T, 0, size) + } + } + + if len(chunk) > 0 { + out <- chunk + } + }() + + return out +} + +func chunkChan[T any](chn <-chan T, size int) <-chan []T { + return chunkChanFiltered(chn, size, nil) +}