wip

Author: paulrosca-snyk

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/muesli/termenv v0.15.2
 	github.com/oapi-codegen/runtime v1.1.1
+	github.com/puzpuzpuz/xsync v1.5.2
 	github.com/rs/zerolog v1.34.0
 	github.com/snyk/code-client-go v1.21.3
 	github.com/snyk/error-catalog-golang-public v0.0.0-20250625135845-2d6f9a31f318
@@ -81,7 +82,6 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/puzpuzpuz/xsync v1.5.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect

internal/fileupload/client.go

@@ -0,0 +1,265 @@
+package fileupload
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/google/uuid"
+	"github.com/puzpuzpuz/xsync"
+
+	listsources "github.com/snyk/cli-extension-os-flows/internal/files"
+	"github.com/snyk/cli-extension-os-flows/internal/fileupload/lowlevel"
+)
+
+// Config contains configuration for the file upload client.
+type Config struct {
+	BaseURL string
+	OrgID   uuid.UUID
+}
+
+// Client provides high-level file upload functionality.
+type Client struct {
+	httpClient *http.Client
+	lowlevel   lowlevel.SealableClient
+	cfg        Config
+	filters    Filters
+}
+
+// NewClient creates a new high-level file upload client.
+// For production use, consumers only need to provide Config.
+// For testing, use WithLowLevelClient option to inject a mock.
+func NewClient(httpClient *http.Client, cfg Config, opts ...Option) *Client {
+	client := &Client{
+		httpClient: httpClient,
+		cfg:        cfg,
+		filters: Filters{
+			supportedExtensions:  xsync.NewMapOf[bool](),
+			supportedConfigFiles: xsync.NewMapOf[bool](),
+		},
+	}
+
+	for _, opt := range opts {
+		opt(client)
+	}
+
+	if client.lowlevel == nil {
+		client.lowlevel = lowlevel.NewClient(lowlevel.Config{
+			BaseURL: cfg.BaseURL,
+		}, lowlevel.WithHTTPClient(httpClient))
+	}
+
+	return client
+}
+
+func (c *Client) getDeeproxyFilters(ctx context.Context) (FiltersResponse, error) {
+	var filtersResp FiltersResponse
+
+	url := fmt.Sprintf("%s/hidden/orgs/%s/code/filters", c.cfg.BaseURL, c.cfg.OrgID)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
+	if err != nil {
+		return filtersResp, fmt.Errorf("failed to create deeproxy filters request: %w", err)
+	}
+
+	req.Header.Set("snyk-org-name", c.cfg.OrgID.String())
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return filtersResp, fmt.Errorf("error making deeproxy filters request: %w", err)
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return filtersResp, fmt.Errorf("unexpected response code: %s (%s)", resp.Status, resp.Body)
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&filtersResp); err != nil {
+		return filtersResp, fmt.Errorf("failed to decode deeproxy filters response: %w", err)
+	}
+
+	return filtersResp, nil
+}
+
+func (c *Client) loadFilters(ctx context.Context) error {
+	c.filters.once.Do(func() {
+		filtersResp, err := c.getDeeproxyFilters(ctx)
+		if err != nil {
+			c.filters.initErr = err
+			return
+		}
+
+		for _, ext := range filtersResp.Extensions {
+			c.filters.supportedExtensions.Store(ext, true)
+		}
+		for _, configFile := range filtersResp.ConfigFiles {
+			// .gitignore and .dcignore should not be uploaded
+			// (https://github.com/snyk/code-client/blob/d6f6a2ce4c14cb4b05aa03fb9f03533d8cf6ca4a/src/files.ts#L138)
+			if configFile == ".gitignore" || configFile == ".dcignore" {
+				continue
+			}
+			c.filters.supportedConfigFiles.Store(configFile, true)
+		}
+	})
+	return c.filters.initErr
+}
+
+// createFileFilter creates a filter function based on the current filtering configuration.
+// Returns nil if filtering should be skipped, otherwise returns a function that evaluates files.
+func (c *Client) createFileFilter(ctx context.Context, opts UploadOptions) (func(string) bool, error) {
+	if opts.SkipFiltering {
+		return nil, nil
+	}
+
+	if err := c.loadFilters(ctx); err != nil {
+		return nil, fmt.Errorf("failed to load deeproxy filters: %w", err)
+	}
+
+	return func(path string) bool {
+		fileExt := filepath.Ext(path)
+		fileName := filepath.Base(path)
+		_, isSupportedExtension := c.filters.supportedExtensions.Load(fileExt)
+		_, isSupportedConfigFile := c.filters.supportedConfigFiles.Load(fileName)
+		return isSupportedExtension || isSupportedConfigFile
+	}, nil
+}
+
+func (c *Client) uploadPaths(ctx context.Context, revID RevisionID, rootPath string, paths []string) error {
+	files := make([]lowlevel.UploadFile, 0, c.lowlevel.GetLimits().FileCountLimit)
+	defer func() {
+		for _, file := range files {
+			file.File.Close()
+		}
+	}()
+
+	for _, pth := range paths {
+		relPth, err := filepath.Rel(rootPath, pth)
+		if err != nil {
+			return err
+		}
+
+		f, err := os.Open(pth)
+		if err != nil {
+			return err
+		}
+
+		files = append(files, lowlevel.UploadFile{
+			Path: relPth,
+			File: f,
+		})
+	}
+
+	err := c.lowlevel.UploadFiles(ctx, c.cfg.OrgID, revID, files)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// addPathsToRevision adds multiple file paths to an existing revision.
+func (c *Client) addPathsToRevision(ctx context.Context, revisionID RevisionID, rootPath string, pathsChan <-chan string, opts UploadOptions) error {
+	filter, err := c.createFileFilter(ctx, opts)
+	if err != nil {
+		return err
+	}
+
+	var chunks <-chan []string
+	if filter != nil {
+		chunks = chunkChanFiltered(pathsChan, c.lowlevel.GetLimits().FileCountLimit, filter)
+	} else {
+		chunks = chunkChan(pathsChan, c.lowlevel.GetLimits().FileCountLimit)
+	}
+
+	for chunk := range chunks {
+		err := c.uploadPaths(ctx, revisionID, rootPath, chunk)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// CreateRevision creates a new revision and returns its ID.
+func (c *Client) CreateRevision(ctx context.Context) (RevisionID, error) {
+	revision, err := c.lowlevel.CreateRevision(ctx, c.cfg.OrgID)
+	if err != nil {
+		return uuid.Nil, fmt.Errorf("failed to create revision: %w", err)
+	}
+	return revision.Data.ID, nil
+}
+
+// AddFileToRevision adds a single file to an existing revision.
+func (c *Client) AddFileToRevision(ctx context.Context, revisionID RevisionID, filePath string, opts UploadOptions) error {
+	writableChan := make(chan string, 1)
+	writableChan <- filePath
+	close(writableChan)
+
+	return c.addPathsToRevision(ctx, revisionID, filepath.Dir(filePath), writableChan, opts)
+}
+
+// AddDirToRevision adds a directory and all its contents to an existing revision.
+func (c *Client) AddDirToRevision(ctx context.Context, revisionID RevisionID, dirPath string, opts UploadOptions) error {
+	sources, err := listsources.ForPath(dirPath, nil, runtime.NumCPU())
+	if err != nil {
+		return err
+	}
+
+	return c.addPathsToRevision(ctx, revisionID, dirPath, sources, opts)
+}
+
+// SealRevision seals a revision, making it immutable.
+func (c *Client) SealRevision(ctx context.Context, revisionID RevisionID) error {
+	_, err := c.lowlevel.SealRevision(ctx, c.cfg.OrgID, revisionID)
+	if err != nil {
+		return fmt.Errorf("failed to seal revision: %w", err)
+	}
+	return nil
+}
+
+// CreateRevisionFromPaths uploads multiple paths (files or directories), returning a revision ID.
+// This is a convenience method that creates, uploads, and seals a revision.
+func (c *Client) CreateRevisionFromPaths(ctx context.Context, paths []string, opts UploadOptions) (RevisionID, error) {
+	revisionID, err := c.CreateRevision(ctx)
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	for _, pth := range paths {
+		info, err := os.Stat(pth)
+		if err != nil {
+			return uuid.Nil, fmt.Errorf("failed to stat path %s: %w", pth, err)
+		}
+
+		if info.IsDir() {
+			if err := c.AddDirToRevision(ctx, revisionID, pth, opts); err != nil {
+				return uuid.Nil, fmt.Errorf("failed to add directory %s: %w", pth, err)
+			}
+		} else {
+			if err := c.AddFileToRevision(ctx, revisionID, pth, opts); err != nil {
+				return uuid.Nil, fmt.Errorf("failed to add file %s: %w", pth, err)
+			}
+		}
+	}
+
+	if err := c.SealRevision(ctx, revisionID); err != nil {
+		return uuid.Nil, err
+	}
+
+	return revisionID, nil
+}
+
+// CreateRevisionFromDir uploads a directory and all its contents, returning a revision ID.
+// This is a convenience method equivalent to CreateRevisionFromPaths with a single directory.
+func (c *Client) CreateRevisionFromDir(ctx context.Context, dirPath string, opts UploadOptions) (RevisionID, error) {
+	return c.CreateRevisionFromPaths(ctx, []string{dirPath}, opts)
+}
+
+// CreateRevisionFromFile uploads a single file, returning a revision ID.
+// This is a convenience method equivalent to CreateRevisionFromPaths with a single file.
+func (c *Client) CreateRevisionFromFile(ctx context.Context, filePath string, opts UploadOptions) (RevisionID, error) {
+	return c.CreateRevisionFromPaths(ctx, []string{filePath}, opts)
+}