Skip to content

Commit 27fd14b

Browse files
paulrosca-snykpaulrosca-snyk
committed
wip
1 parent 8bc5596 commit 27fd14b

File tree

7 files changed

+539
-1
lines changed

7 files changed

+539
-1
lines changed

‎internal/fileupload/client.go

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package fileupload
2+
3+
import (
4+
"context"
5+
"os"
6+
"path/filepath"
7+
"runtime"
8+
9+
"github.com/google/uuid"
10+
11+
listsources "github.com/snyk/cli-extension-os-flows/internal/files"
12+
"github.com/snyk/cli-extension-os-flows/internal/fileupload/lowlevel"
13+
)
14+
15+
// Config contains configuration for the file upload client.
16+
type Config struct {
17+
BaseURL string
18+
OrgID uuid.UUID
19+
}
20+
21+
// Client provides high-level file upload functionality.
22+
type Client struct {
23+
orgID uuid.UUID
24+
lowlevel lowlevel.SealableClient
25+
}
26+
27+
// Option allows customizing the Client during construction.
28+
type Option func(*Client)
29+
30+
// WithLowLevelClient allows injecting a custom low-level client (primarily for testing).
31+
func WithLowLevelClient(client lowlevel.SealableClient) Option {
32+
return func(c *Client) {
33+
c.lowlevel = client
34+
}
35+
}
36+
37+
// NewClient creates a new high-level file upload client.
38+
// For production use, consumers only need to provide Config.
39+
// For testing, use WithLowLevelClient option to inject a mock.
40+
func NewClient(cfg Config, opts ...Option) *Client {
41+
client := &Client{
42+
orgID: cfg.OrgID,
43+
}
44+
45+
// Apply options first (allows overriding the low-level client)
46+
for _, opt := range opts {
47+
opt(client)
48+
}
49+
50+
// Create default low-level client if none was provided
51+
if client.lowlevel == nil {
52+
client.lowlevel = lowlevel.NewClient(lowlevel.Config{
53+
BaseURL: cfg.BaseURL,
54+
})
55+
}
56+
57+
return client
58+
}
59+
60+
func chunkChan[T any](chn <-chan T, size int) <-chan []T {
61+
out := make(chan []T)
62+
chunk := make([]T, 0, size)
63+
64+
go func() {
65+
defer close(out)
66+
67+
for el := range chn {
68+
chunk = append(chunk, el)
69+
if len(chunk) == size {
70+
out <- chunk
71+
chunk = make([]T, 0, size)
72+
}
73+
}
74+
75+
if len(chunk) > 0 {
76+
out <- chunk
77+
}
78+
}()
79+
80+
return out
81+
}
82+
83+
// UploadDir uploads a directory and all its contents, returning a revision ID.
84+
func (c *Client) UploadDir(ctx context.Context, dir *os.File) (RevisionID, error) {
85+
revision, err := c.lowlevel.CreateRevision(ctx, c.orgID)
86+
if err != nil {
87+
return uuid.Nil, err
88+
}
89+
90+
sources, err := listsources.ForPath(dir.Name(), nil, runtime.NumCPU())
91+
if err != nil {
92+
return uuid.Nil, err
93+
}
94+
95+
for chunk := range chunkChan(sources, c.lowlevel.GetLimits().FileCountLimit) {
96+
files := make([]lowlevel.UploadFile, 0, c.lowlevel.GetLimits().FileCountLimit)
97+
defer func() {
98+
for _, file := range files {
99+
file.File.Close()
100+
}
101+
}()
102+
for _, pth := range chunk {
103+
f, err := os.Open(pth)
104+
if err != nil {
105+
return uuid.Nil, err
106+
}
107+
relPth, err := filepath.Rel(dir.Name(), pth)
108+
if err != nil {
109+
return uuid.Nil, err
110+
}
111+
files = append(files, lowlevel.UploadFile{
112+
Path: relPth,
113+
File: f,
114+
})
115+
}
116+
err = c.lowlevel.UploadFiles(ctx, c.orgID, revision.Data.ID, files)
117+
if err != nil {
118+
return uuid.Nil, err
119+
}
120+
}
121+
122+
_, err = c.lowlevel.SealRevision(ctx, c.orgID, revision.Data.ID)
123+
if err != nil {
124+
return uuid.Nil, err
125+
}
126+
127+
return revision.Data.ID, nil
128+
}

‎internal/fileupload/client_test.go

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
//go:generate go run github.com/golang/mock/mockgen -package=mocks -destination=./mocks/mock_http_client.go github.com/snyk/cli-extension-os-flows/internal/fileupload/lowlevel SealableClient
2+
package fileupload
3+
4+
import (
5+
"context"
6+
"fmt"
7+
"os"
8+
"path/filepath"
9+
"slices"
10+
"strings"
11+
"testing"
12+
13+
"github.com/google/uuid"
14+
"github.com/stretchr/testify/assert"
15+
"github.com/stretchr/testify/require"
16+
17+
"github.com/snyk/cli-extension-os-flows/internal/fileupload/lowlevel"
18+
)
19+
20+
func Test_UploadRevision(t *testing.T) {
21+
ctx := context.Background()
22+
fakeSealeableClient := lowlevel.NewFakeSealableClient(lowlevel.FakeClientConfig{
23+
Limits: lowlevel.Limits{
24+
FileCountLimit: 2,
25+
FileSizeLimit: 100,
26+
},
27+
})
28+
orgID := uuid.New()
29+
client := NewClient(Config{
30+
OrgID: orgID,
31+
}, WithLowLevelClient(fakeSealeableClient))
32+
33+
t.Run("uploading a shallow directory", func(t *testing.T) {
34+
expectedFiles := []lowlevel.LoadedFile{
35+
{
36+
Path: "file1",
37+
Content: "content1",
38+
},
39+
{
40+
Path: "file2",
41+
Content: "content2",
42+
},
43+
}
44+
dir, cleanup := createDirWithFiles(expectedFiles)
45+
defer cleanup()
46+
47+
revID, err := client.UploadDir(ctx, dir)
48+
49+
require.NoError(t, err)
50+
uploadedFiles, err := fakeSealeableClient.GetSealedRevisionFiles(revID)
51+
require.NoError(t, err)
52+
expectEqualFiles(t, expectedFiles, uploadedFiles)
53+
})
54+
55+
t.Run("uploading a directory with nested files", func(t *testing.T) {
56+
expectedFiles := []lowlevel.LoadedFile{
57+
{
58+
Path: "src/main.go",
59+
Content: "package main\n\nfunc main() {}",
60+
},
61+
{
62+
Path: "src/utils/helper.go",
63+
Content: "package utils\n\nfunc Helper() {}",
64+
},
65+
}
66+
dir, cleanup := createDirWithFiles(expectedFiles)
67+
defer cleanup()
68+
69+
revID, err := client.UploadDir(ctx, dir)
70+
71+
require.NoError(t, err)
72+
uploadedFiles, err := fakeSealeableClient.GetSealedRevisionFiles(revID)
73+
require.NoError(t, err)
74+
expectEqualFiles(t, expectedFiles, uploadedFiles)
75+
})
76+
77+
t.Run("uploading a directory exceeding the file count limit for a single upload", func(t *testing.T) {
78+
expectedFiles := []lowlevel.LoadedFile{
79+
{
80+
Path: "file1.txt",
81+
Content: "root level file",
82+
},
83+
{
84+
Path: "src/main.go",
85+
Content: "package main\n\nfunc main() {}",
86+
},
87+
{
88+
Path: "src/utils/helper.go",
89+
Content: "package utils\n\nfunc Helper() {}",
90+
},
91+
{
92+
Path: "docs/README.md",
93+
Content: "# Project Documentation",
94+
},
95+
}
96+
dir, cleanup := createDirWithFiles(expectedFiles)
97+
defer cleanup()
98+
99+
revID, err := client.UploadDir(ctx, dir)
100+
require.NoError(t, err)
101+
102+
uploadedFiles, err := fakeSealeableClient.GetSealedRevisionFiles(revID)
103+
expectEqualFiles(t, expectedFiles, uploadedFiles)
104+
})
105+
}
106+
107+
func expectEqualFiles(t *testing.T, expectedFiles, uploadedFiles []lowlevel.LoadedFile) {
108+
t.Helper()
109+
110+
require.NotEmpty(t, uploadedFiles)
111+
require.Equal(t, len(expectedFiles), len(uploadedFiles))
112+
113+
slices.SortFunc(expectedFiles, func(fileA, fileB lowlevel.LoadedFile) int {
114+
return strings.Compare(fileA.Path, fileB.Path)
115+
})
116+
117+
slices.SortFunc(uploadedFiles, func(fileA, fileB lowlevel.LoadedFile) int {
118+
return strings.Compare(fileA.Path, fileB.Path)
119+
})
120+
121+
for i := range uploadedFiles {
122+
assert.Equal(t, expectedFiles[i].Path, uploadedFiles[i].Path)
123+
assert.Equal(t, expectedFiles[i].Content, uploadedFiles[i].Content)
124+
}
125+
}
126+
127+
func createDirWithFiles(files []lowlevel.LoadedFile) (dir *os.File, cleanup func()) {
128+
tempDir, err := os.MkdirTemp("", "cliuploadtest*")
129+
if err != nil {
130+
panic(err)
131+
}
132+
133+
dir, err = os.Open(tempDir)
134+
if err != nil {
135+
panic(err)
136+
}
137+
138+
for _, file := range files {
139+
fullPath := filepath.Join(tempDir, file.Path)
140+
141+
parentDir := filepath.Dir(fullPath)
142+
if err := os.MkdirAll(parentDir, 0755); err != nil {
143+
panic(err)
144+
}
145+
146+
f, err := os.Create(fullPath)
147+
if err != nil {
148+
panic(err)
149+
}
150+
151+
defer f.Close()
152+
153+
if _, err := f.WriteString(file.Content); err != nil {
154+
panic(err)
155+
}
156+
}
157+
158+
cleanup = func() {
159+
if dir != nil {
160+
dir.Close()
161+
}
162+
if err := os.RemoveAll(tempDir); err != nil {
163+
fmt.Printf("failed to cleanup temp directory: %s\n", err.Error())
164+
}
165+
}
166+
167+
return dir, cleanup
168+
}

‎internal/fileupload/errors.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package fileupload
2+
3+
import "github.com/snyk/cli-extension-os-flows/internal/fileupload/lowlevel"
4+
5+
// Aliasing lowlevel errors so that they're scoped to the fileupload package as well
6+
7+
type FileSizeLimitError = lowlevel.FileSizeLimitError
8+
9+
type FileCountLimitError = lowlevel.FileCountLimitError
10+
11+
type FileAccessError = lowlevel.FileAccessError
12+
13+
type DirectoryError = lowlevel.DirectoryError
14+
15+
type HTTPError = lowlevel.HTTPError
16+
17+
type MultipartError = lowlevel.MultipartError

0 commit comments

Comments
 (0)