Skip to content

Commit 03d2fcf

Browse files
yoonsioyoonsio
authored
handle post signed request (#702)
* handle post signed request * fix tests * fix post request to use valid content-length
1 parent 4a8d9eb commit 03d2fcf

File tree

5 files changed

+135
-19
lines changed

5 files changed

+135
-19
lines changed

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ require (
1414
github.com/go-logr/logr v1.4.3
1515
github.com/go-task/task/v3 v3.44.1
1616
github.com/joho/godotenv v1.5.1
17-
github.com/livekit/protocol v1.42.1-0.20250929175250-2ddfb3ee7f7e
17+
github.com/livekit/protocol v1.42.1-0.20251008181454-49a136864c2d
1818
github.com/livekit/server-sdk-go/v2 v2.11.3
1919
github.com/moby/buildkit v0.23.2
2020
github.com/moby/patternmatcher v0.6.0

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,8 @@ github.com/livekit/mediatransportutil v0.0.0-20250825135402-7bc31f107ade h1:lpxP
276276
github.com/livekit/mediatransportutil v0.0.0-20250825135402-7bc31f107ade/go.mod h1:mSNtYzSf6iY9xM3UX42VEI+STHvMgHmrYzEHPcdhB8A=
277277
github.com/livekit/protocol v1.42.1-0.20250929175250-2ddfb3ee7f7e h1:yytr+uwFXtJ8UxBV2q3j55jVKg7zslECuGeE4F56NoU=
278278
github.com/livekit/protocol v1.42.1-0.20250929175250-2ddfb3ee7f7e/go.mod h1:vhMS30QoEyH2p34vi6X1eWkC4EMV72ZGZwQb74ajY7A=
279+
github.com/livekit/protocol v1.42.1-0.20251008181454-49a136864c2d h1:ofC+CPiYDZ4LD+RgIJg4rUFLvKnWJqg66bn00FALKLI=
280+
github.com/livekit/protocol v1.42.1-0.20251008181454-49a136864c2d/go.mod h1:vhMS30QoEyH2p34vi6X1eWkC4EMV72ZGZwQb74ajY7A=
279281
github.com/livekit/psrpc v0.7.0 h1:rtfqfjYN06WJYloE/S0nmkJ/Y04x4pxLQLe8kQ4FVHU=
280282
github.com/livekit/psrpc v0.7.0/go.mod h1:AuDC5uOoEjQJEc69v4Li3t77Ocz0e0NdjQEuFfO+vfk=
281283
github.com/livekit/server-sdk-go/v2 v2.11.3 h1:k+YDxo8wPCixRrS9fJHcbtlurlXhVLfyPva5Ne4tVH0=

‎pkg/agentfs/client.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,13 @@ func (c *Client) CreateAgent(
111111
if err != nil {
112112
return nil, err
113113
}
114-
if err := c.uploadAndBuild(ctx, resp.AgentId, resp.PresignedUrl, workingDir, excludeFiles); err != nil {
114+
if err := c.uploadAndBuild(ctx,
115+
resp.AgentId,
116+
resp.PresignedUrl,
117+
resp.PresignedPostRequest,
118+
workingDir,
119+
excludeFiles,
120+
); err != nil {
115121
return nil, err
116122
}
117123
return resp, nil
@@ -135,14 +141,15 @@ func (c *Client) DeployAgent(
135141
if !resp.Success {
136142
return fmt.Errorf("failed to deploy agent: %s", resp.Message)
137143
}
138-
return c.uploadAndBuild(ctx, agentID, resp.PresignedUrl, workingDir, excludeFiles)
144+
return c.uploadAndBuild(ctx, agentID, resp.PresignedUrl, resp.PresignedPostRequest, workingDir, excludeFiles)
139145
}
140146

141147
// uploadAndBuild uploads the source and triggers remote build
142148
func (c *Client) uploadAndBuild(
143149
ctx context.Context,
144150
agentID string,
145151
presignedUrl string,
152+
presignedPostRequest *lkproto.PresignedPostRequest,
146153
workingDir string,
147154
excludeFiles []string,
148155
) error {
@@ -153,6 +160,7 @@ func (c *Client) uploadAndBuild(
153160
if err := UploadTarball(
154161
workingDir,
155162
presignedUrl,
163+
presignedPostRequest,
156164
excludeFiles,
157165
projectType,
158166
); err != nil {

‎pkg/agentfs/tar.go‎

Lines changed: 63 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"compress/gzip"
2121
"fmt"
2222
"io"
23+
"mime/multipart"
2324
"net/http"
2425
"os"
2526
"path"
@@ -29,6 +30,7 @@ import (
2930
"github.com/schollz/progressbar/v3"
3031

3132
"github.com/livekit/livekit-cli/v2/pkg/util"
33+
"github.com/livekit/protocol/livekit"
3234
"github.com/livekit/protocol/logger"
3335

3436
"github.com/moby/patternmatcher"
@@ -51,7 +53,13 @@ var (
5153
}
5254
)
5355

54-
func UploadTarball(directory string, presignedUrl string, excludeFiles []string, projectType ProjectType) error {
56+
func UploadTarball(
57+
directory string,
58+
presignedUrl string,
59+
presignedPostRequest *livekit.PresignedPostRequest,
60+
excludeFiles []string,
61+
projectType ProjectType,
62+
) error {
5563
excludeFiles = append(excludeFiles, defaultExcludePatterns...)
5664

5765
loadExcludeFiles := func(filename string) (bool, string, error) {
@@ -274,25 +282,73 @@ func UploadTarball(directory string, presignedUrl string, excludeFiles []string,
274282
}),
275283
)
276284

277-
req, err := http.NewRequest("PUT", presignedUrl, io.TeeReader(&buffer, uploadProgress))
285+
if presignedPostRequest != nil {
286+
if err := multipartUpload(presignedPostRequest.Url, presignedPostRequest.Values, &buffer); err != nil {
287+
return fmt.Errorf("multipart upload failed: %w", err)
288+
}
289+
} else {
290+
if err := upload(presignedUrl, &buffer, uploadProgress); err != nil {
291+
return fmt.Errorf("upload failed: %w", err)
292+
}
293+
}
294+
295+
return nil
296+
}
297+
298+
func upload(presignedUrl string, buffer *bytes.Buffer, uploadProgress *progressbar.ProgressBar) error {
299+
req, err := http.NewRequest("PUT", presignedUrl, io.TeeReader(buffer, uploadProgress))
278300
if err != nil {
279301
return fmt.Errorf("failed to create request: %w", err)
280302
}
281303
req.Header.Set("Content-Type", "application/gzip")
282304
req.ContentLength = int64(buffer.Len())
283-
284-
client := &http.Client{}
285-
resp, err := client.Do(req)
305+
resp, err := http.DefaultClient.Do(req)
286306
if err != nil {
287307
return fmt.Errorf("failed to upload tarball: %w", err)
288308
}
289309
defer resp.Body.Close()
290-
291310
if resp.StatusCode != http.StatusOK {
292311
body, _ := io.ReadAll(resp.Body)
293312
return fmt.Errorf("failed to upload tarball: %d: %s", resp.StatusCode, body)
294313
}
314+
return nil
315+
}
295316

296-
fmt.Println()
317+
func multipartUpload(presignedURL string, fields map[string]string, buf *bytes.Buffer) error {
318+
var b bytes.Buffer
319+
w := multipart.NewWriter(&b)
320+
fileName, ok := fields["key"]
321+
if !ok {
322+
fileName = "upload.tar.gz"
323+
}
324+
for k, v := range fields {
325+
if err := w.WriteField(k, v); err != nil {
326+
return err
327+
}
328+
}
329+
part, err := w.CreateFormFile("file", fileName)
330+
if err != nil {
331+
return err
332+
}
333+
if _, err := io.Copy(part, buf); err != nil {
334+
return err
335+
}
336+
if err := w.Close(); err != nil {
337+
return err
338+
}
339+
req, err := http.NewRequest("POST", presignedURL, &b)
340+
if err != nil {
341+
return err
342+
}
343+
req.Header.Set("Content-Type", w.FormDataContentType())
344+
resp, err := http.DefaultClient.Do(req)
345+
if err != nil {
346+
return err
347+
}
348+
defer resp.Body.Close()
349+
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent {
350+
respBody, _ := io.ReadAll(resp.Body)
351+
return fmt.Errorf("failed to upload tarball: %d: %s", resp.StatusCode, respBody)
352+
}
297353
return nil
298354
}

‎pkg/agentfs/tar_test.go‎

Lines changed: 59 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"strings"
1414
"testing"
1515

16+
"github.com/livekit/protocol/livekit"
1617
"github.com/stretchr/testify/require"
1718
)
1819

@@ -45,11 +46,60 @@ func TestUploadTarball(t *testing.T) {
4546
f.Close()
4647

4748
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
49+
body, err := io.ReadAll(r.Body)
50+
require.NoError(t, err)
51+
require.NotZero(t, len(body))
52+
require.Greater(t, fileSize, int64(len(body)))
4853
w.WriteHeader(http.StatusOK)
4954
}))
5055
defer mockServer.Close()
5156

52-
err = UploadTarball(tmpDir, mockServer.URL, []string{}, ProjectTypePythonPip)
57+
err = UploadTarball(tmpDir, mockServer.URL, nil, []string{}, ProjectTypePythonPip)
58+
require.NoError(t, err)
59+
}
60+
61+
func TestUploadTarballPost(t *testing.T) {
62+
tmpDir, err := os.MkdirTemp("", "tarball-test-*")
63+
require.NoError(t, err)
64+
defer os.RemoveAll(tmpDir)
65+
66+
subDir := filepath.Join(tmpDir, "subdir")
67+
err = os.MkdirAll(subDir, 0755)
68+
require.NoError(t, err)
69+
70+
normalFiles := []string{
71+
filepath.Join(subDir, "normal1.txt"),
72+
filepath.Join(subDir, "normal2.txt"),
73+
filepath.Join(tmpDir, "root.txt"),
74+
}
75+
for _, path := range normalFiles {
76+
err = os.WriteFile(path, []byte("normal content"), 0644)
77+
require.NoError(t, err)
78+
}
79+
80+
largeFile := filepath.Join(tmpDir, "large.bin")
81+
f, err := os.Create(largeFile)
82+
require.NoError(t, err)
83+
84+
fileSize := int64(1024 * 1024 * 1024) // 1GB
85+
err = f.Truncate(fileSize)
86+
require.NoError(t, err)
87+
f.Close()
88+
89+
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
90+
err := r.ParseMultipartForm(1024 * 1024 * 1024)
91+
require.NoError(t, err)
92+
require.NotZero(t, len(r.MultipartForm.File["file"]))
93+
require.Equal(t, "upload.tar.gz", r.MultipartForm.File["file"][0].Filename)
94+
require.Greater(t, fileSize, r.MultipartForm.File["file"][0].Size)
95+
w.WriteHeader(http.StatusCreated)
96+
}))
97+
defer mockServer.Close()
98+
99+
err = UploadTarball(tmpDir, "", &livekit.PresignedPostRequest{
100+
Url: mockServer.URL,
101+
Values: map[string]string{},
102+
}, []string{}, ProjectTypePythonPip)
53103
require.NoError(t, err)
54104
}
55105

@@ -86,15 +136,15 @@ func TestUploadTarballFilePermissions(t *testing.T) {
86136
// https://learn.microsoft.com/en-us/windows/win32/secauthz/access-control-lists
87137
//
88138
if runtime.GOOS != "windows" {
89-
err = UploadTarball(tmpDir, mockServer.URL, []string{}, ProjectTypePythonPip)
139+
err = UploadTarball(tmpDir, mockServer.URL, nil, []string{}, ProjectTypePythonPip)
90140
require.Error(t, err)
91141
require.Contains(t, err.Error(), "permission denied")
92142
}
93143

94144
err = os.Remove(restrictedFile)
95145
require.NoError(t, err)
96146

97-
err = UploadTarball(tmpDir, mockServer.URL, []string{}, ProjectTypePythonPip)
147+
err = UploadTarball(tmpDir, mockServer.URL, nil, []string{}, ProjectTypePythonPip)
98148
require.NoError(t, err)
99149
}
100150

@@ -189,7 +239,7 @@ func TestUploadTarballDotfiles(t *testing.T) {
189239
}))
190240
defer mockServer.Close()
191241

192-
err = UploadTarball(tmpDir, mockServer.URL, []string{}, ProjectTypePythonPip)
242+
err = UploadTarball(tmpDir, mockServer.URL, nil, []string{}, ProjectTypePythonPip)
193243
require.NoError(t, err)
194244

195245
contents := readTarContents(t, tarBuffer.Bytes())
@@ -273,7 +323,7 @@ func TestUploadTarballDeepDirectories(t *testing.T) {
273323
}))
274324
defer mockServer.Close()
275325

276-
err = UploadTarball(tmpDir, mockServer.URL, []string{}, ProjectTypePythonPip)
326+
err = UploadTarball(tmpDir, mockServer.URL, nil, []string{}, ProjectTypePythonPip)
277327
require.NoError(t, err)
278328

279329
contents := readTarContents(t, tarBuffer.Bytes())
@@ -370,7 +420,7 @@ venv/
370420
}))
371421
defer mockServer.Close()
372422

373-
err = UploadTarball(tmpDir, mockServer.URL, []string{}, ProjectTypePythonPip)
423+
err = UploadTarball(tmpDir, mockServer.URL, nil, []string{}, ProjectTypePythonPip)
374424
require.NoError(t, err)
375425

376426
contents := readTarContents(t, tarBuffer.Bytes())
@@ -474,7 +524,7 @@ func TestUploadTarballWithPipPythonProject(t *testing.T) {
474524
}))
475525
defer mockServer.Close()
476526

477-
err = UploadTarball(tmpDir, mockServer.URL, []string{"**/livekit.toml"}, ProjectTypePythonPip)
527+
err = UploadTarball(tmpDir, mockServer.URL, nil, []string{"**/livekit.toml"}, ProjectTypePythonPip)
478528
require.NoError(t, err)
479529

480530
contents := readTarContents(t, tarBuffer.Bytes())
@@ -567,7 +617,7 @@ func TestUploadTarballWithUvPythonProject(t *testing.T) {
567617
}))
568618
defer mockServer.Close()
569619

570-
err = UploadTarball(tmpDir, mockServer.URL, []string{"**/livekit.toml"}, ProjectTypePythonUV)
620+
err = UploadTarball(tmpDir, mockServer.URL, nil, []string{"**/livekit.toml"}, ProjectTypePythonUV)
571621
require.NoError(t, err)
572622

573623
contents := readTarContents(t, tarBuffer.Bytes())
@@ -667,7 +717,7 @@ func TestUploadTarballWithNodeProject(t *testing.T) {
667717
}))
668718
defer mockServer.Close()
669719

670-
err = UploadTarball(tmpDir, mockServer.URL, []string{"**/livekit.toml"}, ProjectTypeNode)
720+
err = UploadTarball(tmpDir, mockServer.URL, nil, []string{"**/livekit.toml"}, ProjectTypeNode)
671721
require.NoError(t, err)
672722

673723
contents := readTarContents(t, tarBuffer.Bytes())

0 commit comments

Comments
 (0)