Skip to content

Commit 2f57f07

Browse files
real-danmreal-danm
committed
fix dockerignore for projects without a dockerignore and loading an existing dockerignore
1 parent cb33eeb commit 2f57f07

File tree

7 files changed

+514
-110
lines changed

7 files changed

+514
-110
lines changed

‎cmd/lk/agent.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ func createAgent(ctx context.Context, cmd *cli.Command) error {
432432
return err
433433
}
434434

435-
err = agentfs.UploadTarball(workingDir, resp.PresignedUrl, []string{config.LiveKitTOMLFile})
435+
err = agentfs.UploadTarball(workingDir, resp.PresignedUrl, []string{fmt.Sprintf("**/%s", config.LiveKitTOMLFile)}, projectType)
436436
if err != nil {
437437
return err
438438
}
@@ -595,7 +595,7 @@ func deployAgent(ctx context.Context, cmd *cli.Command) error {
595595
}
596596

597597
presignedUrl := resp.PresignedUrl
598-
err = agentfs.UploadTarball(workingDir, presignedUrl, []string{config.LiveKitTOMLFile})
598+
err = agentfs.UploadTarball(workingDir, presignedUrl, []string{fmt.Sprintf("**/%s", config.LiveKitTOMLFile)}, projectType)
599599
if err != nil {
600600
return err
601601
}

‎pkg/agentfs/docker.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,12 +73,12 @@ func GenerateDockerArtifacts(dir string, projectType ProjectType, settingsMap ma
7373
return nil, nil, fmt.Errorf("unable to fetch client settings from server, please try again later")
7474
}
7575

76-
dockerfileContent, err := fs.ReadFile("examples/" + string(projectType) + ".Dockerfile")
76+
dockerfileContent, err := fs.ReadFile(filepath.Join("examples", string(projectType)+".Dockerfile"))
7777
if err != nil {
7878
return nil, nil, err
7979
}
8080

81-
dockerIgnoreContent, err := fs.ReadFile("examples/" + string(projectType) + ".dockerignore")
81+
dockerIgnoreContent, err := fs.ReadFile(filepath.Join("examples", string(projectType)+".dockerignore"))
8282
if err != nil {
8383
return nil, nil, err
8484
}

‎pkg/agentfs/examples/node.dockerignore‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
11
# Node.js dependencies
2-
node_modules
3-
npm-debug.log
4-
yarn-error.log
5-
pnpm-debug.log
2+
**/node_modules
3+
**/npm-debug.log
4+
**/yarn-error.log
5+
**/pnpm-debug.log
66

77
# Build outputs
8-
dist
9-
build
10-
coverage
8+
**/dist
9+
**/build
10+
**/coverage
1111

1212
# Local environment & config files
13-
.env
14-
.env.local
13+
**/.env
14+
**/.env.local
1515
.DS_Store
1616

1717
# Logs & temp files
18-
*.log
19-
*.gz
20-
*.tgz
21-
.tmp
22-
.cache
18+
**/*.log
19+
**/*.gz
20+
**/*.tgz
21+
**/.tmp
22+
**/.cache
2323

2424
# Docker artifacts
2525
Dockerfile*

‎pkg/agentfs/examples/python.pip.dockerignore‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,32 @@
11
# Python bytecode and artifacts
2-
__pycache__/
3-
*.py[cod]
4-
*.pyo
5-
*.pyd
6-
*.egg-info/
7-
dist/
8-
build/
2+
**/__pycache__/
3+
**/*.py[cod]
4+
**/*.pyo
5+
**/*.pyd
6+
**/*.egg-info/
7+
**/dist/
8+
**/build/
99

1010
# Virtual environments
11-
.venv/
12-
venv/
11+
**/.venv/
12+
**/venv/
1313

1414
# Caches and test output
15-
.cache/
16-
.pytest_cache/
17-
.ruff_cache/
18-
coverage/
15+
**/.cache/
16+
**/.pytest_cache/
17+
**/.ruff_cache/
18+
**/coverage/
1919

2020
# Logs and temp files
21-
*.log
22-
*.gz
23-
*.tgz
24-
.tmp
25-
.cache
21+
**/*.log
22+
**/*.gz
23+
**/*.tgz
24+
**/.tmp
25+
**/.cache
2626

2727
# Environment variables
28-
.env
29-
.env.*
28+
**/.env
29+
**/.env.*
3030

3131
# VCS, editor, OS
3232
.git
@@ -45,4 +45,4 @@ LICENSE
4545
test/
4646
tests/
4747
eval/
48-
evals/
48+
evals/

‎pkg/agentfs/examples/python.uv.dockerignore‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,32 @@
11
# Python bytecode and artifacts
2-
__pycache__/
3-
*.py[cod]
4-
*.pyo
5-
*.pyd
6-
*.egg-info/
7-
dist/
8-
build/
2+
**/__pycache__/
3+
**/*.py[cod]
4+
**/*.pyo
5+
**/*.pyd
6+
**/*.egg-info/
7+
**/dist/
8+
**/build/
99

1010
# Virtual environments
11-
.venv/
12-
venv/
11+
**/.venv/
12+
**/venv/
1313

1414
# Caches and test output
15-
.cache/
16-
.pytest_cache/
17-
.ruff_cache/
18-
coverage/
15+
**/.cache/
16+
**/.pytest_cache/
17+
**/.ruff_cache/
18+
**/coverage/
1919

2020
# Logs and temp files
21-
*.log
22-
*.gz
23-
*.tgz
24-
.tmp
25-
.cache
21+
**/*.log
22+
**/*.gz
23+
**/*.tgz
24+
**/.tmp
25+
**/.cache
2626

2727
# Environment variables
28-
.env
29-
.env.*
28+
**/.env
29+
**/.env.*
3030

3131
# VCS, editor, OS
3232
.git
@@ -45,4 +45,4 @@ LICENSE
4545
test/
4646
tests/
4747
eval/
48-
evals/
48+
evals/

‎pkg/agentfs/tar.go‎

Lines changed: 55 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ import (
2929

3030
"github.com/livekit/livekit-cli/v2/pkg/util"
3131
"github.com/livekit/protocol/logger"
32+
33+
"github.com/moby/patternmatcher"
3234
)
3335

3436
var (
@@ -48,26 +50,65 @@ var (
4850
}
4951
)
5052

51-
func UploadTarball(directory string, presignedUrl string, excludeFiles []string) error {
52-
excludeFiles = append(defaultExcludePatterns, excludeFiles...)
53+
func UploadTarball(directory string, presignedUrl string, excludeFiles []string, projectType ProjectType) error {
54+
excludeFiles = append(excludeFiles, defaultExcludePatterns...)
5355

54-
for _, exclude := range ignoreFilePatterns {
55-
ignore := filepath.Join(directory, exclude)
56-
if _, err := os.Stat(ignore); err == nil {
57-
content, err := os.ReadFile(ignore)
56+
loadExcludeFiles := func(filename string) (bool, string, error) {
57+
if _, err := os.Stat(filename); err == nil {
58+
content, err := os.ReadFile(filename)
5859
if err != nil {
59-
return fmt.Errorf("failed to read %s: %w", ignore, err)
60+
return false, "", err
6061
}
61-
excludeFiles = append(excludeFiles, strings.Split(string(content), "\n")...)
62+
return true, string(content), nil
6263
}
64+
return false, "", nil
65+
}
66+
67+
foundDockerIgnore := false
68+
for _, exclude := range ignoreFilePatterns {
69+
found, content, err := loadExcludeFiles(filepath.Join(directory, exclude))
70+
if err != nil {
71+
logger.Debugw("failed to load exclude file", "filename", exclude, "error", err)
72+
continue
73+
}
74+
if exclude == ".dockerignore" && found {
75+
foundDockerIgnore = true
76+
}
77+
excludeFiles = append(excludeFiles, strings.Split(content, "\n")...)
78+
}
79+
80+
// need to ensure we use a dockerignore file
81+
// if we fail to load a dockerignore file, we have to exit
82+
if !foundDockerIgnore {
83+
dockerIgnoreContent, err := fs.ReadFile(filepath.Join("examples", string(projectType)+".dockerignore"))
84+
if err != nil {
85+
return fmt.Errorf("failed to load exclude file %s: %w", string(projectType), err)
86+
}
87+
excludeFiles = append(excludeFiles, strings.Split(string(dockerIgnoreContent), "\n")...)
88+
}
89+
90+
matcher, err := patternmatcher.New(excludeFiles)
91+
if err != nil {
92+
return fmt.Errorf("failed to create pattern matcher: %w", err)
6393
}
6494

6595
for i, exclude := range excludeFiles {
6696
excludeFiles[i] = strings.TrimSpace(exclude)
6797
}
6898

99+
checkFilesToInclude := func(path string, info os.FileInfo) bool {
100+
if ignored, err := matcher.MatchesOrParentMatches(path); ignored {
101+
return false
102+
} else if err != nil {
103+
return false
104+
}
105+
return true
106+
}
107+
108+
// we walk the directory first to calculate the total size of the tarball
109+
// this lets the progress bar show the correct progress
69110
var totalSize int64
70-
err := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
111+
err = filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
71112
if err != nil {
72113
return err
73114
}
@@ -77,22 +118,8 @@ func UploadTarball(directory string, presignedUrl string, excludeFiles []string)
77118
return nil
78119
}
79120

80-
for _, exclude := range excludeFiles {
81-
if exclude == "" || strings.Contains(exclude, "Dockerfile") {
82-
continue
83-
}
84-
if info.IsDir() {
85-
if strings.HasPrefix(relPath, exclude+"/") || strings.HasPrefix(relPath, exclude) {
86-
return filepath.SkipDir
87-
}
88-
}
89-
matched, err := filepath.Match(exclude, relPath)
90-
if err != nil {
91-
return nil
92-
}
93-
if matched {
94-
return nil
95-
}
121+
if !checkFilesToInclude(relPath, info) {
122+
return nil
96123
}
97124

98125
if !info.IsDir() && info.Mode().IsRegular() {
@@ -134,26 +161,9 @@ func UploadTarball(directory string, presignedUrl string, excludeFiles []string)
134161
return fmt.Errorf("failed to calculate relative path for %s: %w", path, err)
135162
}
136163

137-
for _, exclude := range excludeFiles {
138-
if exclude == "" || strings.Contains(exclude, "Dockerfile") {
139-
continue
140-
}
141-
142-
if info.IsDir() {
143-
if strings.HasPrefix(relPath, exclude+"/") || strings.HasPrefix(relPath, exclude) {
144-
logger.Debugw("excluding directory from tarball", "path", path)
145-
return filepath.SkipDir
146-
}
147-
}
148-
149-
matched, err := filepath.Match(exclude, relPath)
150-
if err != nil {
151-
return nil
152-
}
153-
if matched {
154-
logger.Debugw("excluding file from tarball", "path", path)
155-
return nil
156-
}
164+
if !checkFilesToInclude(relPath, info) {
165+
logger.Debugw("excluding file from tarball", "path", path)
166+
return nil
157167
}
158168

159169
// Follow symlinks and include the actual file contents

0 commit comments

Comments
 (0)