Skip to content

Commit 785f858

Browse files
committed
Add exclude patterns to copy operations
Signed-off-by: Stefan Prodan <[email protected]>
1 parent fd273f1 commit 785f858

File tree

11 files changed

+533
-252
lines changed

11 files changed

+533
-252
lines changed

api/v1beta1/artifactgenerator_types.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,12 @@ type CopyOperation struct {
137137
// +kubebuilder:validation:MaxLength=1024
138138
// +required
139139
To string `json:"to"`
140+
141+
// Exclude specifies a list of glob patterns to exclude
142+
// files and dirs matched by the 'From' field.
143+
// +kubebuilder:validation:MaxItems=100
144+
// +optional
145+
Exclude []string `json:"exclude,omitempty"`
140146
}
141147

142148
// ArtifactGeneratorStatus defines the observed state of ArtifactGenerator.

api/v1beta1/zz_generated.deepcopy.go

Lines changed: 8 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

config/crd/bases/source.extensions.fluxcd.io_artifactgenerators.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,14 @@ spec:
5353
being overwritten by later copy operations.
5454
items:
5555
properties:
56+
exclude:
57+
description: |-
58+
Exclude specifies a list of glob patterns to exclude
59+
files and dirs matched by the 'From' field.
60+
items:
61+
type: string
62+
maxItems: 100
63+
type: array
5664
from:
5765
description: |-
5866
From specifies the source (by alias) and the glob pattern to match files.

config/samples/source_v1beta1_artifactgenerator.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,6 @@ spec:
3131
copy:
3232
- from: "@chart/**"
3333
to: "@artifact/"
34+
exclude: ["*.md"]
3435
- from: "@repo/charts/podinfo/values-prod.yaml"
3536
to: "@artifact/podinfo/values.yaml"

config/testdata/composition/generators.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ spec:
1717
copy:
1818
- from: "@chart/**"
1919
to: "@artifact/"
20+
exclude: ["*.md"]
2021
- from: "@repo/charts/podinfo/values-prod.yaml"
2122
to: "@artifact/podinfo/values.yaml"

docs/spec/v1beta1/artifactgenerators.md

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ spec:
3333
copy:
3434
- from: "@backend/deploy/**"
3535
to: "@artifact/my-app/backend/"
36-
- from: "@frontend/deploy/**/*.yaml"
36+
- from: "@frontend/deploy/*.yaml"
3737
to: "@artifact/my-app/frontend/"
3838
- from: "@config/envs/prod/configmap.yaml"
3939
to: "@artifact/my-app/env.yaml"
@@ -244,9 +244,10 @@ spec:
244244
revision: "@backend"
245245
originRevision: "@frontend"
246246
copy:
247-
- from: "@backend/deploy/*.yaml"
247+
- from: "@backend/deploy/**"
248248
to: "@artifact/backend/"
249-
- from: "@frontend/manifests/**"
249+
exclude: ["**/charts/**"]
250+
- from: "@frontend/manifests/*.yaml"
250251
to: "@artifact/frontend/"
251252
- from: "@config/envs/prod/configmap.yaml"
252253
to: "@artifact/env.yaml"
@@ -260,6 +261,8 @@ Each copy operation specifies how to copy files from sources into the generated
260261
a source and `pattern` is a glob pattern or a specific file/directory path within that source.
261262
- `to`: Destination path in the format `@artifact/path` where `artifact` is
262263
the root of the generated artifact and `path` is the relative path to a file or directory.
264+
- `exclude` (optional): A list of glob patterns to filter out from the source selection.
265+
Any file matched by `from` that also matches an exclude pattern will be ignored.
263266

264267
Copy operations use `cp`-like semantics:
265268

@@ -270,11 +273,11 @@ Copy operations use `cp`-like semantics:
270273
Examples of copy operations:
271274

272275
```yaml
273-
# Copy file to specific path (like `cp source/config.yaml artifact/apps/app.yaml`)
276+
# Copy file to specific path - (like `cp source/config.yaml artifact/apps/app.yaml`)
274277
- from: "@source/config.yaml"
275278
to: "@artifact/apps/app.yaml" # Creates apps/app.yaml file
276279

277-
# Copy file to directory (like `cp source/config.yaml artifact/apps/`)
280+
# Copy file to directory - (like `cp source/config.yaml artifact/apps/`)
278281
- from: "@source/config.yaml"
279282
to: "@artifact/apps/" # Creates apps/config.yaml
280283

@@ -289,6 +292,9 @@ Examples of copy operations:
289292
# Copy files and dirs recursively - (like `cp -r source/configs/** artifact/apps/`)
290293
- from: "@source/configs/**" # All files and sub-dirs under configs/
291294
to: "@artifact/apps/" # Creates apps/file1.yaml, apps/subdir/file2.yaml
295+
exclude:
296+
- "*.md" # Excludes all .md files
297+
- "**/testdata/**" # Excludes all files under any testdata/ dir
292298
```
293299
294300
## Working with ArtifactGenerators

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ go 1.25.0
77
replace github.com/opencontainers/go-digest => github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be
88

99
require (
10+
github.com/bmatcuk/doublestar/v4 v4.9.1
1011
github.com/fluxcd/pkg/apis/meta v1.21.0
1112
github.com/fluxcd/pkg/artifact v0.3.0
1213
github.com/fluxcd/pkg/http/fetch v0.19.0

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
1212
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
1313
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
1414
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
15+
github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE=
16+
github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
1517
github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
1618
github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
1719
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=

internal/builder/builder.go

Lines changed: 73 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"path/filepath"
2525
"strings"
2626

27+
"github.com/bmatcuk/doublestar/v4"
2728
sourcev1 "github.com/fluxcd/source-controller/api/v1"
2829
"golang.org/x/mod/sumdb/dirhash"
2930
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -129,7 +130,7 @@ func applyCopyOperations(ctx context.Context,
129130
// applyCopyOperation applies a single copy operation from the sources to the staging directory.
130131
// This function implements cp-like semantics by first analyzing the source pattern to determine
131132
// if it's a glob, direct file/directory reference, or wildcard pattern, then making copy decisions
132-
// based on the actual source types found.
133+
// based on the actual source types found. Files matching exclude patterns are filtered out.
133134
func applyCopyOperation(ctx context.Context,
134135
op swapi.CopyOperation,
135136
sources map[string]string,
@@ -149,6 +150,12 @@ func applyCopyOperation(ctx context.Context,
149150
return fmt.Errorf("source alias '%s' not found", srcAlias)
150151
}
151152

153+
for _, pattern := range op.Exclude {
154+
if _, err := doublestar.Match(pattern, "."); err != nil {
155+
return fmt.Errorf("invalid exclude pattern '%s'", pattern)
156+
}
157+
}
158+
152159
// Create secure roots for file operations
153160
srcRoot, err := os.OpenRoot(srcDir)
154161
if err != nil {
@@ -168,7 +175,7 @@ func applyCopyOperation(ctx context.Context,
168175

169176
if !isGlobPattern {
170177
// Direct path reference - check what it actually is first (cp-like behavior)
171-
return applySingleSourceCopy(ctx, srcRoot, srcPattern, stagingRoot, destRelPath, destEndsWithSlash)
178+
return applySingleSourceCopy(ctx, srcRoot, srcPattern, stagingRoot, destRelPath, destEndsWithSlash, op.Exclude)
172179
}
173180

174181
// Glob pattern - find all matches and copy each
@@ -181,15 +188,27 @@ func applyCopyOperation(ctx context.Context,
181188
return fmt.Errorf("no files match pattern '%s' in source '%s'", srcPattern, srcAlias)
182189
}
183190

184-
// For glob patterns, destination should be a directory (like cp *.txt dest/)
191+
// Filter out excluded files
192+
filteredMatches := make([]string, 0, len(matches))
185193
for _, match := range matches {
194+
if !shouldExclude(match, op.Exclude) {
195+
filteredMatches = append(filteredMatches, match)
196+
}
197+
}
198+
199+
if len(filteredMatches) == 0 {
200+
return fmt.Errorf("all files matching pattern '%s' in source '%s' were excluded", srcPattern, srcAlias)
201+
}
202+
203+
// For glob patterns, destination should be a directory (like cp *.txt dest/)
204+
for _, match := range filteredMatches {
186205
if err := ctx.Err(); err != nil {
187206
return err
188207
}
189208

190209
// Calculate destination path based on glob pattern type
191210
destFile := calculateGlobDestination(srcPattern, match, destRelPath)
192-
if err := copyFileWithRoots(ctx, srcRoot, match, stagingRoot, destFile); err != nil {
211+
if err := copyFileWithRoots(ctx, srcRoot, match, stagingRoot, destFile, op.Exclude); err != nil {
193212
return fmt.Errorf("failed to copy file '%s' to '%s': %w", match, destFile, err)
194213
}
195214
}
@@ -204,7 +223,8 @@ func applySingleSourceCopy(ctx context.Context,
204223
srcPath string,
205224
stagingRoot *os.Root,
206225
destPath string,
207-
destEndsWithSlash bool) error {
226+
destEndsWithSlash bool,
227+
excludePatterns []string) error {
208228
// Clean the source path to handle trailing slashes
209229
srcPath = filepath.Clean(srcPath)
210230

@@ -218,9 +238,9 @@ func applySingleSourceCopy(ctx context.Context,
218238
}
219239

220240
if srcInfo.IsDir() {
221-
return applySingleDirectoryCopy(ctx, srcRoot, srcPath, stagingRoot, destPath)
241+
return applySingleDirectoryCopy(ctx, srcRoot, srcPath, stagingRoot, destPath, excludePatterns)
222242
} else {
223-
return applySingleFileCopy(ctx, srcRoot, srcPath, stagingRoot, destPath, destEndsWithSlash)
243+
return applySingleFileCopy(ctx, srcRoot, srcPath, stagingRoot, destPath, destEndsWithSlash, excludePatterns)
224244
}
225245
}
226246

@@ -232,7 +252,12 @@ func applySingleFileCopy(ctx context.Context,
232252
srcPath string,
233253
stagingRoot *os.Root,
234254
destPath string,
235-
destEndsWithSlash bool) error {
255+
destEndsWithSlash bool,
256+
excludePatterns []string) error {
257+
// Check if the file should be excluded
258+
if shouldExclude(srcPath, excludePatterns) {
259+
return nil // Skip excluded file
260+
}
236261
var finalDestPath string
237262

238263
if destEndsWithSlash {
@@ -250,7 +275,7 @@ func applySingleFileCopy(ctx context.Context,
250275
}
251276
}
252277

253-
return copyFileWithRoots(ctx, srcRoot, srcPath, stagingRoot, finalDestPath)
278+
return copyFileWithRoots(ctx, srcRoot, srcPath, stagingRoot, finalDestPath, excludePatterns)
254279
}
255280

256281
// applySingleDirectoryCopy handles copying a single directory using cp-like semantics.
@@ -260,11 +285,12 @@ func applySingleDirectoryCopy(ctx context.Context,
260285
srcRoot *os.Root,
261286
srcPath string,
262287
stagingRoot *os.Root,
263-
destPath string) error {
288+
destPath string,
289+
excludePatterns []string) error {
264290
srcDirName := filepath.Base(srcPath)
265291
finalDestPath := filepath.Join(destPath, srcDirName)
266292

267-
return copyFileWithRoots(ctx, srcRoot, srcPath, stagingRoot, finalDestPath)
293+
return copyFileWithRoots(ctx, srcRoot, srcPath, stagingRoot, finalDestPath, excludePatterns)
268294
}
269295

270296
// containsGlobChars returns true if the path contains glob metacharacters
@@ -318,19 +344,21 @@ func parseCopyDestinationRelative(to string) (string, error) {
318344
return strings.TrimPrefix(to, "@artifact/"), nil
319345
}
320346

321-
// copyFileWithRoots copies a file from srcRoot to stagingRoot os.Root.
347+
// copyFileWithRoots copies a file from srcRoot to stagingRoot os.Root,
348+
// excluding files matching exclude patterns.
322349
func copyFileWithRoots(ctx context.Context,
323350
srcRoot *os.Root,
324351
srcPath string,
325352
stagingRoot *os.Root,
326-
destPath string) error {
353+
destPath string,
354+
excludePatterns []string) error {
327355
srcInfo, err := srcRoot.Stat(srcPath)
328356
if err != nil {
329357
return err
330358
}
331359

332360
if srcInfo.IsDir() {
333-
return copyDirWithRoots(ctx, srcRoot, srcPath, stagingRoot, destPath)
361+
return copyDirWithRoots(ctx, srcRoot, srcPath, stagingRoot, destPath, excludePatterns)
334362
}
335363

336364
return copyRegularFileWithRoots(ctx, srcRoot, srcPath, stagingRoot, destPath)
@@ -383,12 +411,14 @@ func copyRegularFileWithRoots(ctx context.Context,
383411
return destFile.Chmod(srcInfo.Mode())
384412
}
385413

386-
// copyDirWithRoots copies a directory recursively using os.Root.
414+
// copyDirWithRoots copies a directory recursively using os.Root,
415+
// skipping files and sub-dirs matching exclude patterns.
387416
func copyDirWithRoots(ctx context.Context,
388417
srcRoot *os.Root,
389418
srcPath string,
390419
stagingRoot *os.Root,
391-
destPath string) error {
420+
destPath string,
421+
excludePatterns []string) error {
392422
return fs.WalkDir(srcRoot.FS(), srcPath, func(path string, d fs.DirEntry, err error) error {
393423
if err := ctx.Err(); err != nil {
394424
return err
@@ -410,6 +440,16 @@ func copyDirWithRoots(ctx context.Context,
410440
return createDirRecursive(stagingRoot, destPath)
411441
}
412442

443+
// Check if this path should be excluded
444+
if shouldExclude(relPath, excludePatterns) {
445+
if d.IsDir() {
446+
// Skip entire directory
447+
return fs.SkipDir
448+
}
449+
// Skip file
450+
return nil
451+
}
452+
413453
destFilePath := filepath.Join(destPath, relPath)
414454

415455
if d.IsDir() {
@@ -450,6 +490,23 @@ func createDirRecursive(root *os.Root, path string) error {
450490
return err
451491
}
452492

493+
// shouldExclude checks if a path matches any of the exclude patterns.
494+
func shouldExclude(filePath string, excludePatterns []string) bool {
495+
if len(excludePatterns) == 0 {
496+
return false
497+
}
498+
499+
for _, pattern := range excludePatterns {
500+
// We validate the patterns when parsing the copy operation,
501+
// so it's safe to use MatchUnvalidated here.
502+
if doublestar.MatchUnvalidated(pattern, filePath) {
503+
return true
504+
}
505+
}
506+
507+
return false
508+
}
509+
453510
// MkdirTempAbs creates a tmp dir and returns the absolute path to the dir.
454511
// This is required since certain OSes like MacOS create temporary files in
455512
// e.g. `/private/var`, to which `/var` is a symlink.

0 commit comments

Comments
 (0)