Skip to content

Commit ca24bad

Browse files
cardilCopilot
cardil
and
Copilot
authored
🎁 Copy/Delete filters (#45)
* Copy/Delete filters * Tests for filters * Extract all files * Filter only files, not dirs * Checkout test * Delete files by filters * Start checkout with empty dir Co-authored-by: Copilot <[email protected]> --------- Co-authored-by: Copilot <[email protected]>
1 parent 1e5dd62 commit ca24bad

14 files changed

+350
-51
lines changed

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ require (
88
github.com/fatih/color v1.18.0
99
github.com/go-git/go-billy/v5 v5.6.2
1010
github.com/go-git/go-git/v5 v5.13.2
11+
github.com/gobwas/glob v0.2.3
1112
github.com/kelseyhightower/envconfig v1.4.0
1213
github.com/mitchellh/go-homedir v1.1.0
1314
github.com/openshift-knative/hack v0.0.0-20250530124021-bed35e443a23

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
390390
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
391391
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
392392
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
393+
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
394+
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
393395
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
394396
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
395397
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

‎pkg/config/defaults.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package config
22

3-
import "github.com/openshift-knative/hack/pkg/dockerfilegen"
3+
import (
4+
"github.com/openshift-knative/deviate/pkg/files"
5+
"github.com/openshift-knative/hack/pkg/dockerfilegen"
6+
)
47

58
// newDefaults creates a new default configuration.
69
func newDefaults(project Project) Config {
@@ -9,7 +12,14 @@ func newDefaults(project Project) Config {
912
releaseSearch = `^release-(\d+)\.(\d+)$`
1013
)
1114
return Config{
12-
GithubWorkflowsRemovalGlob: "knative-*.y?ml",
15+
DeleteFromUpstream: files.Filters{
16+
Include: []string{
17+
".github/workflows/knative-*.y?ml",
18+
},
19+
},
20+
CopyFromMidstream: files.Filters{
21+
Include: []string{"**"},
22+
},
1323
Branches: Branches{
1424
Main: "main",
1525
ReleaseNext: "release-next",

‎pkg/config/git/checkout.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package git
22

3+
import (
4+
"github.com/openshift-knative/deviate/pkg/files"
5+
)
6+
37
type Checkout interface {
48
As(branch string) error
5-
OntoWorkspace() error
9+
OntoWorkspace(filters files.Filters) error
610
}

‎pkg/config/structure.go‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,23 @@
11
package config
22

3-
import "github.com/openshift-knative/hack/pkg/dockerfilegen"
3+
import (
4+
"github.com/openshift-knative/deviate/pkg/files"
5+
"github.com/openshift-knative/hack/pkg/dockerfilegen"
6+
)
47

58
// Config for a deviate to operate.
69
type Config struct {
7-
Upstream string `json:"upstream" valid:"required"`
8-
Downstream string `json:"downstream" valid:"required"`
9-
DryRun bool `json:"dryRun"`
10-
GithubWorkflowsRemovalGlob string `json:"githubWorkflowsRemovalGlob" valid:"required"`
11-
SyncLabels []string `json:"syncLabels" valid:"required"`
12-
DockerfileGen DockerfileGen `json:"dockerfileGen"`
13-
ResyncReleases `json:"resyncReleases"`
14-
Branches `json:"branches"`
15-
Tags `json:"tags"`
16-
Messages `json:"messages"`
10+
Upstream string `json:"upstream" valid:"required"`
11+
Downstream string `json:"downstream" valid:"required"`
12+
DryRun bool `json:"dryRun"`
13+
CopyFromMidstream files.Filters `json:"copyFromMidstream" valid:"required"`
14+
DeleteFromUpstream files.Filters `json:"deleteFromUpstream" valid:"required"`
15+
SyncLabels []string `json:"syncLabels" valid:"required"`
16+
DockerfileGen DockerfileGen `json:"dockerfileGen"`
17+
ResyncReleases `json:"resyncReleases"`
18+
Branches `json:"branches"`
19+
Tags `json:"tags"`
20+
Messages `json:"messages"`
1721
}
1822

1923
// ResyncReleases holds configuration for resyncing past releases.

‎pkg/files/delete_by_filters.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package files
2+
3+
import (
4+
"io/fs"
5+
"os"
6+
"path/filepath"
7+
"strings"
8+
9+
"github.com/openshift-knative/deviate/pkg/errors"
10+
)
11+
12+
// ErrCantDeleteFiles when cannot delete files.
13+
var ErrCantDeleteFiles = errors.New("cannot delete files")
14+
15+
// DeleteFiles will delete all matching files starting at given root directory.
16+
func (f Filters) DeleteFiles(root string) error {
17+
matcher := f.Matcher()
18+
err := filepath.WalkDir(root, func(pth string, de fs.DirEntry, err error) error {
19+
if err != nil {
20+
return err
21+
}
22+
if de.IsDir() {
23+
// continue
24+
return nil
25+
}
26+
pth = filepath.ToSlash(pth)
27+
relPath := strings.TrimPrefix(pth, root+"/")
28+
if matcher.Matches(relPath) {
29+
return os.Remove(pth)
30+
}
31+
return nil
32+
})
33+
return errors.Wrap(err, ErrCantDeleteFiles)
34+
}

‎pkg/files/delete_by_filters_test.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package files_test
2+
3+
import (
4+
"os"
5+
"path"
6+
"slices"
7+
"testing"
8+
9+
gitv5 "github.com/go-git/go-git/v5"
10+
"github.com/openshift-knative/deviate/pkg/files"
11+
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
13+
)
14+
15+
func TestDeleteByFilters(t *testing.T) {
16+
root := t.TempDir()
17+
paths := []string{
18+
"a.txt",
19+
"a/b.txt",
20+
"a/b/c.md",
21+
"a/b/c.txt",
22+
"a/b/d.md",
23+
"a/b/d.txt",
24+
"a/b/e/f.md",
25+
"a/b/e/f.txt",
26+
"a/c.txt",
27+
"c.txt",
28+
}
29+
for _, file := range paths {
30+
fp := path.Join(root, file)
31+
dir := path.Dir(fp)
32+
require.NoError(t, os.MkdirAll(dir, 0o755))
33+
require.NoError(t, os.WriteFile(fp, []byte("test"), 0o600))
34+
}
35+
36+
filters := files.Filters{
37+
Include: []string{"a/b/**"},
38+
Exclude: []string{"**.txt"},
39+
}
40+
require.NoError(t, filters.DeleteFiles(root))
41+
42+
repo, err := gitv5.PlainInit(root, false)
43+
require.NoError(t, err)
44+
wt, err := repo.Worktree()
45+
require.NoError(t, err)
46+
47+
st, serr := wt.Status()
48+
require.NoError(t, serr)
49+
got := make([]string, 0, len(st))
50+
for f := range st {
51+
got = append(got, f)
52+
}
53+
slices.Sort(got)
54+
want := []string{
55+
"a.txt",
56+
"a/b.txt",
57+
"a/b/c.txt",
58+
"a/b/d.txt",
59+
"a/b/e/f.txt",
60+
"a/c.txt",
61+
"c.txt",
62+
}
63+
assert.Equal(t, want, got)
64+
}

‎pkg/files/filters.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package files
2+
3+
import "github.com/gobwas/glob"
4+
5+
// Filters represents what files to include, and which to exclude from copying operations.
6+
type Filters struct {
7+
Include []string `json:"include"`
8+
Exclude []string `json:"exclude"`
9+
}
10+
11+
func (f Filters) Matcher() Matcher {
12+
m := Matcher{
13+
Include: make([]glob.Glob, 0, len(f.Include)),
14+
Exclude: make([]glob.Glob, 0, len(f.Exclude)),
15+
}
16+
separators := []rune{'/'}
17+
for _, p := range f.Include {
18+
g := glob.MustCompile(p, separators...)
19+
m.Include = append(m.Include, g)
20+
}
21+
for _, p := range f.Exclude {
22+
g := glob.MustCompile(p, separators...)
23+
m.Exclude = append(m.Exclude, g)
24+
}
25+
return m
26+
}
27+
28+
type Matcher struct {
29+
Include []glob.Glob
30+
Exclude []glob.Glob
31+
}
32+
33+
func (m Matcher) Matches(pth string) bool {
34+
result := false
35+
for _, pattern := range m.Include {
36+
if pattern.Match(pth) {
37+
result = true
38+
break
39+
}
40+
}
41+
if !result {
42+
return false
43+
}
44+
for _, pattern := range m.Exclude {
45+
if pattern.Match(pth) {
46+
return false
47+
}
48+
}
49+
return true
50+
}

‎pkg/files/filters_test.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package files_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/openshift-knative/deviate/pkg/files"
7+
"github.com/stretchr/testify/assert"
8+
)
9+
10+
func TestFilters_Match(t *testing.T) {
11+
filelist := []string{
12+
"a/b/c.txt",
13+
"a/d.txt",
14+
"a/b/d.md",
15+
"a/b/d.txt",
16+
"a.txt",
17+
"b.md",
18+
}
19+
tcs := []testFiltersMatchCases{{
20+
name: "all md files",
21+
Filters: files.Filters{
22+
Include: []string{"**/*.md", "*.md"},
23+
},
24+
want: []string{
25+
"a/b/d.md",
26+
"b.md",
27+
},
28+
}, {
29+
name: "root level md files",
30+
Filters: files.Filters{
31+
Include: []string{"*.md"},
32+
},
33+
want: []string{
34+
"b.md",
35+
},
36+
}, {
37+
"just one txt",
38+
files.Filters{
39+
Include: []string{"**.txt"},
40+
Exclude: []string{"a/b**", "a.txt"},
41+
},
42+
[]string{
43+
"a/d.txt",
44+
},
45+
}}
46+
for _, tc := range tcs {
47+
t.Run(tc.name, func(t *testing.T) {
48+
matcher := tc.Matcher()
49+
got := make([]string, 0, len(tc.want))
50+
for _, f := range filelist {
51+
if matcher.Matches(f) {
52+
got = append(got, f)
53+
}
54+
}
55+
56+
assert.Equal(t, tc.want, got)
57+
})
58+
}
59+
}
60+
61+
type testFiltersMatchCases struct {
62+
name string
63+
files.Filters
64+
want []string
65+
}

‎pkg/git/checkout.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/go-git/go-git/v5/storage/memory"
1717
"github.com/openshift-knative/deviate/pkg/config/git"
1818
"github.com/openshift-knative/deviate/pkg/errors"
19+
"github.com/openshift-knative/deviate/pkg/files"
1920
)
2021

2122
func (r Repository) Checkout(remote git.Remote, branch string) git.Checkout { //nolint:ireturn
@@ -86,7 +87,7 @@ func (o onGoingCheckout) As(branch string) error {
8687
return nil
8788
}
8889

89-
func (o onGoingCheckout) OntoWorkspace() error {
90+
func (o onGoingCheckout) OntoWorkspace(filters files.Filters) error {
9091
coOpts := &gitv5.CloneOptions{
9192
URL: "file://" + o.repo.Project.Path,
9293
ReferenceName: plumbing.NewBranchReferenceName(o.branch),
@@ -98,23 +99,27 @@ func (o onGoingCheckout) OntoWorkspace() error {
9899
if err != nil {
99100
return errors.Wrap(err, ErrLocalOperationFailed)
100101
}
101-
return o.applyTree(wt, "/")
102+
matcher := filters.Matcher()
103+
return o.applyTree(wt, "", matcher)
102104
}
103105

104-
func (o onGoingCheckout) applyTree(fs billy.Filesystem, dir string) error {
105-
files, err := fs.ReadDir(dir)
106+
func (o onGoingCheckout) applyTree(fs billy.Filesystem, dir string, matcher files.Matcher) error {
107+
infos, err := fs.ReadDir(dir)
106108
if err != nil {
107109
return errors.Wrap(err, ErrLocalOperationFailed)
108110
}
109-
for _, f := range files {
111+
for _, f := range infos {
110112
fp := path.Join(dir, f.Name())
111113
if f.IsDir() {
112-
err = o.applyTree(fs, fp)
114+
err = o.applyTree(fs, fp, matcher)
113115
if err != nil {
114116
return err
115117
}
116118
continue
117119
}
120+
if !matcher.Matches(fp) {
121+
continue
122+
}
118123
err = o.applyFile(fs, fp, f.Mode())
119124
if err != nil {
120125
return err

0 commit comments

Comments
 (0)