diff --git a/go.mod b/go.mod index a891b1b..bdfd00e 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/fatih/color v1.18.0 github.com/go-git/go-billy/v5 v5.6.2 github.com/go-git/go-git/v5 v5.13.2 + github.com/gobwas/glob v0.2.3 github.com/kelseyhightower/envconfig v1.4.0 github.com/mitchellh/go-homedir v1.1.0 github.com/openshift-knative/hack v0.0.0-20250530124021-bed35e443a23 diff --git a/go.sum b/go.sum index 0dc38f0..59e8299 100644 --- a/go.sum +++ b/go.sum @@ -390,6 +390,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 4db6201..658310a 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -1,6 +1,9 @@ package config -import "github.com/openshift-knative/hack/pkg/dockerfilegen" +import ( + "github.com/openshift-knative/deviate/pkg/files" + "github.com/openshift-knative/hack/pkg/dockerfilegen" +) // newDefaults creates a new default configuration. func newDefaults(project Project) Config { @@ -9,7 +12,14 @@ func newDefaults(project Project) Config { releaseSearch = `^release-(\d+)\.(\d+)$` ) return Config{ - GithubWorkflowsRemovalGlob: "knative-*.y?ml", + DeleteFromUpstream: files.Filters{ + Include: []string{ + ".github/workflows/knative-*.y?ml", + }, + }, + CopyFromMidstream: files.Filters{ + Include: []string{"**"}, + }, Branches: Branches{ Main: "main", ReleaseNext: "release-next", diff --git a/pkg/config/git/checkout.go b/pkg/config/git/checkout.go index d4e4fd1..b8064a1 100644 --- a/pkg/config/git/checkout.go +++ b/pkg/config/git/checkout.go @@ -1,6 +1,10 @@ package git +import ( + "github.com/openshift-knative/deviate/pkg/files" +) + type Checkout interface { As(branch string) error - OntoWorkspace() error + OntoWorkspace(filters files.Filters) error } diff --git a/pkg/config/structure.go b/pkg/config/structure.go index a9aea2b..0393790 100644 --- a/pkg/config/structure.go +++ b/pkg/config/structure.go @@ -1,19 +1,23 @@ package config -import "github.com/openshift-knative/hack/pkg/dockerfilegen" +import ( + "github.com/openshift-knative/deviate/pkg/files" + "github.com/openshift-knative/hack/pkg/dockerfilegen" +) // Config for a deviate to operate. type Config struct { - Upstream string `json:"upstream" valid:"required"` - Downstream string `json:"downstream" valid:"required"` - DryRun bool `json:"dryRun"` - GithubWorkflowsRemovalGlob string `json:"githubWorkflowsRemovalGlob" valid:"required"` - SyncLabels []string `json:"syncLabels" valid:"required"` - DockerfileGen DockerfileGen `json:"dockerfileGen"` - ResyncReleases `json:"resyncReleases"` - Branches `json:"branches"` - Tags `json:"tags"` - Messages `json:"messages"` + Upstream string `json:"upstream" valid:"required"` + Downstream string `json:"downstream" valid:"required"` + DryRun bool `json:"dryRun"` + CopyFromMidstream files.Filters `json:"copyFromMidstream" valid:"required"` + DeleteFromUpstream files.Filters `json:"deleteFromUpstream" valid:"required"` + SyncLabels []string `json:"syncLabels" valid:"required"` + DockerfileGen DockerfileGen `json:"dockerfileGen"` + ResyncReleases `json:"resyncReleases"` + Branches `json:"branches"` + Tags `json:"tags"` + Messages `json:"messages"` } // ResyncReleases holds configuration for resyncing past releases. diff --git a/pkg/files/delete_by_filters.go b/pkg/files/delete_by_filters.go new file mode 100644 index 0000000..6bcff13 --- /dev/null +++ b/pkg/files/delete_by_filters.go @@ -0,0 +1,34 @@ +package files + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/openshift-knative/deviate/pkg/errors" +) + +// ErrCantDeleteFiles when cannot delete files. +var ErrCantDeleteFiles = errors.New("cannot delete files") + +// DeleteFiles will delete all matching files starting at given root directory. +func (f Filters) DeleteFiles(root string) error { + matcher := f.Matcher() + err := filepath.WalkDir(root, func(pth string, de fs.DirEntry, err error) error { + if err != nil { + return err + } + if de.IsDir() { + // continue + return nil + } + pth = filepath.ToSlash(pth) + relPath := strings.TrimPrefix(pth, root+"/") + if matcher.Matches(relPath) { + return os.Remove(pth) + } + return nil + }) + return errors.Wrap(err, ErrCantDeleteFiles) +} diff --git a/pkg/files/delete_by_filters_test.go b/pkg/files/delete_by_filters_test.go new file mode 100644 index 0000000..37931fb --- /dev/null +++ b/pkg/files/delete_by_filters_test.go @@ -0,0 +1,64 @@ +package files_test + +import ( + "os" + "path" + "slices" + "testing" + + gitv5 "github.com/go-git/go-git/v5" + "github.com/openshift-knative/deviate/pkg/files" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDeleteByFilters(t *testing.T) { + root := t.TempDir() + paths := []string{ + "a.txt", + "a/b.txt", + "a/b/c.md", + "a/b/c.txt", + "a/b/d.md", + "a/b/d.txt", + "a/b/e/f.md", + "a/b/e/f.txt", + "a/c.txt", + "c.txt", + } + for _, file := range paths { + fp := path.Join(root, file) + dir := path.Dir(fp) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(fp, []byte("test"), 0o600)) + } + + filters := files.Filters{ + Include: []string{"a/b/**"}, + Exclude: []string{"**.txt"}, + } + require.NoError(t, filters.DeleteFiles(root)) + + repo, err := gitv5.PlainInit(root, false) + require.NoError(t, err) + wt, err := repo.Worktree() + require.NoError(t, err) + + st, serr := wt.Status() + require.NoError(t, serr) + got := make([]string, 0, len(st)) + for f := range st { + got = append(got, f) + } + slices.Sort(got) + want := []string{ + "a.txt", + "a/b.txt", + "a/b/c.txt", + "a/b/d.txt", + "a/b/e/f.txt", + "a/c.txt", + "c.txt", + } + assert.Equal(t, want, got) +} diff --git a/pkg/files/filters.go b/pkg/files/filters.go new file mode 100644 index 0000000..e64322f --- /dev/null +++ b/pkg/files/filters.go @@ -0,0 +1,50 @@ +package files + +import "github.com/gobwas/glob" + +// Filters represents what files to include, and which to exclude from copying operations. +type Filters struct { + Include []string `json:"include"` + Exclude []string `json:"exclude"` +} + +func (f Filters) Matcher() Matcher { + m := Matcher{ + Include: make([]glob.Glob, 0, len(f.Include)), + Exclude: make([]glob.Glob, 0, len(f.Exclude)), + } + separators := []rune{'/'} + for _, p := range f.Include { + g := glob.MustCompile(p, separators...) + m.Include = append(m.Include, g) + } + for _, p := range f.Exclude { + g := glob.MustCompile(p, separators...) + m.Exclude = append(m.Exclude, g) + } + return m +} + +type Matcher struct { + Include []glob.Glob + Exclude []glob.Glob +} + +func (m Matcher) Matches(pth string) bool { + result := false + for _, pattern := range m.Include { + if pattern.Match(pth) { + result = true + break + } + } + if !result { + return false + } + for _, pattern := range m.Exclude { + if pattern.Match(pth) { + return false + } + } + return true +} diff --git a/pkg/files/filters_test.go b/pkg/files/filters_test.go new file mode 100644 index 0000000..a9e1e6f --- /dev/null +++ b/pkg/files/filters_test.go @@ -0,0 +1,65 @@ +package files_test + +import ( + "testing" + + "github.com/openshift-knative/deviate/pkg/files" + "github.com/stretchr/testify/assert" +) + +func TestFilters_Match(t *testing.T) { + filelist := []string{ + "a/b/c.txt", + "a/d.txt", + "a/b/d.md", + "a/b/d.txt", + "a.txt", + "b.md", + } + tcs := []testFiltersMatchCases{{ + name: "all md files", + Filters: files.Filters{ + Include: []string{"**/*.md", "*.md"}, + }, + want: []string{ + "a/b/d.md", + "b.md", + }, + }, { + name: "root level md files", + Filters: files.Filters{ + Include: []string{"*.md"}, + }, + want: []string{ + "b.md", + }, + }, { + "just one txt", + files.Filters{ + Include: []string{"**.txt"}, + Exclude: []string{"a/b**", "a.txt"}, + }, + []string{ + "a/d.txt", + }, + }} + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + matcher := tc.Matcher() + got := make([]string, 0, len(tc.want)) + for _, f := range filelist { + if matcher.Matches(f) { + got = append(got, f) + } + } + + assert.Equal(t, tc.want, got) + }) + } +} + +type testFiltersMatchCases struct { + name string + files.Filters + want []string +} diff --git a/pkg/git/checkout.go b/pkg/git/checkout.go index 3e8d415..1f74a1f 100644 --- a/pkg/git/checkout.go +++ b/pkg/git/checkout.go @@ -16,6 +16,7 @@ import ( "github.com/go-git/go-git/v5/storage/memory" "github.com/openshift-knative/deviate/pkg/config/git" "github.com/openshift-knative/deviate/pkg/errors" + "github.com/openshift-knative/deviate/pkg/files" ) func (r Repository) Checkout(remote git.Remote, branch string) git.Checkout { //nolint:ireturn @@ -86,7 +87,7 @@ func (o onGoingCheckout) As(branch string) error { return nil } -func (o onGoingCheckout) OntoWorkspace() error { +func (o onGoingCheckout) OntoWorkspace(filters files.Filters) error { coOpts := &gitv5.CloneOptions{ URL: "file://" + o.repo.Project.Path, ReferenceName: plumbing.NewBranchReferenceName(o.branch), @@ -98,23 +99,27 @@ func (o onGoingCheckout) OntoWorkspace() error { if err != nil { return errors.Wrap(err, ErrLocalOperationFailed) } - return o.applyTree(wt, "/") + matcher := filters.Matcher() + return o.applyTree(wt, "", matcher) } -func (o onGoingCheckout) applyTree(fs billy.Filesystem, dir string) error { - files, err := fs.ReadDir(dir) +func (o onGoingCheckout) applyTree(fs billy.Filesystem, dir string, matcher files.Matcher) error { + infos, err := fs.ReadDir(dir) if err != nil { return errors.Wrap(err, ErrLocalOperationFailed) } - for _, f := range files { + for _, f := range infos { fp := path.Join(dir, f.Name()) if f.IsDir() { - err = o.applyTree(fs, fp) + err = o.applyTree(fs, fp, matcher) if err != nil { return err } continue } + if !matcher.Matches(fp) { + continue + } err = o.applyFile(fs, fp, f.Mode()) if err != nil { return err diff --git a/pkg/git/checkout_test.go b/pkg/git/checkout_test.go new file mode 100644 index 0000000..4177600 --- /dev/null +++ b/pkg/git/checkout_test.go @@ -0,0 +1,76 @@ +package git_test + +import ( + "context" + "slices" + "testing" + + gitv5 "github.com/go-git/go-git/v5" + gitv5config "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/openshift-knative/deviate/pkg/config" + configgit "github.com/openshift-knative/deviate/pkg/config/git" + "github.com/openshift-knative/deviate/pkg/files" + "github.com/openshift-knative/deviate/pkg/git" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckout_OntoWorkspace(t *testing.T) { + projectPath := t.TempDir() + remote := configgit.Remote{ + Name: "origin", + URL: "https://github.com/cardil/ghet", + } + mainBranch := "main" + gr, err := gitv5.PlainClone(projectPath, false, &gitv5.CloneOptions{ + URL: remote.URL, + RemoteName: "origin", + Depth: 1, + SingleBranch: true, + ReferenceName: plumbing.NewBranchReferenceName(mainBranch), + }) + require.NoError(t, err) + wt, gerr := gr.Worktree() + require.NoError(t, gerr) + initCommit := "9ab17a360b240c506cf98dfa83997563aa6d9a28" + require.NoError(t, gr.Fetch(&gitv5.FetchOptions{ + RemoteName: "origin", + RefSpecs: []gitv5config.RefSpec{ + gitv5config.RefSpec(initCommit + ":refs/heads/target"), + }, + Depth: 1, + })) + require.NoError(t, wt.Checkout(&gitv5.CheckoutOptions{ + Branch: plumbing.NewBranchReferenceName("target"), + })) + repo := &git.Repository{ + Context: context.TODO(), + Project: config.Project{ + Path: projectPath, + }, + Repository: gr, + } + filters := files.Filters{ + Include: []string{ + "**.go", + }, + Exclude: []string{ + "**internal**", + "**pkg**", + "**Magefile**", + "**_test.go", + }, + } + err = repo.Checkout(remote, mainBranch).OntoWorkspace(filters) + require.NoError(t, err) + + st, serr := wt.Status() + require.NoError(t, serr) + got := make([]string, 0, len(st)) + for f := range st { + got = append(got, f) + } + slices.Sort(got) + assert.Equal(t, []string{"build/mage.go", "cmd/ght/main.go"}, got) +} diff --git a/pkg/sync/fork_files.go b/pkg/sync/fork_files.go index 5de79f1..dd36878 100644 --- a/pkg/sync/fork_files.go +++ b/pkg/sync/fork_files.go @@ -7,7 +7,7 @@ import ( func (o Operation) addForkFiles(rel release) step { return multiStep([]step{ - o.removeGithubWorkflows, + o.removeUnwantedUpstreamFiles, o.unpackForkOntoWorkspace, o.commitChanges(o.Config.Messages.ApplyForkFiles), o.generateImages(rel), @@ -19,6 +19,6 @@ func (o Operation) unpackForkOntoWorkspace() error { o.Println("- Add fork's files") upstream := git.Remote{Name: "upstream", URL: o.Config.Upstream} err := o.Repository.Checkout(upstream, o.Config.Branches.Main). - OntoWorkspace() + OntoWorkspace(o.Config.CopyFromMidstream) return errors.Wrap(err, ErrSyncFailed) } diff --git a/pkg/sync/remove_github_workflows.go b/pkg/sync/remove_github_workflows.go deleted file mode 100644 index 3b4274e..0000000 --- a/pkg/sync/remove_github_workflows.go +++ /dev/null @@ -1,29 +0,0 @@ -package sync - -import ( - "os" - "path" - "path/filepath" - - "github.com/openshift-knative/deviate/pkg/errors" -) - -func (o Operation) removeGithubWorkflows() error { - o.Println("- Remove upstream Github workflows") - workflows := path.Join(o.State.Project.Path, ".github", "workflows") - - dir, err := os.ReadDir(workflows) - if err != nil { - return errors.Wrap(err, ErrSyncFailed) - } - for _, d := range dir { - fp := path.Join(workflows, d.Name()) - if ok, _ := filepath.Match(o.GithubWorkflowsRemovalGlob, path.Base(fp)); ok { - err = os.RemoveAll(fp) - if err != nil { - return errors.Wrap(err, ErrSyncFailed) - } - } - } - return nil -} diff --git a/pkg/sync/remove_unwanted_upstream_files.go b/pkg/sync/remove_unwanted_upstream_files.go new file mode 100644 index 0000000..0cdcead --- /dev/null +++ b/pkg/sync/remove_unwanted_upstream_files.go @@ -0,0 +1,13 @@ +package sync + +import ( + "github.com/openshift-knative/deviate/pkg/errors" +) + +func (o Operation) removeUnwantedUpstreamFiles() error { + o.Println("- Remove unwanted upstream files") + + return errors.Wrap( + o.Config.DeleteFromUpstream.DeleteFiles(o.State.Project.Path), + ErrSyncFailed) +}