🎁 Copy/Delete filters

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/fatih/color v1.18.0
 	github.com/go-git/go-billy/v5 v5.6.2
 	github.com/go-git/go-git/v5 v5.13.2
+	github.com/gobwas/glob v0.2.3
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/openshift-knative/hack v0.0.0-20250530124021-bed35e443a23

go.sum

@@ -390,6 +390,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
 github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

pkg/config/defaults.go

@@ -1,6 +1,9 @@
 package config
 
-import "github.com/openshift-knative/hack/pkg/dockerfilegen"
+import (
+	"github.com/openshift-knative/deviate/pkg/files"
+	"github.com/openshift-knative/hack/pkg/dockerfilegen"
+)
 
 // newDefaults creates a new default configuration.
 func newDefaults(project Project) Config {
@@ -9,7 +12,14 @@ func newDefaults(project Project) Config {
 		releaseSearch   = `^release-(\d+)\.(\d+)$`
 	)
 	return Config{
-		GithubWorkflowsRemovalGlob: "knative-*.y?ml",
+		DeleteFromUpstream: files.Filters{
+			Include: []string{
+				".github/workflows/knative-*.y?ml",
+			},
+		},
+		CopyFromMidstream: files.Filters{
+			Include: []string{"**"},
+		},
 		Branches: Branches{
 			Main:          "main",
 			ReleaseNext:   "release-next",

pkg/config/git/checkout.go

@@ -1,6 +1,10 @@
 package git
 
+import (
+	"github.com/openshift-knative/deviate/pkg/files"
+)
+
 type Checkout interface {
 	As(branch string) error
-	OntoWorkspace() error
+	OntoWorkspace(filters files.Filters) error
 }

pkg/config/structure.go

@@ -1,19 +1,23 @@
 package config
 
-import "github.com/openshift-knative/hack/pkg/dockerfilegen"
+import (
+	"github.com/openshift-knative/deviate/pkg/files"
+	"github.com/openshift-knative/hack/pkg/dockerfilegen"
+)
 
 // Config for a deviate to operate.
 type Config struct {
-	Upstream                   string        `json:"upstream"                   valid:"required"`
-	Downstream                 string        `json:"downstream"                 valid:"required"`
-	DryRun                     bool          `json:"dryRun"`
-	GithubWorkflowsRemovalGlob string        `json:"githubWorkflowsRemovalGlob" valid:"required"`
-	SyncLabels                 []string      `json:"syncLabels"                 valid:"required"`
-	DockerfileGen              DockerfileGen `json:"dockerfileGen"`
-	ResyncReleases             `json:"resyncReleases"`
-	Branches                   `json:"branches"`
-	Tags                       `json:"tags"`
-	Messages                   `json:"messages"`
+	Upstream           string        `json:"upstream"           valid:"required"`
+	Downstream         string        `json:"downstream"         valid:"required"`
+	DryRun             bool          `json:"dryRun"`
+	CopyFromMidstream  files.Filters `json:"copyFromMidstream"  valid:"required"`
+	DeleteFromUpstream files.Filters `json:"deleteFromUpstream" valid:"required"`
+	SyncLabels         []string      `json:"syncLabels"         valid:"required"`
+	DockerfileGen      DockerfileGen `json:"dockerfileGen"`
+	ResyncReleases     `json:"resyncReleases"`
+	Branches           `json:"branches"`
+	Tags               `json:"tags"`
+	Messages           `json:"messages"`
 }
 
 // ResyncReleases holds configuration for resyncing past releases.

pkg/files/delete_by_filters.go

@@ -0,0 +1,34 @@
+package files
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/openshift-knative/deviate/pkg/errors"
+)
+
+// ErrCantDeleteFiles when cannot delete files.
+var ErrCantDeleteFiles = errors.New("cannot delete files")
+
+// DeleteFiles will delete all matching files starting at given root directory.
+func (f Filters) DeleteFiles(root string) error {
+	matcher := f.Matcher()
+	err := filepath.WalkDir(root, func(pth string, de fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if de.IsDir() {
+			// continue
+			return nil
+		}
+		pth = filepath.ToSlash(pth)
+		relPath := strings.TrimPrefix(pth, root+"/")
+		if matcher.Matches(relPath) {
+			return os.Remove(pth)
+		}
+		return nil
+	})
+	return errors.Wrap(err, ErrCantDeleteFiles)
+}

pkg/files/delete_by_filters_test.go

@@ -0,0 +1,64 @@
+package files_test
+
+import (
+	"os"
+	"path"
+	"slices"
+	"testing"
+
+	gitv5 "github.com/go-git/go-git/v5"
+	"github.com/openshift-knative/deviate/pkg/files"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDeleteByFilters(t *testing.T) {
+	root := t.TempDir()
+	paths := []string{
+		"a.txt",
+		"a/b.txt",
+		"a/b/c.md",
+		"a/b/c.txt",
+		"a/b/d.md",
+		"a/b/d.txt",
+		"a/b/e/f.md",
+		"a/b/e/f.txt",
+		"a/c.txt",
+		"c.txt",
+	}
+	for _, file := range paths {
+		fp := path.Join(root, file)
+		dir := path.Dir(fp)
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(fp, []byte("test"), 0o600))
+	}
+
+	filters := files.Filters{
+		Include: []string{"a/b/**"},
+		Exclude: []string{"**.txt"},
+	}
+	require.NoError(t, filters.DeleteFiles(root))
+
+	repo, err := gitv5.PlainInit(root, false)
+	require.NoError(t, err)
+	wt, err := repo.Worktree()
+	require.NoError(t, err)
+
+	st, serr := wt.Status()
+	require.NoError(t, serr)
+	got := make([]string, 0, len(st))
+	for f := range st {
+		got = append(got, f)
+	}
+	slices.Sort(got)
+	want := []string{
+		"a.txt",
+		"a/b.txt",
+		"a/b/c.txt",
+		"a/b/d.txt",
+		"a/b/e/f.txt",
+		"a/c.txt",
+		"c.txt",
+	}
+	assert.Equal(t, want, got)
+}

pkg/files/filters.go

@@ -0,0 +1,50 @@
+package files
+
+import "github.com/gobwas/glob"
+
+// Filters represents what files to include, and which to exclude from copying operations.
+type Filters struct {
+	Include []string `json:"include"`
+	Exclude []string `json:"exclude"`
+}
+
+func (f Filters) Matcher() Matcher {
+	m := Matcher{
+		Include: make([]glob.Glob, 0, len(f.Include)),
+		Exclude: make([]glob.Glob, 0, len(f.Exclude)),
+	}
+	separators := []rune{'/'}
+	for _, p := range f.Include {
+		g := glob.MustCompile(p, separators...)
+		m.Include = append(m.Include, g)
+	}
+	for _, p := range f.Exclude {
+		g := glob.MustCompile(p, separators...)
+		m.Exclude = append(m.Exclude, g)
+	}
+	return m
+}
+
+type Matcher struct {
+	Include []glob.Glob
+	Exclude []glob.Glob
+}
+
+func (m Matcher) Matches(pth string) bool {
+	result := false
+	for _, pattern := range m.Include {
+		if pattern.Match(pth) {
+			result = true
+			break
+		}
+	}
+	if !result {
+		return false
+	}
+	for _, pattern := range m.Exclude {
+		if pattern.Match(pth) {
+			return false
+		}
+	}
+	return true
+}

pkg/files/filters_test.go

@@ -0,0 +1,65 @@
+package files_test
+
+import (
+	"testing"
+
+	"github.com/openshift-knative/deviate/pkg/files"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilters_Match(t *testing.T) {
+	filelist := []string{
+		"a/b/c.txt",
+		"a/d.txt",
+		"a/b/d.md",
+		"a/b/d.txt",
+		"a.txt",
+		"b.md",
+	}
+	tcs := []testFiltersMatchCases{{
+		name: "all md files",
+		Filters: files.Filters{
+			Include: []string{"**/*.md", "*.md"},
+		},
+		want: []string{
+			"a/b/d.md",
+			"b.md",
+		},
+	}, {
+		name: "root level md files",
+		Filters: files.Filters{
+			Include: []string{"*.md"},
+		},
+		want: []string{
+			"b.md",
+		},
+	}, {
+		"just one txt",
+		files.Filters{
+			Include: []string{"**.txt"},
+			Exclude: []string{"a/b**", "a.txt"},
+		},
+		[]string{
+			"a/d.txt",
+		},
+	}}
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			matcher := tc.Matcher()
+			got := make([]string, 0, len(tc.want))
+			for _, f := range filelist {
+				if matcher.Matches(f) {
+					got = append(got, f)
+				}
+			}
+
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+type testFiltersMatchCases struct {
+	name string
+	files.Filters
+	want []string
+}

pkg/git/checkout.go

@@ -16,6 +16,7 @@ import (
 	"github.com/go-git/go-git/v5/storage/memory"
 	"github.com/openshift-knative/deviate/pkg/config/git"
 	"github.com/openshift-knative/deviate/pkg/errors"
+	"github.com/openshift-knative/deviate/pkg/files"
 )
 
 func (r Repository) Checkout(remote git.Remote, branch string) git.Checkout { //nolint:ireturn
@@ -86,7 +87,7 @@ func (o onGoingCheckout) As(branch string) error {
 	return nil
 }
 
-func (o onGoingCheckout) OntoWorkspace() error {
+func (o onGoingCheckout) OntoWorkspace(filters files.Filters) error {
 	coOpts := &gitv5.CloneOptions{
 		URL:           "file://" + o.repo.Project.Path,
 		ReferenceName: plumbing.NewBranchReferenceName(o.branch),
@@ -98,23 +99,27 @@ func (o onGoingCheckout) OntoWorkspace() error {
 	if err != nil {
 		return errors.Wrap(err, ErrLocalOperationFailed)
 	}
-	return o.applyTree(wt, "/")
+	matcher := filters.Matcher()
+	return o.applyTree(wt, "", matcher)
 }
 
-func (o onGoingCheckout) applyTree(fs billy.Filesystem, dir string) error {
-	files, err := fs.ReadDir(dir)
+func (o onGoingCheckout) applyTree(fs billy.Filesystem, dir string, matcher files.Matcher) error {
+	infos, err := fs.ReadDir(dir)
 	if err != nil {
 		return errors.Wrap(err, ErrLocalOperationFailed)
 	}
-	for _, f := range files {
+	for _, f := range infos {
 		fp := path.Join(dir, f.Name())
 		if f.IsDir() {
-			err = o.applyTree(fs, fp)
+			err = o.applyTree(fs, fp, matcher)
 			if err != nil {
 				return err
 			}
 			continue
 		}
+		if !matcher.Matches(fp) {
+			continue
+		}
 		err = o.applyFile(fs, fp, f.Mode())
 		if err != nil {
 			return err