Copy/Delete filters

Author: cardil

pkg/config/defaults.go

@@ -1,6 +1,9 @@
 package config
 
-import "github.com/openshift-knative/hack/pkg/dockerfilegen"
+import (
+	"github.com/openshift-knative/deviate/pkg/files"
+	"github.com/openshift-knative/hack/pkg/dockerfilegen"
+)
 
 // newDefaults creates a new default configuration.
 func newDefaults(project Project) Config {
@@ -9,7 +12,14 @@ func newDefaults(project Project) Config {
 		releaseSearch   = `^release-(\d+)\.(\d+)$`
 	)
 	return Config{
-		GithubWorkflowsRemovalGlob: "knative-*.y?ml",
+		DeleteFromUpstream: files.Filters{
+			Include: []string{
+				".github/workflows/knative-*.y?ml",
+			},
+		},
+		CopyFromMidstream: files.Filters{
+			Include: []string{"*/*"},
+		},
 		Branches: Branches{
 			Main:        "main",
 			ReleaseNext: "release-next",

pkg/config/git/checkout.go

@@ -1,6 +1,10 @@
 package git
 
+import (
+	"github.com/openshift-knative/deviate/pkg/files"
+)
+
 type Checkout interface {
 	As(branch string) error
-	OntoWorkspace() error
+	OntoWorkspace(filters files.Filters) error
 }

pkg/config/structure.go

@@ -1,19 +1,23 @@
 package config
 
-import "github.com/openshift-knative/hack/pkg/dockerfilegen"
+import (
+	"github.com/openshift-knative/deviate/pkg/files"
+	"github.com/openshift-knative/hack/pkg/dockerfilegen"
+)
 
 // Config for a deviate to operate.
 type Config struct {
-	Upstream                   string               `json:"upstream"                   valid:"required"`
-	Downstream                 string               `json:"downstream"                 valid:"required"`
-	DryRun                     bool                 `json:"dryRun"`
-	GithubWorkflowsRemovalGlob string               `json:"githubWorkflowsRemovalGlob" valid:"required"`
-	SyncLabels                 []string             `json:"syncLabels"                 valid:"required"`
-	DockerfileGen              dockerfilegen.Params `json:"dockerfileGen"`
-	ResyncReleases             `json:"resyncReleases"`
-	Branches                   `json:"branches"`
-	Tags                       `json:"tags"`
-	Messages                   `json:"messages"`
+	Upstream           string               `json:"upstream"           valid:"required"`
+	Downstream         string               `json:"downstream"         valid:"required"`
+	DryRun             bool                 `json:"dryRun"`
+	CopyFromMidstream  files.Filters        `json:"copyFromMidstream"  valid:"required"`
+	DeleteFromUpstream files.Filters        `json:"deleteFromUpstream" valid:"required"`
+	SyncLabels         []string             `json:"syncLabels"         valid:"required"`
+	DockerfileGen      dockerfilegen.Params `json:"dockerfileGen"`
+	ResyncReleases     `json:"resyncReleases"`
+	Branches           `json:"branches"`
+	Tags               `json:"tags"`
+	Messages           `json:"messages"`
 }
 
 // ResyncReleases holds configuration for resyncing past releases.

pkg/files/filters.go

@@ -0,0 +1,29 @@
+package files
+
+import "path"
+
+// Filters represents what files to include, and which to exclude from copying operations.
+type Filters struct {
+	Include []string `json:"include"`
+	Exclude []string `json:"exclude"`
+}
+
+func (f Filters) Matches(pth string) bool {
+	result := false
+	for _, pattern := range f.Include {
+		matched, _ := path.Match(pattern, pth)
+		if matched {
+			result = true
+		}
+	}
+	if !result {
+		return false
+	}
+	for _, pattern := range f.Exclude {
+		matched, _ := path.Match(pattern, pth)
+		if matched {
+			return false
+		}
+	}
+	return true
+}

pkg/git/checkout.go

@@ -16,6 +16,7 @@ import (
 	"github.com/go-git/go-git/v5/storage/memory"
 	"github.com/openshift-knative/deviate/pkg/config/git"
 	"github.com/openshift-knative/deviate/pkg/errors"
+	"github.com/openshift-knative/deviate/pkg/files"
 )
 
 func (r Repository) Checkout(remote git.Remote, branch string) git.Checkout { //nolint:ireturn
@@ -86,7 +87,7 @@ func (o onGoingCheckout) As(branch string) error {
 	return nil
 }
 
-func (o onGoingCheckout) OntoWorkspace() error {
+func (o onGoingCheckout) OntoWorkspace(filters files.Filters) error {
 	coOpts := &gitv5.CloneOptions{
 		URL:           "file://" + o.repo.Project.Path,
 		ReferenceName: plumbing.NewBranchReferenceName(o.branch),
@@ -98,18 +99,21 @@ func (o onGoingCheckout) OntoWorkspace() error {
 	if err != nil {
 		return errors.Wrap(err, ErrLocalOperationFailed)
 	}
-	return o.applyTree(wt, "/")
+	return o.applyTree(wt, "/", filters)
 }
 
-func (o onGoingCheckout) applyTree(fs billy.Filesystem, dir string) error {
-	files, err := fs.ReadDir(dir)
+func (o onGoingCheckout) applyTree(fs billy.Filesystem, dir string, filters files.Filters) error {
+	infos, err := fs.ReadDir(dir)
 	if err != nil {
 		return errors.Wrap(err, ErrLocalOperationFailed)
 	}
-	for _, f := range files {
+	for _, f := range infos {
 		fp := path.Join(dir, f.Name())
+		if !filters.Matches(fp) {
+			continue
+		}
 		if f.IsDir() {
-			err = o.applyTree(fs, fp)
+			err = o.applyTree(fs, fp, filters)
 			if err != nil {
 				return err
 			}

pkg/sync/fork_files.go

@@ -7,7 +7,7 @@ import (
 
 func (o Operation) addForkFiles(rel release) step {
 	return multiStep([]step{
-		o.removeGithubWorkflows,
+		o.removeUnwantedUpstreamFiles,
 		o.unpackForkOntoWorkspace,
 		o.commitChanges(o.Config.Messages.ApplyForkFiles),
 		o.generateImages(rel),
@@ -19,6 +19,6 @@ func (o Operation) unpackForkOntoWorkspace() error {
 	o.Println("- Add fork's files")
 	upstream := git.Remote{Name: "upstream", URL: o.Config.Upstream}
 	err := o.Repository.Checkout(upstream, o.Config.Branches.Main).
-		OntoWorkspace()
+		OntoWorkspace(o.Config.CopyFromMidstream)
 	return errors.Wrap(err, ErrSyncFailed)
 }

pkg/sync/remove_github_workflows.go

@@ -0,0 +1,25 @@
+package sync
+
+import (
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/openshift-knative/deviate/pkg/errors"
+)
+
+func (o Operation) removeUnwantedUpstreamFiles() error {
+	o.Println("- Remove unwanted upstream files")
+
+	return errors.Wrap(filepath.WalkDir(o.State.Project.Path, func(pth string, _ fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if o.Config.DeleteFromUpstream.Matches(pth) {
+			fp := path.Join(o.State.Project.Path, pth)
+			return os.RemoveAll(fp)
+		}
+		return nil
+	}), ErrSyncFailed)
+}