Filter out repositories with .batchignore at root (#509)

This implements https://github.com/sourcegraph/sourcegraph/issues/18330 by querying the locations of `.batchignore` files in each repository yielded by the `on` attribute in a batch spec. If locations were found, the repository is ignored.

This can be overwritten by using the `-force-override-ignore` flag.

Example: given my instance has the following repositories:

- `github.com/sourcegraph-testing/zap`
- `github.com/sourcegraph-testing/titan`
- `github.com/sourcegraph-testing/tidb`
- `github.com/sourcegraph-testing/etcd`
- `github.com/sourcegraph-testing/batch-changes-testing-ignore`

And I use the following batch spec:

```yaml
on:
  - repositoriesMatchingQuery: repohasfile:README.md repo:sourcegraph-testing

steps:
  - run: echo "a horse says 'hello'" >> README.md
    container: alpine:3
```

with this change the `batch-changes-testing-ignore` repository will be ignored:

- no archive will be downloaded
- no steps executed

A message is printed that says it's ignored.

Author: mrnugget

CHANGELOG.md

@@ -14,6 +14,7 @@ All notable changes to `src-cli` are documented in this file.
 ### Added
 
 - Extension publishing will now add a `gitHead` property to the extension's manifest. [#500](https://github.com/sourcegraph/src-cli/pull/500)
+- `src batch [apply|preview]` now ignore repositories in which a `.batchignore` file exists. The `-force-override-ignore` flag can be used to turn that behaviour off. [#509](https://github.com/sourcegraph/src-cli/pull/509)
 
 ### Changed
 

cmd/src/batch_apply.go

@@ -69,6 +69,7 @@ Examples:
 
 		svc := batches.NewService(&batches.ServiceOpts{
 			AllowUnsupported: flags.allowUnsupported,
+			AllowIgnored:     flags.allowIgnored,
 			Client:           cfg.apiClient(flags.api, flagSet.Output()),
 			Workspace:        flags.workspace,
 		})

cmd/src/batch_common.go

@@ -31,6 +31,7 @@ var (
 
 type batchApplyFlags struct {
 	allowUnsupported bool
+	allowIgnored     bool
 	api              *api.Flags
 	apply            bool
 	cacheDir         string
@@ -55,6 +56,10 @@ func newBatchApplyFlags(flagSet *flag.FlagSet, cacheDir, tempDir string) *batchA
 		&caf.allowUnsupported, "allow-unsupported", false,
 		"Allow unsupported code hosts.",
 	)
+	flagSet.BoolVar(
+		&caf.allowIgnored, "force-override-ignore", false,
+		"Do not ignore repositories that have a .batchignore file.",
+	)
 	flagSet.BoolVar(
 		&caf.apply, "apply", false,
 		"Ignored.",
@@ -238,6 +243,14 @@ func batchExecute(ctx context.Context, out *output.Output, svc *batches.Service,
 				block.Write(repo.Name)
 			}
 			block.Close()
+		} else if repoSet, ok := err.(batches.IgnoredRepoSet); ok {
+			batchCompletePending(pending, "Resolved repositories")
+
+			block := out.Block(output.Line(" ", output.StyleWarning, "The repositories listed below contain .batchignore files and will be skipped. Use the -force-override-ignore flag to avoid skipping them."))
+			for repo := range repoSet {
+				block.Write(repo.Name)
+			}
+			block.Close()
 		} else {
 			return "", "", errors.Wrap(err, "resolving repositories")
 		}

cmd/src/batch_preview.go

@@ -44,6 +44,7 @@ Examples:
 
 		svc := batches.NewService(&batches.ServiceOpts{
 			AllowUnsupported: flags.allowUnsupported,
+			AllowIgnored:     flags.allowIgnored,
 			Client:           cfg.apiClient(flags.api, flagSet.Output()),
 			Workspace:        flags.workspace,
 		})

internal/batches/errors.go

@@ -44,3 +44,33 @@ func (e UnsupportedRepoSet) appendRepo(repo *graphql.Repository) {
 func (e UnsupportedRepoSet) hasUnsupported() bool {
 	return len(e) > 0
 }
+
+// IgnoredRepoSet provides a set to manage repositories that are on
+// unsupported code hosts. This type implements error to allow it to be
+// returned directly as an error value if needed.
+type IgnoredRepoSet map[*graphql.Repository]struct{}
+
+func (e IgnoredRepoSet) includes(r *graphql.Repository) bool {
+	_, ok := e[r]
+	return ok
+}
+
+func (e IgnoredRepoSet) Error() string {
+	repos := []string{}
+	for repo := range e {
+		repos = append(repos, repo.Name)
+	}
+
+	return fmt.Sprintf(
+		"found repositories containing .batchignore files:\n\t%s",
+		strings.Join(repos, "\n\t"),
+	)
+}
+
+func (e IgnoredRepoSet) appendRepo(repo *graphql.Repository) {
+	e[repo] = struct{}{}
+}
+
+func (e IgnoredRepoSet) hasIgnored() bool {
+	return len(e) > 0
+}

internal/batches/service.go

@@ -23,6 +23,7 @@ import (
 
 type Service struct {
 	allowUnsupported bool
+	allowIgnored     bool
 	client           api.Client
 	features         featureFlags
 	imageCache       *docker.ImageCache
@@ -31,6 +32,7 @@ type Service struct {
 
 type ServiceOpts struct {
 	AllowUnsupported bool
+	AllowIgnored     bool
 	Client           api.Client
 	Workspace        string
 }
@@ -42,6 +44,7 @@ var (
 func NewService(opts *ServiceOpts) *Service {
 	return &Service{
 		allowUnsupported: opts.AllowUnsupported,
+		allowIgnored:     opts.AllowIgnored,
 		client:           opts.Client,
 		imageCache:       docker.NewImageCache(),
 		workspace:        opts.Workspace,
@@ -529,6 +532,7 @@ func (svc *Service) ResolveNamespace(ctx context.Context, namespace string) (str
 func (svc *Service) ResolveRepositories(ctx context.Context, spec *BatchSpec) ([]*graphql.Repository, error) {
 	seen := map[string]*graphql.Repository{}
 	unsupported := UnsupportedRepoSet{}
+	ignored := IgnoredRepoSet{}
 
 	// TODO: this could be trivially parallelised in the future.
 	for _, on := range spec.On {
@@ -537,6 +541,14 @@ func (svc *Service) ResolveRepositories(ctx context.Context, spec *BatchSpec) ([
 			return nil, errors.Wrapf(err, "resolving %q", on.String())
 		}
 
+		var repoBatchIgnores map[*graphql.Repository][]string
+		if !svc.allowIgnored {
+			repoBatchIgnores, err = svc.FindDirectoriesInRepos(ctx, ".batchignore", repos...)
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		for _, repo := range repos {
 			if !repo.HasBranch() {
 				continue
@@ -552,6 +564,12 @@ func (svc *Service) ResolveRepositories(ctx context.Context, spec *BatchSpec) ([
 						unsupported.appendRepo(repo)
 					}
 				}
+
+				if !svc.allowIgnored {
+					if locations, ok := repoBatchIgnores[repo]; ok && len(locations) > 0 {
+						ignored.appendRepo(repo)
+					}
+				}
 			} else {
 				// If we've already seen this repository, we overwrite the
 				// Commit/Branch fields with the latest value we have
@@ -563,7 +581,7 @@ func (svc *Service) ResolveRepositories(ctx context.Context, spec *BatchSpec) ([
 
 	final := make([]*graphql.Repository, 0, len(seen))
 	for _, repo := range seen {
-		if !unsupported.includes(repo) {
+		if !unsupported.includes(repo) && !ignored.includes(repo) {
 			final = append(final, repo)
 		}
 	}
@@ -572,6 +590,10 @@ func (svc *Service) ResolveRepositories(ctx context.Context, spec *BatchSpec) ([
 		return final, unsupported
 	}
 
+	if ignored.hasIgnored() {
+		return final, ignored
+	}
+
 	return final, nil
 }
 
@@ -678,6 +700,7 @@ func (svc *Service) resolveRepositorySearch(ctx context.Context, query string) (
 			}
 		}
 	}
+
 	if ok, err := svc.client.NewRequest(repositorySearchQuery, map[string]interface{}{
 		"query":       setDefaultQueryCount(query),
 		"queryCommit": false,
@@ -728,7 +751,7 @@ type findDirectoriesResult map[string]struct {
 // files matching the given file name in the repository.
 // The locations are paths relative to the root of the directory.
 // No "/" at the beginning.
-// An empty path ("") represents the root directory.
+// A dot (".") represents the root directory.
 func (svc *Service) FindDirectoriesInRepos(ctx context.Context, fileName string, repos ...*graphql.Repository) (map[*graphql.Repository][]string, error) {
 	const batchSize = 50
 

internal/batches/service_test.go

@@ -118,11 +118,18 @@ const testResolveRepositorySearchResult = `{
 }
 `
 
-func mockGraphQLClient(response string) (client api.Client, done func()) {
+func mockGraphQLClient(responses ...string) (client api.Client, done func()) {
 	mux := http.NewServeMux()
+
+	var count int
 	mux.HandleFunc("/.api/graphql", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
-		_, _ = w.Write([]byte(response))
+
+		_, _ = w.Write([]byte(responses[count]))
+
+		if count < len(responses)-1 {
+			count += 1
+		}
 	})
 
 	ts := httptest.NewServer(mux)
@@ -144,7 +151,7 @@ func TestResolveRepositories_Unsupported(t *testing.T) {
 	}
 
 	t.Run("allowUnsupported:true", func(t *testing.T) {
-		svc := &Service{client: client, allowUnsupported: true}
+		svc := &Service{client: client, allowUnsupported: true, allowIgnored: true}
 
 		repos, err := svc.ResolveRepositories(context.Background(), spec)
 		if err != nil {
@@ -156,7 +163,7 @@ func TestResolveRepositories_Unsupported(t *testing.T) {
 	})
 
 	t.Run("allowUnsupported:false", func(t *testing.T) {
-		svc := &Service{client: client, allowUnsupported: false}
+		svc := &Service{client: client, allowUnsupported: false, allowIgnored: true}
 
 		repos, err := svc.ResolveRepositories(context.Background(), spec)
 		repoSet, ok := err.(UnsupportedRepoSet)
@@ -167,7 +174,7 @@ func TestResolveRepositories_Unsupported(t *testing.T) {
 			t.Fatalf("wrong number of repos. want=%d, have=%d", 1, len(repoSet))
 		}
 		if len(repos) != 3 {
-			t.Fatalf("wrong number of repos. want=%d, have=%d", 4, len(repos))
+			t.Fatalf("wrong number of repos. want=%d, have=%d", 3, len(repos))
 		}
 	})
 }
@@ -216,6 +223,104 @@ const testResolveRepositoriesUnsupported = `{
 }
 `
 
+func TestResolveRepositories_Ignored(t *testing.T) {
+	spec := &BatchSpec{
+		On: []OnQueryOrRepository{
+			{RepositoriesMatchingQuery: "testquery"},
+		},
+	}
+
+	t.Run("allowIgnored:true", func(t *testing.T) {
+		client, done := mockGraphQLClient(testResolveRepositories, testBatchIgnoreInRepos)
+		defer done()
+
+		svc := &Service{client: client, allowIgnored: true}
+
+		repos, err := svc.ResolveRepositories(context.Background(), spec)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if len(repos) != 3 {
+			t.Fatalf("wrong number of repos. want=%d, have=%d", 3, len(repos))
+		}
+	})
+
+	t.Run("allowIgnored:false", func(t *testing.T) {
+		client, done := mockGraphQLClient(testResolveRepositories, testBatchIgnoreInRepos)
+		defer done()
+
+		svc := &Service{client: client, allowIgnored: false}
+
+		repos, err := svc.ResolveRepositories(context.Background(), spec)
+		ignored, ok := err.(IgnoredRepoSet)
+		if !ok {
+			t.Fatalf("err is not IgnoredRepoSet: %s", err)
+		}
+		if len(ignored) != 1 {
+			t.Fatalf("wrong number of ignored repos. want=%d, have=%d", 1, len(ignored))
+		}
+		if len(repos) != 2 {
+			t.Fatalf("wrong number of repos. want=%d, have=%d", 2, len(repos))
+		}
+	})
+}
+
+const testResolveRepositories = `{
+  "data": {
+    "search": {
+      "results": {
+        "results": [
+          {
+            "__typename": "Repository",
+            "id": "UmVwb3NpdG9yeToxMw==",
+            "name": "bitbucket.sgdev.org/SOUR/automation-testing",
+            "url": "/bitbucket.sgdev.org/SOUR/automation-testing",
+            "externalRepository": { "serviceType": "bitbucketserver" },
+            "defaultBranch": { "name": "refs/heads/master", "target": { "oid": "b978d56de5578a935ca0bf07b56528acc99d5a61" } }
+          },
+          {
+            "__typename": "Repository",
+            "id": "UmVwb3NpdG9yeTo0",
+            "name": "github.com/sourcegraph/automation-testing",
+            "url": "/github.com/sourcegraph/automation-testing",
+            "externalRepository": { "serviceType": "github" },
+            "defaultBranch": { "name": "refs/heads/master", "target": { "oid": "6ac8a32ecaf6c4dc5ce050b9af51bce3db8efd63" } }
+          },
+          {
+            "__typename": "Repository",
+            "id": "UmVwb3NpdG9yeTo2MQ==",
+            "name": "gitlab.sgdev.org/sourcegraph/automation-testing",
+            "url": "/gitlab.sgdev.org/sourcegraph/automation-testing",
+            "externalRepository": { "serviceType": "gitlab" },
+            "defaultBranch": { "name": "refs/heads/master", "target": { "oid": "3b79a5d479d2af9cfe91e0aad4e9dddca7278150" } }
+          }
+        ]
+      }
+    }
+  }
+}
+`
+
+const testBatchIgnoreInRepos = `{
+    "data": {
+        "repo_0": {
+            "results": {
+                "results": [
+                    {
+                        "__typename": "FileMatch",
+                        "file": {
+                            "path": ".batchignore"
+                        }
+                    }
+                ]
+            }
+        },
+        "repo_1": { "results": { "results": [] } },
+        "repo_2": { "results": { "results": [] } }
+    }
+}
+`
+
 func TestService_FindDirectoriesInRepos(t *testing.T) {
 	client, done := mockGraphQLClient(testFindDirectoriesInRepos)
 	defer done()