Sparse Checkout Directories in GitRepositories.

    - Add `.spec.sparseCheckout` and `.status.observedSparseCheckout` fields to `GitRepository`.
    - Add controller support to send the sparse checkout directories to go-git via pkg methods.
    - Add tests for testing the observed sparse checkout behavior.
    - Add docs describing the new fields.

Signed-off-by: Dipti Pai <[email protected]>

Author: dipti-pai

api/v1/gitrepository_types.go

@@ -148,6 +148,12 @@ type GitRepositorySpec struct {
 	// should be included in the Artifact produced for this GitRepository.
 	// +optional
 	Include []GitRepositoryInclude `json:"include,omitempty"`
+
+	// SparseCheckout specifies a list of directories to checkout when cloning
+	// the repository. If specified, only these directories are included in the
+	// Artifact produced for this GitRepository.
+	// +optional
+	SparseCheckout []string `json:"sparseCheckout,omitempty"`
 }
 
 // GitRepositoryInclude specifies a local reference to a GitRepository which
@@ -266,6 +272,11 @@ type GitRepositoryStatus struct {
 	// +optional
 	ObservedInclude []GitRepositoryInclude `json:"observedInclude,omitempty"`
 
+	// ObservedSparseCheckout is the observed list of directories used to
+	// produce the current Artifact.
+	// +optional
+	ObservedSparseCheckout []string `json:"observedSparseCheckout,omitempty"`
+
 	// SourceVerificationMode is the last used verification mode indicating
 	// which Git object(s) have been verified.
 	// +optional

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -174,6 +174,15 @@ spec:
                 required:
                 - name
                 type: object
+              sparseCheckout:
+                description: |-
+                  SparseCheckout specifies a list of directories to checkout
+                  when cloning the repository. If specified, only these
+                  directories are included in the Artifact produced for this
+                  GitRepository.
+                items:
+                  type: string
+                type: array
               suspend:
                 description: |-
                   Suspend tells the controller to suspend the reconciliation of this
@@ -443,6 +452,13 @@ spec:
                   ObservedRecurseSubmodules is the observed resource submodules
                   configuration used to produce the current Artifact.
                 type: boolean
+              observedSparseCheckout:
+                description: |-
+                  ObservedSparseCheckout is the observed list of directories used to
+                  produce the current Artifact.
+                items:
+                  type: string
+                type: array
               sourceVerificationMode:
                 description: |-
                   SourceVerificationMode is the last used verification mode indicating

docs/spec/v1/gitrepositories.md

@@ -590,6 +590,13 @@ list](#default-exclusions), and may overrule the [`.sourceignore` file
 exclusions](#sourceignore-file). See [excluding files](#excluding-files)
 for more information.
 
+### Sparse checkout
+
+`.spec.sparseCheckout` is an optional field to specify list of directories to
+checkout when cloning the repository. If specified, only the specified directory
+contents will be present in the artifact produced for this repository.
+
+
 ### Suspend
 
 `.spec.suspend` is an optional field to suspend the reconciliation of a
@@ -1132,6 +1139,27 @@ status:
   ...
 ```
 
+### Observed Sparse Checkout
+
+The source-controller reports observed sparse checkout in the GitRepository's
+`.status.observedSparseCheckout`. The observed sparse checkout is the latest
+`.spec.sparseCheckout` value which resulted in a [ready
+state](#ready-gitrepository), or stalled due to error it can not recover from
+without human intervention. The value is the same as the [sparseCheckout in
+spec](#sparse-checkout). It indicates the sparse checkout configuration used in
+building the current artifact in storage. It is also used by the controller to
+determine if an artifact needs to be rebuilt.
+
+Example:
+```yaml
+status:
+  ...
+  observedSparseCheckout:
+  - kustomize
+  - charts
+  ...
+```
+
 ### Source Verification Mode
 
 The source-controller reports the Git object(s) it verified in the Git

internal/controller/gitrepository_controller.go

@@ -812,6 +812,7 @@ func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context, sp *pat
 	obj.Status.ObservedIgnore = obj.Spec.Ignore
 	obj.Status.ObservedRecurseSubmodules = obj.Spec.RecurseSubmodules
 	obj.Status.ObservedInclude = obj.Spec.Include
+	obj.Status.ObservedSparseCheckout = obj.Spec.SparseCheckout
 
 	// Remove the deprecated symlink.
 	// TODO(hidde): remove 2 minor versions from introduction of v1.
@@ -886,8 +887,9 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context, obj *sourcev1
 	authOpts *git.AuthOptions, proxyOpts *transport.ProxyOptions, dir string, optimized bool) (*git.Commit, error) {
 	// Configure checkout strategy.
 	cloneOpts := repository.CloneConfig{
-		RecurseSubmodules: obj.Spec.RecurseSubmodules,
-		ShallowClone:      true,
+		RecurseSubmodules:         obj.Spec.RecurseSubmodules,
+		SparseCheckoutDirectories: obj.Spec.SparseCheckout,
+		ShallowClone:              true,
 	}
 	if ref := obj.Spec.Reference; ref != nil {
 		cloneOpts.Branch = ref.Branch
@@ -1172,6 +1174,14 @@ func gitContentConfigChanged(obj *sourcev1.GitRepository, includes *artifactSet)
 	if requiresVerification(obj) {
 		return true
 	}
+	if len(obj.Spec.SparseCheckout) != len(obj.Status.ObservedSparseCheckout) {
+		return true
+	}
+	for index, dir := range obj.Spec.SparseCheckout {
+		if dir != obj.Status.ObservedSparseCheckout[index] {
+			return true
+		}
+	}
 
 	// Convert artifactSet to index addressable artifacts and ensure that it and
 	// the included artifacts include all the include from the spec.

internal/controller/gitrepository_controller_test.go

@@ -3073,6 +3073,30 @@ func TestGitContentConfigChanged(t *testing.T) {
 			},
 			want: false,
 		},
+		{
+			name: "unobserved sparse checkout",
+			obj: sourcev1.GitRepository{
+				Spec:   sourcev1.GitRepositorySpec{SparseCheckout: []string{"a/b/c", "x/y/z"}},
+				Status: sourcev1.GitRepositoryStatus{ObservedSparseCheckout: []string{"a/b/c"}},
+			},
+			want: true,
+		},
+		{
+			name: "unobserved case sensitive sparse checkout",
+			obj: sourcev1.GitRepository{
+				Spec:   sourcev1.GitRepositorySpec{SparseCheckout: []string{"a/b/c", "x/y/Z"}},
+				Status: sourcev1.GitRepositoryStatus{ObservedSparseCheckout: []string{"a/b/c", "x/y/z"}},
+			},
+			want: true,
+		},
+		{
+			name: "observed sparse checkout",
+			obj: sourcev1.GitRepository{
+				Spec:   sourcev1.GitRepositorySpec{SparseCheckout: []string{"a/b/c", "x/y/z"}},
+				Status: sourcev1.GitRepositoryStatus{ObservedSparseCheckout: []string{"a/b/c", "x/y/z"}},
+			},
+			want: false,
+		},
 		{
 			name: "unobserved include",
 			obj: sourcev1.GitRepository{