storage: only store relative path in artifact

As the storage base directory is determined during runtime, and
artifacts may live longer than that if they are e.g. stored in a
persistent volume but the mount path configuration changes.

Author: hiddeco

api/v1alpha1/source.go

@@ -1,6 +1,8 @@
 package v1alpha1
 
-import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
 
 // Source interface must be supported by all API types.
 // +k8s:deepcopy-gen=false

controllers/helmchart_controller.go

@@ -188,7 +188,8 @@ func (r *HelmChartReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts
 
 func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
 	repository sourcev1.HelmRepository, chart sourcev1.HelmChart) (sourcev1.HelmChart, error) {
-	cv, err := helm.GetDownloadableChartVersionFromIndex(repository.Status.Artifact.Path, chart.Spec.Chart, chart.Spec.Version)
+	cv, err := helm.GetDownloadableChartVersionFromIndex(r.Storage.LocalPath(*repository.GetArtifact()),
+		chart.Spec.Chart, chart.Spec.Version)
 	if err != nil {
 		return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
 	}
@@ -333,7 +334,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
 	defer os.RemoveAll(tmpDir)
 
 	// open file
-	f, err := os.Open(repository.GetArtifact().Path)
+	f, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact()))
 	if err != nil {
 		err = fmt.Errorf("artifact open error: %w", err)
 		return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
@@ -384,7 +385,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
 
 	// package chart
 	pkg := action.NewPackage()
-	pkg.Destination = filepath.Dir(artifact.Path)
+	pkg.Destination = filepath.Dir(r.Storage.LocalPath(artifact))
 	_, err = pkg.Run(chartPath, nil)
 	if err != nil {
 		err = fmt.Errorf("chart package error: %w", err)

controllers/storage.go

@@ -69,7 +69,7 @@ func NewStorage(basePath string, hostname string, timeout time.Duration) (*Stora
 	}, nil
 }
 
-// ArtifactFor returns an artifact for the given Kubernetes object
+// ArtifactFor returns an artifact for the v1alpha1.Source.
 func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision string) sourcev1.Artifact {
 	path := sourcev1.ArtifactPath(kind, metadata.GetNamespace(), metadata.GetName(), fileName)
 	localPath := filepath.Join(s.BasePath, path)
@@ -83,29 +83,30 @@ func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, rev
 	}
 }
 
-// MkdirAll calls os.MkdirAll for the given artifact base dir
+// MkdirAll calls os.MkdirAll for the given v1alpha1.Artifact base dir.
 func (s *Storage) MkdirAll(artifact sourcev1.Artifact) error {
-	dir := filepath.Dir(artifact.Path)
+	dir := filepath.Dir(s.LocalPath(artifact))
 	return os.MkdirAll(dir, 0777)
 }
 
-// RemoveAll calls os.RemoveAll for the given artifact base dir
+// RemoveAll calls os.RemoveAll for the given v1alpha1.Artifact base dir.
 func (s *Storage) RemoveAll(artifact sourcev1.Artifact) error {
-	dir := filepath.Dir(artifact.Path)
+	dir := filepath.Dir(s.LocalPath(artifact))
 	return os.RemoveAll(dir)
 }
 
 // RemoveAllButCurrent removes all files for the given artifact base dir excluding the current one
 func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
-	dir := filepath.Dir(artifact.Path)
+	localPath := s.LocalPath(artifact)
+	dir := filepath.Dir(localPath)
 	var errors []string
 	_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			errors = append(errors, err.Error())
 			return nil
 		}
 
-		if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
+		if path != localPath && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
 			if err := os.Remove(path); err != nil {
 				errors = append(errors, info.Name())
 			}
@@ -122,7 +123,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
 // ArtifactExist returns a boolean indicating whether the artifact exists in storage and is a
 // regular file.
 func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool {
-	fi, err := os.Lstat(artifact.Path)
+	fi, err := os.Lstat(s.LocalPath(artifact))
 	if err != nil {
 		return false
 	}
@@ -143,7 +144,7 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
 
 	matcher := gitignore.NewMatcher(ps)
 
-	gzFile, err := os.Create(artifact.Path)
+	gzFile, err := os.Create(s.LocalPath(artifact))
 	if err != nil {
 		return err
 	}
@@ -204,28 +205,30 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
 
 // WriteFile writes the given bytes to the artifact path if the checksum differs
 func (s *Storage) WriteFile(artifact sourcev1.Artifact, data []byte) error {
+	localPath := s.LocalPath(artifact)
 	sum := s.Checksum(data)
-	if file, err := os.Stat(artifact.Path); !os.IsNotExist(err) && !file.IsDir() {
-		if fb, err := ioutil.ReadFile(artifact.Path); err == nil && sum == s.Checksum(fb) {
+	if file, err := os.Stat(localPath); !os.IsNotExist(err) && !file.IsDir() {
+		if fb, err := ioutil.ReadFile(localPath); err == nil && sum == s.Checksum(fb) {
 			return nil
 		}
 	}
 
-	return ioutil.WriteFile(artifact.Path, data, 0644)
+	return ioutil.WriteFile(localPath, data, 0644)
 }
 
 // Symlink creates or updates a symbolic link for the given artifact
-// and returns the URL for the symlink
+// and returns the URL for the symlink.
 func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string, error) {
-	dir := filepath.Dir(artifact.Path)
+	localPath := s.LocalPath(artifact)
+	dir := filepath.Dir(localPath)
 	link := filepath.Join(dir, linkName)
 	tmpLink := link + ".tmp"
 
 	if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) {
 		return "", err
 	}
 
-	if err := os.Symlink(artifact.Path, tmpLink); err != nil {
+	if err := os.Symlink(localPath, tmpLink); err != nil {
 		return "", err
 	}
 
@@ -245,11 +248,20 @@ func (s *Storage) Checksum(b []byte) string {
 
 // Lock creates a file lock for the given artifact
 func (s *Storage) Lock(artifact sourcev1.Artifact) (unlock func(), err error) {
-	lockFile := artifact.Path + ".lock"
+	lockFile := s.LocalPath(artifact) + ".lock"
 	mutex := lockedfile.MutexAt(lockFile)
 	return mutex.Lock()
 }
 
+// LocalPath returns the local path of the given artifact (that is: relative to
+// the Storage.BasePath).
+func (s *Storage) LocalPath(artifact sourcev1.Artifact) string {
+	if artifact.Path == "" {
+		return ""
+	}
+	return filepath.Join(s.BasePath, artifact.Path)
+}
+
 func getPatterns(reader io.Reader, path []string) []gitignore.Pattern {
 	var ps []gitignore.Pattern
 	scanner := bufio.NewScanner(reader)

controllers/storage_test.go

@@ -113,9 +113,9 @@ func walkTar(tarFile string, match string) (bool, error) {
 	return false, nil
 }
 
-func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) {
+func testPatterns(t *testing.T, dir string, artifact sourcev1.Artifact, table ignoreMap) {
 	for name, expected := range table {
-		res, err := walkTar(artifact.Path, name)
+		res, err := walkTar(filepath.Join(dir, artifact.Path), name)
 		if err != nil {
 			t.Fatalf("while reading tarball: %v", err)
 		}
@@ -130,13 +130,7 @@ func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) {
 	}
 }
 
-func createArchive(t *testing.T, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact {
-	dir, err := createStoragePath()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Cleanup(cleanupStoragePath(dir))
-
+func createArchive(t *testing.T, dir string, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact {
 	storage, err := NewStorage(dir, "hostname", time.Minute)
 	if err != nil {
 		t.Fatalf("Error while bootstrapping storage: %v", err)
@@ -197,7 +191,13 @@ func TestArchiveBasic(t *testing.T) {
 		".gitignore": false,
 	}
 
-	testPatterns(t, createArchive(t, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table)
+	dir, err := createStoragePath()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(cleanupStoragePath(dir))
+
+	testPatterns(t, dir, createArchive(t, dir, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table)
 }
 
 func TestArchiveIgnore(t *testing.T) {
@@ -221,8 +221,14 @@ func TestArchiveIgnore(t *testing.T) {
 		table[item] = false
 	}
 
+	dir, err := createStoragePath()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(cleanupStoragePath(dir))
+
 	t.Run("automatically ignored files", func(t *testing.T) {
-		testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{}), table)
+		testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{}), table)
 	})
 
 	table = ignoreMap{}
@@ -231,15 +237,15 @@ func TestArchiveIgnore(t *testing.T) {
 	}
 
 	t.Run("only vcs ignored files", func(t *testing.T) {
-		testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table)
+		testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table)
 	})
 
 	filenames = append(filenames, "test.txt")
 	table["test.txt"] = false
 	sourceIgnoreFile := "*.txt"
 
 	t.Run("sourceignore injected via CRD", func(t *testing.T) {
-		testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table)
+		testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table)
 	})
 
 	table = ignoreMap{}
@@ -248,7 +254,7 @@ func TestArchiveIgnore(t *testing.T) {
 	}
 
 	t.Run("sourceignore injected via filename", func(t *testing.T) {
-		testPatterns(t, createArchive(t, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
+		testPatterns(t, dir, createArchive(t, dir, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
 	})
 }