Skip to content

Commit ec51eda

Browse files
hiddecohiddeco
authored
Merge pull request #134 from fluxcd/artifact-api-improvements
2 parents ecaa980 + 68947cf commit ec51eda

12 files changed

+138
-76
lines changed

‎api/v1alpha1/artifact.go‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,35 +17,47 @@ limitations under the License.
1717
package v1alpha1
1818

1919
import (
20-
"fmt"
20+
"path"
21+
"strings"
2122

2223
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2324
)
2425

25-
// Artifact represents the output of a source synchronisation
26+
// Artifact represents the output of a source synchronisation.
2627
type Artifact struct {
27-
// Path is the local file path of this artifact.
28+
// Path is the relative file path of this artifact.
2829
// +required
2930
Path string `json:"path"`
3031

3132
// URL is the HTTP address of this artifact.
3233
// +required
3334
URL string `json:"url"`
3435

35-
// Revision is a human readable identifier traceable in the origin source system.
36-
// It can be a commit sha, git tag, a helm index timestamp,
37-
// a helm chart version, a checksum, etc.
36+
// Revision is a human readable identifier traceable in the origin
37+
// source system. It can be a Git commit sha, Git tag, a Helm index
38+
// timestamp, a Helm chart version, etc.
3839
// +optional
3940
Revision string `json:"revision"`
4041

42+
// Checksum is the SHA1 checksum of the artifact.
43+
// +optional
44+
Checksum string `json:"checksum"`
45+
4146
// LastUpdateTime is the timestamp corresponding to the last
4247
// update of this artifact.
4348
// +required
4449
LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
4550
}
4651

52+
// ArtifactDir returns the artifact dir path in the form of
53+
// <source-kind>/<source-namespace>/<source-name>.
54+
func ArtifactDir(kind, namespace, name string) string {
55+
kind = strings.ToLower(kind)
56+
return path.Join(kind, namespace, name)
57+
}
58+
4759
// ArtifactPath returns the artifact path in the form of
48-
// <source-kind>/<source-namespace>/<source-name>/<artifact-filename>
60+
// <source-kind>/<source-namespace>/<source-name>/<artifact-filename>.
4961
func ArtifactPath(kind, namespace, name, filename string) string {
50-
return fmt.Sprintf("%s/%s/%s/%s", kind, namespace, name, filename)
62+
return path.Join(ArtifactDir(kind, namespace, name), filename)
5163
}

‎api/v1alpha1/source.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package v1alpha1
22

3-
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3+
import (
4+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
5+
)
46

57
// Source interface must be supported by all API types.
68
// +k8s:deepcopy-gen=false

‎config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,18 +129,21 @@ spec:
129129
description: Artifact represents the output of the last successful
130130
repository sync.
131131
properties:
132+
checksum:
133+
description: Checksum is the SHA1 checksum of the artifact.
134+
type: string
132135
lastUpdateTime:
133136
description: LastUpdateTime is the timestamp corresponding to
134137
the last update of this artifact.
135138
format: date-time
136139
type: string
137140
path:
138-
description: Path is the local file path of this artifact.
141+
description: Path is the relative file path of this artifact.
139142
type: string
140143
revision:
141144
description: Revision is a human readable identifier traceable
142-
in the origin source system. It can be a commit sha, git tag,
143-
a helm index timestamp, a helm chart version, a checksum, etc.
145+
in the origin source system. It can be a Git commit sha, Git
146+
tag, a Helm index timestamp, a Helm chart version, etc.
144147
type: string
145148
url:
146149
description: URL is the HTTP address of this artifact.

‎config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,18 +101,21 @@ spec:
101101
description: Artifact represents the output of the last successful
102102
chart sync.
103103
properties:
104+
checksum:
105+
description: Checksum is the SHA1 checksum of the artifact.
106+
type: string
104107
lastUpdateTime:
105108
description: LastUpdateTime is the timestamp corresponding to
106109
the last update of this artifact.
107110
format: date-time
108111
type: string
109112
path:
110-
description: Path is the local file path of this artifact.
113+
description: Path is the relative file path of this artifact.
111114
type: string
112115
revision:
113116
description: Revision is a human readable identifier traceable
114-
in the origin source system. It can be a commit sha, git tag,
115-
a helm index timestamp, a helm chart version, a checksum, etc.
117+
in the origin source system. It can be a Git commit sha, Git
118+
tag, a Helm index timestamp, a Helm chart version, etc.
116119
type: string
117120
url:
118121
description: URL is the HTTP address of this artifact.

‎config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -81,18 +81,21 @@ spec:
8181
description: Artifact represents the output of the last successful
8282
repository sync.
8383
properties:
84+
checksum:
85+
description: Checksum is the SHA1 checksum of the artifact.
86+
type: string
8487
lastUpdateTime:
8588
description: LastUpdateTime is the timestamp corresponding to
8689
the last update of this artifact.
8790
format: date-time
8891
type: string
8992
path:
90-
description: Path is the local file path of this artifact.
93+
description: Path is the relative file path of this artifact.
9194
type: string
9295
revision:
9396
description: Revision is a human readable identifier traceable
94-
in the origin source system. It can be a commit sha, git tag,
95-
a helm index timestamp, a helm chart version, a checksum, etc.
97+
in the origin source system. It can be a Git commit sha, Git
98+
tag, a Helm index timestamp, a Helm chart version, etc.
9699
type: string
97100
url:
98101
description: URL is the HTTP address of this artifact.

‎controllers/gitrepository_controller.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -212,8 +212,10 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
212212
}
213213
}
214214

215+
// TODO(hidde): implement checksum when https://github.com/fluxcd/source-controller/pull/133
216+
// has been merged.
215217
artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(),
216-
fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision)
218+
fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision, "")
217219

218220
// create artifact dir
219221
err = r.Storage.MkdirAll(artifact)
@@ -300,10 +302,10 @@ func (r *GitRepositoryReconciler) verify(ctx context.Context, publicKeySecret ty
300302
// gc performs a garbage collection on all but current artifacts of
301303
// the given repository.
302304
func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository, all bool) error {
305+
if all {
306+
return r.Storage.RemoveAll(r.Storage.ArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "", ""))
307+
}
303308
if repository.Status.Artifact != nil {
304-
if all {
305-
return r.Storage.RemoveAll(*repository.Status.Artifact)
306-
}
307309
return r.Storage.RemoveAllButCurrent(*repository.Status.Artifact)
308310
}
309311
return nil

‎controllers/helmchart_controller.go‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,8 @@ func (r *HelmChartReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts
188188

189189
func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
190190
repository sourcev1.HelmRepository, chart sourcev1.HelmChart) (sourcev1.HelmChart, error) {
191-
cv, err := helm.GetDownloadableChartVersionFromIndex(repository.Status.Artifact.Path, chart.Spec.Chart, chart.Spec.Version)
191+
cv, err := helm.GetDownloadableChartVersionFromIndex(r.Storage.LocalPath(*repository.GetArtifact()),
192+
chart.Spec.Chart, chart.Spec.Version)
192193
if err != nil {
193194
return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
194195
}
@@ -260,7 +261,7 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
260261

261262
sum := r.Storage.Checksum(chartBytes)
262263
artifact := r.Storage.ArtifactFor(chart.Kind, chart.GetObjectMeta(),
263-
fmt.Sprintf("%s-%s-%s.tgz", cv.Name, cv.Version, sum), cv.Version)
264+
fmt.Sprintf("%s-%s-%s.tgz", cv.Name, cv.Version, sum), cv.Version, sum)
264265

265266
// create artifact dir
266267
err = r.Storage.MkdirAll(artifact)
@@ -333,7 +334,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
333334
defer os.RemoveAll(tmpDir)
334335

335336
// open file
336-
f, err := os.Open(repository.GetArtifact().Path)
337+
f, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact()))
337338
if err != nil {
338339
err = fmt.Errorf("artifact open error: %w", err)
339340
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
@@ -364,8 +365,10 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
364365
return chart, nil
365366
}
366367

368+
// TODO(hidde): implement checksum when https://github.com/fluxcd/source-controller/pull/133
369+
// has been merged.
367370
artifact := r.Storage.ArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(),
368-
fmt.Sprintf("%s-%s.tgz", chartMetadata.Name, chartMetadata.Version), chartMetadata.Version)
371+
fmt.Sprintf("%s-%s.tgz", chartMetadata.Name, chartMetadata.Version), chartMetadata.Version, "")
369372

370373
// create artifact dir
371374
err = r.Storage.MkdirAll(artifact)
@@ -384,7 +387,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
384387

385388
// package chart
386389
pkg := action.NewPackage()
387-
pkg.Destination = filepath.Dir(artifact.Path)
390+
pkg.Destination = filepath.Dir(r.Storage.LocalPath(artifact))
388391
_, err = pkg.Run(chartPath, nil)
389392
if err != nil {
390393
err = fmt.Errorf("chart package error: %w", err)
@@ -432,10 +435,10 @@ func (r *HelmChartReconciler) getGitRepositoryWithArtifact(ctx context.Context,
432435
// gc performs a garbage collection on all but current artifacts of
433436
// the given chart.
434437
func (r *HelmChartReconciler) gc(chart sourcev1.HelmChart, all bool) error {
438+
if all {
439+
return r.Storage.RemoveAll(r.Storage.ArtifactFor(chart.Kind, chart.GetObjectMeta(), "", "", ""))
440+
}
435441
if chart.Status.Artifact != nil {
436-
if all {
437-
return r.Storage.RemoveAll(*chart.Status.Artifact)
438-
}
439442
return r.Storage.RemoveAllButCurrent(*chart.Status.Artifact)
440443
}
441444
return nil

‎controllers/helmrepository_controller.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
229229

230230
sum := r.Storage.Checksum(index)
231231
artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(),
232-
fmt.Sprintf("index-%s.yaml", sum), sum)
232+
fmt.Sprintf("index-%s.yaml", sum), i.Generated.Format(time.RFC3339Nano), sum)
233233

234234
// create artifact dir
235235
err = r.Storage.MkdirAll(artifact)
@@ -294,10 +294,10 @@ func (r *HelmRepositoryReconciler) shouldResetStatus(repository sourcev1.HelmRep
294294
// gc performs a garbage collection on all but current artifacts of
295295
// the given repository.
296296
func (r *HelmRepositoryReconciler) gc(repository sourcev1.HelmRepository, all bool) error {
297+
if all {
298+
return r.Storage.RemoveAll(r.Storage.ArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "", ""))
299+
}
297300
if repository.Status.Artifact != nil {
298-
if all {
299-
return r.Storage.RemoveAll(*repository.Status.Artifact)
300-
}
301301
return r.Storage.RemoveAllButCurrent(*repository.Status.Artifact)
302302
}
303303
return nil

‎controllers/storage.go‎

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,8 @@ func NewStorage(basePath string, hostname string, timeout time.Duration) (*Stora
6969
}, nil
7070
}
7171

72-
// ArtifactFor returns an artifact for the given Kubernetes object
73-
func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision string) sourcev1.Artifact {
74-
kind = strings.ToLower(kind)
72+
// ArtifactFor returns an artifact for the v1alpha1.Source.
73+
func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision, checksum string) sourcev1.Artifact {
7574
path := sourcev1.ArtifactPath(kind, metadata.GetNamespace(), metadata.GetName(), fileName)
7675
localPath := filepath.Join(s.BasePath, path)
7776
url := fmt.Sprintf("http://%s/%s", s.Hostname, path)
@@ -80,33 +79,35 @@ func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, rev
8079
Path: localPath,
8180
URL: url,
8281
Revision: revision,
82+
Checksum: checksum,
8383
LastUpdateTime: metav1.Now(),
8484
}
8585
}
8686

87-
// MkdirAll calls os.MkdirAll for the given artifact base dir
87+
// MkdirAll calls os.MkdirAll for the given v1alpha1.Artifact base dir.
8888
func (s *Storage) MkdirAll(artifact sourcev1.Artifact) error {
89-
dir := filepath.Dir(artifact.Path)
89+
dir := filepath.Dir(s.LocalPath(artifact))
9090
return os.MkdirAll(dir, 0777)
9191
}
9292

93-
// RemoveAll calls os.RemoveAll for the given artifact base dir
93+
// RemoveAll calls os.RemoveAll for the given v1alpha1.Artifact base dir.
9494
func (s *Storage) RemoveAll(artifact sourcev1.Artifact) error {
95-
dir := filepath.Dir(artifact.Path)
95+
dir := filepath.Dir(s.LocalPath(artifact))
9696
return os.RemoveAll(dir)
9797
}
9898

9999
// RemoveAllButCurrent removes all files for the given artifact base dir excluding the current one
100100
func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
101-
dir := filepath.Dir(artifact.Path)
101+
localPath := s.LocalPath(artifact)
102+
dir := filepath.Dir(localPath)
102103
var errors []string
103104
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
104105
if err != nil {
105106
errors = append(errors, err.Error())
106107
return nil
107108
}
108109

109-
if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
110+
if path != localPath && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
110111
if err := os.Remove(path); err != nil {
111112
errors = append(errors, info.Name())
112113
}
@@ -123,7 +124,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
123124
// ArtifactExist returns a boolean indicating whether the artifact exists in storage and is a
124125
// regular file.
125126
func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool {
126-
fi, err := os.Lstat(artifact.Path)
127+
fi, err := os.Lstat(s.LocalPath(artifact))
127128
if err != nil {
128129
return false
129130
}
@@ -144,7 +145,7 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
144145

145146
matcher := gitignore.NewMatcher(ps)
146147

147-
gzFile, err := os.Create(artifact.Path)
148+
gzFile, err := os.Create(s.LocalPath(artifact))
148149
if err != nil {
149150
return err
150151
}
@@ -205,28 +206,30 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
205206

206207
// WriteFile writes the given bytes to the artifact path if the checksum differs
207208
func (s *Storage) WriteFile(artifact sourcev1.Artifact, data []byte) error {
209+
localPath := s.LocalPath(artifact)
208210
sum := s.Checksum(data)
209-
if file, err := os.Stat(artifact.Path); !os.IsNotExist(err) && !file.IsDir() {
210-
if fb, err := ioutil.ReadFile(artifact.Path); err == nil && sum == s.Checksum(fb) {
211+
if file, err := os.Stat(localPath); !os.IsNotExist(err) && !file.IsDir() {
212+
if fb, err := ioutil.ReadFile(localPath); err == nil && sum == s.Checksum(fb) {
211213
return nil
212214
}
213215
}
214216

215-
return ioutil.WriteFile(artifact.Path, data, 0644)
217+
return ioutil.WriteFile(localPath, data, 0644)
216218
}
217219

218220
// Symlink creates or updates a symbolic link for the given artifact
219-
// and returns the URL for the symlink
221+
// and returns the URL for the symlink.
220222
func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string, error) {
221-
dir := filepath.Dir(artifact.Path)
223+
localPath := s.LocalPath(artifact)
224+
dir := filepath.Dir(localPath)
222225
link := filepath.Join(dir, linkName)
223226
tmpLink := link + ".tmp"
224227

225228
if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) {
226229
return "", err
227230
}
228231

229-
if err := os.Symlink(artifact.Path, tmpLink); err != nil {
232+
if err := os.Symlink(localPath, tmpLink); err != nil {
230233
return "", err
231234
}
232235

@@ -246,11 +249,20 @@ func (s *Storage) Checksum(b []byte) string {
246249

247250
// Lock creates a file lock for the given artifact
248251
func (s *Storage) Lock(artifact sourcev1.Artifact) (unlock func(), err error) {
249-
lockFile := artifact.Path + ".lock"
252+
lockFile := s.LocalPath(artifact) + ".lock"
250253
mutex := lockedfile.MutexAt(lockFile)
251254
return mutex.Lock()
252255
}
253256

257+
// LocalPath returns the local path of the given artifact (that is: relative to
258+
// the Storage.BasePath).
259+
func (s *Storage) LocalPath(artifact sourcev1.Artifact) string {
260+
if artifact.Path == "" {
261+
return ""
262+
}
263+
return filepath.Join(s.BasePath, artifact.Path)
264+
}
265+
254266
func getPatterns(reader io.Reader, path []string) []gitignore.Pattern {
255267
var ps []gitignore.Pattern
256268
scanner := bufio.NewScanner(reader)

0 commit comments

Comments
 (0)