Skip to content

Commit 983480b

Browse files
tonistiigitonistiigi
authored
Merge pull request moby#3409 from jedevc/ociindex-refactor
ociindex utility package refactor
2 parents 30cd3b4 + 9623017 commit 983480b

File tree

2 files changed

+129
-82
lines changed

2 files changed

+129
-82
lines changed

‎client/ociindex/ociindex.go

Lines changed: 99 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -4,76 +4,94 @@ import (
44
"encoding/json"
55
"io"
66
"os"
7+
"path"
78

89
"github.com/gofrs/flock"
910
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
1011
"github.com/pkg/errors"
1112
)
1213

1314
const (
14-
// IndexJSONLockFileSuffix is the suffix of the lock file
15-
IndexJSONLockFileSuffix = ".lock"
15+
// indexFile is the name of the index file
16+
indexFile = "index.json"
17+
18+
// lockFileSuffix is the suffix of the lock file
19+
lockFileSuffix = ".lock"
1620
)
1721

18-
// PutDescToIndex puts desc to index with tag.
19-
// Existing manifests with the same tag will be removed from the index.
20-
func PutDescToIndex(index *ocispecs.Index, desc ocispecs.Descriptor, tag string) error {
21-
if index == nil {
22-
index = &ocispecs.Index{}
22+
type StoreIndex struct {
23+
indexPath string
24+
lockPath string
25+
}
26+
27+
func NewStoreIndex(storePath string) StoreIndex {
28+
indexPath := path.Join(storePath, indexFile)
29+
return StoreIndex{
30+
indexPath: indexPath,
31+
lockPath: indexPath + lockFileSuffix,
2332
}
24-
if index.SchemaVersion == 0 {
25-
index.SchemaVersion = 2
33+
}
34+
35+
func (s StoreIndex) Read() (*ocispecs.Index, error) {
36+
lock := flock.New(s.lockPath)
37+
locked, err := lock.TryRLock()
38+
if err != nil {
39+
return nil, errors.Wrapf(err, "could not lock %s", s.lockPath)
2640
}
27-
if tag != "" {
28-
if desc.Annotations == nil {
29-
desc.Annotations = make(map[string]string)
30-
}
31-
desc.Annotations[ocispecs.AnnotationRefName] = tag
32-
// remove existing manifests with the same tag
33-
var manifests []ocispecs.Descriptor
34-
for _, m := range index.Manifests {
35-
if m.Annotations[ocispecs.AnnotationRefName] != tag {
36-
manifests = append(manifests, m)
37-
}
38-
}
39-
index.Manifests = manifests
41+
if !locked {
42+
return nil, errors.Errorf("could not lock %s", s.lockPath)
4043
}
41-
index.Manifests = append(index.Manifests, desc)
42-
return nil
44+
defer func() {
45+
lock.Unlock()
46+
os.RemoveAll(s.lockPath)
47+
}()
48+
49+
b, err := os.ReadFile(s.indexPath)
50+
if err != nil {
51+
return nil, errors.Wrapf(err, "could not read %s", s.indexPath)
52+
}
53+
var idx ocispecs.Index
54+
if err := json.Unmarshal(b, &idx); err != nil {
55+
return nil, errors.Wrapf(err, "could not unmarshal %s (%q)", s.indexPath, string(b))
56+
}
57+
return &idx, nil
4358
}
4459

45-
func PutDescToIndexJSONFileLocked(indexJSONPath string, desc ocispecs.Descriptor, tag string) error {
46-
lockPath := indexJSONPath + IndexJSONLockFileSuffix
47-
lock := flock.New(lockPath)
60+
func (s StoreIndex) Put(tag string, desc ocispecs.Descriptor) error {
61+
lock := flock.New(s.lockPath)
4862
locked, err := lock.TryLock()
4963
if err != nil {
50-
return errors.Wrapf(err, "could not lock %s", lockPath)
64+
return errors.Wrapf(err, "could not lock %s", s.lockPath)
5165
}
5266
if !locked {
53-
return errors.Errorf("could not lock %s", lockPath)
67+
return errors.Errorf("could not lock %s", s.lockPath)
5468
}
5569
defer func() {
5670
lock.Unlock()
57-
os.RemoveAll(lockPath)
71+
os.RemoveAll(s.lockPath)
5872
}()
59-
f, err := os.OpenFile(indexJSONPath, os.O_RDWR|os.O_CREATE, 0644)
73+
74+
f, err := os.OpenFile(s.indexPath, os.O_RDWR|os.O_CREATE, 0644)
6075
if err != nil {
61-
return errors.Wrapf(err, "could not open %s", indexJSONPath)
76+
return errors.Wrapf(err, "could not open %s", s.indexPath)
6277
}
6378
defer f.Close()
79+
6480
var idx ocispecs.Index
6581
b, err := io.ReadAll(f)
6682
if err != nil {
67-
return errors.Wrapf(err, "could not read %s", indexJSONPath)
83+
return errors.Wrapf(err, "could not read %s", s.indexPath)
6884
}
6985
if len(b) > 0 {
7086
if err := json.Unmarshal(b, &idx); err != nil {
71-
return errors.Wrapf(err, "could not unmarshal %s (%q)", indexJSONPath, string(b))
87+
return errors.Wrapf(err, "could not unmarshal %s (%q)", s.indexPath, string(b))
7288
}
7389
}
74-
if err = PutDescToIndex(&idx, desc, tag); err != nil {
90+
91+
if err = insertDesc(&idx, desc, tag); err != nil {
7592
return err
7693
}
94+
7795
b, err = json.Marshal(idx)
7896
if err != nil {
7997
return err
@@ -87,27 +105,56 @@ func PutDescToIndexJSONFileLocked(indexJSONPath string, desc ocispecs.Descriptor
87105
return nil
88106
}
89107

90-
func ReadIndexJSONFileLocked(indexJSONPath string) (*ocispecs.Index, error) {
91-
lockPath := indexJSONPath + IndexJSONLockFileSuffix
92-
lock := flock.New(lockPath)
93-
locked, err := lock.TryRLock()
108+
func (s StoreIndex) Get(tag string) (*ocispecs.Descriptor, error) {
109+
idx, err := s.Read()
94110
if err != nil {
95-
return nil, errors.Wrapf(err, "could not lock %s", lockPath)
111+
return nil, err
96112
}
97-
if !locked {
98-
return nil, errors.Errorf("could not lock %s", lockPath)
113+
114+
for _, m := range idx.Manifests {
115+
if t, ok := m.Annotations[ocispecs.AnnotationRefName]; ok && t == tag {
116+
return &m, nil
117+
}
99118
}
100-
defer func() {
101-
lock.Unlock()
102-
os.RemoveAll(lockPath)
103-
}()
104-
b, err := os.ReadFile(indexJSONPath)
119+
return nil, nil
120+
}
121+
122+
func (s StoreIndex) GetSingle() (*ocispecs.Descriptor, error) {
123+
idx, err := s.Read()
105124
if err != nil {
106-
return nil, errors.Wrapf(err, "could not read %s", indexJSONPath)
125+
return nil, err
107126
}
108-
var idx ocispecs.Index
109-
if err := json.Unmarshal(b, &idx); err != nil {
110-
return nil, errors.Wrapf(err, "could not unmarshal %s (%q)", indexJSONPath, string(b))
127+
128+
if len(idx.Manifests) == 1 {
129+
return &idx.Manifests[0], nil
111130
}
112-
return &idx, nil
131+
return nil, nil
132+
}
133+
134+
// insertDesc puts desc to index with tag.
135+
// Existing manifests with the same tag will be removed from the index.
136+
func insertDesc(index *ocispecs.Index, desc ocispecs.Descriptor, tag string) error {
137+
if index == nil {
138+
return nil
139+
}
140+
141+
if index.SchemaVersion == 0 {
142+
index.SchemaVersion = 2
143+
}
144+
if tag != "" {
145+
if desc.Annotations == nil {
146+
desc.Annotations = make(map[string]string)
147+
}
148+
desc.Annotations[ocispecs.AnnotationRefName] = tag
149+
// remove existing manifests with the same tag
150+
var manifests []ocispecs.Descriptor
151+
for _, m := range index.Manifests {
152+
if m.Annotations[ocispecs.AnnotationRefName] != tag {
153+
manifests = append(manifests, m)
154+
}
155+
}
156+
index.Manifests = manifests
157+
}
158+
index.Manifests = append(index.Manifests, desc)
159+
return nil
113160
}

‎client/solve.go

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
129129
ex = opt.Exports[0]
130130
}
131131

132-
indicesToUpdate := []string{}
132+
storesToUpdate := []string{}
133133

134134
if !opt.SessionPreInitialized {
135135
if len(syncedDirs) > 0 {
@@ -194,7 +194,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
194194
return nil, err
195195
}
196196
contentStores["export"] = cs
197-
indicesToUpdate = append(indicesToUpdate, filepath.Join(ex.OutputDir, "index.json"))
197+
storesToUpdate = append(storesToUpdate, ex.OutputDir)
198198
default:
199199
s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir))
200200
}
@@ -327,8 +327,9 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
327327
if err = json.Unmarshal([]byte(manifestDescJSON), &manifestDesc); err != nil {
328328
return nil, err
329329
}
330-
for indexJSONPath, tag := range cacheOpt.indicesToUpdate {
331-
if err = ociindex.PutDescToIndexJSONFileLocked(indexJSONPath, manifestDesc, tag); err != nil {
330+
for storePath, tag := range cacheOpt.storesToUpdate {
331+
idx := ociindex.NewStoreIndex(storePath)
332+
if err := idx.Put(tag, manifestDesc); err != nil {
332333
return nil, err
333334
}
334335
}
@@ -342,12 +343,13 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
342343
if err = json.Unmarshal([]byte(manifestDescDt), &manifestDesc); err != nil {
343344
return nil, err
344345
}
345-
for _, indexJSONPath := range indicesToUpdate {
346+
for _, storePath := range storesToUpdate {
346347
tag := "latest"
347348
if t, ok := res.ExporterResponse["image.name"]; ok {
348349
tag = t
349350
}
350-
if err = ociindex.PutDescToIndexJSONFileLocked(indexJSONPath, manifestDesc, tag); err != nil {
351+
idx := ociindex.NewStoreIndex(storePath)
352+
if err := idx.Put(tag, manifestDesc); err != nil {
351353
return nil, err
352354
}
353355
}
@@ -406,10 +408,10 @@ func defaultSessionName() string {
406408
}
407409

408410
type cacheOptions struct {
409-
options controlapi.CacheOptions
410-
contentStores map[string]content.Store // key: ID of content store ("local:" + csDir)
411-
indicesToUpdate map[string]string // key: index.JSON file name, value: tag
412-
frontendAttrs map[string]string
411+
options controlapi.CacheOptions
412+
contentStores map[string]content.Store // key: ID of content store ("local:" + csDir)
413+
storesToUpdate map[string]string // key: path to content store, value: tag
414+
frontendAttrs map[string]string
413415
}
414416

415417
func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cacheOptions, error) {
@@ -418,7 +420,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
418420
cacheImports []*controlapi.CacheOptionsEntry
419421
)
420422
contentStores := make(map[string]content.Store)
421-
indicesToUpdate := make(map[string]string) // key: index.JSON file name, value: tag
423+
storesToUpdate := make(map[string]string)
422424
frontendAttrs := make(map[string]string)
423425
for _, ex := range opt.CacheExports {
424426
if ex.Type == "local" {
@@ -440,8 +442,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
440442
tag = t
441443
}
442444
// TODO(AkihiroSuda): support custom index JSON path and tag
443-
indexJSONPath := filepath.Join(csDir, "index.json")
444-
indicesToUpdate[indexJSONPath] = tag
445+
storesToUpdate[csDir] = tag
445446
}
446447
if ex.Type == "registry" {
447448
regRef := ex.Attrs["ref"]
@@ -465,27 +466,26 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
465466
bklog.G(ctx).Warning("local cache import at " + csDir + " not found due to err: " + err.Error())
466467
continue
467468
}
468-
// if digest is not specified, load from "latest" tag
469+
// if digest is not specified, attempt to load from tag
469470
if im.Attrs["digest"] == "" {
470-
idx, err := ociindex.ReadIndexJSONFileLocked(filepath.Join(csDir, "index.json"))
471+
tag := "latest"
472+
if t, ok := im.Attrs["tag"]; ok {
473+
tag = t
474+
}
475+
476+
idx := ociindex.NewStoreIndex(csDir)
477+
desc, err := idx.Get(tag)
471478
if err != nil {
472479
bklog.G(ctx).Warning("local cache import at " + csDir + " not found due to err: " + err.Error())
473480
continue
474481
}
475-
for _, m := range idx.Manifests {
476-
tag := "latest"
477-
if t, ok := im.Attrs["tag"]; ok {
478-
tag = t
479-
}
480-
if m.Annotations[ocispecs.AnnotationRefName] == tag {
481-
im.Attrs["digest"] = string(m.Digest)
482-
break
483-
}
484-
}
485-
if im.Attrs["digest"] == "" {
486-
return nil, errors.New("local cache importer requires either explicit digest, \"latest\" tag or custom tag on index.json")
482+
if desc != nil {
483+
im.Attrs["digest"] = desc.Digest.String()
487484
}
488485
}
486+
if im.Attrs["digest"] == "" {
487+
return nil, errors.New("local cache importer requires either explicit digest, \"latest\" tag or custom tag on index.json")
488+
}
489489
contentStores["local:"+csDir] = cs
490490
}
491491
if im.Type == "registry" {
@@ -513,9 +513,9 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
513513
Exports: cacheExports,
514514
Imports: cacheImports,
515515
},
516-
contentStores: contentStores,
517-
indicesToUpdate: indicesToUpdate,
518-
frontendAttrs: frontendAttrs,
516+
contentStores: contentStores,
517+
storesToUpdate: storesToUpdate,
518+
frontendAttrs: frontendAttrs,
519519
}
520520
return &res, nil
521521
}

0 commit comments

Comments
 (0)