Skip to content

Commit 23837e5

Browse files
tonistiigitonistiigi
authored
Merge pull request moby#3061 from jedevc/force-ocimediatypes
Force oci-mediatypes for containerimage outputs that utilize annotations
2 parents cdc17fe + 0a5265a commit 23837e5

File tree

7 files changed

+249
-106
lines changed

7 files changed

+249
-106
lines changed

‎client/client_test.go

Lines changed: 170 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ func TestIntegration(t *testing.T) {
171171
testCallInfo,
172172
testPullWithLayerLimit,
173173
testExportAnnotations,
174+
testExportAnnotationsMediaTypes,
174175
testExportAttestations,
175176
)
176177
tests = append(tests, diffOpTestCases()...)
@@ -6098,6 +6099,12 @@ func testExportAnnotations(t *testing.T, sb integration.Sandbox) {
60986099
require.NoError(t, err)
60996100
defer c.Close()
61006101

6102+
registry, err := sb.NewRegistry()
6103+
if errors.Is(err, integration.ErrRequirements) {
6104+
t.Skip(err.Error())
6105+
}
6106+
require.NoError(t, err)
6107+
61016108
amd64 := platforms.MustParse("linux/amd64")
61026109
arm64 := platforms.MustParse("linux/arm64")
61036110
ps := []ocispecs.Platform{amd64, arm64}
@@ -6163,18 +6170,19 @@ func testExportAnnotations(t *testing.T, sb integration.Sandbox) {
61636170

61646171
// testing for image exporter
61656172

6166-
target := "testannotations:latest"
6173+
target := registry + "/buildkit/testannotations:latest"
61676174

61686175
_, err = c.Build(sb.Context(), SolveOpt{
61696176
Exports: []ExportEntry{
61706177
{
61716178
Type: ExporterImage,
61726179
Attrs: map[string]string{
6173-
"name": target,
6174-
"annotation-index.gio": "generic index opt",
6175-
"annotation-manifest.gmo": "generic manifest opt",
6176-
"annotation-manifest-descriptor.gmdo": "generic manifest descriptor opt",
6177-
"annotation-manifest[linux/amd64].mo": "amd64 manifest opt",
6180+
"name": target,
6181+
"push": "true",
6182+
"annotation-index.gio": "generic index opt",
6183+
"annotation-manifest.gmo": "generic manifest opt",
6184+
"annotation-manifest-descriptor.gmdo": "generic manifest descriptor opt",
6185+
"annotation-manifest[linux/amd64].mo": "amd64 manifest opt",
61786186
"annotation-manifest-descriptor[linux/amd64].mdo": "amd64 manifest descriptor opt",
61796187
"annotation-manifest[linux/arm64].mo": "arm64 manifest opt",
61806188
"annotation-manifest-descriptor[linux/arm64].mdo": "arm64 manifest descriptor opt",
@@ -6184,53 +6192,42 @@ func testExportAnnotations(t *testing.T, sb integration.Sandbox) {
61846192
}, "", frontend, nil)
61856193
require.NoError(t, err)
61866194

6187-
ctx := namespaces.WithNamespace(sb.Context(), "buildkit")
6188-
cdAddress := sb.ContainerdAddress()
6189-
if cdAddress != "" {
6190-
client, err := newContainerd(cdAddress)
6191-
require.NoError(t, err)
6192-
defer client.Close()
6193-
6194-
img, err := client.GetImage(ctx, target)
6195-
require.NoError(t, err)
6195+
desc, provider, err := contentutil.ProviderFromRef(target)
6196+
require.NoError(t, err)
6197+
imgs, err := testutil.ReadImages(sb.Context(), provider, desc)
6198+
require.NoError(t, err)
6199+
require.Equal(t, 2, len(imgs.Images))
61966200

6197-
var index ocispecs.Index
6198-
indexBytes, err := content.ReadBlob(ctx, client.ContentStore(), img.Target())
6199-
require.NoError(t, err)
6200-
require.NoError(t, json.Unmarshal(indexBytes, &index))
6201-
6202-
require.Equal(t, "generic index", index.Annotations["gi"])
6203-
require.Equal(t, "generic index opt", index.Annotations["gio"])
6204-
for _, desc := range index.Manifests {
6205-
require.Equal(t, "generic manifest descriptor", desc.Annotations["gmd"])
6206-
require.Equal(t, "generic manifest descriptor opt", desc.Annotations["gmdo"])
6207-
switch {
6208-
case platforms.Only(amd64).Match(*desc.Platform):
6209-
require.Equal(t, "amd64 manifest descriptor", desc.Annotations["md"])
6210-
require.Equal(t, "amd64 manifest descriptor opt", desc.Annotations["mdo"])
6211-
case platforms.Only(arm64).Match(*desc.Platform):
6212-
require.Equal(t, "arm64 manifest descriptor", desc.Annotations["md"])
6213-
require.Equal(t, "arm64 manifest descriptor opt", desc.Annotations["mdo"])
6214-
default:
6215-
require.Fail(t, "unrecognized platform")
6216-
}
6201+
require.Equal(t, "generic index", imgs.Index.Annotations["gi"])
6202+
require.Equal(t, "generic index opt", imgs.Index.Annotations["gio"])
6203+
for _, desc := range imgs.Index.Manifests {
6204+
require.Equal(t, "generic manifest descriptor", desc.Annotations["gmd"])
6205+
require.Equal(t, "generic manifest descriptor opt", desc.Annotations["gmdo"])
6206+
switch {
6207+
case platforms.Only(amd64).Match(*desc.Platform):
6208+
require.Equal(t, "amd64 manifest descriptor", desc.Annotations["md"])
6209+
require.Equal(t, "amd64 manifest descriptor opt", desc.Annotations["mdo"])
6210+
case platforms.Only(arm64).Match(*desc.Platform):
6211+
require.Equal(t, "arm64 manifest descriptor", desc.Annotations["md"])
6212+
require.Equal(t, "arm64 manifest descriptor opt", desc.Annotations["mdo"])
6213+
default:
6214+
require.Fail(t, "unrecognized platform")
62176215
}
6216+
}
62186217

6219-
mfst, err := images.Manifest(ctx, client.ContentStore(), img.Target(), platforms.Only(amd64))
6220-
require.NoError(t, err)
6221-
require.Equal(t, "generic default", mfst.Annotations["gd"])
6222-
require.Equal(t, "generic manifest", mfst.Annotations["gm"])
6223-
require.Equal(t, "generic manifest opt", mfst.Annotations["gmo"])
6224-
require.Equal(t, "amd64 manifest", mfst.Annotations["m"])
6225-
require.Equal(t, "amd64 manifest opt", mfst.Annotations["mo"])
6218+
amdImage := imgs.Find(platforms.Format(amd64))
6219+
require.Equal(t, "generic default", amdImage.Manifest.Annotations["gd"])
6220+
require.Equal(t, "generic manifest", amdImage.Manifest.Annotations["gm"])
6221+
require.Equal(t, "generic manifest opt", amdImage.Manifest.Annotations["gmo"])
6222+
require.Equal(t, "amd64 manifest", amdImage.Manifest.Annotations["m"])
6223+
require.Equal(t, "amd64 manifest opt", amdImage.Manifest.Annotations["mo"])
62266224

6227-
mfst, err = images.Manifest(ctx, client.ContentStore(), img.Target(), platforms.Only(arm64))
6228-
require.NoError(t, err)
6229-
require.Equal(t, "generic manifest", mfst.Annotations["gm"])
6230-
require.Equal(t, "generic manifest opt", mfst.Annotations["gmo"])
6231-
require.Equal(t, "arm64 manifest", mfst.Annotations["m"])
6232-
require.Equal(t, "arm64 manifest opt", mfst.Annotations["mo"])
6233-
}
6225+
armImage := imgs.Find(platforms.Format(arm64))
6226+
require.Equal(t, "generic default", armImage.Manifest.Annotations["gd"])
6227+
require.Equal(t, "generic manifest", armImage.Manifest.Annotations["gm"])
6228+
require.Equal(t, "generic manifest opt", armImage.Manifest.Annotations["gmo"])
6229+
require.Equal(t, "arm64 manifest", armImage.Manifest.Annotations["m"])
6230+
require.Equal(t, "arm64 manifest opt", armImage.Manifest.Annotations["mo"])
62346231

62356232
// testing for oci exporter
62366233

@@ -6306,6 +6303,119 @@ func testExportAnnotations(t *testing.T, sb integration.Sandbox) {
63066303
}
63076304
}
63086305

6306+
func testExportAnnotationsMediaTypes(t *testing.T, sb integration.Sandbox) {
6307+
requiresLinux(t)
6308+
c, err := New(sb.Context(), sb.Address())
6309+
require.NoError(t, err)
6310+
defer c.Close()
6311+
6312+
registry, err := sb.NewRegistry()
6313+
if errors.Is(err, integration.ErrRequirements) {
6314+
t.Skip(err.Error())
6315+
}
6316+
require.NoError(t, err)
6317+
6318+
p := platforms.DefaultSpec()
6319+
ps := []ocispecs.Platform{p}
6320+
6321+
frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
6322+
res := gateway.NewResult()
6323+
expPlatforms := &exptypes.Platforms{
6324+
Platforms: make([]exptypes.Platform, len(ps)),
6325+
}
6326+
for i, p := range ps {
6327+
st := llb.Scratch().File(
6328+
llb.Mkfile("platform", 0600, []byte(platforms.Format(p))),
6329+
)
6330+
6331+
def, err := st.Marshal(ctx)
6332+
if err != nil {
6333+
return nil, err
6334+
}
6335+
6336+
r, err := c.Solve(ctx, gateway.SolveRequest{
6337+
Definition: def.ToPB(),
6338+
})
6339+
if err != nil {
6340+
return nil, err
6341+
}
6342+
6343+
ref, err := r.SingleRef()
6344+
if err != nil {
6345+
return nil, err
6346+
}
6347+
6348+
_, err = ref.ToState()
6349+
if err != nil {
6350+
return nil, err
6351+
}
6352+
6353+
k := platforms.Format(p)
6354+
res.AddRef(k, ref)
6355+
6356+
expPlatforms.Platforms[i] = exptypes.Platform{
6357+
ID: k,
6358+
Platform: p,
6359+
}
6360+
}
6361+
dt, err := json.Marshal(expPlatforms)
6362+
if err != nil {
6363+
return nil, err
6364+
}
6365+
res.AddMeta(exptypes.ExporterPlatformsKey, dt)
6366+
6367+
return res, nil
6368+
}
6369+
6370+
target := registry + "/buildkit/testannotationsmedia:1"
6371+
_, err = c.Build(sb.Context(), SolveOpt{
6372+
Exports: []ExportEntry{
6373+
{
6374+
Type: ExporterImage,
6375+
Attrs: map[string]string{
6376+
"name": target,
6377+
"push": "true",
6378+
"annotation-manifest.a": "b",
6379+
},
6380+
},
6381+
},
6382+
}, "", frontend, nil)
6383+
require.NoError(t, err)
6384+
6385+
desc, provider, err := contentutil.ProviderFromRef(target)
6386+
require.NoError(t, err)
6387+
imgs, err := testutil.ReadImages(sb.Context(), provider, desc)
6388+
require.NoError(t, err)
6389+
require.Equal(t, 1, len(imgs.Images))
6390+
6391+
target2 := registry + "/buildkit/testannotationsmedia:2"
6392+
_, err = c.Build(sb.Context(), SolveOpt{
6393+
Exports: []ExportEntry{
6394+
{
6395+
Type: ExporterImage,
6396+
Attrs: map[string]string{
6397+
"name": target2,
6398+
"push": "true",
6399+
"annotation-index.c": "d",
6400+
},
6401+
},
6402+
},
6403+
}, "", frontend, nil)
6404+
require.NoError(t, err)
6405+
6406+
desc, provider, err = contentutil.ProviderFromRef(target2)
6407+
require.NoError(t, err)
6408+
imgs2, err := testutil.ReadImages(sb.Context(), provider, desc)
6409+
require.NoError(t, err)
6410+
require.Equal(t, 1, len(imgs2.Images))
6411+
6412+
require.Equal(t, "b", imgs.Images[0].Manifest.Annotations["a"])
6413+
require.Equal(t, "d", imgs2.Index.Annotations["c"])
6414+
6415+
require.Equal(t, images.MediaTypeDockerSchema2ManifestList, imgs.Index.MediaType)
6416+
require.Equal(t, ocispecs.MediaTypeImageIndex, imgs2.Index.MediaType)
6417+
}
6418+
63096419
func testExportAttestations(t *testing.T, sb integration.Sandbox) {
63106420
requiresLinux(t)
63116421
c, err := New(sb.Context(), sb.Address())
@@ -6425,28 +6535,29 @@ func testExportAttestations(t *testing.T, sb integration.Sandbox) {
64256535
desc, provider, err := contentutil.ProviderFromRef(target)
64266536
require.NoError(t, err)
64276537

6428-
index, err := testutil.ReadIndex(sb.Context(), provider, desc)
6538+
imgs, err := testutil.ReadImages(sb.Context(), provider, desc)
64296539
require.NoError(t, err)
6430-
require.Equal(t, len(ps)*2, len(index))
6540+
require.Equal(t, len(ps)*2, len(imgs.Images))
64316541

6432-
var imgs []*testutil.ImageInfo
6542+
var bases []*testutil.ImageInfo
64336543
for _, p := range ps {
64346544
pk := platforms.Format(p)
6435-
img := index.Find(pk)
6545+
img := imgs.Find(pk)
64366546
require.NotNil(t, img)
64376547
require.Equal(t, pk, platforms.Format(*img.Desc.Platform))
64386548
require.Equal(t, 1, len(img.Layers))
64396549
require.Equal(t, []byte(fmt.Sprintf("hello %s!", pk)), img.Layers[0]["greeting"].Data)
6440-
imgs = append(imgs, img)
6550+
bases = append(bases, img)
64416551
}
64426552

6443-
atts := index.Filter("unknown/unknown")
6444-
require.Equal(t, len(ps), len(atts))
6445-
for i, att := range atts {
6553+
atts := imgs.Filter("unknown/unknown")
6554+
require.Equal(t, len(ps), len(atts.Images))
6555+
for i, att := range atts.Images {
6556+
require.Equal(t, ocispecs.MediaTypeImageManifest, att.Desc.MediaType)
64466557
require.Equal(t, "unknown/unknown", platforms.Format(*att.Desc.Platform))
64476558
require.Equal(t, "unknown/unknown", att.Img.OS+"/"+att.Img.Architecture)
64486559
require.Equal(t, attestation.DockerAnnotationReferenceTypeDefault, att.Desc.Annotations[attestation.DockerAnnotationReferenceType])
6449-
require.Equal(t, imgs[i].Desc.Digest.String(), att.Desc.Annotations[attestation.DockerAnnotationReferenceDigest])
6560+
require.Equal(t, bases[i].Desc.Digest.String(), att.Desc.Annotations[attestation.DockerAnnotationReferenceDigest])
64506561
require.Equal(t, 2, len(att.Layers))
64516562
require.Equal(t, len(att.Layers), len(att.Img.RootFS.DiffIDs))
64526563
require.Equal(t, len(att.Img.History), 0)
@@ -6460,7 +6571,7 @@ func testExportAttestations(t *testing.T, sb integration.Sandbox) {
64606571
subjects := []intoto.Subject{{
64616572
Name: "_",
64626573
Digest: map[string]string{
6463-
"sha256": imgs[i].Desc.Digest.Encoded(),
6574+
"sha256": bases[i].Desc.Digest.Encoded(),
64646575
},
64656576
}}
64666577
require.Equal(t, subjects, attest.Subject)

‎exporter/containerimage/export.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
7878
RefCfg: cacheconfig.RefConfig{
7979
Compression: compression.New(compression.Default),
8080
},
81-
BuildInfo: true,
82-
Annotations: make(AnnotationsGroup),
81+
BuildInfo: true,
8382
},
8483
store: true,
8584
}
@@ -210,7 +209,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source
210209
if err != nil {
211210
return nil, err
212211
}
213-
opts.Annotations = as.Merge(opts.Annotations)
212+
opts.AddAnnotations(as)
214213

215214
ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary)
216215
if err != nil {

‎exporter/containerimage/opts.go

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ func (c *ImageCommitOpts) Load(opt map[string]string) (map[string]string, error)
4242
if err != nil {
4343
return nil, err
4444
}
45-
c.Annotations = as
4645
opt = toStringMap(optb)
4746

4847
for k, v := range opt {
@@ -91,14 +90,42 @@ func (c *ImageCommitOpts) Load(opt map[string]string) (map[string]string, error)
9190
}
9291
}
9392

94-
if esgz && !c.OCITypes {
95-
logrus.Warn("forcibly turning on oci-mediatype mode for estargz")
96-
c.OCITypes = true
93+
if esgz {
94+
c.EnableOCITypes("estargz")
9795
}
9896

97+
c.AddAnnotations(as)
98+
9999
return rest, nil
100100
}
101101

102+
func (c *ImageCommitOpts) AddAnnotations(annotations AnnotationsGroup) {
103+
if annotations == nil {
104+
return
105+
}
106+
if c.Annotations == nil {
107+
c.Annotations = AnnotationsGroup{}
108+
}
109+
c.Annotations = c.Annotations.Merge(annotations)
110+
for _, a := range annotations {
111+
if len(a.Index)+len(a.IndexDescriptor)+len(a.ManifestDescriptor) > 0 {
112+
c.EnableOCITypes("annotations")
113+
}
114+
}
115+
}
116+
117+
func (c *ImageCommitOpts) EnableOCITypes(reason string) {
118+
if !c.OCITypes {
119+
message := "forcibly turning on oci-mediatype mode"
120+
if reason != "" {
121+
message += " for " + reason
122+
}
123+
logrus.Warn(message)
124+
125+
c.OCITypes = true
126+
}
127+
}
128+
102129
func parseBool(dest *bool, key string, value string) error {
103130
b, err := strconv.ParseBool(value)
104131
if err != nil {

‎exporter/containerimage/writer.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,12 +99,16 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session
9999
return mfstDesc, nil
100100
}
101101

102-
refCount := len(p.Platforms)
102+
attestCount := 0
103103
for _, attests := range inp.Attestations {
104-
refCount += len(attests)
104+
attestCount += len(attests)
105105
}
106-
if refCount != len(inp.Refs) {
107-
return nil, errors.Errorf("number of required refs does not match references %d %d", refCount, len(inp.Refs))
106+
if count := attestCount + len(p.Platforms); count != len(inp.Refs) {
107+
return nil, errors.Errorf("number of required refs does not match references %d %d", count, len(inp.Refs))
108+
}
109+
110+
if attestCount > 0 {
111+
opts.EnableOCITypes("attestations")
108112
}
109113

110114
refs := make([]cache.ImmutableRef, 0, len(inp.Refs))

0 commit comments

Comments
 (0)