Skip to content

Commit 20c028e

Browse files
piotrjanikpiotrjanik
committed
fix: add --oci-layout flag for ocm download resource
<!-- markdownlint-disable MD041 --> This change adds an optional --oci-layout flag to the `ocm download artifacts` command to store blobs at blobs/<algorithm>/<encoded> per OCI Image Layout Specification instead of the default blobs/<algorithm>.<encoded> format. This enables compatibility with tools that expect OCI-compliant blob paths. <!-- Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`. --> Fixes #1668 Signed-off-by: Piotr Janik <piotr.janik@sap.com>
1 parent 71ba4a5 commit 20c028e

File tree

5 files changed

+335
-1
lines changed

5 files changed

+335
-1
lines changed

‎api/oci/extensions/repositories/artifactset/ctf_roundtrip_test.go‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
//go:build integration
2+
3+
package artifactset_test
4+
5+
import (
6+
"bytes"
7+
"context"
8+
"fmt"
9+
"os"
10+
"path/filepath"
11+
12+
. "github.com/onsi/ginkgo/v2"
13+
. "github.com/onsi/gomega"
14+
15+
"github.com/mandelsoft/vfs/pkg/osfs"
16+
"oras.land/oras-go/v2/content/oci"
17+
18+
envhelper "ocm.software/ocm/api/helper/env"
19+
. "ocm.software/ocm/cmds/ocm/testhelper"
20+
)
21+
22+
const (
23+
componentName = "example.com/hello"
24+
componentVersion = "1.0.0"
25+
resourceName = "hello-image"
26+
resourceVersion = "1.0.0"
27+
imageReference = "hello-world:linux"
28+
)
29+
30+
// This test verifies the CTF-based workflow with --oci-layout flag:
31+
// 1. Create a CTF archive with an OCI image resource
32+
// 2. Transfer CTF to new CTF with --copy-resources
33+
// 3. Verify components and resources in target CTF
34+
// 4. Download resource with --oci-layout flag:
35+
// - Creates OCI Image Layout directory (index.json, oci-layout, blobs/sha256/...)
36+
// - Verifies layout structure is OCI-compliant
37+
// - Resolves artifact by resource version using ORAS
38+
// 5. Download resource without --oci-layout:
39+
// - Creates OCM artifact set format (not OCI-compliant)
40+
// - Verifies layout structure check fails
41+
var _ = Describe("CTF to CTF-with-resource to OCI roundtrip", Ordered, func() {
42+
var (
43+
tempDir string
44+
sourceCTF string
45+
targetCTF string
46+
resourcesOciDir string
47+
resourcesOcmDir string
48+
env *TestEnv
49+
)
50+
51+
BeforeAll(func() {
52+
53+
var err error
54+
tempDir, err = os.MkdirTemp("", "ocm-ctf-oci-layout-*")
55+
Expect(err).To(Succeed())
56+
57+
env = NewTestEnv(envhelper.FileSystem(osfs.New()))
58+
})
59+
60+
AfterAll(func() {
61+
if env != nil {
62+
env.Cleanup()
63+
}
64+
})
65+
66+
It("creates CTF using stable OCM release", func() {
67+
sourceCTF = filepath.Join(tempDir, "ctf-source")
68+
constructorFile := filepath.Join(tempDir, "component-constructor.yaml")
69+
constructorContent := `components:
70+
- name: ` + componentName + `
71+
version: ` + componentVersion + `
72+
provider:
73+
name: example.com
74+
resources:
75+
- name: ` + resourceName + `
76+
type: ociImage
77+
version: ` + resourceVersion + `
78+
relation: external
79+
access:
80+
type: ociArtifact
81+
imageReference: ` + imageReference + `
82+
`
83+
err := os.WriteFile(constructorFile, []byte(constructorContent), 0644)
84+
Expect(err).To(Succeed(), "MUST create constructor file")
85+
86+
// Create CTF directory
87+
err = os.MkdirAll(sourceCTF, 0755)
88+
Expect(err).To(Succeed(), "MUST create CTF directory")
89+
90+
// Use the current OCM version to create the CTF
91+
GinkgoWriter.Printf("Creating CTF using current OCM version\n")
92+
93+
buf := bytes.NewBuffer(nil)
94+
err = env.CatchOutput(buf).Execute(
95+
"add", "componentversions",
96+
"--create",
97+
"--file", sourceCTF,
98+
constructorFile,
99+
)
100+
GinkgoWriter.Printf("OCM output: %s\n", buf.String())
101+
Expect(err).To(Succeed(), "OCM MUST create CTF: %s", buf.String())
102+
})
103+
104+
It("transfers CTF to new CTF with --copy-resources", func() {
105+
targetCTF = filepath.Join(tempDir, "ctf-target")
106+
buf := bytes.NewBuffer(nil)
107+
GinkgoWriter.Printf(" #### transfer componentversions " + sourceCTF + " " + targetCTF + "--copy-resources")
108+
Expect(env.CatchOutput(buf).Execute(
109+
"transfer", "componentversions", sourceCTF, targetCTF, "--copy-resources")).To(Succeed())
110+
GinkgoWriter.Printf("Transfer output: %s\n", buf.String())
111+
})
112+
113+
It("verifies components and resources in target CTF", func() {
114+
buf := bytes.NewBuffer(nil)
115+
Expect(env.CatchOutput(buf).Execute("get", "componentversions", targetCTF)).To(Succeed())
116+
GinkgoWriter.Printf("Components: %s\n", buf.String())
117+
Expect(buf.String()).To(ContainSubstring(componentName))
118+
119+
// List resources
120+
buf.Reset()
121+
Expect(env.CatchOutput(buf).Execute(
122+
"get", "resources",
123+
targetCTF+"//"+componentName+":"+componentVersion,
124+
)).To(Succeed())
125+
GinkgoWriter.Printf("Resources: %s\n", buf.String())
126+
Expect(buf.String()).To(ContainSubstring(resourceName))
127+
})
128+
129+
It("downloads resource from target CTF with --oci-layout", func() {
130+
resourcesOciDir = filepath.Join(tempDir, "resource-oci-layout")
131+
132+
buf := bytes.NewBuffer(nil)
133+
Expect(env.CatchOutput(buf).Execute(
134+
"download", "resources",
135+
"--oci-layout",
136+
"-O", resourcesOciDir,
137+
targetCTF+"//"+componentName+":"+componentVersion,
138+
resourceName,
139+
)).To(Succeed())
140+
Expect(verifyOCILayoutStructure(resourcesOciDir)).To(Succeed())
141+
store, err := oci.New(resourcesOciDir)
142+
Expect(err).To(Succeed(), "ORAS failed to open OCI layout: %w", err)
143+
144+
srcDesc, err := store.Resolve(context.Background(), resourceVersion)
145+
Expect(err).To(Succeed(), "resource MUST be OCI compliant")
146+
GinkgoWriter.Printf("Successfully verified OCI layout with ORAS: digest=%s\n", srcDesc.Digest)
147+
148+
})
149+
150+
It("downloads resource from target CTF without --oci-layout", func() {
151+
resourcesOcmDir = filepath.Join(tempDir, "resource-ocm-layout")
152+
153+
buf := bytes.NewBuffer(nil)
154+
Expect(env.CatchOutput(buf).Execute(
155+
"download", "resources",
156+
"-O", resourcesOcmDir,
157+
targetCTF+"//"+componentName+":"+componentVersion,
158+
resourceName,
159+
)).To(Succeed())
160+
Expect(verifyOCILayoutStructure(resourcesOcmDir)).ToNot(Succeed())
161+
GinkgoWriter.Printf("Resource download output: %s\n", buf.String())
162+
})
163+
})
164+
165+
// verifyOCILayoutStructure checks that the OCI layout has the expected structure.
166+
// Returns an error if any required file or directory is missing.
167+
func verifyOCILayoutStructure(ociDir string) error {
168+
// Check oci-layout file exists
169+
ociLayoutPath := filepath.Join(ociDir, "oci-layout")
170+
if _, err := os.Stat(ociLayoutPath); err != nil {
171+
return fmt.Errorf("oci-layout file MUST exist: %w", err)
172+
}
173+
174+
// Check index.json exists
175+
indexPath := filepath.Join(ociDir, "index.json")
176+
if _, err := os.Stat(indexPath); err != nil {
177+
return fmt.Errorf("index.json MUST exist: %w", err)
178+
}
179+
180+
// Check blobs directory exists
181+
blobsDir := filepath.Join(ociDir, "blobs")
182+
info, err := os.Stat(blobsDir)
183+
if err != nil {
184+
return fmt.Errorf("blobs directory MUST exist: %w", err)
185+
}
186+
if !info.IsDir() {
187+
return fmt.Errorf("blobs MUST be a directory, got file")
188+
}
189+
190+
GinkgoWriter.Printf("OCI layout structure verified: oci-layout, index.json, blobs/\n")
191+
return nil
192+
}

‎api/ocm/extensions/download/handlers/init.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@ import (
66
_ "ocm.software/ocm/api/ocm/extensions/download/handlers/dirtree"
77
_ "ocm.software/ocm/api/ocm/extensions/download/handlers/executable"
88
_ "ocm.software/ocm/api/ocm/extensions/download/handlers/helm"
9+
_ "ocm.software/ocm/api/ocm/extensions/download/handlers/ocilayout"
910
_ "ocm.software/ocm/api/ocm/extensions/download/handlers/ocirepo"
1011
)

‎api/ocm/extensions/download/handlers/ocilayout/handler.go‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package ocilayout
2+
3+
import (
4+
"path/filepath"
5+
"strings"
6+
7+
"github.com/mandelsoft/goutils/errors"
8+
"github.com/mandelsoft/goutils/finalizer"
9+
"github.com/mandelsoft/vfs/pkg/vfs"
10+
11+
"ocm.software/ocm/api/oci/artdesc"
12+
"ocm.software/ocm/api/oci/extensions/repositories/artifactset"
13+
"ocm.software/ocm/api/ocm/cpi"
14+
"ocm.software/ocm/api/ocm/extensions/download"
15+
"ocm.software/ocm/api/utils/accessio"
16+
"ocm.software/ocm/api/utils/accessobj"
17+
"ocm.software/ocm/api/utils/logging"
18+
common "ocm.software/ocm/api/utils/misc"
19+
)
20+
21+
const PRIORITY = 200
22+
23+
type Handler struct{}
24+
25+
func New() download.Handler {
26+
return &Handler{}
27+
}
28+
29+
func (h *Handler) Download(p common.Printer, racc cpi.ResourceAccess, path string, fs vfs.FileSystem) (ok bool, _ string, err error) {
30+
var finalize finalizer.Finalizer
31+
defer finalize.FinalizeWithErrorPropagation(&err)
32+
33+
// Step 1: Get access method to read resource content
34+
m, err := racc.AccessMethod()
35+
if err != nil {
36+
return false, "", err
37+
}
38+
finalize.Close(m)
39+
40+
// Step 2: Check MIME type - only handle OCI artifacts (tar/tar+gzip)
41+
if !isOCIArtifact(m.MimeType()) {
42+
logging.Logger().Debug("skipping non-OCI artifact", "mime", m.MimeType())
43+
return false, "", nil
44+
}
45+
46+
if path == "" {
47+
path = racc.Meta().GetName()
48+
}
49+
50+
// Step 3: Open resource blob as artifact set (contains OCI image)
51+
src, err := artifactset.OpenFromDataAccess(accessobj.ACC_READONLY, m.MimeType(), m)
52+
if err != nil {
53+
return true, "", errors.Wrapf(err, "open artifact set")
54+
}
55+
finalize.Close(src)
56+
57+
// Step 4: Get the main artifact from the set
58+
art, err := src.GetArtifact(src.GetMain().String())
59+
if err != nil {
60+
return true, "", errors.Wrapf(err, "get artifact")
61+
}
62+
finalize.Close(art)
63+
64+
// Step 5: Create target directory with OCI format (index.json + oci-layout)
65+
target, err := artifactset.Create(accessobj.ACC_CREATE, path, 0o755,
66+
accessio.PathFileSystem(fs),
67+
accessobj.FormatDirectory,
68+
artifactset.StructureFormat(artifactset.FORMAT_OCI),
69+
)
70+
if err != nil {
71+
return true, "", errors.Wrapf(err, "create OCI layout")
72+
}
73+
74+
// Step 6: Transfer all manifests and blobs to target with resource version as tag
75+
version := racc.Meta().GetVersion()
76+
if err := artifactset.TransferArtifact(art, target, version); err != nil {
77+
target.Close()
78+
return true, "", errors.Wrapf(err, "transfer artifact")
79+
}
80+
81+
if err := target.Close(); err != nil {
82+
return true, "", errors.Wrapf(err, "close target")
83+
}
84+
85+
// Step 7: Convert blob paths from sha256.DIGEST to sha256/DIGEST
86+
if err := convertBlobPaths(fs, path); err != nil {
87+
return true, "", err
88+
}
89+
90+
p.Printf("%s: downloaded to OCI layout\n", path)
91+
return true, path, nil
92+
}
93+
94+
func isOCIArtifact(mime string) bool {
95+
return artdesc.IsOCIMediaType(mime) &&
96+
(strings.HasSuffix(mime, "+tar") || strings.HasSuffix(mime, "+tar+gzip"))
97+
}
98+
99+
// convertBlobPaths converts blob paths from artifactset format (sha256.DIGEST)
100+
// to OCI Image Layout format (sha256/DIGEST).
101+
//
102+
// This is needed because artifactset uses DigestToFileName which always produces
103+
// "sha256.DIGEST" format. The FORMAT_OCI option only controls the descriptor file
104+
// (index.json) and oci-layout file creation, not the blob path structure.
105+
//
106+
// Call trace where DigestToFileName is invoked:
107+
//
108+
// TransferArtifact() -> TransferManifest() -> set.AddBlob()
109+
// -> accessobj.FileSystemBlobAccess.AddBlob()
110+
// -> path := a.DigestPath(blob.Digest())
111+
// -> common.DigestToFileName(digest) // returns "sha256.DIGEST"
112+
func convertBlobPaths(fs vfs.FileSystem, dir string) error {
113+
blobsDir := filepath.Join(dir, "blobs")
114+
entries, err := vfs.ReadDir(fs, blobsDir)
115+
if err != nil {
116+
return err
117+
}
118+
119+
for _, e := range entries {
120+
if e.IsDir() {
121+
continue
122+
}
123+
name := e.Name()
124+
if algo, dig, ok := strings.Cut(name, "."); ok {
125+
algoDir := filepath.Join(blobsDir, algo)
126+
fs.MkdirAll(algoDir, 0o755)
127+
fs.Rename(filepath.Join(blobsDir, name), filepath.Join(algoDir, dig))
128+
}
129+
}
130+
return nil
131+
}

‎cmds/ocm/commands/ocmcmds/resources/download/action.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"ocm.software/ocm/api/ocm"
1313
v1 "ocm.software/ocm/api/ocm/compdesc/meta/v1"
1414
"ocm.software/ocm/api/ocm/extensions/download"
15+
ocilayouthdlr "ocm.software/ocm/api/ocm/extensions/download/handlers/ocilayout"
1516
"ocm.software/ocm/api/ocm/tools/signing"
1617
"ocm.software/ocm/api/utils/blobaccess"
1718
common2 "ocm.software/ocm/api/utils/misc"
@@ -33,7 +34,14 @@ type Action struct {
3334
}
3435

3536
func NewAction(ctx ocm.ContextProvider, opts *output.Options) *Action {
36-
return &Action{downloaders: download.For(ctx), opts: opts}
37+
downloaders := download.For(ctx).Copy()
38+
39+
local := From(opts)
40+
if local.OCILayout {
41+
downloaders.Register(ocilayouthdlr.New(), download.ForArtifactType(download.ALL), download.WithPrio(ocilayouthdlr.PRIORITY))
42+
}
43+
44+
return &Action{downloaders: downloaders, opts: opts}
3745
}
3846

3947
func (d *Action) AddOptions(opts ...options.Options) {

‎cmds/ocm/commands/ocmcmds/resources/download/options.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ type Option struct {
2121
SilentOption bool
2222
UseHandlers bool
2323
Verify bool
24+
OCILayout bool
2425
}
2526

2627
func (o *Option) SetUseHandlers(ok ...bool) *Option {
@@ -33,6 +34,7 @@ func (o *Option) AddFlags(fs *pflag.FlagSet) {
3334
fs.BoolVarP(&o.UseHandlers, "download-handlers", "d", false, "use download handler if possible")
3435
}
3536
fs.BoolVarP(&o.Verify, "verify", "", false, "verify downloads")
37+
fs.BoolVarP(&o.OCILayout, "oci-layout", "", false, "download OCI artifacts in OCI Image Layout format (blobs/<algorithm>/<encoded>)")
3638
}
3739

3840
func (o *Option) Usage() string {

0 commit comments

Comments
 (0)