Skip to content

Commit cb24803

Browse files
authored
[deckhouse-cli] mirror pull part two (#224)
Signed-off-by: Pavel Okhlopkov <pavel.okhlopkov@flant.com>
1 parent f9dbe10 commit cb24803

File tree

27 files changed

+1197
-172
lines changed

27 files changed

+1197
-172
lines changed

internal/layout.go

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ limitations under the License.
1616

1717
package internal
1818

19+
import "path"
20+
1921
// deckhouse repo structure
2022
// root-segment:<version>
2123
// root-segment/install:<version>
@@ -28,25 +30,37 @@ const (
2830
InstallSegment = "install"
2931
InstallStandaloneSegment = "install-standalone"
3032
ReleaseChannelSegment = "release-channel"
31-
ModulesSegment = "modules"
32-
ModulesExtraSegment = "extra"
33-
ModulesReleasesSegment = "releases"
34-
// TODO: fix?
35-
SecurityDatabasesSegment = "security-databases"
33+
34+
ModulesSegment = "modules"
35+
ModulesExtraSegment = "extra"
36+
ModulesReleasesSegment = "releases"
37+
38+
SecuritySegment = "security"
39+
40+
SecurityTrivyDBSegment = "trivy-db"
41+
SecurityTrivyBDUSegment = "trivy-bdu"
42+
SecurityTrivyJavaDBSegment = "trivy-java-db"
43+
SecurityTrivyChecksSegment = "trivy-checks"
3644
)
3745

38-
var segmentByMirrorType = map[MirrorType]string{
46+
var pathByMirrorType = map[MirrorType]string{
3947
MirrorTypeDeckhouse: "",
4048
MirrorTypeDeckhouseInstall: InstallSegment,
4149
MirrorTypeDeckhouseInstallStandalone: InstallStandaloneSegment,
4250
MirrorTypeDeckhouseReleaseChannels: ReleaseChannelSegment,
43-
MirrorTypeModules: ModulesSegment,
44-
MirrorTypeModulesReleaseChannels: ModulesReleasesSegment,
45-
MirrorTypeModulesExtra: ModulesExtraSegment,
46-
MirrorTypeSecurityDatabases: SecurityDatabasesSegment,
51+
52+
MirrorTypeModules: ModulesSegment,
53+
MirrorTypeModulesReleaseChannels: ModulesReleasesSegment,
54+
MirrorTypeModulesExtra: ModulesExtraSegment,
55+
56+
MirrorTypeSecurity: SecuritySegment,
57+
MirrorTypeSecurityTrivyDBSegment: path.Join(SecuritySegment, SecurityTrivyDBSegment),
58+
MirrorTypeSecurityTrivyBDUSegment: path.Join(SecuritySegment, SecurityTrivyBDUSegment),
59+
MirrorTypeSecurityTrivyJavaDBSegment: path.Join(SecuritySegment, SecurityTrivyJavaDBSegment),
60+
MirrorTypeSecurityTrivyChecksSegment: path.Join(SecuritySegment, SecurityTrivyChecksSegment),
4761
}
4862

49-
// InstallSegmentByMirrorType returns the path segment for install based on the mirror type.
50-
func InstallSegmentByMirrorType(mirrorType MirrorType) string {
51-
return segmentByMirrorType[mirrorType]
63+
// InstallPathByMirrorType returns the path segment for install based on the mirror type.
64+
func InstallPathByMirrorType(mirrorType MirrorType) string {
65+
return pathByMirrorType[mirrorType]
5266
}

internal/mirror.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,9 @@ const (
2626
MirrorTypeModules
2727
MirrorTypeModulesReleaseChannels
2828
MirrorTypeModulesExtra
29-
MirrorTypeSecurityDatabases
29+
MirrorTypeSecurity
30+
MirrorTypeSecurityTrivyDBSegment
31+
MirrorTypeSecurityTrivyBDUSegment
32+
MirrorTypeSecurityTrivyJavaDBSegment
33+
MirrorTypeSecurityTrivyChecksSegment
3034
)

internal/mirror/cmd/pull/flags/flags.go

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ limitations under the License.
1717
package flags
1818

1919
import (
20-
"fmt"
2120
"os"
2221

2322
"github.com/Masterminds/semver/v3"
@@ -63,8 +62,6 @@ var (
6362
NoSecurityDB bool
6463
NoModules bool
6564
OnlyExtraImages bool
66-
67-
IgnoreSuspendedChannels bool
6865
)
6966

7067
func AddFlags(flagSet *pflag.FlagSet) {
@@ -200,15 +197,6 @@ module-name@=v1.3.0+stable → exact tag match: include only v1.3.0 and and publ
200197
false,
201198
"Interact with registries over HTTP.",
202199
)
203-
flagSet.BoolVar(
204-
&IgnoreSuspendedChannels,
205-
"ignore-suspended-channels",
206-
false,
207-
"Ignore suspended release channels instead of failing.",
208-
)
209-
if err := flagSet.MarkHidden("ignore-suspended-channels"); err != nil {
210-
panic(fmt.Sprintf("failed to mark flag as hidden: %v", err))
211-
}
212200
flagSet.StringVar(
213201
&TempDir,
214202
"tmp-dir",

internal/mirror/cmd/pull/pull.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,6 @@ func (p *Puller) Execute(ctx context.Context) error {
264264
registryservice.NewService(c, logger),
265265
pullflags.TempDir,
266266
pullflags.DeckhouseTag,
267-
pullflags.IgnoreSuspendedChannels,
268267
logger.Named("pull"),
269268
p.logger,
270269
)

internal/mirror/layout.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
1+
/*
2+
Copyright 2025 Flant JSC
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
117
package mirror
218

319
import (

internal/mirror/modules/layout.go

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
Copyright 2025 Flant JSC
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package modules
18+
19+
import (
20+
"fmt"
21+
"path/filepath"
22+
23+
v1 "github.com/google/go-containerregistry/pkg/v1"
24+
"github.com/google/go-containerregistry/pkg/v1/layout"
25+
26+
"github.com/deckhouse/deckhouse-cli/internal"
27+
"github.com/deckhouse/deckhouse-cli/internal/mirror/puller"
28+
regimage "github.com/deckhouse/deckhouse-cli/pkg/registry/image"
29+
)
30+
31+
type ImageDownloadList struct {
32+
rootURL string
33+
34+
Modules map[string]*puller.ImageMeta
35+
ModulesReleaseChannels map[string]*puller.ImageMeta
36+
ModulesExtra map[string]*puller.ImageMeta
37+
}
38+
39+
func NewImageDownloadList(rootURL string) *ImageDownloadList {
40+
return &ImageDownloadList{
41+
rootURL: rootURL,
42+
43+
Modules: make(map[string]*puller.ImageMeta),
44+
ModulesReleaseChannels: make(map[string]*puller.ImageMeta),
45+
ModulesExtra: make(map[string]*puller.ImageMeta),
46+
}
47+
}
48+
49+
func (l *ImageDownloadList) FillModulesImages(modules []string) {
50+
for _, module := range modules {
51+
l.Modules[filepath.Join(l.rootURL, internal.ModulesSegment, module)+":latest"] = nil
52+
l.ModulesReleaseChannels[filepath.Join(l.rootURL, internal.ModulesSegment, module, internal.ModulesReleasesSegment)+":latest"] = nil
53+
}
54+
}
55+
56+
func (l *ImageDownloadList) FillForTag(tag string) {
57+
// If we are to pull only the specific requested version, we should not pull any release channels at all.
58+
if tag != "" {
59+
return
60+
}
61+
62+
// For modules, release channels might be handled differently
63+
// TODO: implement if needed
64+
}
65+
66+
type ImageLayouts struct {
67+
platform v1.Platform
68+
workingDir string
69+
70+
Modules *regimage.ImageLayout
71+
ModulesReleaseChannels *regimage.ImageLayout
72+
ModulesExtra *regimage.ImageLayout
73+
}
74+
75+
func NewImageLayouts(rootFolder string) *ImageLayouts {
76+
l := &ImageLayouts{
77+
workingDir: rootFolder,
78+
platform: v1.Platform{Architecture: "amd64", OS: "linux"},
79+
}
80+
81+
return l
82+
}
83+
84+
func (l *ImageLayouts) setLayoutByMirrorType(rootFolder string, mirrorType internal.MirrorType) error {
85+
layoutPath := filepath.Join(rootFolder, internal.InstallPathByMirrorType(mirrorType))
86+
87+
layout, err := regimage.NewImageLayout(layoutPath)
88+
if err != nil {
89+
return fmt.Errorf("failed to create image layout: %w", err)
90+
}
91+
92+
switch mirrorType {
93+
case internal.MirrorTypeModules:
94+
l.Modules = layout
95+
case internal.MirrorTypeModulesReleaseChannels:
96+
l.ModulesReleaseChannels = layout
97+
case internal.MirrorTypeModulesExtra:
98+
l.ModulesExtra = layout
99+
default:
100+
return fmt.Errorf("wrong mirror type in modules image layout: %v", mirrorType)
101+
}
102+
103+
return nil
104+
}
105+
106+
// AsList returns a list of layout.Path's in it. Undefined path's are not included in the list.
107+
func (l *ImageLayouts) AsList() []layout.Path {
108+
paths := make([]layout.Path, 0)
109+
if l.Modules != nil {
110+
paths = append(paths, l.Modules.Path())
111+
}
112+
if l.ModulesReleaseChannels != nil {
113+
paths = append(paths, l.ModulesReleaseChannels.Path())
114+
}
115+
if l.ModulesExtra != nil {
116+
paths = append(paths, l.ModulesExtra.Path())
117+
}
118+
return paths
119+
}

0 commit comments

Comments
 (0)