[deckhouse-cli] feat/new module mechanism draft (#234)

Signed-off-by: Pavel Okhlopkov <[email protected]>

Author: ldmonster

internal/mirror/modules/layout.go

@@ -28,28 +28,45 @@ import (
 	regimage "github.com/deckhouse/deckhouse-cli/pkg/registry/image"
 )
 
+type ModulesDownloadList struct {
+	rootURL string
+	list    map[string]*ImageDownloadList
+}
+
+func NewModulesDownloadList(rootURL string) *ModulesDownloadList {
+	return &ModulesDownloadList{
+		rootURL: rootURL,
+		list:    make(map[string]*ImageDownloadList),
+	}
+}
+
+func (l *ModulesDownloadList) Module(moduleName string) *ImageDownloadList {
+	return l.list[moduleName]
+}
+
+func (l *ModulesDownloadList) FillModulesImages(modules []string) {
+	for _, moduleName := range modules {
+		list := NewImageDownloadList(filepath.Join(l.rootURL, moduleName))
+		list.FillForTag("")
+		l.list[moduleName] = list
+	}
+}
+
 type ImageDownloadList struct {
 	rootURL string
 
-	Modules                map[string]*puller.ImageMeta
-	ModulesReleaseChannels map[string]*puller.ImageMeta
-	ModulesExtra           map[string]*puller.ImageMeta
+	Module                map[string]*puller.ImageMeta
+	ModuleReleaseChannels map[string]*puller.ImageMeta
+	ModuleExtra           map[string]*puller.ImageMeta
 }
 
 func NewImageDownloadList(rootURL string) *ImageDownloadList {
 	return &ImageDownloadList{
 		rootURL: rootURL,
 
-		Modules:                make(map[string]*puller.ImageMeta),
-		ModulesReleaseChannels: make(map[string]*puller.ImageMeta),
-		ModulesExtra:           make(map[string]*puller.ImageMeta),
-	}
-}
-
-func (l *ImageDownloadList) FillModulesImages(modules []string) {
-	for _, module := range modules {
-		l.Modules[filepath.Join(l.rootURL, internal.ModulesSegment, module)+":latest"] = nil
-		l.ModulesReleaseChannels[filepath.Join(l.rootURL, internal.ModulesSegment, module, internal.ModulesReleasesSegment)+":latest"] = nil
+		Module:                make(map[string]*puller.ImageMeta),
+		ModuleReleaseChannels: make(map[string]*puller.ImageMeta),
+		ModuleExtra:           make(map[string]*puller.ImageMeta),
 	}
 }
 
@@ -59,8 +76,41 @@ func (l *ImageDownloadList) FillForTag(tag string) {
 		return
 	}
 
-	// For modules, release channels might be handled differently
-	// TODO: implement if needed
+	for _, channel := range internal.GetAllDefaultReleaseChannels() {
+		l.ModuleReleaseChannels[l.rootURL+":"+channel] = nil
+	}
+}
+
+type ModulesImageLayouts struct {
+	platform   v1.Platform
+	workingDir string
+
+	list map[string]*ImageLayouts
+}
+
+func NewModulesImageLayouts(rootFolder string) *ModulesImageLayouts {
+	l := &ModulesImageLayouts{
+		workingDir: rootFolder,
+		platform:   v1.Platform{Architecture: "amd64", OS: "linux"},
+		list:       make(map[string]*ImageLayouts),
+	}
+
+	return l
+}
+
+func (l *ModulesImageLayouts) Module(moduleName string) *ImageLayouts {
+	return l.list[moduleName]
+}
+
+// AsList returns a list of layout.Path's from all modules. Undefined path's are not included in the list.
+func (l *ModulesImageLayouts) AsList() []layout.Path {
+	var paths []layout.Path
+	for _, imgLayout := range l.list {
+		if imgLayout != nil {
+			paths = append(paths, imgLayout.AsList()...)
+		}
+	}
+	return paths
 }
 
 type ImageLayouts struct {

internal/mirror/modules/modules.go

@@ -18,13 +18,15 @@ package modules
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"time"
 
 	dkplog "github.com/deckhouse/deckhouse/pkg/log"
+	"github.com/deckhouse/deckhouse/pkg/registry/client"
 
 	"github.com/deckhouse/deckhouse-cli/internal"
 	"github.com/deckhouse/deckhouse-cli/internal/mirror/chunked"
@@ -33,17 +35,18 @@ import (
 	"github.com/deckhouse/deckhouse-cli/pkg/libmirror/bundle"
 	"github.com/deckhouse/deckhouse-cli/pkg/libmirror/layouts"
 	"github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log"
-	"github.com/deckhouse/deckhouse-cli/pkg/libmirror/validation"
 	registryservice "github.com/deckhouse/deckhouse-cli/pkg/registry/service"
 )
 
 type Service struct {
+	workingDir string
+
 	// modulesService handles Deckhouse platform registry operations
 	modulesService *registryservice.ModulesService
 	// layout manages the OCI image layouts for different components
-	layout *ImageLayouts
-	// downloadList manages the list of images to be downloaded
-	downloadList *ImageDownloadList
+	layout *ModulesImageLayouts
+	// modulesDownloadList manages the list of images to be downloaded
+	modulesDownloadList *ModulesDownloadList
 	// pullerService handles the pulling of images
 	pullerService *puller.PullerService
 
@@ -64,36 +67,28 @@ func NewService(
 ) *Service {
 	userLogger.Infof("Creating OCI Image Layouts for Modules")
 
-	tmpDir := filepath.Join(workingDir, "modules")
-
-	layout, err := createOCIImageLayoutsForModules(tmpDir)
-	if err != nil {
-		//TODO: handle error
-		userLogger.Warnf("Create OCI Image Layouts: %v", err)
-	}
-
 	rootURL := registryService.GetRoot()
 
 	return &Service{
-		modulesService: registryService.ModuleService(),
-		layout:         layout,
-		downloadList:   NewImageDownloadList(rootURL),
-		pullerService:  puller.NewPullerService(logger, userLogger),
-		rootURL:        rootURL,
-		logger:         logger,
-		userLogger:     userLogger,
+		workingDir:          workingDir,
+		modulesService:      registryService.ModuleService(),
+		modulesDownloadList: NewModulesDownloadList(rootURL),
+		pullerService:       puller.NewPullerService(logger, userLogger),
+		rootURL:             rootURL,
+		logger:              logger,
+		userLogger:          userLogger,
 	}
 }
 
 // PullModules pulls the Deckhouse modules
 // It validates access to the registry and pulls the module images
-func (svc *Service) PullModules(ctx context.Context, modules []string) error {
+func (svc *Service) PullModules(ctx context.Context) error {
 	err := svc.validateModulesAccess(ctx)
 	if err != nil {
 		return fmt.Errorf("validate modules access: %w", err)
 	}
 
-	err = svc.pullModules(ctx, modules)
+	err = svc.pullModules(ctx)
 	if err != nil {
 		return fmt.Errorf("pull modules: %w", err)
 	}
@@ -110,56 +105,96 @@ func (svc *Service) validateModulesAccess(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
 	defer cancel()
 
-	modulesRepo := filepath.Join(svc.rootURL, internal.ModulesSegment)
-	validator := validation.NewRemoteRegistryAccessValidator()
-	err := validator.ValidateListAccessForRepo(ctx, modulesRepo, []validation.Option{}...) // TODO: add options
+	// For specific tags, check if the tag exists
+	_, err := svc.modulesService.ListTags(ctx)
+	if errors.Is(err, client.ErrImageNotFound) {
+		svc.userLogger.Warnf("Skipping pull of modules: %v", err)
+
+		return nil
+	}
+
 	if err != nil {
-		return fmt.Errorf("modules registry is not accessible: %w", err)
+		return fmt.Errorf("failed to check modules lists: %w", err)
 	}
 
 	return nil
 }
 
-func (svc *Service) pullModules(ctx context.Context, modules []string) error {
+func (svc *Service) pullModules(ctx context.Context) error {
 	logger := svc.userLogger
 
-	// Fill download list with modules images
-	svc.downloadList.FillModulesImages(modules)
-
-	// Pull modules images
-	err := logger.Process("Pull Modules", func() error {
-		config := puller.PullConfig{
-			Name:             "Modules",
-			ImageSet:         svc.downloadList.Modules,
-			Layout:           svc.layout.Modules,
-			AllowMissingTags: true, // Allow missing module images
-			GetterService:    svc.modulesService,
-		}
+	tmpDir := filepath.Join(svc.workingDir, "modules")
 
-		return svc.pullerService.PullImages(ctx, config)
-	})
+	modules, err := svc.modulesService.ListTags(ctx)
 	if err != nil {
-		return err
+		return fmt.Errorf("list modules: %w", err)
+	}
+
+	for _, module := range modules {
+		logger.Infof("Module found: %s", module)
+	}
+
+	moduleImagesLayout, err := createOCIImageLayoutsForModules(tmpDir, modules)
+	if err != nil {
+		return fmt.Errorf("create OCI image layouts for modules: %w", err)
 	}
+	svc.layout = moduleImagesLayout
+
+	// Fill download list with modules images
+	svc.modulesDownloadList.FillModulesImages(modules)
+
+	err = logger.Process("Pull Modules", func() error {
+		for _, module := range modules {
+			config := puller.PullConfig{
+				Name:             module + " release channels",
+				ImageSet:         svc.modulesDownloadList.Module(module).ModuleReleaseChannels,
+				Layout:           svc.layout.Module(module).ModulesReleaseChannels,
+				AllowMissingTags: true,
+				GetterService:    svc.modulesService.Module(module).ReleaseChannels(),
+			}
+
+			err = svc.pullerService.PullImages(ctx, config)
+			if err != nil {
+				return err
+			}
+
+			// TODO:
+			// we must extract module images tags from release channels before pulling module images
+
+			// Pull modules images
+			config = puller.PullConfig{
+				Name:             module,
+				ImageSet:         svc.modulesDownloadList.Module(module).Module,
+				Layout:           svc.layout.Module(module).Modules,
+				AllowMissingTags: true, // Allow missing module images
+				GetterService:    svc.modulesService.Module(module),
+			}
+
+			err := svc.pullerService.PullImages(ctx, config)
+			if err != nil {
+				return err
+			}
+
+			config = puller.PullConfig{
+				Name:             module + " extra",
+				ImageSet:         svc.modulesDownloadList.Module(module).ModuleExtra,
+				Layout:           svc.layout.Module(module).ModulesExtra,
+				AllowMissingTags: true,
+				GetterService:    svc.modulesService.Module(module).Extra(),
+			}
 
-	// Pull modules release channels
-	err = logger.Process("Pull Modules Release Channels", func() error {
-		config := puller.PullConfig{
-			Name:             "Modules Release Channels",
-			ImageSet:         svc.downloadList.ModulesReleaseChannels,
-			Layout:           svc.layout.ModulesReleaseChannels,
-			AllowMissingTags: true,
-			GetterService:    svc.modulesService,
+			err = svc.pullerService.PullImages(ctx, config)
+			if err != nil {
+				return err
+			}
 		}
 
-		return svc.pullerService.PullImages(ctx, config)
+		return nil
 	})
 	if err != nil {
 		return err
 	}
 
-	// TODO: Pull modules extra images if needed
-
 	err = logger.Process("Processing modules image indexes", func() error {
 		for _, l := range svc.layout.AsList() {
 			err = layouts.SortIndexManifests(l)
@@ -204,6 +239,25 @@ func (svc *Service) pullModules(ctx context.Context, modules []string) error {
 
 func createOCIImageLayoutsForModules(
 	rootFolder string,
+	modules []string,
+) (*ModulesImageLayouts, error) {
+	layouts := NewModulesImageLayouts(rootFolder)
+
+	for _, moduleName := range modules {
+		moduleLayouts, err := createOCIImageLayoutsForModule(
+			filepath.Join(rootFolder, moduleName),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("create OCI image layouts for module %s: %w", moduleName, err)
+		}
+		layouts.list[moduleName] = moduleLayouts
+	}
+
+	return layouts, nil
+}
+
+func createOCIImageLayoutsForModule(
+	rootFolder string,
 ) (*ImageLayouts, error) {
 	layouts := NewImageLayouts(rootFolder)
 

internal/mirror/pull.go

@@ -78,7 +78,7 @@ func (svc *PullService) Pull(ctx context.Context) error {
 		return fmt.Errorf("pull security databases: %w", err)
 	}
 
-	err = svc.modulesService.PullModules(ctx, nil)
+	err = svc.modulesService.PullModules(ctx)
 	if err != nil {
 		return fmt.Errorf("pull modules: %w", err)
 	}