feat: new push/pull (#243)

Signed-off-by: Timur Tuktamyshev <[email protected]>
Signed-off-by: Pavel Okhlopkov <[email protected]>
Co-authored-by: Pavel Okhlopkov <[email protected]>

Author: timmilesdw

internal/layout.go

@@ -18,22 +18,32 @@ package internal
 
 import "path"
 
-// deckhouse repo structure
-// root-segment:<version>
-// root-segment/install:<version>
-// root-segment/install-standalone:<version>
-// root-segment/release-channel:<version>
-// root-segment/modules/<module-name>:<version>
-// root-segment/modules/<module-name>/releases:<version>
-// root-segment/modules/<module-name>/extra/<module-extra-name>:<version>
+// deckhouse repo structure (relative to root path like registry.deckhouse.io/deckhouse/fe)
+//
+// Platform:
+//
+//	<root>:<version>                                    - Deckhouse main image
+//	<root>/release-channel:<channel>                    - Release channel metadata
+//	<root>/install:<version>                            - Installer image
+//	<root>/install-standalone:<version>                 - Standalone installer
+//
+// Security:
+//
+//	<root>/security/<security-name>:<version>           - Security databases (trivy-db, trivy-bdu, etc.)
+//
+// Modules:
+//
+//	<root>/modules/<module-name>:<version>                    - Module main image
+//	<root>/modules/<module-name>/release:<channel>            - Module release channel metadata
+//	<root>/modules/<module-name>/extra/<extra-name>:<version> - Module extra images
 const (
 	InstallSegment           = "install"
 	InstallStandaloneSegment = "install-standalone"
 	ReleaseChannelSegment    = "release-channel"
 
-	ModulesSegment         = "modules"
-	ModulesExtraSegment    = "extra"
-	ModulesReleasesSegment = "releases"
+	ModulesSegment        = "modules"
+	ModulesReleaseSegment = "release"
+	ModulesExtraSegment   = "extra"
 
 	SecuritySegment = "security"
 
@@ -49,9 +59,9 @@ var pathByMirrorType = map[MirrorType]string{
 	MirrorTypeDeckhouseInstallStandalone: InstallStandaloneSegment,
 	MirrorTypeDeckhouseReleaseChannels:   ReleaseChannelSegment,
 
-	MirrorTypeModules:                ModulesSegment,
-	MirrorTypeModulesReleaseChannels: ModulesReleasesSegment,
-	MirrorTypeModulesExtra:           ModulesExtraSegment,
+	// Module paths are relative to modules/<module-name>/ directory
+	MirrorTypeModules:                "",                    // Module main image at root of module dir
+	MirrorTypeModulesReleaseChannels: ModulesReleaseSegment, // modules/<name>/release
 
 	MirrorTypeSecurity:                   SecuritySegment,
 	MirrorTypeSecurityTrivyDBSegment:     path.Join(SecuritySegment, SecurityTrivyDBSegment),

internal/mirror.go

@@ -25,7 +25,6 @@ const (
 	MirrorTypeDeckhouseReleaseChannels
 	MirrorTypeModules
 	MirrorTypeModulesReleaseChannels
-	MirrorTypeModulesExtra
 	MirrorTypeSecurity
 	MirrorTypeSecurityTrivyDBSegment
 	MirrorTypeSecurityTrivyBDUSegment

internal/mirror/cmd/pull/flags/flags.go

@@ -55,8 +55,9 @@ var (
 	SourceRegistryPassword string
 	DeckhouseLicenseToken  string
 
-	DoGOSTDigest bool
-	NoPullResume bool
+	DoGOSTDigest  bool
+	NoPullResume  bool
+	IgnoreSuspend bool
 
 	NoPlatform      bool
 	NoSecurityDB    bool
@@ -161,6 +162,12 @@ module-name@=v1.3.0+stable → exact tag match: include only v1.3.0 and and publ
 		false,
 		"Do not continue last unfinished pull operation and start from scratch.",
 	)
+	flagSet.BoolVar(
+		&IgnoreSuspend,
+		"ignore-suspend",
+		false,
+		"Ignore suspended release channels and continue mirroring. Use with caution.",
+	)
 	flagSet.BoolVar(
 		&NoPlatform,
 		"no-platform",

internal/mirror/cmd/pull/pull.go

@@ -23,8 +23,10 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"os/signal"
 	"path"
 	"path/filepath"
+	"syscall"
 	"time"
 
 	"github.com/Masterminds/semver/v3"
@@ -102,11 +104,23 @@ func NewCommand() *cobra.Command {
 }
 
 func pull(cmd *cobra.Command, _ []string) error {
+	// Set up graceful cancellation on Ctrl+C
+	parentCtx := cmd.Context()
+	if parentCtx == nil {
+		parentCtx = context.Background()
+	}
+	ctx, cancel := signal.NotifyContext(parentCtx, syscall.SIGINT, syscall.SIGTERM)
+	defer cancel()
+
 	puller := NewPuller(cmd)
 
 	puller.logger.Infof("d8 version: %s", version.Version)
 
-	if err := puller.Execute(cmd.Context()); err != nil {
+	if err := puller.Execute(ctx); err != nil {
+		if errors.Is(err, context.Canceled) {
+			puller.logger.WarnLn("Operation cancelled by user")
+			return nil
+		}
 		return ErrPullFailed
 	}
 
@@ -164,6 +178,7 @@ func buildPullParams(logger params.Logger) *params.PullParams {
 		SkipSecurityDatabases: pullflags.NoSecurityDB,
 		SkipModules:           pullflags.NoModules,
 		OnlyExtraImages:       pullflags.OnlyExtraImages,
+		IgnoreSuspend:         pullflags.IgnoreSuspend,
 		DeckhouseTag:          pullflags.DeckhouseTag,
 		SinceVersion:          pullflags.SinceVersion,
 	}
@@ -276,6 +291,7 @@ func (p *Puller) Execute(ctx context.Context) error {
 				SkipSecurity:    pullflags.NoSecurityDB,
 				SkipModules:     pullflags.NoModules,
 				OnlyExtraImages: pullflags.OnlyExtraImages,
+				IgnoreSuspend:   pullflags.IgnoreSuspend,
 				ModuleFilter:    filter,
 				BundleDir:       pullflags.ImagesBundlePath,
 				BundleChunkSize: pullflags.ImagesBundleChunkSizeGB * 1000 * 1000 * 1000,

internal/mirror/cmd/push/push.go

@@ -23,9 +23,11 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	"os/signal"
 	"path"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/google/go-containerregistry/pkg/authn"
@@ -36,6 +38,7 @@ import (
 	"github.com/deckhouse/deckhouse/pkg/registry"
 	regclient "github.com/deckhouse/deckhouse/pkg/registry/client"
 
+	"github.com/deckhouse/deckhouse-cli/internal/mirror"
 	"github.com/deckhouse/deckhouse-cli/internal/mirror/chunked"
 	"github.com/deckhouse/deckhouse-cli/internal/mirror/operations"
 	"github.com/deckhouse/deckhouse-cli/internal/version"
@@ -188,7 +191,7 @@ func pushStaticPackages(pushParams *params.PushParams, logger params.Logger, cli
 		}
 
 		if err = pkg.Close(); err != nil {
-			logger.Warnf("Could not close bundle package %s: %w", pkgName, err)
+			logger.Warnf("Could not close bundle package %s: %v", pkgName, err)
 		}
 	}
 	return nil
@@ -289,6 +292,11 @@ func (p *Pusher) Execute() error {
 		return err
 	}
 
+	// Use new push service when NEW_PULL env is set
+	if os.Getenv("NEW_PULL") == "true" {
+		return p.executeNewPush()
+	}
+
 	if err := p.pushStaticPackages(); err != nil {
 		return err
 	}
@@ -300,6 +308,63 @@ func (p *Pusher) Execute() error {
 	return nil
 }
 
+// executeNewPush runs the push using the push service.
+// This service expects the bundle to have the exact same structure as the registry:
+// - Each OCI layout's relative path becomes its registry segment
+// - Works with unified bundles where pull saved the structure as-is
+func (p *Pusher) executeNewPush() error {
+	// Set up graceful cancellation on Ctrl+C
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer cancel()
+
+	logger := dkplog.NewNop()
+
+	if log.DebugLogLevel() >= 3 {
+		logger = dkplog.NewLogger(dkplog.WithLevel(slog.LevelDebug))
+	}
+
+	// Create registry client
+	clientOpts := &regclient.Options{
+		Insecure:      p.pushParams.Insecure,
+		TLSSkipVerify: p.pushParams.SkipTLSVerification,
+		Logger:        logger,
+	}
+
+	if p.pushParams.RegistryAuth != nil {
+		clientOpts.Auth = p.pushParams.RegistryAuth
+	}
+
+	var client registry.Client
+	client = regclient.NewClientWithOptions(p.pushParams.RegistryHost, clientOpts)
+
+	// Scope to the registry path
+	if p.pushParams.RegistryPath != "" {
+		client = client.WithSegment(p.pushParams.RegistryPath)
+	}
+
+	svc := mirror.NewPushService(
+		client,
+		&mirror.PushServiceOptions{
+			BundleDir:  p.pushParams.BundleDir,
+			WorkingDir: p.pushParams.WorkingDir,
+		},
+		logger.Named("push"),
+		p.logger.(*log.SLogger),
+	)
+
+	err := svc.Push(ctx)
+	if err != nil {
+		// Handle context cancellation gracefully
+		if errors.Is(err, context.Canceled) {
+			p.logger.WarnLn("Operation cancelled by user")
+			return nil
+		}
+		return err
+	}
+
+	return nil
+}
+
 // validateRegistryAccess validates access to the registry
 func (p *Pusher) validateRegistryAccess() error {
 	p.logger.InfoLn("Validating registry access")

internal/mirror/modules/layout.go

@@ -117,15 +117,20 @@ type ImageLayouts struct {
 	platform   v1.Platform
 	workingDir string
 
-	Modules                *regimage.ImageLayout
+	// Modules is the main module image layout (modules/<name>/)
+	Modules *regimage.ImageLayout
+	// ModulesReleaseChannels is the release channel layout (modules/<name>/release/)
 	ModulesReleaseChannels *regimage.ImageLayout
-	ModulesExtra           *regimage.ImageLayout
+	// ExtraImages holds layouts for each extra image (modules/<name>/extra/<extra-name>/)
+	// Key is the extra image name (e.g., "scanner", "enforcer")
+	ExtraImages map[string]*regimage.ImageLayout
 }
 
 func NewImageLayouts(rootFolder string) *ImageLayouts {
 	l := &ImageLayouts{
-		workingDir: rootFolder,
-		platform:   v1.Platform{Architecture: "amd64", OS: "linux"},
+		workingDir:  rootFolder,
+		platform:    v1.Platform{Architecture: "amd64", OS: "linux"},
+		ExtraImages: make(map[string]*regimage.ImageLayout),
 	}
 
 	return l
@@ -144,15 +149,31 @@ func (l *ImageLayouts) setLayoutByMirrorType(rootFolder string, mirrorType inter
 		l.Modules = layout
 	case internal.MirrorTypeModulesReleaseChannels:
 		l.ModulesReleaseChannels = layout
-	case internal.MirrorTypeModulesExtra:
-		l.ModulesExtra = layout
 	default:
 		return fmt.Errorf("wrong mirror type in modules image layout: %v", mirrorType)
 	}
 
 	return nil
 }
 
+// GetOrCreateExtraLayout returns or creates a layout for a specific extra image.
+// Extra images are stored under: modules/<name>/extra/<extra-name>/
+func (l *ImageLayouts) GetOrCreateExtraLayout(extraName string) (*regimage.ImageLayout, error) {
+	if existing, ok := l.ExtraImages[extraName]; ok {
+		return existing, nil
+	}
+
+	// Create layout at modules/<module-name>/extra/<extra-name>/
+	layoutPath := filepath.Join(l.workingDir, "extra", extraName)
+	layout, err := regimage.NewImageLayout(layoutPath)
+	if err != nil {
+		return nil, fmt.Errorf("create extra image layout for %s: %w", extraName, err)
+	}
+
+	l.ExtraImages[extraName] = layout
+	return layout, nil
+}
+
 // AsList returns a list of layout.Path's in it. Undefined path's are not included in the list.
 func (l *ImageLayouts) AsList() []layout.Path {
 	paths := make([]layout.Path, 0)
@@ -162,8 +183,11 @@ func (l *ImageLayouts) AsList() []layout.Path {
 	if l.ModulesReleaseChannels != nil {
 		paths = append(paths, l.ModulesReleaseChannels.Path())
 	}
-	if l.ModulesExtra != nil {
-		paths = append(paths, l.ModulesExtra.Path())
+	// Add all extra image layouts
+	for _, extraLayout := range l.ExtraImages {
+		if extraLayout != nil {
+			paths = append(paths, extraLayout.Path())
+		}
 	}
 	return paths
 }