feat: support build with push and pull with extract from remote (#108)

Signed-off-by: chlins <[email protected]>

Author: chlins

.golangci.yml

@@ -4,7 +4,6 @@ run:
   skip-dirs:
     - test/mocks
 
-
 linters-settings:
   gocyclo:
     min-complexity: 100

.mockery.yaml

@@ -19,3 +19,11 @@ packages:
       Modelfile:
         config:
           dir: test/mocks/modelfile
+  github.com/CloudNativeAI/modctl/pkg/backend/build:
+    interfaces:
+      Builder:
+        config:
+          dir: test/mocks/backend/build
+      OutputStrategy:
+        config:
+          dir: test/mocks/backend/build

cmd/build.go

@@ -52,6 +52,9 @@ func init() {
 	flags.IntVarP(&buildConfig.Concurrency, "concurrency", "c", buildConfig.Concurrency, "specify the number of concurrent build operations")
 	flags.StringVarP(&buildConfig.Target, "target", "t", "", "target model artifact name")
 	flags.StringVarP(&buildConfig.Modelfile, "modelfile", "f", "Modelfile", "model file path")
+	flags.BoolVarP(&buildConfig.OutputRemote, "output-remote", "", false, "turning on this flag will output model artifact to remote registry directly")
+	flags.BoolVarP(&buildConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
+	flags.BoolVarP(&buildConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")
 
 	if err := viper.BindPFlags(flags); err != nil {
 		panic(fmt.Errorf("bind cache list flags to viper: %w", err))
@@ -65,11 +68,7 @@ func runBuild(ctx context.Context, workDir string) error {
 		return err
 	}
 
-	opts := []backend.Option{
-		backend.WithConcurrency(buildConfig.Concurrency),
-	}
-
-	if err := b.Build(ctx, buildConfig.Modelfile, workDir, buildConfig.Target, opts...); err != nil {
+	if err := b.Build(ctx, buildConfig.Modelfile, workDir, buildConfig.Target, buildConfig); err != nil {
 		return err
 	}
 

cmd/login.go

@@ -89,11 +89,7 @@ func runLogin(ctx context.Context, registry string) error {
 
 	fmt.Println("\nLogging In...")
 
-	opts := []backend.Option{
-		backend.WithPlainHTTP(loginConfig.PlainHTTP),
-	}
-
-	if err := b.Login(ctx, registry, loginConfig.Username, loginConfig.Password, opts...); err != nil {
+	if err := b.Login(ctx, registry, loginConfig.Username, loginConfig.Password, loginConfig); err != nil {
 		return err
 	}
 

cmd/pull.go

@@ -54,6 +54,7 @@ func init() {
 	flags.BoolVar(&pullConfig.Insecure, "insecure", false, "use insecure connection for the pull operation and skip TLS verification")
 	flags.StringVar(&pullConfig.Proxy, "proxy", "", "use proxy for the pull operation")
 	flags.StringVar(&pullConfig.ExtractDir, "extract-dir", "", "specify the extract dir for extracting the model artifact")
+	flags.BoolVar(&pullConfig.ExtractFromRemote, "extract-from-remote", false, "turning on this flag will pull and extract the data from remote registry and no longer store model artifact locally, so user must specify extract-dir as the output directory")
 
 	if err := viper.BindPFlags(flags); err != nil {
 		panic(fmt.Errorf("bind cache pull flags to viper: %w", err))
@@ -71,21 +72,7 @@ func runPull(ctx context.Context, target string) error {
 		return fmt.Errorf("target is required")
 	}
 
-	opts := []backend.Option{
-		backend.WithInsecure(pullConfig.Insecure),
-		backend.WithPlainHTTP(pullConfig.PlainHTTP),
-		backend.WithConcurrency(pullConfig.Concurrency),
-	}
-
-	if pullConfig.Proxy != "" {
-		opts = append(opts, backend.WithProxy(pullConfig.Proxy))
-	}
-
-	if pullConfig.ExtractDir != "" {
-		opts = append(opts, backend.WithOutput(pullConfig.ExtractDir))
-	}
-
-	if err := b.Pull(ctx, target, opts...); err != nil {
+	if err := b.Pull(ctx, target, pullConfig); err != nil {
 		return err
 	}
 

cmd/push.go

@@ -64,12 +64,7 @@ func runPush(ctx context.Context, target string) error {
 		return err
 	}
 
-	opts := []backend.Option{
-		backend.WithPlainHTTP(pushConfig.PlainHTTP),
-		backend.WithConcurrency(pushConfig.Concurrency),
-	}
-
-	if err := b.Push(ctx, target, opts...); err != nil {
+	if err := b.Push(ctx, target, pushConfig); err != nil {
 		return err
 	}
 

pkg/backend/backend.go

@@ -19,25 +19,26 @@ package backend
 import (
 	"context"
 
+	"github.com/CloudNativeAI/modctl/pkg/config"
 	"github.com/CloudNativeAI/modctl/pkg/storage"
 )
 
 // Backend is the interface to represent the backend.
 type Backend interface {
 	// Login logs into a registry.
-	Login(ctx context.Context, registry, username, password string, opts ...Option) error
+	Login(ctx context.Context, registry, username, password string, cfg *config.Login) error
 
 	// Logout logs out from a registry.
 	Logout(ctx context.Context, registry string) error
 
 	// Build builds the user materials into the OCI image which follows the Model Spec.
-	Build(ctx context.Context, modelfilePath, workDir, target string, opts ...Option) error
+	Build(ctx context.Context, modelfilePath, workDir, target string, cfg *config.Build) error
 
 	// Pull pulls an artifact from a registry.
-	Pull(ctx context.Context, target string, opts ...Option) error
+	Pull(ctx context.Context, target string, cfg *config.Pull) error
 
 	// Push pushes the image to the registry.
-	Push(ctx context.Context, target string, opts ...Option) error
+	Push(ctx context.Context, target string, cfg *config.Push) error
 
 	// List lists all the model artifacts.
 	List(ctx context.Context) ([]*ModelArtifact, error)

pkg/backend/build.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/CloudNativeAI/modctl/pkg/backend/build"
 	"github.com/CloudNativeAI/modctl/pkg/backend/processor"
+	"github.com/CloudNativeAI/modctl/pkg/config"
 	"github.com/CloudNativeAI/modctl/pkg/modelfile"
 
 	modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1"
@@ -30,13 +31,7 @@ import (
 )
 
 // Build builds the user materials into the OCI image which follows the Model Spec.
-func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target string, opts ...Option) error {
-	// apply options.
-	options := &Options{}
-	for _, opt := range opts {
-		opt(options)
-	}
-
+func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target string, cfg *config.Build) error {
 	// parse the repo name and tag name from target.
 	ref, err := ParseReference(target)
 	if err != nil {
@@ -53,24 +48,39 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
 		return fmt.Errorf("tag is required")
 	}
 
+	// using the local output by default.
+	outputType := build.OutputTypeLocal
+	if cfg.OutputRemote {
+		outputType = build.OutputTypeRemote
+	}
+
+	opts := []build.Option{
+		build.WithPlainHTTP(cfg.PlainHTTP),
+		build.WithInsecure(cfg.Insecure),
+	}
+	builder, err := build.NewBuilder(outputType, b.store, modelfile, repo, tag, opts...)
+	if err != nil {
+		return fmt.Errorf("failed to create builder: %w", err)
+	}
+
 	layers := []ocispec.Descriptor{}
-	layerDescs, err := b.process(ctx, workDir, repo, options, b.getProcessors(modelfile)...)
+	layerDescs, err := b.process(ctx, builder, workDir, cfg, b.getProcessors(modelfile)...)
 	if err != nil {
 		return fmt.Errorf("failed to process files: %w", err)
 	}
 
 	layers = append(layers, layerDescs...)
 
 	// build the image config.
-	configDesc, err := build.BuildConfig(ctx, b.store, modelfile, repo)
+	configDesc, err := builder.BuildConfig(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to build image config: %w", err)
 	}
 
 	fmt.Printf("%s => %s (%s)\n", "Built config", configDesc.Digest, humanize.IBytes(uint64(configDesc.Size)))
 
 	// build the image manifest.
-	manifestDesc, err := build.BuildManifest(ctx, b.store, repo, tag, layers, configDesc, manifestAnnotation())
+	manifestDesc, err := builder.BuildManifest(ctx, layers, configDesc, manifestAnnotation())
 	if err != nil {
 		return fmt.Errorf("failed to build image manifest: %w", err)
 	}
@@ -102,10 +112,10 @@ func (b *backend) getProcessors(modelfile modelfile.Modelfile) []processor.Proce
 }
 
 // process walks the user work directory and process the identified files.
-func (b *backend) process(ctx context.Context, workDir string, repo string, opts *Options, processors ...processor.Processor) ([]ocispec.Descriptor, error) {
+func (b *backend) process(ctx context.Context, builder build.Builder, workDir string, cfg *config.Build, processors ...processor.Processor) ([]ocispec.Descriptor, error) {
 	descriptors := []ocispec.Descriptor{}
 	for _, p := range processors {
-		descs, err := p.Process(ctx, workDir, repo, processor.WithConcurrency(opts.concurrency))
+		descs, err := p.Process(ctx, builder, workDir, processor.WithConcurrency(cfg.Concurrency))
 		if err != nil {
 			return nil, err
 		}