feat: support output the log to file

Signed-off-by: chlins <chlins.zhang@gmail.com>

Author: chlins

cmd/attach.go

@@ -84,7 +84,7 @@ func runAttach(ctx context.Context, filepath string) error {
 		sp.Start()
 		defer sp.Stop()
 
-		nydusName, err := b.Nydusify(ctx, attachConfig.Target)
+		nydusName, err := b.Nydusify(ctx, attachConfig.Target, rootConfig)
 		if err != nil {
 			err = fmt.Errorf("failed to nydusify %s: %w", attachConfig.Target, err)
 			sp.FinalMSG = err.Error()

cmd/build.go

@@ -21,12 +21,12 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/CloudNativeAI/modctl/pkg/backend"
-	"github.com/CloudNativeAI/modctl/pkg/config"
 	"github.com/briandowns/spinner"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"github.com/CloudNativeAI/modctl/pkg/backend"
+	"github.com/CloudNativeAI/modctl/pkg/config"
 )
 
 var buildConfig = config.NewBuild()
@@ -86,7 +86,7 @@ func runBuild(ctx context.Context, workDir string) error {
 		sp.Start()
 		defer sp.Stop()
 
-		nydusName, err := b.Nydusify(ctx, buildConfig.Target)
+		nydusName, err := b.Nydusify(ctx, buildConfig.Target, rootConfig)
 		if err != nil {
 			err = fmt.Errorf("failed to nydusify %s: %w", buildConfig.Target, err)
 			sp.FinalMSG = err.Error()

cmd/push.go

@@ -81,7 +81,7 @@ func runPush(ctx context.Context, target string) error {
 		sp.Start()
 		defer sp.Stop()
 
-		nydusName, err := b.Nydusify(ctx, target)
+		nydusName, err := b.Nydusify(ctx, target, rootConfig)
 		if err != nil {
 			err = fmt.Errorf("failed to nydusify %s: %w", target, err)
 			sp.FinalMSG = err.Error()

cmd/root.go

@@ -24,11 +24,12 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/CloudNativeAI/modctl/cmd/modelfile"
-	"github.com/CloudNativeAI/modctl/pkg/config"
-
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"github.com/CloudNativeAI/modctl/cmd/modelfile"
+	"github.com/CloudNativeAI/modctl/pkg/config"
 )
 
 var rootConfig *config.Root
@@ -51,6 +52,23 @@ var rootCmd = &cobra.Command{
 				}
 			}()
 		}
+
+		// Initialize logger.
+		logFile, err := os.OpenFile(rootConfig.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		if err != nil {
+			logrus.Fatalf("Failed to open log file: %v", err)
+		}
+		// defer logFile.Close()
+
+		logLevel, err := logrus.ParseLevel(rootConfig.LogLevel)
+		if err != nil {
+			logrus.Fatalf("Failed to parse log level: %v", err)
+		}
+
+		logrus.SetOutput(logFile)
+		logrus.SetLevel(logLevel)
+		logrus.SetFormatter(&logrus.TextFormatter{})
+
 		return nil
 	},
 }
@@ -83,6 +101,8 @@ func init() {
 	flags.StringVar(&rootConfig.StoargeDir, "storage-dir", rootConfig.StoargeDir, "specify the storage directory for modctl")
 	flags.BoolVar(&rootConfig.Pprof, "pprof", rootConfig.Pprof, "enable pprof")
 	flags.StringVar(&rootConfig.PprofAddr, "pprof-addr", rootConfig.PprofAddr, "specify the address for pprof")
+	flags.StringVar(&rootConfig.LogLevel, "log-level", rootConfig.LogLevel, "specify the log level")
+	flags.StringVar(&rootConfig.LogFile, "log-file", rootConfig.LogFile, "specify the log file")
 
 	// Bind common flags.
 	if err := viper.BindPFlags(flags); err != nil {

pkg/archiver/archiver.go

@@ -23,12 +23,15 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/sirupsen/logrus"
 )
 
 // Tar creates a tar archive of the specified path (file or directory)
 // and returns the content as a stream. For individual files, it preserves
 // the directory structure relative to the working directory.
 func Tar(srcPath string, workDir string) (io.Reader, error) {
+	logrus.Infof("Taring %s, work dir: %s", srcPath, workDir)
 	pr, pw := io.Pipe()
 
 	go func() {
@@ -38,6 +41,7 @@ func Tar(srcPath string, workDir string) (io.Reader, error) {
 
 		info, err := os.Stat(srcPath)
 		if err != nil {
+			logrus.Errorf("failed to stat source path: %v", err)
 			pw.CloseWithError(fmt.Errorf("failed to stat source path: %w", err))
 			return
 		}
@@ -53,28 +57,33 @@ func Tar(srcPath string, workDir string) (io.Reader, error) {
 				// Create a relative path for the tar file header.
 				relPath, err := filepath.Rel(workDir, path)
 				if err != nil {
+					logrus.Errorf("failed to get relative path: %v", err)
 					return fmt.Errorf("failed to get relative path: %w", err)
 				}
 
 				header, err := tar.FileInfoHeader(info, "")
 				if err != nil {
+					logrus.Errorf("failed to create tar header: %v", err)
 					return fmt.Errorf("failed to create tar header: %w", err)
 				}
 
 				// Set the header name to preserve directory structure.
 				header.Name = relPath
 				if err := tw.WriteHeader(header); err != nil {
+					logrus.Errorf("failed to write header: %v", err)
 					return fmt.Errorf("failed to write header: %w", err)
 				}
 
 				if !info.IsDir() {
 					file, err := os.Open(path)
 					if err != nil {
+						logrus.Errorf("failed to open file %s: %v", path, err)
 						return fmt.Errorf("failed to open file %s: %w", path, err)
 					}
 					defer file.Close()
 
 					if _, err := io.Copy(tw, file); err != nil {
+						logrus.Errorf("failed to write file %s to tar: %v", path, err)
 						return fmt.Errorf("failed to write file %s to tar: %w", path, err)
 					}
 				}
@@ -83,20 +92,23 @@ func Tar(srcPath string, workDir string) (io.Reader, error) {
 			})
 
 			if err != nil {
+				logrus.Errorf("failed to walk directory: %v", err)
 				pw.CloseWithError(fmt.Errorf("failed to walk directory: %w", err))
 				return
 			}
 		} else {
 			// For a single file, include the directory structure.
 			file, err := os.Open(srcPath)
 			if err != nil {
+				logrus.Errorf("failed to open file %s: %v", srcPath, err)
 				pw.CloseWithError(fmt.Errorf("failed to open file: %w", err))
 				return
 			}
 			defer file.Close()
 
 			header, err := tar.FileInfoHeader(info, "")
 			if err != nil {
+				logrus.Errorf("failed to create tar header for file %s: %v", srcPath, err)
 				pw.CloseWithError(fmt.Errorf("failed to create tar header: %w", err))
 				return
 			}
@@ -105,18 +117,21 @@ func Tar(srcPath string, workDir string) (io.Reader, error) {
 			// This keeps the directory structure as part of the file path in the tar.
 			relPath, err := filepath.Rel(workDir, srcPath)
 			if err != nil {
+				logrus.Errorf("failed to get relative path for file %s: %v", srcPath, err)
 				pw.CloseWithError(fmt.Errorf("failed to get relative path: %w", err))
 				return
 			}
 
 			// Use the relative path (including directories) as the header name.
 			header.Name = relPath
 			if err := tw.WriteHeader(header); err != nil {
+				logrus.Errorf("failed to write header for file %s: %v", srcPath, err)
 				pw.CloseWithError(fmt.Errorf("failed to write header: %w", err))
 				return
 			}
 
 			if _, err := io.Copy(tw, file); err != nil {
+				logrus.Errorf("failed to copy file to tar for file %s: %v", srcPath, err)
 				pw.CloseWithError(fmt.Errorf("failed to copy file to tar: %w", err))
 				return
 			}
@@ -129,10 +144,12 @@ func Tar(srcPath string, workDir string) (io.Reader, error) {
 // Untar extracts the contents of a tar archive from the provided reader
 // to the specified destination path.
 func Untar(reader io.Reader, destPath string) error {
+	logrus.Infof("Untaring archive to %s", destPath)
 	tarReader := tar.NewReader(reader)
 
 	// Ensure destination directory exists.
 	if err := os.MkdirAll(destPath, 0755); err != nil {
+		logrus.Errorf("failed to create destination directory: %v", err)
 		return fmt.Errorf("failed to create destination directory: %w", err)
 	}
 
@@ -142,12 +159,14 @@ func Untar(reader io.Reader, destPath string) error {
 			break
 		}
 		if err != nil {
+			logrus.Errorf("error reading tar: %v", err)
 			return fmt.Errorf("error reading tar: %w", err)
 		}
 
 		// Sanitize file paths to prevent directory traversal.
 		cleanPath := filepath.Clean(header.Name)
 		if strings.Contains(cleanPath, "..") || strings.HasPrefix(cleanPath, "/") || strings.HasPrefix(cleanPath, ":\\") {
+			logrus.Errorf("tar file contains invalid path: %s", cleanPath)
 			return fmt.Errorf("tar file contains invalid path: %s", cleanPath)
 		}
 
@@ -156,12 +175,14 @@ func Untar(reader io.Reader, destPath string) error {
 		// Create directories for all path components.
 		dirPath := filepath.Dir(targetPath)
 		if err := os.MkdirAll(dirPath, 0755); err != nil {
+			logrus.Errorf("failed to create directory %s: %v", dirPath, err)
 			return fmt.Errorf("failed to create directory %s: %w", dirPath, err)
 		}
 
 		switch header.Typeflag {
 		case tar.TypeDir:
 			if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil {
+				logrus.Errorf("failed to create directory %s: %v", targetPath, err)
 				return fmt.Errorf("failed to create directory %s: %w", targetPath, err)
 			}
 
@@ -172,21 +193,25 @@ func Untar(reader io.Reader, destPath string) error {
 				os.FileMode(header.Mode),
 			)
 			if err != nil {
+				logrus.Errorf("failed to create file %s: %v", targetPath, err)
 				return fmt.Errorf("failed to create file %s: %w", targetPath, err)
 			}
 
 			if _, err := io.Copy(file, tarReader); err != nil {
 				file.Close()
+				logrus.Errorf("failed to write to file %s: %v", targetPath, err)
 				return fmt.Errorf("failed to write to file %s: %w", targetPath, err)
 			}
 			file.Close()
 
 		case tar.TypeSymlink:
 			if isRel(header.Linkname, destPath) && isRel(header.Name, destPath) {
 				if err := os.Symlink(header.Linkname, targetPath); err != nil {
+					logrus.Errorf("failed to create symlink %s -> %s: %v", targetPath, header.Linkname, err)
 					return fmt.Errorf("failed to create symlink %s -> %s: %w", targetPath, header.Linkname, err)
 				}
 			} else {
+				logrus.Errorf("symlink %s -> %s points outside of destination directory", targetPath, header.Linkname)
 				return fmt.Errorf("symlink %s -> %s points outside of destination directory", targetPath, header.Linkname)
 			}
 

pkg/backend/attach.go

@@ -27,6 +27,7 @@ import (
 	modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1"
 	godigest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/sirupsen/logrus"
 
 	internalpb "github.com/CloudNativeAI/modctl/internal/pb"
 	"github.com/CloudNativeAI/modctl/pkg/backend/build"
@@ -58,13 +59,17 @@ var (
 
 // Attach attaches user materials into the model artifact which follows the Model Spec.
 func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attach) error {
+	logrus.Infof("Attaching file %s to %s", filepath, cfg.Source)
+
 	srcManifest, err := b.getManifest(ctx, cfg.Source, cfg)
 	if err != nil {
+		logrus.Errorf("failed to get source manifest: %v", err)
 		return fmt.Errorf("failed to get source manifest: %w", err)
 	}
 
 	srcModelConfig, err := b.getModelConfig(ctx, cfg.Source, srcManifest.Config, cfg)
 	if err != nil {
+		logrus.Errorf("failed to get source model config: %v", err)
 		return fmt.Errorf("failed to get source model config: %w", err)
 	}
 
@@ -95,11 +100,13 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 
 	proc := b.getProcessor(filepath)
 	if proc == nil {
+		logrus.Errorf("failed to get processor for file %s", filepath)
 		return fmt.Errorf("failed to get processor for file %s", filepath)
 	}
 
 	builder, err := b.getBuilder(cfg.Target, cfg)
 	if err != nil {
+		logrus.Errorf("failed to create builder: %v", err)
 		return fmt.Errorf("failed to create builder: %w", err)
 	}
 
@@ -109,6 +116,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 
 	newLayers, err := proc.Process(ctx, builder, ".", processor.WithProgressTracker(pb))
 	if err != nil {
+		logrus.Errorf("failed to process layers: %v", err)
 		return fmt.Errorf("failed to process layers: %w", err)
 	}
 
@@ -122,6 +130,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 	}
 	// Return earlier if the diffID has no changed, which means the artifact has not changed.
 	if reflect.DeepEqual(diffIDs, srcModelConfig.ModelFS.DiffIDs) {
+		logrus.Info("The artifact has not changed after attaching layer, skip to attach")
 		return nil
 	}
 
@@ -147,6 +156,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 		}),
 	))
 	if err != nil {
+		logrus.Errorf("failed to build model config: %v", err)
 		return fmt.Errorf("failed to build model config: %w", err)
 	}
 
@@ -163,9 +173,12 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 		}),
 	))
 	if err != nil {
+		logrus.Errorf("failed to build model manifest: %v", err)
 		return fmt.Errorf("failed to build model manifest: %w", err)
 	}
 
+	logrus.Infof("Attched %s to %s successfully", filepath, cfg.Source)
+
 	return nil
 }
 

pkg/backend/backend.go

@@ -65,7 +65,7 @@ type Backend interface {
 	Tag(ctx context.Context, source, target string) error
 
 	// Nydusify converts the model artifact to nydus format.
-	Nydusify(ctx context.Context, target string) (string, error)
+	Nydusify(ctx context.Context, target string, cfg *config.Root) (string, error)
 }
 
 // backend is the implementation of Backend.