diff --git a/README.md b/README.md index 7f21604698..a08bd1c456 100644 --- a/README.md +++ b/README.md @@ -1143,6 +1143,11 @@ Ignore /var/run when taking image snapshot. Set it to false to preserve Set this flag as `--ignore-path=` to ignore path when taking an image snapshot. Set it multiple times for multiple ignore paths. +#### Flag `--image-fs-extract-progres` + +Set this flag to show the progress of extracting an image filesystem. Defaults +to `false`. + #### Flag `--image-fs-extract-retry` Set this flag to the number of retries that should happen for the extracting an diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 7f0339c5df..89743d2ea0 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -239,6 +239,7 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().BoolVarP(&opts.SkipTLSVerifyPull, "skip-tls-verify-pull", "", false, "Pull from insecure registry ignoring TLS verify") RootCmd.PersistentFlags().IntVar(&opts.PushRetry, "push-retry", 0, "Number of retries for the push operation") RootCmd.PersistentFlags().BoolVar(&opts.PushIgnoreImmutableTagErrors, "push-ignore-immutable-tag-errors", false, "If true, known tag immutability errors are ignored and the push finishes with success.") + RootCmd.PersistentFlags().BoolVar(&opts.ImageFSExtractProgress, "image-fs-extract-progress", false, "Show progress for image FS extraction") RootCmd.PersistentFlags().IntVar(&opts.ImageFSExtractRetry, "image-fs-extract-retry", 0, "Number of retries for image FS extraction") RootCmd.PersistentFlags().IntVar(&opts.ImageDownloadRetry, "image-download-retry", 0, "Number of retries for downloading the remote image") RootCmd.PersistentFlags().StringVarP(&opts.KanikoDir, "kaniko-dir", "", constants.DefaultKanikoPath, "Path to the kaniko directory, this takes precedence over the KANIKO_DIR environment variable.") diff --git a/pkg/config/options.go b/pkg/config/options.go index 6f3b67a197..9afe39457e 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -77,6 +77,7 @@ type KanikoOptions struct { OCILayoutPath string Compression Compression CompressionLevel int + ImageFSExtractProgress bool ImageFSExtractRetry int SingleSnapshot bool Reproducible bool diff --git a/pkg/executor/build.go b/pkg/executor/build.go index d163af6831..dabd5b7f42 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -332,7 +332,7 @@ func (s *stageBuilder) build() error { t := timing.Start("FS Unpacking") retryFunc := func() error { - _, err := getFSFromImage(config.RootDir, s.image, util.ExtractFile) + _, err := getFSFromImage(config.RootDir, s.image, util.ExtractFile, s.opts.ImageFSExtractProgress) return err } @@ -939,7 +939,7 @@ func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) e if err := saveStageAsTarball(c.From, sourceImage); err != nil { return err } - if err := extractImageToDependencyDir(c.From, sourceImage); err != nil { + if err := extractImageToDependencyDir(c.From, sourceImage, opts.ImageFSExtractProgress); err != nil { return err } } @@ -960,7 +960,7 @@ func fromPreviousStage(copyCommand *instructions.CopyCommand, previousStageNames return false } -func extractImageToDependencyDir(name string, image v1.Image) error { +func extractImageToDependencyDir(name string, image v1.Image, extractionProgress bool) error { t := timing.Start("Extracting Image to Dependency Dir") defer timing.DefaultRun.Stop(t) dependencyDir := filepath.Join(config.KanikoDir, name) @@ -968,7 +968,7 @@ func extractImageToDependencyDir(name string, image v1.Image) error { return err } logrus.Debugf("Trying to extract to %s", dependencyDir) - _, err := util.GetFSFromImage(dependencyDir, image, util.ExtractFile) + _, err := util.GetFSFromImage(dependencyDir, image, util.ExtractFile, extractionProgress) return err } diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 4191b13f5d..7586a2c06a 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -923,7 +923,7 @@ func Test_stageBuilder_build(t *testing.T) { config *v1.ConfigFile stage config.KanikoStage crossStageDeps map[int][]string - mockGetFSFromImage func(root string, img v1.Image, extract util.ExtractFunction) ([]string, error) + mockGetFSFromImage func(root string, img v1.Image, extract util.ExtractFunction, extractionProgress bool) ([]string, error) shouldInitSnapshot bool } @@ -1441,7 +1441,7 @@ RUN foobar opts: &config.KanikoOptions{InitialFSUnpacked: true}, stage: config.KanikoStage{Index: 0}, crossStageDeps: map[int][]string{0: {"some-dep"}}, - mockGetFSFromImage: func(root string, img v1.Image, extract util.ExtractFunction) ([]string, error) { + mockGetFSFromImage: func(root string, img v1.Image, extract util.ExtractFunction, extractionPogress bool) ([]string, error) { return nil, fmt.Errorf("getFSFromImage shouldn't be called if fs is already unpacked") }, }, diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index e968d62e8e..968783822c 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -96,9 +96,9 @@ type FileContext struct { type ExtractFunction func(string, *tar.Header, string, io.Reader) error type FSConfig struct { - includeWhiteout bool - printExtractionProgress bool - extractFunc ExtractFunction + includeWhiteout bool + extractFunc ExtractFunction + extractionProgress bool } type FSOpt func(*FSConfig) @@ -127,21 +127,21 @@ func IncludeWhiteout() FSOpt { } } -func PrintExtractionProgress() FSOpt { +func ExtractFunc(extractFunc ExtractFunction) FSOpt { return func(opts *FSConfig) { - opts.printExtractionProgress = true + opts.extractFunc = extractFunc } } -func ExtractFunc(extractFunc ExtractFunction) FSOpt { +func ExtractionProgress() FSOpt { return func(opts *FSConfig) { - opts.extractFunc = extractFunc + opts.extractionProgress = true } } // GetFSFromImage extracts the layers of img to root // It returns a list of all files extracted -func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]string, error) { +func GetFSFromImage(root string, img v1.Image, extract ExtractFunction, extractionProgress bool) ([]string, error) { if img == nil { return nil, errors.New("image cannot be nil") } @@ -151,7 +151,11 @@ func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]strin return nil, err } - return GetFSFromLayers(root, layers, ExtractFunc(extract), PrintExtractionProgress()) + opts := []FSOpt{ExtractFunc(extract)} + if extractionProgress { + opts = append(opts, ExtractionProgress()) + } + return GetFSFromLayers(root, layers, opts...) } func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, error) { @@ -180,12 +184,9 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e layerSizes = append(layerSizes, layerSize) totalSize += layerSize } - printExtractionProgress := cfg.printExtractionProgress - if totalSize == 0 { - printExtractionProgress = false - } - if printExtractionProgress { + showProgress := totalSize > 0 && cfg.extractionProgress + if showProgress { logrus.Infof("Extracting image layers to %s", root) } @@ -193,30 +194,27 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e var extractedBytes int64 for i, l := range layers { if mediaType, err := l.MediaType(); err == nil { - logrus.Tracef("Extracting layer %d/%d of media type %s", i+1, len(layers), mediaType) + logrus.Tracef("Extracting layer %d (%d/%d) of media type %s", i, i+1, len(layers), mediaType) } else { - logrus.Tracef("Extracting layer %d/%d", i+1, len(layers)) + logrus.Tracef("Extracting layer %d (%d/%d)", i, i+1, len(layers)) } progressPerc := float64(extractedBytes) / float64(totalSize) * 100 - if printExtractionProgress { + if showProgress { logrus.Infof("Extracting layer %d/%d (%.1f%%)", i+1, len(layers), progressPerc) } - r, err := l.Uncompressed() + cr, err := l.Uncompressed() if err != nil { return nil, err } - defer r.Close() + defer cr.Close() - if printExtractionProgress { - r = &printAfterReader{ - ReadCloser: r, - after: time.Second, - print: func(n int) { - logrus.Infof("Extracting layer %d/%d (%.1f%%) %s", i+1, len(layers), progressPerc, strings.Repeat(".", n)) - }, - } + var r io.Reader = cr + if showProgress { + r = newDoAfterReader(r, func(count int) { + logrus.Infof("Extracting layer %d/%d (%.1f%%) %s", i+1, len(layers), progressPerc, strings.Repeat(".", count)) + }, time.Second) } tr := tar.NewReader(r) @@ -271,32 +269,40 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e extractedBytes += layerSizes[i] } - if printExtractionProgress { + if showProgress { logrus.Infof("Extraction complete") } return extractedFiles, nil } -type printAfterReader struct { - io.ReadCloser +type doAfterReader struct { + r io.Reader t time.Time after time.Duration count int - print func(int) + do func(int) +} + +func newDoAfterReader(r io.Reader, do func(int), after time.Duration) *doAfterReader { + return &doAfterReader{ + r: r, + do: do, + after: after, + } } -func (r *printAfterReader) Read(p []byte) (n int, err error) { - n, err = r.ReadCloser.Read(p) +func (r *doAfterReader) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) if r.t.IsZero() { r.t = time.Now() } if time.Since(r.t) >= r.after { r.count++ - r.print(r.count) + r.do(r.count) r.t = time.Now() } - return + return n, err } // DeleteFilesystem deletes the extracted image file system