Introduce option for image expiration grace period and fix log-level. (#5)

Author: Gerrit91

Makefile

@@ -17,6 +17,7 @@ release:: all;
 .PHONY: start
 start: all
 	bin/metal-image-cache-sync \
+	#   --log-level debug \
 	  --metal-api-endpoint $(METAL_API_ENDPOINT) \
 	  --metal-api-hmac $(METAL_API_HMAC) \
 	  --max-cache-size 10G \

cmd/internal/determine-sync-images/lister.go

@@ -17,22 +17,20 @@ import (
 )
 
 type SyncLister struct {
-	logger       *zap.SugaredLogger
-	driver       *metalgo.Driver
-	config       *api.Config
-	s3           *s3.S3
-	excludePaths []string
-	collector    *metrics.Collector
+	logger    *zap.SugaredLogger
+	driver    *metalgo.Driver
+	config    *api.Config
+	s3        *s3.S3
+	collector *metrics.Collector
 }
 
 func NewSyncLister(logger *zap.SugaredLogger, driver *metalgo.Driver, s3 *s3.S3, collector *metrics.Collector, config *api.Config) *SyncLister {
 	return &SyncLister{
-		logger:       logger,
-		driver:       driver,
-		config:       config,
-		s3:           s3,
-		excludePaths: config.ExcludePaths,
-		collector:    collector,
+		logger:    logger,
+		driver:    driver,
+		config:    config,
+		s3:        s3,
+		collector: collector,
 	}
 }
 
@@ -49,10 +47,12 @@ func (s *SyncLister) DetermineSyncList() ([]api.OS, error) {
 
 	s.collector.SetMetalAPIImageCount(len(resp.Image))
 
+	expirationGraceDays := 24 * time.Hour * time.Duration(s.config.ExpirationGraceDays)
+
 	images := api.OSImagesByOS{}
 	for _, img := range resp.Image {
 		skip := false
-		for _, exclude := range s.excludePaths {
+		for _, exclude := range s.config.ExcludePaths {
 			if strings.Contains(img.URL, exclude) {
 				skip = true
 				break
@@ -65,7 +65,7 @@ func (s *SyncLister) DetermineSyncList() ([]api.OS, error) {
 		}
 
 		if img.ExpirationDate != nil {
-			if time.Since(time.Time(*img.ExpirationDate)) > 0 {
+			if time.Since(time.Time(*img.ExpirationDate)) > expirationGraceDays {
 				s.logger.Debugw("not considering expired image, skipping", "id", *img.ID)
 				continue
 			}

cmd/main.go

@@ -33,8 +33,6 @@ import (
 const (
 	moduleName  = "metal-image-cache-sync"
 	cfgFileType = "yaml"
-
-	logLevelFlg = "log-level"
 )
 
 var (
@@ -52,8 +50,8 @@ var rootCmd = &cobra.Command{
 	SilenceErrors: true,
 	SilenceUsage:  true,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-		initLogging()
 		initConfig()
+		initLogging()
 		initSignalHandlers()
 		return nil
 	},
@@ -74,6 +72,7 @@ func main() {
 
 func init() {
 	rootCmd.Flags().String("bind-address", "127.0.0.1:3000", "http server bind address")
+	rootCmd.Flags().String("log-level", "info", "sets the application log level")
 
 	rootCmd.Flags().String("image-store", "metal-stack.io", "url to the image store")
 	rootCmd.Flags().String("image-store-bucket", "images", "bucket of the image store")
@@ -88,6 +87,8 @@ func init() {
 	rootCmd.Flags().Int("min-images-per-name", 3, "minimum amount of images to keep of an image variant")
 	rootCmd.Flags().Int("max-images-per-name", -1, "maximum amount of images to cache for an image variant, unlimited if less than zero")
 
+	rootCmd.Flags().Uint("expiration-grace-period", 0, "the amount of days to still sync images even if they have already expired in the metal-api (defaults to zero)")
+
 	rootCmd.Flags().String("root-path", "/var/lib/metal-image-cache-sync/images", "root path of where to store the images")
 	rootCmd.Flags().StringSlice("excludes", []string{"/pull_requests/"}, "url paths to exclude from the sync")
 
@@ -100,8 +101,8 @@ func init() {
 func initLogging() {
 	level := zap.InfoLevel
 
-	if viper.IsSet(logLevelFlg) {
-		err := level.UnmarshalText([]byte(viper.GetString(logLevelFlg)))
+	if viper.IsSet("log-level") {
+		err := level.UnmarshalText([]byte(viper.GetString("log-level")))
 		if err != nil {
 			log.Fatalf("can't initialize zap logger: %v", err)
 		}
@@ -128,7 +129,7 @@ func initConfig() {
 	if cfgFile != "" {
 		viper.SetConfigFile(cfgFile)
 		if err := viper.ReadInConfig(); err != nil {
-			logger.Fatalw("config file path set explicitly, but unreadable", "error", err)
+			log.Fatalf("config file path set explicitly, but unreadable: %v", err)
 		}
 	} else {
 		viper.SetConfigName("config")
@@ -138,15 +139,10 @@ func initConfig() {
 		if err := viper.ReadInConfig(); err != nil {
 			usedCfg := viper.ConfigFileUsed()
 			if usedCfg != "" {
-				logger.Fatalw("config file unreadable", "config-file", usedCfg, "error", err)
+				log.Fatalf("config file %s unreadable: %v", usedCfg, err)
 			}
 		}
 	}
-
-	usedCfg := viper.ConfigFileUsed()
-	if usedCfg != "" {
-		logger.Infow("read config file", "config-file", usedCfg)
-	}
 }
 
 func initSignalHandlers() {

pkg/api/config.go

@@ -27,21 +27,24 @@ type Config struct {
 	SyncSchedule string `validate:"required"`
 	DryRun       bool
 	ExcludePaths []string
+
+	ExpirationGraceDays uint
 }
 
 func NewConfig() (*Config, error) {
 	c := &Config{
-		BindAddress:        viper.GetString("bind-address"),
-		ImageCacheRootPath: viper.GetString("root-path"),
-		MinImagesPerName:   viper.GetInt("min-images-per-name"),
-		MaxImagesPerName:   viper.GetInt("max-images-per-name"),
-		ImageStore:         viper.GetString("image-store"),
-		ImageBucket:        viper.GetString("image-store-bucket"),
-		MetalAPIEndpoint:   viper.GetString("metal-api-endpoint"),
-		MetalAPIHMAC:       viper.GetString("metal-api-hmac"),
-		SyncSchedule:       viper.GetString("schedule"),
-		DryRun:             viper.GetBool("dry-run"),
-		ExcludePaths:       viper.GetStringSlice("excludes"),
+		BindAddress:         viper.GetString("bind-address"),
+		ImageCacheRootPath:  viper.GetString("root-path"),
+		MinImagesPerName:    viper.GetInt("min-images-per-name"),
+		MaxImagesPerName:    viper.GetInt("max-images-per-name"),
+		ImageStore:          viper.GetString("image-store"),
+		ImageBucket:         viper.GetString("image-store-bucket"),
+		MetalAPIEndpoint:    viper.GetString("metal-api-endpoint"),
+		MetalAPIHMAC:        viper.GetString("metal-api-hmac"),
+		SyncSchedule:        viper.GetString("schedule"),
+		DryRun:              viper.GetBool("dry-run"),
+		ExcludePaths:        viper.GetStringSlice("excludes"),
+		ExpirationGraceDays: viper.GetUint("expiration-grace-period"),
 	}
 
 	var err error