Update Precacher to No Longer Fatally Error by Default [2.0] (#14014)

Author: sameluch

toolkit/Makefile

@@ -120,6 +120,10 @@ RPMS_DIR        ?= $(OUT_DIR)/RPMS
 SRPMS_DIR       ?= $(OUT_DIR)/SRPMS
 IMAGES_DIR      ?= $(OUT_DIR)/images
 
+# Turning on non-fatal mode by default. The precacher is not critical to the build
+# if the user is depending on failures from the precacher, it can be turned off with this option or with the tool directly.
+PRECACHER_NON_FATAL ?= y
+
 # External source server
 SOURCE_URL         ?= https://azurelinuxsrcstorage.blob.core.windows.net/sources/core
 

toolkit/scripts/precache.mk

@@ -49,6 +49,7 @@ $(STATUS_FLAGS_DIR)/precache.flag: $(go-precacher) $(chroot_worker) $(rpms_snaps
 		$(if $(filter y,$(ENABLE_CPU_PROFILE)),--enable-cpu-prof) \
 		$(if $(filter y,$(ENABLE_MEM_PROFILE)),--enable-mem-prof) \
 		$(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \
+		$(if $(filter y,$(PRECACHER_NON_FATAL)),--non-fatal-mode) \
 		--timestamp-file=$(TIMESTAMP_DIR)/precacher.jsonl && \
 	if [ ! -f $@ ] || [ -s "$(precache_downloaded_files)" ]; then \
 		touch $@; \

toolkit/tools/precacher/precacher.go

@@ -62,6 +62,7 @@ var (
 	buildDir          = app.Flag("worker-dir", "Directory to store chroot while running repo query.").Required().String()
 
 	concurrentNetOps = app.Flag("concurrent-net-ops", "Number of concurrent network operations to perform.").Default(defaultNetOpsCount).Uint()
+	nonFatalMode     = app.Flag("non-fatal-mode", "Run in non-fatal mode, where errors are logged but do not cause the program to exit with a non-zero code.").Bool()
 )
 
 func main() {
@@ -80,11 +81,22 @@ func main() {
 
 	rpmSnapshot, err := rpmSnapshotFromFile(*snapshot)
 	if err != nil {
-		logger.PanicOnError(err)
+		if *nonFatalMode {
+			logger.Log.Errorf("%s", err)
+			return
+		} else {
+			logger.PanicOnError(err)
+		}
+
 	}
 	packagesAvailableFromRepos, err := repoutils.GetAllRepoData(*repoUrls, *repoFiles, *workerTar, *buildDir, *repoUrlsFile)
 	if err != nil {
-		logger.PanicOnError(err)
+		if *nonFatalMode {
+			logger.Log.Errorf("%s", err)
+			return
+		} else {
+			logger.PanicOnError(err)
+		}
 	}
 
 	logger.Log.Infof("Found %d available packages", len(packagesAvailableFromRepos))
@@ -96,13 +108,22 @@ func main() {
 
 	downloadedPackages, err := downloadMissingPackages(rpmSnapshot, packagesAvailableFromRepos, *outDir, *concurrentNetOps)
 	if err != nil {
-		logger.PanicOnError(err)
+		logger.Log.Warnf("Package download failed")
+		logger.Log.Warnf("Missing package download failed: %s", err)
+		// reset the error to nil so we can still write the summary file
+		// packages which are not able to be downloaded are not considered a failure of the tool, just a failure to download some packages
+		err = nil
 	}
 
 	logger.Log.Infof("Downloaded %d packages into the cache", len(downloadedPackages))
 	err = writeSummaryFile(*outputSummaryFile, downloadedPackages)
 	if err != nil {
-		logger.PanicOnError(err)
+		if *nonFatalMode {
+			logger.Log.Errorf("%s", err)
+			return
+		} else {
+			logger.PanicOnError(err)
+		}
 	}
 }