Merge pull request #490 from nix-community/refactoring

disable cached failed builds by default

Author: Mic92

buildbot_nix/buildbot_nix/__init__.py

@@ -80,7 +80,7 @@ class NixEvalConfig:
     max_memory_size: int
 
     eval_lock: MasterLock
-    failed_builds_db: FailedBuildDB
+    failed_builds_db: FailedBuildDB | None
     gcroots_user: str = "buildbot-worker"
 
     show_trace: bool = False
@@ -1533,7 +1533,7 @@ def _configure_projects(
                             or multiprocessing.cpu_count(),
                             max_memory_size=self.config.eval_max_memory_size,
                             eval_lock=eval_lock,
-                            failed_builds_db=DB,  # type: ignore[arg-type]  # DB is guaranteed to be initialized above
+                            failed_builds_db=DB,
                             gcroots_user=self.config.gcroots_user,
                             show_trace=self.config.show_trace_on_failure,
                         ),
@@ -1646,14 +1646,10 @@ def configure(self, config: dict[str, Any]) -> None:
         eval_lock = util.MasterLock("nix-eval")
 
         global DB  # noqa: PLW0603
-        if DB is None:
+        if DB is None and self.config.cache_failed_builds:
             DB = FailedBuildDB(Path("failed_builds.dbm"))
             atexit.register(lambda: DB.close() if DB is not None else None)
 
-        if DB is None:
-            msg = "Database initialization failed"
-            raise RuntimeError(msg)
-
         # Configure projects
         succeeded_projects = self._configure_projects(
             config, projects, worker_names, eval_lock

buildbot_nix/buildbot_nix/build_trigger.py

@@ -50,7 +50,7 @@ class JobsConfig:
     successful_jobs: list[NixEvalJobSuccess]
     failed_jobs: list[NixEvalJobError]
     combine_builds: bool
-    failed_builds_db: FailedBuildDB
+    failed_builds_db: FailedBuildDB | None
 
 
 class BuildTrigger(buildstep.ShellMixin, steps.BuildStep):
@@ -72,7 +72,7 @@ class BuildTrigger(buildstep.ShellMixin, steps.BuildStep):
     wait_for_finish_deferred: defer.Deferred[tuple[list[int], int]] | None
     brids: list[int]
     consumers: dict[int, Any]
-    failed_builds_db: FailedBuildDB
+    failed_builds_db: FailedBuildDB | None
 
     @dataclass
     class ScheduledJob:
@@ -372,51 +372,54 @@ async def _schedule_failed_evaluations(
                 scheduled.append(
                     BuildTrigger.ScheduledJob(failed_job, brids, results_deferred)
                 )
-                self.brids.extend(brids)
+                self.brids.extend(brids.values())
 
         return overall_result
 
     async def _process_build_for_scheduling(
         self,
         build: NixEvalJobSuccess,
-        context: SchedulingContext,
+        ctx: SchedulingContext,
     ) -> None:
         """Process a single build to determine if it should be scheduled."""
-        failed_build = self.jobs_config.failed_builds_db.check_build(build.drvPath)
+        failed_build = None
+        if self.jobs_config.failed_builds_db is not None:
+            failed_build = self.jobs_config.failed_builds_db.check_build(build.drvPath)
 
-        if context.job_closures.get(build.drvPath):
+        if ctx.job_closures.get(build.drvPath):
             # Has dependencies, skip for now
             return
 
         if failed_build is not None and self.build:
             if self.build.reason != "rebuild":
                 # Skip due to cached failure
-                context.scheduler_log.addStdout(
+                ctx.scheduler_log.addStdout(
                     f"\t- skipping {build.attr} due to cached failure, first failed at {failed_build.time}\n"
                     f"\t  see build at {failed_build.url}\n"
                 )
-                context.build_schedule_order.remove(build)
+                ctx.build_schedule_order.remove(build)
 
                 brids, results_deferred = await self.schedule(
-                    context.ss_for_trigger,
+                    ctx.ss_for_trigger,
                     *self.schedule_cached_failure(build, failed_build),
                 )
-                context.scheduled.append(
+                ctx.scheduled.append(
                     BuildTrigger.ScheduledJob(build, brids, results_deferred)
                 )
                 self.brids.extend(brids.values())
             else:
                 # Rebuild requested, remove from cache and schedule
-                self.jobs_config.failed_builds_db.remove_build(build.drvPath)
-                context.scheduler_log.addStdout(
+                if self.jobs_config.failed_builds_db is not None:
+                    self.jobs_config.failed_builds_db.remove_build(build.drvPath)
+                ctx.scheduler_log.addStdout(
                     f"\t- not skipping {build.attr} with cached failure due to rebuild, first failed at {failed_build.time}\n"
                 )
-                context.build_schedule_order.remove(build)
-                context.schedule_now.append(build)
+                ctx.build_schedule_order.remove(build)
+                ctx.schedule_now.append(build)
         else:
             # No cached failure, schedule normally
-            context.build_schedule_order.remove(build)
-            context.schedule_now.append(build)
+            ctx.build_schedule_order.remove(build)
+            ctx.schedule_now.append(build)
 
     async def _get_failed_build_url(self, brids: dict[str, Any]) -> str:
         """Get the URL of the actual failed build."""
@@ -435,7 +438,7 @@ async def _get_failed_build_url(self, brids: dict[str, Any]) -> str:
     async def _handle_failed_job(
         self,
         job: NixEvalJob,
-        context: SchedulingContext,
+        ctx: SchedulingContext,
         brids: dict[str, Any],
         result: int,
     ) -> None:
@@ -444,7 +447,7 @@ async def _handle_failed_job(
             return
 
         # Update failed builds cache if needed
-        if result == util.FAILURE:
+        if result == util.FAILURE and self.jobs_config.failed_builds_db is not None:
             should_add_to_cache = (
                 self.build and self.build.reason == "rebuild"
             ) or not self.jobs_config.failed_builds_db.check_build(job.drvPath)
@@ -456,28 +459,28 @@ async def _handle_failed_job(
 
         # Schedule dependent failures
         removed = self.get_failed_dependents(
-            job, context.build_schedule_order, context.job_closures
+            job, ctx.build_schedule_order, ctx.job_closures
         )
         for removed_job in removed:
             scheduler, props = self.schedule_dependency_failed(removed_job, job)
             dep_brids, results_deferred = await self.schedule(
-                context.ss_for_trigger, scheduler, props
+                ctx.ss_for_trigger, scheduler, props
             )
-            context.build_schedule_order.remove(removed_job)
-            context.scheduled.append(
+            ctx.build_schedule_order.remove(removed_job)
+            ctx.scheduled.append(
                 BuildTrigger.ScheduledJob(removed_job, dep_brids, results_deferred)
             )
             self.brids.extend(dep_brids.values())
 
         if removed:
-            context.scheduler_log.addStdout(
+            ctx.scheduler_log.addStdout(
                 "\t- removed jobs: "
                 + ", ".join([job.drvPath for job in removed])
                 + "\n"
             )
 
         # Update job closures
-        for job_closure in context.job_closures.values():
+        for job_closure in ctx.job_closures.values():
             if job.drvPath in job_closure:
                 job_closure.remove(job.drvPath)
 
@@ -523,7 +526,7 @@ async def run(self) -> int:
         )
 
         # Create scheduling context that will be reused throughout the loop
-        context = BuildTrigger.SchedulingContext(
+        ctx = BuildTrigger.SchedulingContext(
             build_schedule_order=build_schedule_order,
             job_closures=job_closures,
             ss_for_trigger=ss_for_trigger,
@@ -533,34 +536,33 @@ async def run(self) -> int:
         )
 
         # Main scheduling loop
-        while build_schedule_order or scheduled:
-            scheduler_log.addStdout("Scheduling...\n")
+        while ctx.build_schedule_order or ctx.scheduled:
+            ctx.scheduler_log.addStdout("Scheduling...\n")
 
             # Determine which jobs to schedule now
-            context.schedule_now = []
-            for build in list(build_schedule_order):
-                await self._process_build_for_scheduling(build, context)
+            ctx.schedule_now = []
+            for build in list(ctx.build_schedule_order):
+                await self._process_build_for_scheduling(build, ctx)
 
-            if not context.schedule_now:
-                scheduler_log.addStdout("\tNo builds to schedule found.\n")
+            if not ctx.schedule_now:
+                ctx.scheduler_log.addStdout("\tNo builds to schedule found.\n")
 
             # Schedule ready jobs
-            for job in context.schedule_now:
-                scheduler_log.addStdout(f"\t- {job.attr}\n")
+            for job in ctx.schedule_now:
+                ctx.scheduler_log.addStdout(f"\t- {job.attr}\n")
                 brids, results_deferred = await self.schedule(
-                    ss_for_trigger,
+                    ctx.ss_for_trigger,
                     *self.schedule_success(build_props, job),
                 )
-                scheduled.append(
+                ctx.scheduled.append(
                     BuildTrigger.ScheduledJob(job, brids, results_deferred)
                 )
                 self.brids.extend(brids.values())
 
-            scheduler_log.addStdout("Waiting...\n")
+            ctx.scheduler_log.addStdout("Waiting...\n")
 
-            # Wait for a job to complete
             self.wait_for_finish_deferred = defer.DeferredList(
-                [job.results for job in scheduled],
+                [job.results for job in ctx.scheduled],
                 fireOnOneCallback=True,
                 fireOnOneErrback=True,
             )
@@ -570,12 +572,12 @@ async def run(self) -> int:
             results, index = await self.wait_for_finish_deferred  # type: ignore[assignment]
 
             # Process completed job
-            job, brids, _ = scheduled[index]
+            job, brids, _ = ctx.scheduled[index]
             done.append(BuildTrigger.DoneJob(job, brids, results))
-            del scheduled[index]
+            del ctx.scheduled[index]
             result = results[0]
 
-            scheduler_log.addStdout(
+            ctx.scheduler_log.addStdout(
                 f"Found finished build {job.attr}, result {util.Results[result].upper()}\n"
             )
 
@@ -588,10 +590,10 @@ async def run(self) -> int:
                 )
 
             # Handle failed jobs and their dependents
-            await self._handle_failed_job(job, context, brids, result)
+            await self._handle_failed_job(job, ctx, brids, result)
 
             overall_result = worst_status(result, overall_result)
-            scheduler_log.addStdout(
+            ctx.scheduler_log.addStdout(
                 f"\t- new result: {util.Results[overall_result].upper()} \n"
             )
 
@@ -605,7 +607,7 @@ async def run(self) -> int:
                 self.build,
                 overall_result,
             )
-        scheduler_log.addStdout("Done!\n")
+        ctx.scheduler_log.addStdout("Done!\n")
         return overall_result
 
     def getCurrentSummary(self) -> dict[str, str]:  # noqa: N802

buildbot_nix/buildbot_nix/models.py

@@ -300,6 +300,7 @@ class BuildbotNixConfig(BaseModel):
     nix_workers_secret_file: Path | None = None
     effects_per_repo_secrets: dict[str, str] = {}
     show_trace_on_failure: bool = False
+    cache_failed_builds: bool = False
 
     def nix_worker_secrets(self) -> WorkerConfig:
         if self.nix_workers_secret_file is None:

nixosModules/master.nix

@@ -569,6 +569,13 @@ in
         description = "Show stack traces on failed evaluations";
       };
 
+      cacheFailedBuilds = lib.mkEnableOption ''
+        cache failed builds in local database to avoid retrying them.
+        When enabled, failed builds will be remembered and skipped in subsequent evaluations
+        unless explicitly rebuilt. When disabled (the default), all builds will be attempted
+        regardless of previous failures
+      '';
+
       outputsPath = lib.mkOption {
         type = lib.types.nullOr lib.types.path;
         description = "Path where we store the latest build store paths names for nix attributes as text files. This path will be exposed via nginx at \${domain}/nix-outputs";
@@ -742,15 +749,17 @@ in
                     null
                   else
                     {
-                      user_allowlist = cfg.gitea.userAllowlist;
-                      repo_allowlist = cfg.gitea.repoAllowlist;
+                      filters = {
+                        user_allowlist = cfg.gitea.userAllowlist;
+                        repo_allowlist = cfg.gitea.repoAllowlist;
+                        topic = cfg.gitea.topic;
+                      };
                       token_file = "gitea-token";
                       webhook_secret_file = "gitea-webhook-secret";
                       project_cache_file = "gitea-project-cache.json";
                       oauth_secret_file = "gitea-oauth-secret";
                       instance_url = cfg.gitea.instanceUrl;
                       oauth_id = cfg.gitea.oauthId;
-                      topic = cfg.gitea.topic;
                       ssh_private_key_file = cfg.gitea.sshPrivateKeyFile;
                       ssh_known_hosts_file = cfg.gitea.sshKnownHostsFile;
                     };
@@ -759,8 +768,11 @@ in
                     null
                   else
                     {
-                      user_allowlist = cfg.github.userAllowlist;
-                      repo_allowlist = cfg.github.repoAllowlist;
+                      filters = {
+                        user_allowlist = cfg.github.userAllowlist;
+                        repo_allowlist = cfg.github.repoAllowlist;
+                        topic = cfg.github.topic;
+                      };
                       auth_type =
                         if (cfg.github.authType ? "legacy") then
                           { token_file = "github-token"; }
@@ -778,7 +790,6 @@ in
                       webhook_secret_file = "github-webhook-secret";
                       oauth_secret_file = "github-oauth-secret";
                       oauth_id = cfg.github.oauthId;
-                      topic = cfg.github.topic;
                     };
                 pull_based =
                   if cfg.pullBased.repositories == [ ] then
@@ -815,6 +826,7 @@ in
                 branches = cfg.branches;
                 nix_workers_secret_file = "buildbot-nix-workers";
                 show_trace_on_failure = cfg.showTrace;
+                cache_failed_builds = cfg.cacheFailedBuilds;
               }
             }").read_text()))
           )