Merge pull request #2641 from locriandev/okd

ART-14594: Apply OKD config to late-resolved parent images

Author: openshift-merge-bot[bot]

doozer/doozerlib/cli/images_konflux.py

@@ -220,14 +220,32 @@ def __init__(
     @start_as_current_span_async(TRACER, "images:konflux:build")
     async def run(self):
         runtime = self.runtime
-        runtime.initialize(mode='images', clone_distgits=False)
 
+        # IMPORTANT: For OKD builds, set up OKD configuration BEFORE loading images
+        # This ensures that when images (including parent images) are loaded during initialization,
+        # they use the correct OKD-specific branch (e.g., rhel-10 instead of rhel-9 for OKD 5.0)
         if self.variant is BuildVariant.OKD:
-            group_config = self.runtime.group_config.copy()
+            # Step 1: Initialize only group and assembly configs (config_only=True)
+            # This loads group_config but doesn't load images yet
+            runtime.initialize(mode='images', clone_distgits=False, config_only=True)
+
+            # Step 2: Merge OKD configuration into group config BEFORE loading images
+            group_config = runtime.group_config.copy()
             if group_config.get('okd'):
                 LOGGER.info('Build images using OKD group configuration')
                 group_config = deep_merge(group_config, group_config['okd'])
-                runtime.group_config = Model(group_config)
+                runtime._group_config = Model(group_config)
+                # Update runtime.branch to use the OKD branch override
+                # This ensures parent images use the correct rhel-10 branch instead of rhel-9
+                if group_config.get('branch'):
+                    runtime.branch = group_config['branch']
+                    LOGGER.info(f'Updated runtime branch to OKD variant: {runtime.branch}')
+
+            # Step 3: Now do full initialization with OKD config already applied
+            runtime.initialize(mode='images', clone_distgits=False)
+        else:
+            # For non-OKD builds, initialize normally
+            runtime.initialize(mode='images', clone_distgits=False)
 
         runtime.konflux_db.bind(KonfluxBuildRecord)
         assert runtime.source_resolver is not None, "source_resolver is not initialized. Doozer bug?"

doozer/doozerlib/cli/images_okd.py

@@ -94,13 +94,55 @@ def __init__(
         self.state = {}
 
     async def run(self):
-        self.runtime.initialize(mode='images', clone_distgits=False, build_system='konflux')
+        # IMPORTANT: Set up OKD configuration BEFORE loading images
+        # This ensures that when images (including parent images) are loaded during initialization,
+        # they use the correct OKD-specific branch (e.g., rhel-10 instead of rhel-9 for OKD 5.0)
 
+        # Step 1: Initialize only group and assembly configs (config_only=True)
+        # This loads group_config but doesn't load images yet
+        self.runtime.initialize(mode='images', clone_distgits=False, build_system='konflux', config_only=True)
+
+        # Step 2: Merge OKD configuration into group config BEFORE loading images
         group_config = self.runtime.group_config.copy()
         if group_config.get('okd', None):
             self.logger.info('Using OKD group configuration')
             group_config = deep_merge(group_config, group_config['okd'])
-            self.runtime.group_config = Model(group_config)
+            # Set the merged config on runtime BEFORE loading images
+            self.runtime._group_config = Model(group_config)
+            # Update runtime.branch to use the OKD branch override
+            # This ensures parent images use the correct rhel-10 branch instead of rhel-9
+            if group_config.get('branch'):
+                self.runtime.branch = group_config['branch']
+                self.logger.info(f'Updated runtime branch to OKD variant: {self.runtime.branch}')
+
+        # Step 3: Now do full initialization with OKD config already applied
+        # Initialize with disabled=True to load all images (including those with mode: disabled)
+        # This is necessary because some parent images may be disabled for OCP but enabled for OKD
+        self.runtime.initialize(mode='images', clone_distgits=False, build_system='konflux', disabled=True)
+
+        # Step 4: Apply OKD config to ALL loaded images
+        # This ensures that parent images also get the OKD branch overrides
+        for image_meta in self.runtime.image_metas():
+            if image_meta.config.okd is not Missing:
+                # Apply the OKD configuration to this image
+                image_meta.config = self.get_okd_image_config(image_meta)
+                self.logger.debug(f'Applied OKD config to {image_meta.distgit_key}')
+
+        # Step 5: Wrap late_resolve_image to apply OKD config to any images loaded during rebase
+        # This handles parent images that are loaded on-demand when not explicitly included via --images
+        original_late_resolve_image = self.runtime.late_resolve_image
+
+        def okd_late_resolve_image(distgit_name, add=False, required=True):
+            # Call the original method to load the image
+            meta = original_late_resolve_image(distgit_name, add=add, required=required)
+            # Apply OKD config if this image has one
+            if meta and meta.config.okd is not Missing:
+                meta.config = self.get_okd_image_config(meta)
+                self.logger.debug(f'Applied OKD config to late-resolved image {meta.distgit_key}')
+            return meta
+
+        # Replace the method on the runtime instance
+        self.runtime.late_resolve_image = okd_late_resolve_image
 
         # For OKD, we need to use the OKD group variant (e.g., okd-4.20 instead of openshift-4.20)
         # This ensures the Konflux DB cache is loaded for the correct group
@@ -119,6 +161,7 @@ async def run(self):
             image_repo=self.image_repo,
         )
 
+        # Rebase all loaded images (filtered by --images flag if specified)
         metas = self.runtime.ordered_image_metas()
         tasks = [self.rebase_image(image_meta, rebaser) for image_meta in metas]
         await asyncio.gather(*tasks, return_exceptions=False)

doozer/doozerlib/cli/scan_sources_konflux.py

@@ -1715,6 +1715,22 @@ async def config_scan_source_changes_konflux(runtime: Runtime, ci_kubeconfig, as
 
     # Initialize group config: we need this to determine the canonical builders behavior
     runtime.initialize(config_only=True)
+
+    # IMPORTANT: For OKD scans, merge OKD configuration BEFORE loading images
+    # This ensures that when images (including parent images) are loaded during initialization,
+    # they use the correct OKD-specific branch (e.g., rhel-10 instead of rhel-9 for OKD 5.0)
+    if variant == BuildVariant.OKD.value:
+        group_config = runtime.group_config.copy()
+        if group_config.get('okd'):
+            runtime._logger.info('Scanning sources using OKD group configuration')
+            group_config = deep_merge(group_config, group_config['okd'])
+            runtime._group_config = Model(group_config)
+            # Update runtime.branch to use the OKD branch override
+            # This ensures parent images use the correct rhel-10 branch instead of rhel-9
+            if group_config.get('branch'):
+                runtime.branch = group_config['branch']
+                runtime._logger.info(f'Updated runtime branch to OKD variant: {runtime.branch}')
+
     runtime.initialize(mode='both' if variant != BuildVariant.OKD.value else 'images', clone_distgits=False)
 
     async with aiohttp.ClientSession() as session:

pyartcd/pyartcd/pipelines/okd.py

@@ -126,8 +126,11 @@ async def init_build_plan(self):
             load_okd_only=True,
         )
 
-        # Filter to only include payload images
-        self.group_images = await self._filter_payload_images(all_group_images)
+        # For OKD, do NOT filter to payload-only images
+        # We need to include base/parent images (like openshift-enterprise-base-rhel9)
+        # even though they don't have for_payload=true, because they're needed to build payload images.
+        # Doozer's images:okd rebase/build commands will process whatever images are loaded.
+        self.group_images = all_group_images
         self.build_plan.active_image_count = len(self.group_images)
 
         # Set the image list based on the build strategies
@@ -136,46 +139,6 @@ async def init_build_plan(self):
         # Log the initial build plan
         self.logger.info('Initial build plan:\n%s', self.build_plan)
 
-    async def _filter_payload_images(self, image_names: list) -> list:
-        """
-        Filter image list to only include images marked for_payload.
-        Non-payload images will be excluded from rebase, build, and imagestream operations.
-
-        Arg(s):
-            image_names (list): List of image names to filter
-        Return Value(s):
-            list: Filtered list containing only payload images
-        """
-        ocp_build_data_path = Path(self.runtime.doozer_working) / 'ocp-build-data' / 'images'
-        payload_images = []
-        non_payload_images = []
-
-        for image_name in image_names:
-            yaml_file = ocp_build_data_path / f'{image_name}.yml'
-
-            try:
-                with open(yaml_file) as f:
-                    image_metadata = yaml.safe_load(f)
-
-                if image_metadata.get('for_payload', False):
-                    payload_images.append(image_name)
-                else:
-                    non_payload_images.append(image_name)
-
-            except Exception as e:
-                self.logger.warning('Failed to load metadata for %s: %s. Excluding from build.', image_name, e)
-                non_payload_images.append(image_name)
-
-        if non_payload_images:
-            self.logger.info(
-                'Excluding %d non-payload images from pipeline: %s',
-                len(non_payload_images),
-                ', '.join(non_payload_images),
-            )
-
-        self.logger.info('Pipeline will operate on %d payload images', len(payload_images))
-        return payload_images
-
     def check_building_images(self):
         if self.build_plan.image_build_strategy == BuildStrategy.NONE:
             jenkins.update_title('[NO IMAGES]')
@@ -187,16 +150,16 @@ def check_building_images(self):
             jenkins.update_title(f'[{self.build_plan.active_image_count}] images')
 
         elif self.build_plan.image_build_strategy == BuildStrategy.ONLY:
-            # Filter image_list to only include payload images (from self.group_images)
+            # Filter image_list to only include images that exist in the group
             self.build_plan.images_included = [img for img in self.image_list if img in self.group_images]
             self.build_plan.images_excluded = []
 
-            # Warn if any requested images are not payload images
-            non_payload_requested = [img for img in self.image_list if img not in self.group_images]
-            if non_payload_requested:
+            # Warn if any requested images don't exist in the group
+            non_existent_requested = [img for img in self.image_list if img not in self.group_images]
+            if non_existent_requested:
                 self.logger.warning(
-                    'Requested images are not payload images and will be skipped: %s',
-                    ', '.join(non_payload_requested),
+                    'Requested images do not exist in the group and will be skipped: %s',
+                    ', '.join(non_existent_requested),
                 )
 
             n_images = len(self.build_plan.images_included)
@@ -622,7 +585,11 @@ async def update_imagestreams(self):
                 continue
 
             # Determine payload tag name
-            # Note: All images in the pipeline are already filtered to be payload images
+            # Note: Only images with for_payload=true should be tagged into imagestreams
+            if not image_metadata.get('for_payload', False):
+                self.logger.debug('Skipping non-payload image %s from imagestream tagging', image_name)
+                continue
+
             payload_tag = self._get_payload_tag_name(image_name, image_metadata)
             image_pullspec = image.get('image_pullspec')
             image_tag = image.get('image_tag')
@@ -773,9 +740,9 @@ def building_images(self):
     def include_exclude_param(self):
         """
         Returns the include/exclude parameters for the Doozer command based on the image build strategy.
-        Note: self.group_images contains only payload images, and images_included/images_excluded
-        have already been filtered in check_building_images() to only contain payload images.
-        We must always pass explicit --images parameter to enforce payload-only filtering to doozer.
+        Note: self.group_images contains all enabled OKD images (including base/parent images).
+        images_included/images_excluded have already been filtered in check_building_images()
+        to only contain images that exist in the group.
         """
 
         build_strategy = self.build_plan.image_build_strategy