Cheery-picking to update to latest

ubuntu/build.py

@@ -30,9 +30,10 @@
 from build_kernel import build_kernel, reorganize_kernel_debs
 from build_dtb import build_dtb
 from build_deb import PackageBuilder, PackageNotFoundError, PackageBuildError
+from release_debian_changelog_update import process_debian_trees
 from constants import *
 from datetime import date
-from helpers import create_new_directory, umount_dir, check_if_root, check_and_append_line_in_file, cleanup_file, cleanup_directory, change_folder_perm_read_write, print_build_logs, start_local_apt_server, build_deb_package_gz, pull_debs_wget
+from helpers import create_new_directory, umount_dir, check_if_root, check_and_append_line_in_file, cleanup_file, cleanup_directory, change_folder_perm_read_write, print_build_logs, start_local_apt_server, build_deb_package_gz, pull_debs_wget, extract_vmlinux
 from deb_organize import generate_manifest_map
 from pack_deb import PackagePacker
 from flat_meta import create_flat_meta
@@ -94,6 +95,9 @@ def parse_arguments():
                         help='Generate Debian binary (default: False)')
     parser.add_argument('--pack-image', action='store_true', default=False,
                         help='Pack system.img with generated debians (default: False)')
+    parser.add_argument('--release-prep-url', type=str, required=False,
+                        help='prepares workspace for release (default: False)',
+                        )
     parser.add_argument('--pack-variant', type=str, choices=['base', 'qcom'], default='qcom',
                         help='Pack variant (only base or qcom, default: qcom)')
     parser.add_argument('--packages-manifest-path', type=str, required=False,
@@ -177,6 +181,7 @@ def parse_arguments():
 IF_BUILD_KERNEL = args.build_kernel
 IF_GEN_DEBIANS = args.gen_debians
 IF_PACK_IMAGE = args.pack_image
+IF_RELEASE_PREP_URL = args.release_prep_url
 IF_FLAT_META = args.flat_meta
 IS_CLEANUP_ENABLED = not args.nocleanup
 IS_PREPARE_SOURCE = args.prepare_sources
@@ -235,6 +240,8 @@ def parse_arguments():
         reorganize_kernel_debs(WORKSPACE_DIR, KERNEL_DEB_OUT_DIR)
 
         build_dtb(KERNEL_DEB_OUT_DIR, LINUX_MODULES_DEB, COMBINED_DTB_FILE, OUT_DIR)
+        logger.info("Building vmlinux as requested")
+        extract_vmlinux(DEB_OUT_DIR, LINUX_IMAGE_DBGSYM_DEB, VMLINUX_QCOM_FILE, OUT_DIR)
 
     except Exception as e:
         logger.critical(f"Exception during kernel build : {e}")
@@ -246,6 +253,26 @@ def parse_arguments():
             logger.critical("Kernel build failed. Exiting.")
             exit(1)
 
+if IF_RELEASE_PREP_URL:
+    logger.info("Running the release preparation phase")
+    try:
+        statuses = process_debian_trees(
+            input_root=SOURCES_DIR,
+            apt_source_line=IF_RELEASE_PREP_URL,
+            prefer_debian_changelog=False,
+            dry_run=False,
+        )
+    except Exception as e:
+        logger.error(f"[FATAL] {e}")
+        #return 1
+
+    logger.info("\nSUMMARY")
+    for debian_dir, st in statuses.items():
+        logger.info("-" * 80)
+        logger.info(f"debian dir: {debian_dir}")
+        logger.info(f"action: {st.get('action')}")
+        logger.info(f"details: {st.get('details')}")
+
 if IF_GEN_DEBIANS or IS_PREPARE_SOURCE :
     error_during_packages_build = False
 
@@ -346,19 +373,22 @@ def parse_arguments():
             cleanup_directory(MOUNT_DIR)
 
         create_new_directory(MOUNT_DIR)
-
+        packer = PackagePacker(MOUNT_DIR, IMAGE_TYPE, PACK_VARIANT, OUT_DIR, OUT_SYSTEM_IMG, APT_SERVER_CONFIG, DEB_OUT_TEMP_DIR, DEB_OUT_DIR, DEBIAN_INSTALL_DIR, IS_CLEANUP_ENABLED, PACKAGES_MANIFEST_PATH,QC_FOLDER)
         files_check = glob.glob(os.path.join(KERNEL_DEB_OUT_DIR, LINUX_MODULES_DEB))
         if len(files_check) == 0:
             logger.warning(f"No files matching {LINUX_MODULES_DEB} exist in {KERNEL_DEB_OUT_DIR}. Pulling it from pkg.qualcomm.com")
-            cur_file = os.path.dirname(os.path.realpath(__file__))
-            manifest_file_path = os.path.join(cur_file, "packages", "base", f"{IMAGE_TYPE}.manifest")
+
+            #Get the merged manifest path
+            manifest_file_path = packer.get_merged_manifest()
+
             pull_debs_wget(manifest_file_path, KERNEL_DEB_OUT_DIR,KERNEL_DEBS,KERNEL_DEB_URL)
         else:
             logger.info("Linux modules found locally. Skipping pull from pkg.qualcomm.com")
 
         build_dtb(KERNEL_DEB_OUT_DIR, LINUX_MODULES_DEB, COMBINED_DTB_FILE, OUT_DIR)
-
-        packer = PackagePacker(MOUNT_DIR, IMAGE_TYPE, PACK_VARIANT, OUT_DIR, OUT_SYSTEM_IMG, APT_SERVER_CONFIG, DEB_OUT_TEMP_DIR, DEB_OUT_DIR, DEBIAN_INSTALL_DIR, IS_CLEANUP_ENABLED, PACKAGES_MANIFEST_PATH,QC_FOLDER)
+        if not IF_BUILD_KERNEL: #this is needed when user runs both build kernel and pack image extravtion dhouldnt run twice
+            logger.info("Building vmlinux as requested")
+            extract_vmlinux(DEB_OUT_DIR, LINUX_IMAGE_DBGSYM_DEB, VMLINUX_QCOM_FILE, OUT_DIR)
 
         packer.build_image()
 
@@ -404,4 +434,4 @@ def parse_arguments():
             exit(1)
 
 logger.info("Script execution sucessful")
-exit(0)
+exit(0)

ubuntu/build_deb.py

@@ -62,17 +62,15 @@ def __init__(self, CHROOT_NAME, CHROOT_DIR, SOURCE_DIR, APT_SERVER_CONFIG, \
         self.ARCH = ARCH
         self.CHROOT_SUFFIX = CHROOT_SUFFIX
         self.SOURCE_DIR = SOURCE_DIR
-        self.DEB_OUT_DIR = DEB_OUT_DIR
         self.APT_SERVER_CONFIG = APT_SERVER_CONFIG
-        self.CHROOT_NAME = CHROOT_NAME
         self.MANIFEST_MAP = MANIFEST_MAP
         self.DEB_OUT_TEMP_DIR = DEB_OUT_TEMP_DIR
         self.IS_CLEANUP_ENABLED = IS_CLEANUP_ENABLED
         self.DEB_OUT_DIR = DEB_OUT_DIR
         self.DEB_OUT_DIR_APT = DEB_OUT_DIR_APT
         self.DEBIAN_INSTALL_DIR_APT = DEBIAN_INSTALL_DIR_APT
         self.IS_PREPARE_SOURCE = IS_PREPARE_SOURCE
-        self.DEBIAN_MIRROR  = "http://ports.ubuntu.com"
+        self.DEBIAN_MIRROR = f"http://ports-ubuntu.qualcomm.com/ports.ubuntu.com/{SNAP_SHOT_DATE}"
         self.packages = {}
 
         self.generate_schroot_config()
@@ -102,16 +100,18 @@ def generate_schroot_config(self):
         # this command creates a chroot environment that will be named "{DIST}-{ARCH}-{SUFFIX}"
         # We supply our own suffix, otherwise sbuild will use 'sbuild'
         cmd = f"sbuild-createchroot --arch={self.ARCH}" \
+                                 f" --keyring="" " \
                                  f" --chroot-suffix=-{self.CHROOT_SUFFIX}" \
                                  f" --components=main,universe" \
                                  f" {self.DIST}" \
                                  f" {self.CHROOT_DIR}/{self.CHROOT_NAME}" \
                                  f" {self.DEBIAN_MIRROR}"
-
+        
         logger.debug(f"Creating schroot environment with command: {cmd}")
-
         result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
 
+        subprocess.run(["chroot", f"{self.CHROOT_DIR}/{self.CHROOT_NAME}", "bash", "-c", f"sed -i 's|{self.DEBIAN_MIRROR}|[trusted=yes] {self.DEBIAN_MIRROR}|' /etc/apt/sources.list"]) 
+
         if result.returncode != 0:
             raise Exception(f"Error creating schroot environment: {result.stderr}")
         else:
@@ -337,6 +337,18 @@ def reorganize_outputs_in_oss_prop(self, repo_source_path, repo_build_tmp_dir):
             else:
                 logger.debug(f"No .dsc file found for {package_name}")
 
+    def reorganize_dsc_in_oss_prop(self, repo_path):
+        oss_or_prop = search_manifest_map_for_path(self.MANIFEST_MAP, self.SOURCE_DIR, repo_path)
+        parent_dir = repo_path.parent
+
+        for file in os.listdir(parent_dir):
+            file_path = parent_dir / file
+            if file_path.is_file() and file.endswith('.dsc'):
+                pkg_name = file.split('_')[0]
+                pkg_dir = os.path.join(self.DEB_OUT_DIR, oss_or_prop, pkg_name)
+                create_new_directory(pkg_dir, delete_if_exists=False)
+                shutil.move(str(file_path), os.path.join(pkg_dir, file))
+
     def build_package(self, package):
         """
         Builds a package inside the chroot environment.
@@ -394,7 +406,8 @@ def build_package(self, package):
             print_build_logs(package_temp_dir)
             raise PackageBuildError(f"Failed to build {packages}: {e}")
 
-        self.reorganize_outputs_in_oss_prop(repo_path, package_temp_dir)
+        self.reorganize_dsc_in_oss_prop(repo_path)
+        self.reorganize_deb_in_oss_prop(repo_path)
 
         logger.info(f"{packages} built successfully!")
 

ubuntu/constants.py

@@ -4,7 +4,13 @@
 
 import os
 
+LINUX_IMAGE_DBGSYM_DEB = "oss/linux-qcom-tools*/linux-qcom-tools*_arm64.deb"
 LINUX_MODULES_DEB = "linux-modules-*-qcom/linux-modules-*_arm64.deb"
+<<<<<<< HEAD
+SNAP_SHOT_DATE = "2025-09-23"  #update date for snapshot date from https://ports-ubuntu.qualcomm.com/ports.ubuntu.com/
+=======
+SNAP_SHOT_DATE = "2025-09-12"  #update date for snapshot date from https://ports-ubuntu.qualcomm.com/ports.ubuntu.com/
+>>>>>>> 965bfda (Debian package:use a Qualcomm-specific Ubuntu mirror with a snapshot date.)
 
 KERNEL_DEBS = [
     "linux-modules",
@@ -20,6 +26,7 @@
 ]
 
 COMBINED_DTB_FILE  = "combined-dtb.dtb"
+VMLINUX_QCOM_FILE  = "vmlinux"
 IMAGE_NAME         = "system.img"
 
 IMAGE_SIZE_IN_G     = 8

ubuntu/helpers.py

@@ -23,6 +23,7 @@
 from apt_server import AptServer
 from constants import TERMINAL, HOST_FS_MOUNT
 from color_logger import logger
+import tempfile
 
 def check_if_root() -> bool:
     """
@@ -66,6 +67,72 @@ def check_and_append_line_in_file(file_path, line_to_check, append_if_missing=Fa
 
     return False
 
+
+def extract_vmlinux(deb_dir, deb_file_regex, vmlinux_filename, out_dir):
+    """
+    Extracts the vmlinux file from a Debian package and places it in the output directory.
+
+    Args:
+    -----
+    - deb_dir (str): Directory containing the .deb files.
+    - deb_file_regex (str): Regex pattern to match the .deb file.
+    - vmlinux_filename (str): Name of the vmlinux file to extract.
+    - out_dir (str): Directory to copy the extracted vmlinux file to.
+
+    Raises:
+    -------
+    - SystemExit: If not run as root, if no matching .deb files found, or if errors occur.
+    """
+    if not check_if_root():
+        logger.error('Please run this script as root user.')
+        raise Exception('Root privileges required')
+
+    vmlinux_path = os.path.join(out_dir, vmlinux_filename)
+    if os.path.exists(vmlinux_path):
+        logger.info(f"Removing existing vmlinux at {vmlinux_path}")
+        os.remove(vmlinux_path)
+
+    # Step 0: Check if the .deb file exists
+    files = glob.glob(os.path.join(deb_dir, deb_file_regex))
+    if len(files) == 0:
+        logger.error(f"Error: No files matching {deb_file_regex} exist in {deb_dir}")
+        raise Exception(f"No files matching {deb_file_regex} found")
+
+    # Step 1: Extract the .deb package to a temporary directory
+    deb_file = files[0]  # Assuming only one file matches the regex
+    try:
+        temp_dir = tempfile.mkdtemp()
+        logger.debug(f'Temp path for vmlinux extraction: {temp_dir}')
+        subprocess.run(["dpkg-deb", '-x', deb_file, temp_dir], check=True)
+
+        # Step 2: Find the vmlinux file within the temporary directory
+        file_path = None
+        for root, _, files in os.walk(temp_dir):
+            if vmlinux_filename in files:
+                file_path = os.path.join(root, vmlinux_filename)
+                break
+
+        # Step 3: Copy the vmlinux file to the output directory
+        if file_path:
+            try:
+                shutil.copy(file_path, vmlinux_path)
+                os.chmod(vmlinux_path, 0o644)
+                logger.info(f"{vmlinux_filename} has been copied to {vmlinux_path}")
+            except Exception as e:
+                logger.error(f"Error copying file {file_path}")
+                logger.error(f"Resulted in error: {e}")
+        else:
+            logger.error(f"{vmlinux_filename} not found in {deb_file}")
+
+    except Exception as e:
+        logger.error(f"Error extracting vmlinux: {e}")
+        raise
+    finally:
+        # Step 4: Clean up the temporary directory
+        if temp_dir:
+            shutil.rmtree(temp_dir)
+            logger.info(f"Cleaned up temporary directory {temp_dir}")
+
 def parse_debs_manifest(manifest_path):
     """
     Parses a manifest file and returns a dictionary of module names and their corresponding versions.
@@ -431,7 +498,8 @@ def build_deb_package_gz(dir, start_server=True) -> str:
 
             raise Exception(result.stderr)
 
-        # Even with a successful exit code, dpkg-scanpackages still outputs the number of entries written to stderr        logger.debug(result.stderr.strip())
+        # Even with a successful exit code, dpkg-scanpackages still outputs the number of entries written to stderr
+        logger.debug(result.stderr.strip())
 
 
         cmd = f"gzip -k -f {packages_path}"
@@ -454,7 +522,6 @@ def build_deb_package_gz(dir, start_server=True) -> str:
         return start_local_apt_server(dir)
     return None
 
-
 def pull_debs_wget(manifest_file_path, out_dir,DEBS_to_download_list,base_url):
     """
     Downloads Debian packages from a remote repository using wget.
@@ -492,6 +559,18 @@ def pull_debs_wget(manifest_file_path, out_dir,DEBS_to_download_list,base_url):
 
     # Generate wget links and download
     os.makedirs(out_dir, exist_ok=True)
+    #logger.info(f"version map {version_map}...")
+
+    matches = {k: v for k, v in version_map.items() if 'linux-modules' in k}
+
+    # Get the first match (you can change this logic if needed)
+    first_match_value = next(iter(matches.values()))
+    first_match_key = next(iter(matches))
+    name_suffix = first_match_value.rsplit('.', 1)[0]
+
+    # Construct new key and update version_map
+    linux_qcom_tools_suffix = 'linux-qcom-tools-' + name_suffix
+    version_map[linux_qcom_tools_suffix] = version_map[first_match_key]
     for module in DEBS_to_download_list:
         for name, version in version_map.items():
             if name.startswith(module):
@@ -509,3 +588,5 @@ def pull_debs_wget(manifest_file_path, out_dir,DEBS_to_download_list,base_url):
                 except subprocess.CalledProcessError as e:
                     logger.error(f"error: Failed to download {url}: {e}")
                 break  # Stop after first match
+
+

ubuntu/pack_deb.py

@@ -140,8 +140,6 @@ def merge_manifests_from_folder(self, folder_path, image_type, subdir_filter=Non
             logger.error(f"Failed to create or write to merged manifest file: {e}")
             return None
 
-
-
     def parse_manifests(self):
         """
         Parses the base and QCOM manifests to gather the list of packages to include in the image.
@@ -242,7 +240,17 @@ def build_image(self):
                 if config.strip():
                     bash_command += f" \"{config.strip()}\""
 
+<<<<<<< HEAD
+<<<<<<< HEAD
         bash_command += f" \"deb [arch=arm64 trusted=yes] http://ports.ubuntu.com/ubuntu-ports noble main universe multiverse restricted\""
+=======
+        bash_command += f" \"deb [arch=arm64 trusted=yes] http://ports-ubuntu.qualcomm.com/ports.ubuntu.com/{SNAP_SHOT_DATE} noble main universe multiverse restricted\""
+        bash_command += f" \"deb [arch=arm64 trusted=yes] http://ports-ubuntu.qualcomm.com/ports.ubuntu.com/{SNAP_SHOT_DATE} noble-updates main universe multiverse restricted\""
+>>>>>>> 965bfda (Debian package:use a Qualcomm-specific Ubuntu mirror with a snapshot date.)
+=======
+        bash_command += f" \"deb [arch=arm64 trusted=yes] http://ports.ubuntu.com noble main universe multiverse restricted\""
+        bash_command += f" \"deb [arch=arm64 trusted=yes] https://ports-ubuntu.qualcomm.com/x04_initial_release noble main\""
+>>>>>>> bfec10e (update to x04_initial_release )
 
         out = run_command_for_result(bash_command)
         if out['returncode'] != 0:
@@ -296,3 +304,40 @@ def extract_manifest(self, flavor):
         else:
             logger.info(f"Manifest for {flavor} saved to {manifest_path}")
 
+
+    def get_merged_manifest(self):
+        if self.PACKAGES_MANIFEST_PATH:
+            logger.info(f"User provided manifest path: {self.PACKAGES_MANIFEST_PATH}")
+            return self.PACKAGES_MANIFEST_PATH
+
+        manifest_paths = []
+
+        # Always merge from qc_folder and packages
+        search_dirs = [
+            (self.qc_folder, "base"),
+            (os.path.join(self.cur_file, "packages"), "base")
+        ]
+
+        if self.VARIANT == "qcom":
+            search_dirs += [
+                (self.qc_folder, "qcom"),
+                (os.path.join(self.cur_file, "packages"), "qcom")
+            ]
+
+        for folder, subdir in search_dirs:
+            if folder and os.path.isdir(folder):
+                for root, _, files in os.walk(folder):
+                    # Only include manifests from subdirectories matching subdir
+                    if subdir in os.path.relpath(root, folder).split(os.sep):
+                        for file in files:
+                            if file == f"{self.IMAGE_TYPE}.manifest":
+                                manifest_paths.append(os.path.join(root, file))
+
+        # Merge all found manifest files into one
+        merged_manifest_path = os.path.join(self.TEMP_DIR, f"{self.IMAGE_TYPE}_merged.manifest")
+        with open(merged_manifest_path, 'w') as merged_file:
+            for manifest in manifest_paths:
+                with open(manifest, 'r') as f:
+                    merged_file.write(f.read())
+        logger.info(f"Final merged manifest saved to: {merged_manifest_path}")
+        return merged_manifest_path