diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0782a39..cb3bd58 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,11 +15,11 @@ In general, contributors should develop on branches based off of `main` and pull ```bash git clone https://github.com//qcom-build-utils.git - ``` + ``` 1. Create a new branch based on `main`: - ```bash + ```bash git checkout -b main ``` diff --git a/ubuntu/apt_server.py b/ubuntu/apt_server.py index b824be5..6ef744b 100644 --- a/ubuntu/apt_server.py +++ b/ubuntu/apt_server.py @@ -9,20 +9,18 @@ The server is built using Python's built-in `http.server` and `socketserver` modules, and it can be run in a separate thread to allow for concurrent operations. ''' -import http.server +from http.server import SimpleHTTPRequestHandler import socketserver import threading import os import socket -import logging +from color_logger import logger -logger = logging.getLogger("APT-LOCAL") - -logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s || %(levelname)s || %(message)s", - datefmt="%H:%M:%S" -) +class QuietHTTPRequestHandler(SimpleHTTPRequestHandler): + def log_message(self, format, *args): + # Silently drop all the logging. This is a bit of a hack, but the HTTP server's + # thread is a bit noisy and we don't want to clutter the output. + pass class AptServer: def __init__(self, port=8000, directory="debian_packages", max_retries=10): @@ -61,17 +59,18 @@ def start(self): for attempt in range(self.max_retries): self.port = self.port + 1 try: - handler = lambda *args, **kwargs: http.server.SimpleHTTPRequestHandler(*args, directory=self.directory, **kwargs) + # Use the quiet log handler + handler = lambda *args, **kwargs: QuietHTTPRequestHandler(*args, directory=self.directory, **kwargs) httpd = socketserver.TCPServer(("", self.port), handler) - logger.info(f"Serving {self.directory} as HTTP on port {self.port}...") + logger.debug(f"Serving {self.directory} as HTTP on port {self.port}...") server_thread = threading.Thread(target=httpd.serve_forever, daemon=True) server_thread.start() return server_thread except OSError as e: - print(e) if isinstance(e, socket.error) or "Address already in use" in str(e): logger.warning(f"Port {self.port} is in use. Retrying... ({attempt + 1}/{self.max_retries})") else: - raise Exception(e) + logger.error(f"Error starting server on port {self.port}: {e}") + raise e raise RuntimeError(f"Could not start server on port {self.port} after {self.max_retries} retries.") diff --git a/ubuntu/build.py b/ubuntu/build.py old mode 100644 new mode 100755 index 81a3ebd..02b8c60 --- a/ubuntu/build.py +++ b/ubuntu/build.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. # # SPDX-License-Identifier: BSD-3-Clause-Clear @@ -23,14 +24,32 @@ import random import shutil import argparse +import traceback +import glob + from build_kernel import build_kernel, reorganize_kernel_debs from build_dtb import build_dtb -from build_deb import PackageBuilder +from build_deb import PackageBuilder, PackageNotFoundError, PackageBuildError from constants import * from datetime import date -from helpers import create_new_directory, umount_dir, check_if_root, check_and_append_line_in_file, cleanup_file, logger, cleanup_directory, change_folder_perm_read_write, print_build_logs, start_local_apt_server, build_deb_package_gz, mount_img +from helpers import create_new_directory, umount_dir, check_if_root, check_and_append_line_in_file, cleanup_file, cleanup_directory, change_folder_perm_read_write, print_build_logs, start_local_apt_server, build_deb_package_gz, pull_debs_wget from deb_organize import generate_manifest_map from pack_deb import PackagePacker +from flat_meta import create_flat_meta +from deb_abi_checker import multiple_repo_deb_abi_checker +from color_logger import logger + +# Check for root privileges +if not check_if_root(): + logger.critical('Please run this script as root user.') + #exit(1) + +DIST = "noble" +ARCH = "arm64" +CHROOT_SUFFIX = "ubuntu" +CHROOT_NAME = DIST + "-" + ARCH + "-" + CHROOT_SUFFIX +CHROOT_DIR = "/srv/chroot" + def parse_arguments(): """ @@ -46,20 +65,27 @@ def parse_arguments(): """ parser = argparse.ArgumentParser(description="Process command line arguments.") - parser.add_argument('--apt-server-config', type=str, required=False, default="deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main", + parser.add_argument('--apt-server-config', type=str, required=False, + default="deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main", help='APT Server configuration to use') parser.add_argument('--mount_dir', type=str, required=False, - help='Mount directoryfor builds (default: /build)') - parser.add_argument('--workspace', type=str, required=True, - help='Workspace directory (mandatory)') + help='Mount directory for builds (default: /build/mount)', + default="build/mount") + parser.add_argument('--workspace', type=str, required=False, + default=".", + help='Workspace directory, defaults to pwd') parser.add_argument('--build-kernel', action='store_true', default=False, help='Build kernel') parser.add_argument('--kernel-src-dir', type=str, required=False, - help='Kernel directory (default: /kernel)') + help='Kernel directory (default: /kernel)', + default="kernel") parser.add_argument('--kernel-dest-dir', type=str, required=False, help='Kernel out directory (default: /debian_packages/oss)') - parser.add_argument('--kernel-deb-in', type=str, required=False, + parser.add_argument('--kernel-deb-path', type=str, required=False, help='directory with built kernel debians (default: /debian_packages/oss)') + parser.add_argument('--kernel-deb-url', type=str, required=False, + help='directory with built kernel debians', + default="https://pkg.qualcomm.com/pool/stable/main") parser.add_argument('--flavor', type=str, choices=['server', 'desktop'], default='server', help='Image flavor (only server or desktop, default: server)') parser.add_argument('--debians-path', type=str, required=False, @@ -73,37 +99,69 @@ def parse_arguments(): parser.add_argument('--packages-manifest-path', type=str, required=False, help='Absolute path to the package manifest file') parser.add_argument('--output-image-file', type=str, required=False, - help='Path for output system.img (default: /out/system.img)') - parser.add_argument('--chroot-name', type=str, required=False, - help='chroot name to use') + help='Output file name in /out/system.img', + default="out/system.img") parser.add_argument('--package', type=str, required=False, help='Package to build') parser.add_argument("--nocleanup", action="store_true", help="Cleanup workspace after build", default=False) parser.add_argument("--prepare-sources", action="store_true", help="Prepares sources, does not build", default=False) + parser.add_argument("--no-abi-check", action="store_true", + help="Skip ABI compatibility check", default=False) + parser.add_argument("--force-abi-check", action="store_true", + help="Skip ABI compatibility check", default=False) # Deprecated + parser.add_argument('--chroot-name', type=str, required=False, + help='chroot name to use') parser.add_argument('--skip-starter-image', action='store_true', default=False, help='Build starter image (deprecated)') parser.add_argument('--input-image-file', type=str, required=False, help='Path for input system.img (deprecated)') + parser.add_argument('--flat-meta', type=str, required=False,help='Flat meta') args = parser.parse_args() + # Make workspace absolute path. If no value was passed, resolve the '.' default value to the current pwd + if not os.path.isabs(args.workspace): + args.workspace = os.path.abspath(args.workspace) + + # If not overriden with an absolute path, resolve the relative path to the workspace : /out/system.img + if not os.path.isabs(args.output_image_file): + args.output_image_file = os.path.join(args.workspace, args.output_image_file) + + # If not overriden with an absolute path, resolve the repative path to the workspace : /build/mount + if not os.path.isabs(args.mount_dir): + args.mount_dir = os.path.join(args.workspace, args.mount_dir) + + # If not overriden with an absolute path, resolve the repative path to the workspace : /kernel + if not os.path.isabs(args.kernel_src_dir): + args.kernel_src_dir = os.path.join(args.workspace, args.kernel_src_dir) + + if 'lnxbuild' in args.workspace: + logger.disable_color() + logger.info("the string 'lnxbuild' was detected in the workspace path, which indicates a CI build. Turning off the color encoding for the logging to avoid polluting the log with special characters") + # Absolute path checks for path_arg, path_value in { - '--workspace': args.workspace, '--kernel-dest-dir': args.kernel_dest_dir, - '--kernel-deb-in' : args.kernel_deb_in , + '--kernel-deb-path' : args.kernel_deb_path , '--debians-path': args.debians_path, - '--output-image-file': args.output_image_file, '--packages-manifest-path': args.packages_manifest_path, }.items(): if path_value and not os.path.isabs(path_value): - logger.error(f"Error: {path_arg} must be an absolute path.") + logger.critical(f"Error: {path_arg} must be an absolute path.") exit(1) + # Check for conflicting arguments + if args.kernel_deb_path and IF_BUILD_KERNEL: + logger.critical('Error: --kernel-deb-path and --build-kernel cannot be used together.') + exit(1) + + if args.chroot_name: + logger.warning("The argument --chroot-name is not used anymore. Take it out to silence this warning.") + return args # Parse command-line arguments @@ -114,30 +172,26 @@ def parse_arguments(): IMAGE_TYPE = args.flavor PACKAGES_MANIFEST_PATH = args.packages_manifest_path -# Generate a unique chroot name if not provided -CHROOT_NAME = args.chroot_name if args.chroot_name else f"ubuntu-{date.today()}-{random.randint(0, 10000)}" - -OUT_SYSTEM_IMG = args.output_image_file - BUILD_PACKAGE_NAME = args.package - DEBIAN_INSTALL_DIR = args.debians_path # Process Flags IF_BUILD_KERNEL = args.build_kernel IF_GEN_DEBIANS = args.gen_debians IF_PACK_IMAGE = args.pack_image +IF_FLAT_META = args.flat_meta IS_CLEANUP_ENABLED = not args.nocleanup IS_PREPARE_SOURCE = args.prepare_sources PACK_VARIANT = args.pack_variant -# Define mount directory -MOUNT_DIR = args.mount_dir if args.mount_dir else os.path.join(WORKSPACE_DIR, "build") -MOUNT_DIR = os.path.join(MOUNT_DIR, CHROOT_NAME) +TARGET_HW = args.flat_meta +NO_ABI_CHECK = args.no_abi_check +FORCE_ABI_CHECK = args.force_abi_check # Define kernel and output directories -KERNEL_DIR = args.kernel_src_dir if args.kernel_src_dir else os.path.join(WORKSPACE_DIR, "kernel") +KERNEL_DIR = args.kernel_src_dir +KERNEL_DEB_URL = args.kernel_deb_url SOURCES_DIR = os.path.join(WORKSPACE_DIR, "sources") OUT_DIR = os.path.join(WORKSPACE_DIR, "out") DEB_OUT_DIR = os.path.join(WORKSPACE_DIR, "debian_packages") @@ -146,25 +200,17 @@ def parse_arguments(): KERNEL_DEB_OUT_DIR = ( args.kernel_dest_dir if args.kernel_dest_dir - else args.kernel_deb_in if args.kernel_deb_in + else args.kernel_deb_path if args.kernel_deb_path else OSS_DEB_OUT_DIR ) PROP_DEB_OUT_DIR = os.path.join(DEB_OUT_DIR, "prop") -TEMP_DIR = os.path.join(DEB_OUT_DIR, "temp") +DEB_OUT_TEMP_DIR = os.path.join(DEB_OUT_DIR, "temp") -# Check for conflicting arguments -if args.kernel_deb_in and IF_BUILD_KERNEL: - logger.error('Error: --kernel-deb-in and --build-kernel cannot be used together.') - exit(1) - -# Check for root privileges -if not check_if_root(): - logger.error('Please run this script as root user.') - exit(1) +# Set up APT server configuration and generate manifest map +APT_SERVER_CONFIG = [config.strip() for config in args.apt_server_config.split(',')] if args.apt_server_config else None +APT_SERVER_CONFIG = list(set(APT_SERVER_CONFIG)) if APT_SERVER_CONFIG else None # Create necessary directories for the build process -create_new_directory(WORKSPACE_DIR, delete_if_exists=False) -create_new_directory(MOUNT_DIR, delete_if_exists=False) create_new_directory(KERNEL_DIR, delete_if_exists=False) create_new_directory(KERNEL_DEB_OUT_DIR, delete_if_exists=False) create_new_directory(SOURCES_DIR, delete_if_exists=False) @@ -172,10 +218,7 @@ def parse_arguments(): create_new_directory(DEB_OUT_DIR, delete_if_exists=False) create_new_directory(OSS_DEB_OUT_DIR, delete_if_exists=False) create_new_directory(PROP_DEB_OUT_DIR, delete_if_exists=False) -create_new_directory(TEMP_DIR, delete_if_exists=True) - -# Set up APT server configuration and generate manifest map -APT_SERVER_CONFIG = [config.strip() for config in args.apt_server_config.split(',')] if args.apt_server_config else None +create_new_directory(DEB_OUT_TEMP_DIR, delete_if_exists=False) # Don't clear all the temp folders try: MANIFEST_MAP = generate_manifest_map(WORKSPACE_DIR) @@ -183,28 +226,33 @@ def parse_arguments(): logger.error(f"Failed to generate manifest map: {e}") MANIFEST_MAP = {} -APT_SERVER_CONFIG = list(set(APT_SERVER_CONFIG)) if APT_SERVER_CONFIG else None - -ERROR_EXIT_BUILD = False - # Build the kernel if specified if IF_BUILD_KERNEL: + error_during_kernel_build = False + + logger.info("Running the kernel build phase") + try: os.chdir(WORKSPACE_DIR) build_kernel(KERNEL_DIR) reorganize_kernel_debs(WORKSPACE_DIR, KERNEL_DEB_OUT_DIR) build_dtb(KERNEL_DEB_OUT_DIR, LINUX_MODULES_DEB, COMBINED_DTB_FILE, OUT_DIR) + except Exception as e: - logger.error(e) - ERROR_EXIT_BUILD = True + logger.critical(f"Exception during kernel build : {e}") + traceback.print_exc() + error_during_kernel_build = True -# Exit if there was an error during kernel build -if ERROR_EXIT_BUILD: - exit(1) + finally: + if error_during_kernel_build: + logger.critical("Kernel build failed. Exiting.") + exit(1) if IF_GEN_DEBIANS or IS_PREPARE_SOURCE : - builder = None + error_during_packages_build = False + + logger.info("Running the debian packages generation phase") try: DEB_OUT_DIR_APT = None @@ -216,63 +264,149 @@ def parse_arguments(): DEBIAN_INSTALL_DIR_APT = build_deb_package_gz(DEBIAN_INSTALL_DIR, start_server=True) # Initialize the PackageBuilder to load packages - builder = PackageBuilder(MOUNT_DIR, SOURCES_DIR, APT_SERVER_CONFIG, CHROOT_NAME, MANIFEST_MAP, TEMP_DIR, DEB_OUT_DIR, DEB_OUT_DIR_APT, DEBIAN_INSTALL_DIR, DEBIAN_INSTALL_DIR_APT, IS_CLEANUP_ENABLED, IS_PREPARE_SOURCE) + builder = PackageBuilder(CHROOT_NAME, CHROOT_DIR, SOURCES_DIR, APT_SERVER_CONFIG, MANIFEST_MAP, DEB_OUT_TEMP_DIR, DEB_OUT_DIR, DEB_OUT_DIR_APT, DEBIAN_INSTALL_DIR_APT, IS_CLEANUP_ENABLED, IS_PREPARE_SOURCE) builder.load_packages() # Build a specific package if provided, otherwise build all packages if BUILD_PACKAGE_NAME: - # TODO: Check if package is available - can_build = builder.build_specific_package(BUILD_PACKAGE_NAME) - if not can_build: - raise Exception(f"Unable to build {BUILD_PACKAGE_NAME}") + logger.debug(f"Building specific package: {BUILD_PACKAGE_NAME}") + builder.build_specific_package(BUILD_PACKAGE_NAME) else: + logger.debug("Building all packages") builder.build_all_packages() except Exception as e: - logger.error(e) - print_build_logs(TEMP_DIR) - ERROR_EXIT_BUILD = True + error_during_packages_build = True + + logger.critical(f"Exception during debian package(s) generation : {e}") + + if not isinstance(e, PackageBuildError): + # Dont clog the output with the stack trace if it just a package build exception, the full build log is + # already printed in build function if build fails.= + traceback.print_exc() finally: - if ERROR_EXIT_BUILD: + if error_during_packages_build: + logger.critical("Debian package generation error. Exiting.") exit(1) -# Set output system image path if not provided -if OUT_SYSTEM_IMG is None: - OUT_SYSTEM_IMG = os.path.join(OUT_DIR, IMAGE_NAME) + +if NO_ABI_CHECK: + logger.warning("ABI check is explicitely disabled. Skipping ABI check.") +elif (not IF_GEN_DEBIANS and not IS_PREPARE_SOURCE) and not FORCE_ABI_CHECK: + logger.debug("Skipping ABI check since no debian packages generated") +else: + if FORCE_ABI_CHECK and (not IF_GEN_DEBIANS and not IS_PREPARE_SOURCE): + logger.info("Forcing ABI check even if no debian package were built") + + error_during_abi_check = False + + logger.info("Running the ABI checking phase") + + try: + if not APT_SERVER_CONFIG: + raise Exception("No apt server config provided") + + if len(APT_SERVER_CONFIG) > 1: + logger.warning("Multiple apt server configs are not supported yet, picking the first one in the list") + + logger.debug("Running the package ABI checker over the temp folder containing all the repo outputs") + check_passed = multiple_repo_deb_abi_checker(DEB_OUT_TEMP_DIR, APT_SERVER_CONFIG[0]) + + if check_passed: + logger.info("ABI check passed.") + else: + logger.critical("ABI check failed.") + + except Exception as e: + logger.critical(f"Exception during the ABI checking : {e}") + traceback.print_exc() + error_during_abi_check = True + + finally: + if error_during_abi_check: + logger.critical("ABI check failed. Exiting.") + exit(1) # Pack the image if specified if IF_PACK_IMAGE: + error_during_image_packing = False packer = None - cleanup_file(OUT_SYSTEM_IMG) - create_new_directory(MOUNT_DIR) + + logger.info("Running the image packing phase") + + # Define mount directory + MOUNT_DIR = args.mount_dir + OUT_SYSTEM_IMG = args.output_image_file + + logger.debug(f"mount dir {MOUNT_DIR}") + logger.debug(f"out system img {OUT_SYSTEM_IMG}") + try: - build_dtb(KERNEL_DEB_OUT_DIR, LINUX_MODULES_DEB, COMBINED_DTB_FILE, OUT_DIR) + if os.path.isfile(OUT_SYSTEM_IMG): + cleanup_file(OUT_SYSTEM_IMG) - packer = PackagePacker(MOUNT_DIR, IMAGE_TYPE, PACK_VARIANT, OUT_DIR, OUT_SYSTEM_IMG, APT_SERVER_CONFIG, TEMP_DIR, DEB_OUT_DIR, DEBIAN_INSTALL_DIR, IS_CLEANUP_ENABLED, PACKAGES_MANIFEST_PATH) + if os.path.exists(MOUNT_DIR): + # Make sure no leftovers from a previous run are present, especially in terms of mouted directories. + umount_dir(MOUNT_DIR, UMOUNT_HOST_FS=True) + cleanup_directory(MOUNT_DIR) + create_new_directory(MOUNT_DIR) + + files_check = glob.glob(os.path.join(KERNEL_DEB_OUT_DIR, LINUX_MODULES_DEB)) + if len(files_check) == 0: + logger.warning(f"No files matching {LINUX_MODULES_DEB} exist in {KERNEL_DEB_OUT_DIR}. Pulling it from pkg.qualcomm.com") + cur_file = os.path.dirname(os.path.realpath(__file__)) + manifest_file_path = os.path.join(cur_file, "packages", "base", f"{IMAGE_TYPE}.manifest") + pull_debs_wget(manifest_file_path, KERNEL_DEB_OUT_DIR,KERNEL_DEBS,KERNEL_DEB_URL) + else: + logger.info("Linux modules found locally. Skipping pull from pkg.qualcomm.com") + + build_dtb(KERNEL_DEB_OUT_DIR, LINUX_MODULES_DEB, COMBINED_DTB_FILE, OUT_DIR) + + packer = PackagePacker(MOUNT_DIR, IMAGE_TYPE, PACK_VARIANT, OUT_DIR, OUT_SYSTEM_IMG, APT_SERVER_CONFIG, DEB_OUT_TEMP_DIR, DEB_OUT_DIR, DEBIAN_INSTALL_DIR, IS_CLEANUP_ENABLED, PACKAGES_MANIFEST_PATH) packer.build_image() + except Exception as e: - logger.error(e) - print_build_logs(TEMP_DIR) - ERROR_EXIT_BUILD = True - umount_dir(MOUNT_DIR, UMOUNT_HOST_FS=True) + error_during_image_packing = True + + logger.critical(f"Exception during packaging : {e}") + traceback.print_exc() + + print_build_logs(DEB_OUT_TEMP_DIR) finally: + umount_dir(MOUNT_DIR, UMOUNT_HOST_FS=True) + if IS_CLEANUP_ENABLED: cleanup_directory(MOUNT_DIR) - if ERROR_EXIT_BUILD: + if error_during_image_packing: + logger.critical("Image packing failed. Exiting.") exit(1) +if IF_FLAT_META: + try: + create_flat_meta(PACK_VARIANT, IMAGE_TYPE, TARGET_HW, WORKSPACE_DIR) + except Exception as e: + logger.error(e) + ERROR_EXIT_BUILD = True + # Change permissions for output directories if cleanup is enabled if IS_CLEANUP_ENABLED: + error_during_cleanup = False + try: change_folder_perm_read_write(OSS_DEB_OUT_DIR) change_folder_perm_read_write(PROP_DEB_OUT_DIR) change_folder_perm_read_write(DEB_OUT_DIR) change_folder_perm_read_write(OUT_DIR) except Exception: - ERROR_EXIT_BUILD = True + error_during_cleanup = True + + finally: + if error_during_cleanup: + logger.critical("Cleanup failed. Exiting.") + exit(1) -if ERROR_EXIT_BUILD: - exit(1) +logger.info("Script execution sucessful") +exit(0) \ No newline at end of file diff --git a/ubuntu/build_deb.py b/ubuntu/build_deb.py index 4d6c13d..1581b11 100644 --- a/ubuntu/build_deb.py +++ b/ubuntu/build_deb.py @@ -19,55 +19,61 @@ from queue import Queue from collections import defaultdict, deque from constants import * -from helpers import check_if_root, logger, run_command, check_and_append_line_in_file, create_new_directory, build_deb_package_gz, run_command_for_result +from helpers import check_if_root, run_command, check_and_append_line_in_file, create_new_directory, build_deb_package_gz, run_command_for_result, print_build_logs from deb_organize import search_manifest_map_for_path +from color_logger import logger + +class PackageNotFoundError(Exception): + """ + Exception raised when a package is not found. + """ + pass + +class PackageBuildError(Exception): + """ + Exception raised when there is an error during package building. + """ + pass class PackageBuilder: - def __init__(self, MOUNT_DIR, SOURCE_DIR, APT_SERVER_CONFIG, CHROOT_NAME, \ - MANIFEST_MAP=None, TEMP_DIR=None, DEB_OUT_DIR=None, DEB_OUT_DIR_APT=None, DEBIAN_INSTALL_DIR=None, \ - DEBIAN_INSTALL_DIR_APT=None, IS_CLEANUP_ENABLED=True, IS_PREPARE_SOURCE=False): + def __init__(self, CHROOT_NAME, CHROOT_DIR, SOURCE_DIR, APT_SERVER_CONFIG, \ + MANIFEST_MAP=None, DEB_OUT_TEMP_DIR=None, DEB_OUT_DIR=None, DEB_OUT_DIR_APT=None, \ + DEBIAN_INSTALL_DIR_APT=None, IS_CLEANUP_ENABLED=True, IS_PREPARE_SOURCE=False, DIST= "noble", ARCH="arm64", CHROOT_SUFFIX="ubuntu"): """ Initializes the PackageBuilder instance. Args: ----- - - MOUNT_DIR (str): The directory where the chroot environment will be mounted. + - CHROOT_NAME (str): The name of the chroot environment. + - CHROOT_DIR (str): The directory where the chroot environment is found, or created if it doesnt already exist. - SOURCE_DIR (str): The source directory containing the packages to build. - APT_SERVER_CONFIG (list): Configuration for the APT server. - - CHROOT_NAME (str): The name of the chroot environment. - MANIFEST_MAP (dict, optional): A mapping of package paths to their properties. - - TEMP_DIR (str, optional): Temporary directory for building packages. + - DEB_OUT_TEMP_DIR (str, optional): Temporary directory for building packages. - DEB_OUT_DIR (str, optional): Output directory for built Debian packages. - DEB_OUT_DIR_APT (str, optional): Output directory for APT repository. - - DEBIAN_INSTALL_DIR (str, optional): Directory for Debian installation files. - DEBIAN_INSTALL_DIR_APT (str, optional): Directory for APT installation files. - IS_CLEANUP_ENABLED (bool, optional): Flag to enable cleanup of the mount directory. - IS_PREPARE_SOURCE (bool, optional): If True, prepares the source directory before building. Defaults to False. """ - if not check_if_root(): - logger.error('Please run this script as root user.') - exit(1) + self.CHROOT_NAME = CHROOT_NAME + self.CHROOT_DIR = CHROOT_DIR + self.DIST = DIST + self.ARCH = ARCH + self.CHROOT_SUFFIX = CHROOT_SUFFIX self.SOURCE_DIR = SOURCE_DIR self.DEB_OUT_DIR = DEB_OUT_DIR - self.MOUNT_DIR = Path(MOUNT_DIR) self.APT_SERVER_CONFIG = APT_SERVER_CONFIG self.CHROOT_NAME = CHROOT_NAME - - self.DIST = "noble" - - self.packages = {} - self.MANIFEST_MAP = MANIFEST_MAP - - self.TEMP_DIR = TEMP_DIR - + self.DEB_OUT_TEMP_DIR = DEB_OUT_TEMP_DIR self.IS_CLEANUP_ENABLED = IS_CLEANUP_ENABLED - self.DEB_OUT_DIR = DEB_OUT_DIR - self.DEBIAN_INSTALL_DIR = DEBIAN_INSTALL_DIR self.DEB_OUT_DIR_APT = DEB_OUT_DIR_APT self.DEBIAN_INSTALL_DIR_APT = DEBIAN_INSTALL_DIR_APT self.IS_PREPARE_SOURCE = IS_PREPARE_SOURCE + self.DEBIAN_MIRROR = "http://ports.ubuntu.com" + self.packages = {} self.generate_schroot_config() @@ -79,29 +85,51 @@ def generate_schroot_config(self): ------- - Exception: If there is an error creating the schroot environment. """ - logger.info(f"Generating schroot configuration for {self.CHROOT_NAME} at {self.MOUNT_DIR}") - if not os.path.exists(os.path.join(self.MOUNT_DIR, "root")): - out = run_command_for_result(f"sbuild-createchroot --arch=arm64 --chroot-suffix={self.CHROOT_NAME} --components=main,universe {self.DIST} {self.MOUNT_DIR} http://ports.ubuntu.com") - if out['returncode'] != 0: - if self.IS_CLEANUP_ENABLED: - cleanup_directory(self.MOUNT_DIR) - raise Exception(f"Error creating schroot environment: {out['output']}") - else: - logger.info(f"Schroot environment {self.CHROOT_NAME} created successfully.") + + logger.debug(f"Checking if chroot container '{self.CHROOT_NAME}' is already registered") + + cmd = f"schroot -l | grep chroot:{self.CHROOT_NAME}" + result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + + if result.returncode == 0: + logger.info(f"Schroot container {self.CHROOT_NAME} already exists. Skipping creation.") + return + + logger.warning(f"Schroot container '{self.CHROOT_NAME}' does not exist, creating it for the first time.") + logger.warning(f"The chroot will be created in {self.CHROOT_DIR}/{self.CHROOT_NAME}") + logger.warning(f"Its config will be stored as /etc/schroot/chroot.d/{self.CHROOT_NAME}.conf") + + # this command creates a chroot environment that will be named "{DIST}-{ARCH}-{SUFFIX}" + # We supply our own suffix, otherwise sbuild will use 'sbuild' + cmd = f"sbuild-createchroot --arch={self.ARCH}" \ + f" --chroot-suffix=-{self.CHROOT_SUFFIX}" \ + f" --components=main,universe" \ + f" {self.DIST}" \ + f" {self.CHROOT_DIR}/{self.CHROOT_NAME}" \ + f" {self.DEBIAN_MIRROR}" + + logger.debug(f"Creating schroot environment with command: {cmd}") + + result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + + if result.returncode != 0: + raise Exception(f"Error creating schroot environment: {result.stderr}") else: - logger.warning(f"Schroot environment {self.CHROOT_NAME} already exists at {self.MOUNT_DIR}. Skipping creation.") + logger.info(f"Schroot environment {self.CHROOT_NAME} created successfully.") def load_packages(self): """Load package metadata from build_config.py and fetch dependencies from control files.""" for root, dirs, files in os.walk(self.SOURCE_DIR): dirs[:] = [d for d in dirs if d != '.git'] if 'debian' in dirs: + root_name = Path(root).name debian_dir = Path(os.path.join(root, 'debian')) pkg_names, dependencies = self.get_packages_from_control(debian_dir / "control") self.packages[str(debian_dir)] = { "debian_dir": debian_dir, "repo_path": Path(root), + "repo_name": root_name, "dependencies": dependencies, "packages": pkg_names, "visited": False @@ -223,41 +251,91 @@ def detect_cycle(self): return sorted_order - def reorganize_deb_in_oss_prop(self, repo_path): + def reorganize_outputs_in_oss_prop(self, repo_source_path, repo_build_tmp_dir): """ - Reorganizes built .deb files into the appropriate output directory based on the manifest map. + Reorganizes built packages files into the appropriate output directory based on the manifest map. + Also reorganizes the 'dsc' file form the repo source path. - Args: - ----- - - repo_path (Path): The path to the repository containing the built packages. - """ - oss_or_prop = search_manifest_map_for_path(self.MANIFEST_MAP, self.SOURCE_DIR, repo_path) - for root, dirs, files in os.walk(self.TEMP_DIR): - for file in files: - if file.endswith('.deb'): - pkg_name = file.split('_')[0] - pkg_dir = os.path.join(self.DEB_OUT_DIR, oss_or_prop, pkg_name) - create_new_directory(pkg_dir, delete_if_exists=False) - shutil.move(os.path.join(root, file), os.path.join(pkg_dir, file)) - - def reorganize_dsc_in_oss_prop(self, repo_path): - """ - Reorganizes .dsc files into the appropriate output directory based on the manifest map. + A given 'repo_build_tmp_dir' is a folder containing all the built packages for a given repository. + In most cases it contains one .deb package, but it is possible that building one repo yields more than one package + For any .deb package, there is also almost always an associated -dev.deb and -dbgsym.ddeb package. + In some cases, it is possible that for a given package, only a -dev.deb file exists, no .deb. Args: ----- - - repo_path (Path): The path to the repository containing the .dsc files. + - repo_source_path (Path): The path to the repository containing the sources of the built packages. + - repo_build_tmp_dir (Path): The path to the temporary directory where the built packages are stored. """ - oss_or_prop = search_manifest_map_for_path(self.MANIFEST_MAP, self.SOURCE_DIR, repo_path) - parent_dir = repo_path.parent - for file in os.listdir(parent_dir): - file_path = parent_dir / file - if file_path.is_file() and file.endswith('.dsc'): - pkg_name = file.split('_')[0] - pkg_dir = os.path.join(self.DEB_OUT_DIR, oss_or_prop, pkg_name) - create_new_directory(pkg_dir, delete_if_exists=False) - shutil.move(str(file_path), os.path.join(pkg_dir, file)) + # Look back into the source directory manifest to determine if the package is OSS (open source) or PROP (proprietary). + oss_or_prop = search_manifest_map_for_path(self.MANIFEST_MAP, self.SOURCE_DIR, repo_source_path) + + repo_parent_path = repo_source_path.parent + + # Create a list of all the packages (.deb, -dev.deb, -dbgsym.ddeb) + files = os.listdir(repo_build_tmp_dir) + deb_files = [f for f in files if f.endswith('.deb') and "-dev" not in f] + dev_files = [f for f in files if f.endswith('.deb') and "-dev" in f] + dbg_files = [f for f in files if f.endswith('.ddeb') and "-dbgsym" in f] + + # Isolate all the canonical package names (i.e. remove the version and architecture from the filenames) + deb_pkg_names = [f.split('_')[0] for f in deb_files] + dev_pkg_names = [f.split('_')[0].removesuffix("-dev") for f in dev_files] + dbg_pkg_names = [f.split('_')[0].removesuffix("-dbgsym") for f in dbg_files] + + # Second pass to remove all the major version that often suffix the package names + # The norm is that packages that include the major in the deb name DO NOT include it in the dev + # this ensures we deal with root package name and not doubles when we combine the lists below + deb_pkg_names = [(f[:-1] if f[-1].isdigit() else f) for f in deb_pkg_names] + dev_pkg_names = [(f[:-1] if f[-1].isdigit() else f) for f in dev_pkg_names] + dbg_pkg_names = [(f[:-1] if f[-1].isdigit() else f) for f in dbg_pkg_names] + + package_names = list(set(deb_pkg_names) | set(dev_pkg_names) | set(dbg_pkg_names)) + + # Important that the list be sorted from the longest package name to the shortest + # Starting with the longest and removing it from the _files lists ensures we deal + # properly specificaly with the edge case or qcom-adreno/qcom-adreno-cl + package_names.sort(reverse=True, key=lambda x: len(x)) + + for package_name in package_names: + output_dir = os.path.join(self.DEB_OUT_DIR, oss_or_prop, package_name) + create_new_directory(output_dir, delete_if_exists=False) + + logger.debug(f"Re-organizing outputs of package: {package_name} (oss/prop: {oss_or_prop})") + + deb_package = next((file for file in deb_files if package_name in file), None) + dev_package = next((file for file in dev_files if package_name in file), None) + dbg_package = next((file for file in dbg_files if package_name in file), None) + + if deb_package is not None: + shutil.copy(os.path.join(repo_build_tmp_dir, deb_package), os.path.join(output_dir, deb_package)) + logger.info(f'Copied {deb_package} to {output_dir}') + deb_files.remove(deb_package) + else: + logger.debug(f"No .deb package found for {package_name}") + + if dev_package is not None: + shutil.copy(os.path.join(repo_build_tmp_dir, dev_package), os.path.join(output_dir, dev_package)) + logger.info(f'Copied {dev_package} to {output_dir}') + dev_files.remove(dev_package) + else: + logger.debug(f"No -dev.deb package found for {package_name}") + + if dbg_package is not None: + shutil.copy(os.path.join(repo_build_tmp_dir, dbg_package), os.path.join(output_dir, dbg_package)) + logger.info(f'Copied {dbg_package} to {output_dir}') + dbg_files.remove(dbg_package) + else: + logger.debug(f"No -dbgsym.ddeb package found for {package_name}") + + # Deal with the .dsc file + dsc_package = next((f for f in os.listdir(repo_parent_path) if f.endswith('.dsc') and package_name in f), None) + + if dsc_package is not None: + shutil.move(os.path.join(repo_parent_path, dsc_package), os.path.join(output_dir, dsc_package)) + logger.info(f'Moved {dsc_package} to {output_dir}') + else: + logger.debug(f"No .dsc file found for {package_name}") def build_package(self, package): """ @@ -265,29 +343,37 @@ def build_package(self, package): Args: ----- - - package (str): The name of the package to build. + - package (str): The name of the package to build, this is a 'debian' folder Raises: ------- - Exception: If there is an error during the build process. """ - package_info = self.packages[package] + logger.debug(f"Building debian folder: {package}") + + if not Path(package).is_dir() or Path(package).name != "debian": + raise ValueError ("'package' argument must be a debian folder: {package}") + + package_info = self.packages[package] repo_path = package_info["repo_path"] + repo_name = package_info["repo_name"] debian_dir = package_info["debian_dir"] packages = package_info['packages'] - logger.info(f"Building {packages}...") + package_temp_dir = os.path.join(self.DEB_OUT_TEMP_DIR, repo_name) + + create_new_directory(package_temp_dir, delete_if_exists=True) + + logger.debug(f"Building deb packages : {packages} listed in the Control file") os.chdir(repo_path) - create_new_directory(self.TEMP_DIR) - if self.IS_PREPARE_SOURCE: - logger.info(f"generating dsc for {packages}...") - cmd = f"sbuild --source --no-arch-all --no-arch-any -d {self.DIST}-arm64{self.CHROOT_NAME} --build-dir {self.TEMP_DIR} " + if self.IS_PREPARE_SOURCE: + logger.debug(f"generating dsc for {packages}...") + cmd = f"sbuild --source --no-arch-all --no-arch-any -d {self.CHROOT_NAME} --build-dir {package_temp_dir}" else: - cmd = f"sbuild -A --arch=arm64 -d {self.DIST}-arm64{self.CHROOT_NAME} --no-run-lintian \ - --build-dir {self.TEMP_DIR} --build-dep-resolver=apt" + cmd = f"sbuild -A --arch=arm64 -d {self.CHROOT_NAME} --no-run-lintian --build-dir {package_temp_dir} --build-dep-resolver=apt" if self.DEB_OUT_DIR_APT: build_deb_package_gz(self.DEB_OUT_DIR, start_server=False) # Rebuild Packages file @@ -301,10 +387,14 @@ def build_package(self, package): if config.strip(): cmd += f" --extra-repository=\"{config.strip()}\"" - run_command(cmd, cwd=repo_path) + try: + run_command(cmd, cwd=repo_path) + except Exception as e: + logger.error(f"Failed to build {packages}: {e}") + print_build_logs(package_temp_dir) + raise PackageBuildError(f"Failed to build {packages}: {e}") - self.reorganize_dsc_in_oss_prop(repo_path) - self.reorganize_deb_in_oss_prop(repo_path) + self.reorganize_outputs_in_oss_prop(repo_path, package_temp_dir) logger.info(f"{packages} built successfully!") @@ -329,18 +419,34 @@ def build_specific_package(self, package_name): Returns: -------- - - bool: True if the package was found and built, False otherwise. + + Raises: + ------- + - PackageNotFoundError: If the package is not found in the packages list. + - PackageBuildError: If the package fails to build (raising up from the build_package function). """ - found = False + for package in self.packages: if not self.packages[package]['visited']: if package_name in self.packages[package]['packages']: for dep in self.packages[package]['dependencies']: - self.build_specific_package(dep) self.packages[package]['visited'] = True + logger.debug(f"Building dependency: {dep}") + + try: + self.build_specific_package(dep) + except PackageNotFoundError as e: + # Its possible that a dependency is not found in the packages list, + # yet the build is successful. Catch the exception, and continue. + logger.error(f"Failed to find dependency: {e}") + except PackageBuildError as e: + # If the dependency build fails, raise the exception up. + logger.error(f"Failed to build dependency: {e}") + raise e + + # let any potential exception from build_package raise up to the caller self.build_package(package) - found = True + return - if not found: - logger.error(f"Package '{package_name}' not found.") - return False + # If we reach here, the package was not found in the packages list. + raise PackageNotFoundError(f"Package '{package_name}' not found.") \ No newline at end of file diff --git a/ubuntu/build_dtb.py b/ubuntu/build_dtb.py index 4d5653f..b97607b 100644 --- a/ubuntu/build_dtb.py +++ b/ubuntu/build_dtb.py @@ -16,7 +16,8 @@ import shlex import tempfile import subprocess -from helpers import logger, cleanup_directory, check_if_root +from helpers import cleanup_directory, check_if_root +from color_logger import logger def build_dtb(deb_dir, deb_file_regex, combined_dtb_filename, out_dir): """ @@ -55,7 +56,7 @@ def build_dtb(deb_dir, deb_file_regex, combined_dtb_filename, out_dir): deb_file = files[0] # Assuming only one file matches the regex try: temp_dir = tempfile.mkdtemp() - logger.info(f'Temp path for dtb extraction: {temp_dir}') + logger.debug(f'Temp path for dtb extraction: {temp_dir}') subprocess.run(["dpkg-deb", '-x', deb_file, temp_dir], check=True) except Exception as e: logger.error(f"Error extracting .deb file: {e}") @@ -67,7 +68,7 @@ def build_dtb(deb_dir, deb_file_regex, combined_dtb_filename, out_dir): if combined_dtb_filename in files: file_path = os.path.join(root, combined_dtb_filename) break - + # Step 3: Process the combined-dtb.dtb file if file_path: # Step 4: Use a hardcoded block size diff --git a/ubuntu/build_kernel.py b/ubuntu/build_kernel.py index ccd7330..4dacf3f 100644 --- a/ubuntu/build_kernel.py +++ b/ubuntu/build_kernel.py @@ -10,7 +10,8 @@ logging, and file management. """ -from helpers import check_if_root, logger, check_and_append_line_in_file, set_env, create_new_directory +from helpers import check_if_root, check_and_append_line_in_file, set_env, create_new_directory +from color_logger import logger import os import shutil import subprocess diff --git a/ubuntu/color_logger.py b/ubuntu/color_logger.py new file mode 100644 index 0000000..66819ee --- /dev/null +++ b/ubuntu/color_logger.py @@ -0,0 +1,72 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# +# SPDX-License-Identifier: BSD-3-Clause-Clear + +""" +color_logger.py + +This module provides a color logger class, allowing users to log messages with colored text. +The class includes methods for logging messages at different levels, the same as the standard +python 'logging' mdule : debug, info, warning, error, and critical. + +Usage: + from color_logger import logger + + logger.debug('This is a debug message') + logger.info('This is an info message') + logger.warning('This is a warning message') + logger.error('This is an error message') + logger.critical('This is a critical message') +""" + +import logging +import datetime + +class ColorLogger: + LEVEL_STRING = { + logging.DEBUG: 'DEBG', + logging.INFO: 'INFO', + logging.WARNING: 'WARN', + logging.ERROR: 'ERR ', + logging.CRITICAL: 'CRIT' + } + + LEVEL_COLORS = { + logging.DEBUG: '\033[94m', #CYAN + logging.INFO: '\033[92m', #GREEN + logging.WARNING: '\033[93m', #YELLOW + logging.ERROR: '\033[91m', #RED + logging.CRITICAL: '\033[95m' #MAGENTA + } + + def __init__(self, name: str, level=logging.DEBUG): + self.logger = logging.getLogger(name) + self.logger.setLevel(level) + self.color_enabled = True + + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(message)s')) + self.logger.addHandler(handler) + + def log(self, level, message): + reset = "\033[0m" + color = self.LEVEL_COLORS.get(level, "") + level_str = self.LEVEL_STRING.get(level, ' ') + colored_message = f"{color}{message}{reset}" + timestamp = datetime.datetime.now().strftime("%H:%M:%S") + + self.logger.log(level, f"[{timestamp}] {level_str} : {colored_message if self.color_enabled else message}") + + def debug(self, msg): self.log(logging.DEBUG, msg) + def info(self, msg): self.log(logging.INFO, msg) + def warning(self, msg): self.log(logging.WARNING, msg) + def error(self, msg): self.log(logging.ERROR, msg) + def critical(self, msg): self.log(logging.CRITICAL, msg) + + def disable_color(self): + self.color_enabled = False + + def enable_color(self): + self.color_enabled = True + +logger = ColorLogger("BUILD") diff --git a/ubuntu/deb_abi_checker.py b/ubuntu/deb_abi_checker.py new file mode 100755 index 0000000..d219c13 --- /dev/null +++ b/ubuntu/deb_abi_checker.py @@ -0,0 +1,782 @@ +#!/usr/bin/env python3 +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# +# SPDX-License-Identifier: BSD-3-Clause-Clear + +""" +deb_abi_checker.py: ABI Comparison Tool + +This script compares two Debian binary packages (.deb) to detect ABI (Application Binary Interface) changes. +It usses the abipkgdiff tool (Package-level ABI comparison) + -------------------------------------------------- + - Compares the entire old .deb and new .deb packages directly. + - Internally extracts and analyzes binary symbols and type information using libabigail. + - Reports any changes in ABI that may cause incompatibility (e.g., removed or modified symbols). + + Advantages: + - Simple interface: only requires two .deb files as input. + - Ideal for high-level package comparison. + + Limitations: + - Depends on symbol/debug info availability. + - Does not show per-library granularity. + +Usage: + + +Options: + --report-dir Directory to save logs (default: ./reports) + --keep-temp Preserve extracted .deb directories for inspection +""" + +import os +import sys +import subprocess +import tempfile +import shutil +import argparse +import glob +import re +import urllib.request +import urllib.parse +import traceback +from helpers import create_new_directory +from color_logger import logger + +RETURN_ABI_NO_DIFF = 0b00000 +RETURN_ABI_COMPATIBLE_DIFF = 0b00001 +RETURN_ABI_INCOMPATIBLE_DIFF = 0b00010 +RETURN_ABI_STRIPPED_PACKAGE = 0b00100 +RETURN_PPA_PACKAGE_NOT_FOUND = 0b01000 +RETURN_PPA_ERROR = 0b10000 + +class ABI_DIFF_Result: + def __init__(self, package_name): + self.package_name = package_name + self.repo_name = None + + self.new_deb_name=None + self.new_dev_name=None + self.new_ddeb_name=None + self.new_deb_version=None + + self.old_deb_name=None + self.old_dev_name=None + self.old_ddeb_name=None + self.old_deb_version=None + + self.abi_pkg_diff_result = None + self.abi_pkg_diff_remark = None + self.abi_pkg_diff_output = None + +# package_name - result +global_checker_results: dict[str, ABI_DIFF_Result] = {} + + + +def print_results(log_file=None): + + log = "ABI Check results\n" + + log += ("-" * 100 + "\n") + + for package_name, result in global_checker_results.items(): + log += f"Package Name: {package_name}\n" + log += f"Repository Name: {result.repo_name}\n" + log += f"New Package:\n" + log += f" - DEB Name: {result.new_deb_name}\n" + log += f" - DEV Name: {result.new_dev_name}\n" + log += f" - DDEB Name: {result.new_ddeb_name}\n" + log += f" - Version: {result.new_deb_version}\n" + log += f"Old Package:\n" + log += f" - DEB Name: {result.old_deb_name}\n" + log += f" - DEV Name: {result.old_dev_name}\n" + log += f" - DDEB Name: {result.old_ddeb_name}\n" + log += f" - Version: {result.old_deb_version}\n" + log += f"ABI Package Diff:\n" + log += f" - Result: {result.abi_pkg_diff_result}\n" + log += f" - Remark: {result.abi_pkg_diff_remark}\n" + log += f" - Output: {"" if result.abi_pkg_diff_output is not None else result.abi_pkg_diff_output}\n" + if result.abi_pkg_diff_output is not None: + cmd = f"echo \"{result.abi_pkg_diff_output}\" | sed 's/^/ /'" + output = subprocess.run(cmd, capture_output=True, text=True, shell=True) + + log += f"{output.stdout}\n" + + log += ("-" * 100 + "\n") + + if log_file is not None: + with open(log_file, 'w') as f: + f.write(log) + + logger.debug(log) + +def parse_arguments(): + parser = argparse.ArgumentParser(description="Compare two .deb packages using abipkgdiff or abidiff.") + parser.add_argument("--apt-server-config", + default="deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main", + help="APT server configuration to download the old package to compare against") + + parser.add_argument("--new-package-dir", + required=True, + help="Path to the folder containing the new package to compare. (.deb, optional -dev.deb, optional -dbgsym.ddeb)") + + parser.add_argument("--delete-temp", + action="store_true", + help="Keep temp extracted folders for debugging.") + + parser.add_argument("--old-version", + required=False, + help="Specific version of the old package to compare against. (optional)") + args = parser.parse_args() + + return args + +def main(): + args = parse_arguments() + + logger.debug(f"args: {args}") + + if not os.path.isabs(args.new_package_dir): + args.new_package_dir = os.path.abspath(args.new_package_dir) + + print_debug_tree = True + + + + ret = single_repo_deb_abi_checker(args.new_package_dir, + args.apt_server_config, + True if args.delete_temp is False else False, + None if not args.old_version else args.old_version, + print_debug_tree=print_debug_tree) + + print_results(None) + + sys.exit(ret) + +def multiple_repo_deb_abi_checker(package_dir, apt_server_config, keep_temp=True, specific_apt_version=None) -> bool: + """ + Runs the ABI check in a folder containing multiple package folders. + + Note: For a single package, use the function single_repo_deb_abi_checker() + + Args: + package_dir (str): Path to the temporary directory containing the packages. + Must have a structure like the following, where the core deb package is placed alongside + its development and debug package: + . + └── my_package + ├── my_package_1.0.0_arm64.deb + ├── my_package-dbgsym_1.0.0_arm64.ddeb + └── my_package-dev_1.0.0_arm64.deb + + apt_server_config (str): APT server configuration to download the old package to compare against. + Must be in the format "deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main". + + keep_temp (bool): Whether to keep the temporary directory after the comparison. + + specific_apt_version (str): Specific version of the old package to compare against. (optional) + + + Returns: + -------- + - bool: True if the package ABI diff was performed sucessfully, False otherwise. + Note that this does not mean that the ABI diff passed, only that it was performed successfully. + """ + + all_repos_successful = True + + for folder in os.listdir(package_dir): + folder_path = os.path.join(package_dir, folder) + if os.path.isdir(folder_path): + + try: + success = single_repo_deb_abi_checker(folder_path, apt_server_config, keep_temp, specific_apt_version) + except Exception as e: + logger.critical(f"Function single_repo_deb_abi_checker threw an exception: {e}") + success = False + + traceback.print_exc() + + finally: + if not success: + all_repos_successful = False + + log_file = os.path.join(package_dir, "abi_checker.log") + + print_results(log_file) + + return all_repos_successful + +def single_repo_deb_abi_checker(repo_package_dir, apt_server_config, keep_temp=True, specific_apt_version=None, print_debug_tree=False) -> int: + """ + Runs the ABI check for all the packages in a single repo output directory + + Note: For running the ABI check accross multiple repo folders, use the function multiple_repo_deb_abi_checker(), which + will run the ABI check for all the packages in all the repo folders. + + Args: + repo_package_dir (str): Path to the directory where a repo has build its packages. This directory + would typically be named after the repo name. For example, if the repo is named "my_package", + then the directory would be named "my_package". + + Must have a structure like the following, where the core deb package is placed alongside + its development and debug package: + . + └── repo_package_dir + ├── my_package_1.0.0_arm64.deb + ├── my_package-dbgsym_1.0.0_arm64.ddeb + └── my_package-dev_1.0.0_arm64.deb + + Note: It is possible for a repo to produce multiple core packages, in which case the directory + would contain multiple core packages. For example: + . + └── repo_package_dir + ├── my_package_1.0.0_arm64.deb + ├── my_package_2.0.0_arm64.deb + ├── my_package-dbgsym_1.0.0_arm64.ddeb + ├── my_package-dbgsym_2.0.0_arm64.ddeb + └── my_package-dev_1.0.0_arm64.deb + └── my_package-dev_2.0.0_arm64.deb + + If this is the case, this function will handle all the core packages in the directory. + + apt_server_config (str): APT server configuration to download the old package to compare against. + Must be in the format "deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main". + + keep_temp (bool): Whether to keep the temporary directory after the comparison. + + specific_apt_version (str): Specific version of the old package to compare against. (optional) + + Returns: + -------- + - bool: True if the package ABI diff was performed sucessfully, False otherwise. + Note that this does not mean that the ABI diff passed, only that it was performed successfully. + """ + + logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: Checking {repo_package_dir}") + + basedir = os.path.basename(repo_package_dir) + + logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: performing abi checking for repo '{basedir}'") + + if print_debug_tree: + tree_cmd = f"tree -a {repo_package_dir} | sed 's/^/ /'" + tree_output = subprocess.run(tree_cmd, capture_output=True, text=True, shell=True) + if tree_output.returncode == 0: + logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: Content :\n{tree_output.stdout}") + else: + logger.error(f"[ABI_CHECKER]/[SINGLE_REPO]: Failed to run 'tree' command: {tree_output.stderr}") + + abi_check_temp_dir = os.path.join(repo_package_dir, "abi_check_tmp") + + create_new_directory(abi_check_temp_dir, delete_if_exists=True) # <-- !delete the directory if it already exists + + # Find the .deb file(s) in the abi_check_temp_dir that represents the core packages + # We filter out the -dev and -dbgsym packages as we are interested in building the list of core packages + # that are built from the repo. + deb_files = [f for f in os.listdir(repo_package_dir) if f.endswith('.deb') and '-dev' not in f and '-dbgsym' not in f] + + if not deb_files: + logger.warning(f"[ABI_CHECKER]/[SINGLE_REPO]: No .deb file found, nothing to compare, returning success") + return RETURN_ABI_NO_DIFF + + logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: Found {len(deb_files)} package{"s" if len(deb_files) > 1 else ""}") + + final_ret = 0 + + for deb_file in deb_files: + logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: core deb file detected: {deb_file}") + package_name = os.path.splitext(os.path.basename(deb_file))[0].split('_')[0] + logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: package name: {package_name}") + + package_abi_check_temp_dir = os.path.join(abi_check_temp_dir, package_name) + create_new_directory(package_abi_check_temp_dir) + + global_checker_results[package_name] = ABI_DIFF_Result(package_name) + global_checker_results[package_name].repo_name = basedir + + # Run the single_package_abi_checker function for the package + ret = single_package_abi_checker(repo_package_dir=repo_package_dir, + package_abi_check_temp_dir=package_abi_check_temp_dir, + package_name=package_name, + package_file=deb_file, + apt_server_config=apt_server_config, + keep_temp=keep_temp, + specific_apt_version=specific_apt_version, + print_debug_tree=print_debug_tree) + + final_ret = final_ret | ret + + return final_ret + +def single_package_abi_checker(repo_package_dir, + package_abi_check_temp_dir, + package_name, + package_file, + apt_server_config, + keep_temp=True, + specific_apt_version=None, + print_debug_tree=False) -> bool: + """ + Runs the ABI check in a folder containing a single package. + """ + + result = global_checker_results[package_name] + + logger.debug(f"[ABI_CHECKER]/{package_name}: running single_package_abi_checker") + + old_extract_dir = os.path.join(package_abi_check_temp_dir, "old") + new_extract_dir = os.path.join(package_abi_check_temp_dir, "new") + + create_new_directory(old_extract_dir) + create_new_directory(new_extract_dir) + + new_version = os.path.splitext(package_file)[0].split('_')[1] + logger.info(f"[ABI_CHECKER]/{package_name}: New package version: {new_version}") + + new_deb_path = os.path.join(repo_package_dir, package_file) + + result.new_deb_name = package_file + result.new_deb_version = new_version + + # -dev.deb package is optional, but if it exists, we need to extract it too + # The package name may contain the major version number at the end, but by canonical convention, dev packages shall not contain that + # major version, so deal with this to make sure the dev package not containing it is found + package_name_without_major = (package_name[:-1] if package_name[-1].isdigit() else package_name) + + + deb_dev_files = [f for f in os.listdir(repo_package_dir) if f.endswith('.deb') and package_name_without_major in f and "-dev" in f] + + if not deb_dev_files: + logger.warning(f"[ABI_CHECKER]/{package_name}: No -dev.deb package found") + new_dev_path = None + elif len(deb_dev_files) == 1: + logger.info(f"[ABI_CHECKER]/{package_name}: -dev.deb package found: {deb_dev_files[0]}") + new_dev_path = os.path.join(repo_package_dir, deb_dev_files[0]) + result.new_dev_name = deb_dev_files[0] + else: + deb_dev_file = [f for f in deb_dev_files if f"{package_name_without_major}-dev" in f] + if len(deb_dev_file) > 1: + logger.critical(f"[ABI_CHECKER]/{package_name}: Multiple -dev.deb files found") + result.new_dev_name = "ERROR : multiple detected" + return False + new_dev_path = os.path.join(repo_package_dir, deb_dev_file[0]) + result.new_dev_name = deb_dev_file[0] + + # -dbgsym.ddeb package is optional, but if it exists, we need to extract it too + + deb_ddeb_files = [f for f in os.listdir(repo_package_dir) if f.endswith('.ddeb') and f"{package_name}-dbgsym" in f] + + if not deb_ddeb_files: + logger.warning(f"[ABI_CHECKER]/{package_name}: No -dbgsym.ddeb package found") + new_ddeb_path = None + elif len(deb_ddeb_files) == 1: + logger.info(f"[ABI_CHECKER]/{package_name}: -dbgsym.ddeb debug package found: {deb_ddeb_files[0]}") + new_ddeb_path = os.path.join(repo_package_dir, deb_ddeb_files[0]) + result.new_ddeb_name = deb_ddeb_files[0] + else: + logger.critical(f"[ABI_CHECKER]/{package_name}: Multiple -dev-dbgsym.ddeb files found") + result.new_ddeb_name = "ERROR : multiple detected" + return False + + # Extract all the packages in the 'new' directory + extract_deb(new_deb_path, new_dev_path, new_ddeb_path, new_extract_dir) + + if print_debug_tree: + # Run the 'tree' command to list files in a tree structure + tree_cmd = f"tree -a {new_extract_dir} | sed 's/^/ /'" + tree_output = subprocess.run(tree_cmd, capture_output=True, text=True, shell=True) + if tree_output.returncode == 0: + logger.debug(f"[ABI_CHECKER]/{package_name}: Tree structure of new_extract_dir:\n{tree_output.stdout}") + else: + logger.error(f"[ABI_CHECKER]/{package_name}: Failed to run 'tree' command: {tree_output.stderr}") + + # ******* OLD DEB PACKAGE fetching ********************************************************* + + logger.debug(f"[ABI_CHECKER]/{package_name}: Fetching old deb package from APT server") + logger.debug(f"[ABI_CHECKER]/{package_name}: APT Server Config: {apt_server_config}") + + old_download_dir = os.path.join(package_abi_check_temp_dir, "old_download") + + create_new_directory(old_download_dir) + + apt_dir = os.path.join(old_download_dir, "apt") + create_new_directory(apt_dir) + + # Use apt-get to download the latest version of the package + if specific_apt_version is None: + logger.debug(f"[ABI_CHECKER]/{package_name}: Using apt-get to download the *latest* version of {package_name}") + else: + logger.warning(f"[ABI_CHECKER]/{package_name}: Using apt-get to download the *specific* version {specific_apt_version} of {package_name}") + + # Create a temporary sources.list file + temp_sources_list = os.path.join(apt_dir, "sources.list") + with open(temp_sources_list, "w") as f: + f.write(apt_server_config) + + cache_dir = os.path.join(apt_dir, "cache") + create_new_directory(cache_dir) + + opt = f" -o Dir::Etc::sourcelist={temp_sources_list}" + opt += f" -o Dir::Etc::sourceparts=/dev/null" + opt += f" -o Dir::State={cache_dir}" + opt += f" -o Dir::Cache={cache_dir}" + + # Update the package list + cmd = "apt-get update" + opt + + logger.debug(f"[ABI_CHECKER]/{package_name}: Running: {cmd}") + apt_ret = subprocess.run(cmd, cwd=old_download_dir, shell=True, capture_output=True) + if apt_ret.returncode != 0: + logger.critical(f"[ABI_CHECKER]/{package_name}: Failed to update package list: {apt_ret.stderr}") + return RETURN_PPA_ERROR + + # download the .deb package + pkg = package_name + (("=" + specific_apt_version) if specific_apt_version else "") + cmd = f"apt-get download {pkg}" + opt + apt_ret = subprocess.run(cmd, cwd=old_download_dir, shell=True, capture_output=True) + if apt_ret.returncode != 0: + logger.error(f"[ABI_CHECKER]/{package_name}: Failed to download {pkg}: {apt_ret.stderr}") + return RETURN_PPA_PACKAGE_NOT_FOUND + else: + logger.info(f"[ABI_CHECKER]/{package_name}: Downloaded {pkg}") + + # download the -dev.deb package + pkg = package_name + "-dev" + (("=" + specific_apt_version) if specific_apt_version else "") + cmd = f"apt-get download {pkg}" + opt + apt_ret = subprocess.run(cmd, cwd=old_download_dir, shell=True, capture_output=True) + if apt_ret.returncode != 0: + logger.warning(f"[ABI_CHECKER]/{package_name}: Failed to download {pkg}: {apt_ret.stderr}") + else: + logger.info(f"[ABI_CHECKER]/{package_name}: Downloaded {pkg}") + + # download the -dbgsym.deb package + pkg = package_name + "-dbgsym" + (("=" + specific_apt_version) if specific_apt_version else "") + cmd = f"apt-get download {pkg}" + opt + apt_ret = subprocess.run(cmd, cwd=old_download_dir, shell=True, capture_output=True) + if apt_ret.returncode != 0: + logger.warning(f"[ABI_CHECKER]/{package_name}: Failed to download {pkg}: {apt_ret.stderr}") + else: + logger.info(f"[ABI_CHECKER]/{package_name}: Downloaded {pkg}") + + + # Configure the old packages paths + old_deb_file = next((f for f in os.listdir(old_download_dir) if f.endswith('.deb') and '-dev' not in f), None) + if old_deb_file is None: + logger.critical(f"[ABI_CHECKER]/{package_name}: No .deb file found in '{old_download_dir}' that does not contain '-dev' in the name") + result.old_deb_name = "ERROR : None found" + raise Exception("No .deb file found in '{old_download_dir}' that does not contain '-dev' in the name") + + old_deb_path = os.path.join(old_download_dir, old_deb_file) + result.old_deb_name = old_deb_file + result.old_deb_version = os.path.splitext(old_deb_file)[0].split('_')[1] + + + old_version = os.path.splitext(os.path.basename(old_deb_path))[0].split('_')[1] + logger.info(f"[ABI_CHECKER]/{package_name}: Old package version: {old_version}") + result.old_version = old_version + + old_dev_file = next((f for f in os.listdir(old_download_dir) if f.endswith('.deb') and '-dev' in f), None) + if old_dev_file is None: + old_dev_path = None + logger.warning(f"[ABI_CHECKER]/{package_name}: No -dev.deb file that does contains '-dev' in the name") + else: + old_dev_path = os.path.join(old_download_dir, old_dev_file) + result.old_dev_name = old_dev_file + + old_ddeb_file = next((f for f in os.listdir(old_download_dir) if f.endswith('.ddeb') and '-dbgsym' in f), None) + if old_ddeb_file is None: + old_ddeb_path = None + logger.warning(f"[ABI_CHECKER]/{package_name}: No -dbgsym.ddeb file found that does contains '-dbgsym' in the name") + else: + old_ddeb_path = os.path.join(old_download_dir, old_ddeb_file) + result.old_ddeb_name = old_ddeb_file + + extract_deb(old_deb_path, old_dev_path, old_ddeb_path, old_extract_dir) + + if print_debug_tree: + # Run the 'tree' command to list files in a tree structure + tree_cmd = f"tree -a {old_extract_dir} | sed 's/^/ /'" + tree_output = subprocess.run(tree_cmd, capture_output=True, text=True, shell=True) + if tree_output.returncode == 0: + logger.debug(f"[ABI_CHECKER]: Tree structure of old_extract_dir:\n{tree_output.stdout}") + else: + logger.error(f"[ABI_CHECKER]: Failed to run 'tree' command: {tree_output.stderr}") + + # ******* ABI CHECKING ********************************************************************** + + report_dir = os.path.join(package_abi_check_temp_dir,"report") + + abidiff_result = compare_with_abipkgdiff(old_deb_path, old_dev_path, old_ddeb_path, + new_deb_path, new_dev_path, new_ddeb_path, + report_dir, include_non_reachable_types=True) + + # The return value between abidiff and abipkgdiff has the same meaning, so we can use the same analysis + if abidiff_result != 0: + + cmd =f"cat {report_dir}/abipkgdiff_output.txt" + + log = subprocess.run(cmd, shell=True, capture_output=True, text=True) + result.abi_pkg_diff_output = log.stdout + + + # Analyze the first 4 bits of the return value + bit1 = (abidiff_result & 0b0001) + bit2 = (abidiff_result & 0b0010) >> 1 + bit3 = (abidiff_result & 0b0100) >> 2 + bit4 = (abidiff_result & 0b1000) >> 3 + + # Determine the overall result based on the bit analysis + if bit1: + logger.critical(f"[ABI_CHECKER]: abipkgdiff encountered an error") + result.abi_pkg_diff_result = "ERROR" + raise Exception("abipkgdiff encountered an error") + if bit2: + logger.error(f"[ABI_CHECKER]: abipkgdiff usage error. This has shown to be true for stripped packages") + result.abi_pkg_diff_result = "STRIPPED-PACKAGE" + return RETURN_ABI_STRIPPED_PACKAGE + if bit3: + result.abi_pkg_diff_result = "COMPATIBLE-DIFF" + logger.warning(f"[ABI_CHECKER]: abipkgdiff detected ABI changes") + if bit4: + # if bit 4 is set, bit 3 must be too, so this fallthrough is ok + result.abi_pkg_diff_result = "INCOMPATIBLE-DIFF" + logger.warning(f"[ABI_CHECKER]: abipkgdiff detected ABI ***INCOMPATIBLE*** changes.") + + # Print the content of all the files in 'report_dir' + for filename in os.listdir(report_dir): + file_path = os.path.join(report_dir, filename) + if os.path.isfile(file_path): + with open(file_path, 'r') as file: + logger.debug(f"Content of {filename}:") + logger.warning(file.read()) + else: + # No ABI DIFF + result.abi_pkg_diff_result = "NO-DIFF" + + logger.info(f"[ABI_CHECKER]/{package_name}: abipkgdiff did not find any differences between old and new packages") + + + msg = "[ABI_CHECKER]/{package_name}: Although, no {pkg} was found for the {version} package, interpret the results with caution" + + if old_dev_path is None: + logger.warning(msg.format(package_name=package_name, pkg="-dev.deb", version="old")) + if new_dev_path is None: + logger.warning(msg.format(package_name=package_name, pkg="-dev.deb", version="new")) + if old_dev_path is None or new_dev_path is None: + result.abi_pkg_diff_remark = "NO-DEV-PACKAGE" + + if old_ddeb_path is None: + logger.warning(msg.format(package_name=package_name, pkg="-dbgsym.ddeb", version="old")) + if new_ddeb_path is None: + logger.warning(msg.format(package_name=package_name, pkg="-dbgsym.ddeb", version="new")) + if old_ddeb_path is None or new_ddeb_path is None: + if result.abi_pkg_diff_remark is not None: + result.abi_pkg_diff_remark += ", NO-DBG-PACKAGE" + else: + result.abi_pkg_diff_remark = "NO-DBG-PACKAGE" + + + if not keep_temp: + logger.debug(f"[ABI_CHECKER]: Removing temporary directory {abi_check_temp_dir}") + shutil.rmtree(abi_check_temp_dir) + + return analyze_abi_diff_result(old_version, new_version, abidiff_result) + +def extract_deb(deb_path, dev_path, ddeb_path, extract_dir): + """Extract the content of a .deb package and its .ddeb to a specified directory.""" + + if deb_path is None: + raise ValueError("deb_path cannot be None") + if not deb_path.endswith(".deb") or not os.path.isfile(deb_path): + raise ValueError(f"Invalid deb_path: {deb_path}. Expected a file with .deb extension") + + cmd = ["dpkg", "-x", deb_path, extract_dir] + subprocess.run(cmd, check=True) + + if dev_path is not None: + cmd = ["dpkg", "-x", dev_path, extract_dir] + subprocess.run(cmd, check=True) + + if ddeb_path is not None: + cmd = ["dpkg", "-x", ddeb_path, extract_dir] + subprocess.run(cmd, check=True) + +def compare_with_abipkgdiff(old_deb_path, old_dev_path, old_ddeb_path, + new_deb_path, new_dev_path, new_ddeb_path, + report_dir, include_non_reachable_types=False): + """Run abipkgdiff on two .deb packages and log the result.""" + + logger.debug("[ABI_CHECKER]/[ABI_PKG_DIFF] : Comparing with abipkgdiff tool") + + os.makedirs(report_dir, exist_ok=True) + log_path = os.path.join(report_dir, "abipkgdiff_output.txt") + + cmd = "abipkgdiff" + + if include_non_reachable_types: + logger.debug("[ABI_CHECKER]/[ABI_PKG_DIFF] : Using --non-reachable-types option") + cmd += " --non-reachable-types" + + if old_dev_path is not None and new_dev_path is not None: + cmd += f" --devel-pkg1 {old_dev_path} --devel-pkg2 {new_dev_path}" + else: + logger.warning("[ABI_CHECKER]/[ABI_PKG_DIFF]: One or both of the -dev packages are missing. Potentially missing on information") + + if old_ddeb_path is not None and new_ddeb_path is not None: + cmd += f" --debug-info-pkg1 {old_ddeb_path} --debug-info-pkg2 {new_ddeb_path}" + else: + logger.warning("[ABI_CHECKER]/[ABI_PKG_DIFF]: One or both of the -dbgsym.ddeb packages are missing. Potentially missing on information") + + cmd += f" {old_deb_path} {new_deb_path}" + + + logger.debug(f"[ABI_CHECKER]/[ABI_PKG_DIFF]: command: {cmd}") + + abidiff_output = subprocess.run(cmd, capture_output=True, text=True, shell=True) + + with open(log_path, "w") as f: + f.write(abidiff_output.stdout) + + rc = abidiff_output.returncode + + return rc + +def version_bumped(old_version, new_version, index): + """ + Checks if the major version has been increased. + + Args: + old_version (str): The old version string (e.g., "1.0.0"). + new_version (str): The new version string (e.g., "2.0.0"). + + Returns: + bool: True if the major version has been increased, False otherwise. + """ + if index not in ["major", "minor", "patch"]: + raise ValueError("Index must be one of 'major', 'minor', or 'patch'") + + # Remove the build number from the version strings, if present + old_version = old_version.split('-')[0] + new_version = new_version.split('-')[0] + + # Split the version strings into their components + old_version_parts = list(map(int, old_version.split('.'))) + new_version_parts = list(map(int, new_version.split('.'))) + + # Determine the index of the version part to check + if index == "major": + index = 0 + elif index == "minor": + index = 1 + elif index == "patch": + index = 2 + + # Check if the version part at the specified index has increased + if new_version_parts[index] > old_version_parts[index]: + return True + else: + return False + +def analyze_abi_diff_result(old_version, new_version, abidiff_result): + import re + + # Keep the first part of the version, before the first '-' or '+' + old_version = re.split('[+-]', old_version)[0] + new_version = re.split('[+-]', new_version)[0] + + # Define a regular expression pattern for a major-minor-patch version + version_pattern = r"^\d+\.\d+\.\d+(-\d+)?$" + + # Check if old_version and new_version match the pattern + if not re.match(version_pattern, old_version): + raise ValueError(f"Invalid old version: {old_version}. Expected a string in the format 'major.minor.patch'") + + if not re.match(version_pattern, new_version): + raise ValueError(f"Invalid new version: {new_version}. Expected a string in the format 'major.minor.patch'") + + logger.debug("[ABI_CHECKER]/[RESULT]: Performing version analysis of the ABI diff result versus the versions") + + # If both versions are valid, proceed with the analysis + # For now, just print the versions and the result + logger.debug(f"[ABI_CHECKER]/[RESULT]: Old version: {old_version}") + logger.debug(f"[ABI_CHECKER]/[RESULT]: New version: {new_version}") + + if (abidiff_result & 0b0011): + raise ValueError("[ABI_CHECKER]/[RESULT]: ASSERT : this scenario should have already been dealt with") + + abi_change = True if (abidiff_result & 0b0100) else False + incompatible_abi_change = True if (abidiff_result & 0b1000) else False + + + if incompatible_abi_change and not abi_change: + raise ValueError("[ABI_CHECKER]/[RESULT]: ASSERT : impossible scenario, if incompatible is set, change has to be set too") + + major_bumped = version_bumped(old_version, new_version, "major") + minor_bumped = version_bumped(old_version, new_version, "minor") + patch_bumped = version_bumped(old_version, new_version, "patch") + + if incompatible_abi_change: # Incompatible change + logger.error(f"[ABI_CHECKER]/[RESULT]: INCOMPATIBLE change detected") + + if major_bumped: + logger.debug(f"[ABI_CHECKER]/[RESULT]: OK : major version bumbed") + logger.debug("[ABI_CHECKER]/[RESULT]: Increasing the major version for an incompatible ABI is what is required") + result_pass = True + elif minor_bumped: + logger.debug(f"[ABI_CHECKER]/[RESULT]: NOT-OK : minor version bumbed") + logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing only the minor version for an incompatible ABI change is not enough") + result_pass = False + elif patch_bumped: + logger.debug(f"[ABI_CHECKER]/[RESULT]: NOT-OK : patch version bumbed") + logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing only the patch version for an incompatible ABI change is not enough") + result_pass = False + else: + logger.debug(f"[ABI_CHECKER]/[RESULT]: NOT-OK : no version bumped") + logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing the version number is required for an ABI change") + result_pass = False + + elif abi_change: # Compatible change + logger.warning(f"[ABI_CHECKER]/[RESULT]: COMPATIBLE change detected") + + if major_bumped: + logger.debug(f"[ABI_CHECKER]/[RESULT]: OK : major version bumbed") + logger.warning(f"[ABI_CHECKER]/[RESULT]: Increasing the major version for a compatible ABI change was probably overkill, but at least it respects version increase") + result_pass = True + elif minor_bumped: + logger.debug(f"[ABI_CHECKER]/[RESULT]: OK : minor version bumbed") + logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing the minor version for a compatible ABI change is what is required") + result_pass = True + elif patch_bumped: + logger.debug(f"[ABI_CHECKER]/[RESULT]: NOT-OK : patch version bumbed") + logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing only the patch number while there is an ABI change, albeit compatible, is not enough") + result_pass = False + else: + logger.debug(f"[ABI_CHECKER]/[RESULT]: NOT-OK : no version bumbed") + logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing at least the minor version number is required for a compatible ABI change") + result_pass = False + + else: # No change + logger.info(f"[ABI_CHECKER]/[RESULT]: No ABI change detected") + + if major_bumped: + logger.debug(f"[ABI_CHECKER]/[RESULT]: OK : major version bumbed") + logger.warning("[ABI_CHECKER]/[RESULT]: Increasing the major version when there is no ABI change is probably overkill, but at least it respects version increase") + result_pass = True + elif minor_bumped: + logger.debug(f"[ABI_CHECKER]/[RESULT]: OK : minor version bumbed") + logger.warning(f"[ABI_CHECKER]/[RESULT]: Increasing the minor version for a compatible ABI change is probably overkill, but at least it respects version increase") + result_pass = True + elif patch_bumped: + logger.debug(f"[ABI_CHECKER]/[RESULT]: OK : patch version bumbed") + logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing only the patch number while there is no ABI change seems reasonable") + result_pass = True + else: + logger.debug(f"[ABI_CHECKER]/[RESULT]: OK : no version bump") + result_pass = True + + return result_pass + +if __name__ == "__main__": + main() diff --git a/ubuntu/deb_organize.py b/ubuntu/deb_organize.py index d12a1d6..d99e429 100644 --- a/ubuntu/deb_organize.py +++ b/ubuntu/deb_organize.py @@ -13,7 +13,7 @@ import os import sys -from helpers import logger +from color_logger import logger from generate_project_info_from_manifest import create_project_info_file diff --git a/ubuntu/flat_meta.py b/ubuntu/flat_meta.py new file mode 100644 index 0000000..3e5bf73 --- /dev/null +++ b/ubuntu/flat_meta.py @@ -0,0 +1,66 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# +# SPDX-License-Identifier: BSD-3-Clause-Clear + +import os +import shutil +import subprocess +import tarfile +from helpers import logger + +def create_flat_meta(pack_variant, flavor, target_hw, workspace): + board_specific_path = os.path.join(workspace, "sources/modem-apis/qclinux/Ubuntu_NHLOS/") + inc_file = os.path.join(board_specific_path, f"firmware-{target_hw}.inc") + if not os.path.isfile(inc_file): + raise FileNotFoundError(f"INC file not found: {inc_file}") + + output_files = ["system.img", "efi.bin", "dtb.bin"] + output_dir = os.path.join(workspace, "out") + # Check for missing files + missing_files = [f for f in output_files if not os.path.isfile(os.path.join(output_dir, f))] + + # Raise an exception if any files are missing + if missing_files: + logger.info("Files missed required for Flatmeta creation, Ensure build.py --pack-image done.") + raise FileNotFoundError(f"The following required files are missing from {output_dir}: {', '.join(missing_files)}") + logger.info(f"Creating Flat meta for {target_hw}") + # ExtractBUILD_ID BUILD_ID and BIN_PATH + build_id = None + bin_path = None + with open(inc_file, 'r') as f: + for line in f: + if 'BUILD_ID' in line: + build_id = line.split('=')[1].strip().strip('"') + elif 'BIN_PATH' in line: + bin_path = line.split('=')[1].strip().strip('"') + + if not build_id or not bin_path: + raise ValueError("BUILD_ID or BIN_PATH not found in the .inc file") + + print(f"Meta SP : {bin_path}") + print(f"Meta ID : {build_id}") + + # Download the tar.gz file + tarball_url = f"https://artifactory-las.qualcomm.com/artifactory/lint-lv-local/ubun_nhlos/{bin_path}/{build_id}.tar.gz" + tarball_name = f"{build_id}.tar.gz" + subprocess.run(["wget", "--no-check-certificate", tarball_url], check=True) + + # Extract the tarball + extract_path = os.path.join(f"{workspace}", f"ub_{pack_variant}_image", flavor, target_hw) + os.makedirs(extract_path, exist_ok=True) + with tarfile.open(tarball_name, "r:gz") as tar: + tar.extractall(path=extract_path) + + # Copy output files + for file in output_files: + src = os.path.join(output_dir, file) + if os.path.exists(src): + shutil.copy(src, extract_path) + # Copy vmlinu* files + for file in os.listdir(output_dir): + if file.startswith("vmlinu"): + shutil.copy(os.path.join(output_dir, file), extract_path) + + # Remove the tarball + os.remove(tarball_name) + print(f"✅ Flat meta created successfully under : {extract_path}.") diff --git a/ubuntu/generate_project_info_from_manifest.py b/ubuntu/generate_project_info_from_manifest.py index 09b5cc4..acfcc27 100644 --- a/ubuntu/generate_project_info_from_manifest.py +++ b/ubuntu/generate_project_info_from_manifest.py @@ -14,7 +14,7 @@ import os import requests from lxml import etree -from helpers import logger +from color_logger import logger def parse_from_string(xml_string): diff --git a/ubuntu/helpers.py b/ubuntu/helpers.py index 6e59807..a9ec028 100644 --- a/ubuntu/helpers.py +++ b/ubuntu/helpers.py @@ -6,7 +6,7 @@ helper.py This module provides utilities for managing Debian package builds and related operations. -It includes functions for executing shell commands, managing files and directories, +It includes functions for executing shell commands, managing files and directories, logging, and setting up a local APT server. """ @@ -17,37 +17,12 @@ import shutil import logging import subprocess +import glob +from pathlib import Path from git import Repo from apt_server import AptServer from constants import TERMINAL, HOST_FS_MOUNT - -class ColorFormatter(logging.Formatter): - COLORS = { - 'DEBUG': '\033[94m', # Blue - 'INFO': '\033[92m', # Green - 'WARNING': '\033[93m', # Yellow - 'ERROR': '\033[91m', # Red - 'CRITICAL': '\033[95m', # Magenta - } - RESET = '\033[0m' - - def format(self, record): - log_color = self.COLORS.get(record.levelname, self.RESET) - message = super().format(record) - return f"{log_color}{message}{self.RESET}" - -logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s || %(levelname)s || %(message)s", - datefmt="%H:%M:%S" -) - -handler = logging.StreamHandler() -formatter = ColorFormatter('%(levelname)s: %(message)s') -handler.setFormatter(formatter) - -logger = logging.getLogger("DEB-BUILD") -logger.addHandler(handler) +from color_logger import logger def check_if_root() -> bool: """ @@ -76,11 +51,10 @@ def check_and_append_line_in_file(file_path, line_to_check, append_if_missing=Fa if not os.path.exists(file_path): logger.error(f"{file_path} does not exist.") exit(1) - - lines = [] + with open(file_path, "r") as file: lines = file.readlines() - + for line in lines: if line.strip() == line_to_check.strip(): return True @@ -92,6 +66,28 @@ def check_and_append_line_in_file(file_path, line_to_check, append_if_missing=Fa return False +def parse_debs_manifest(manifest_path): + """ + Parses a manifest file and returns a dictionary of module names and their corresponding versions. + """ + DEBS = [] + user_manifest = Path(manifest_path) + if not user_manifest.is_file() or not user_manifest.name.endswith('.manifest'): + raise ValueError(f"Provided manifest path '{user_manifest}' is not a valid '.manifest' file.") + if os.path.isfile(manifest_path): + with open(manifest_path, 'r') as f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + parts = list(line.split('\t')) + DEBS.append({ + 'package': parts[0], + 'version': parts[1] if len(parts) > 1 else None, + }) + return DEBS + else: + print(f"Manifest file {manifest_path} not found.") + return None def run_command(command, check=True, get_object=False, cwd=None): """ @@ -112,20 +108,26 @@ def run_command(command, check=True, get_object=False, cwd=None): ------- - Exception: If the command fails and check is True. """ - logger.info(f'Running: {command}') + + logger.debug(f'Running command: {command}') + try: - if not cwd: - result = subprocess.run(command, shell=True, check=check, capture_output=True, text=True) - else: - result = subprocess.run(command, shell=True, check=check, capture_output=True, text=True, cwd=cwd) + result = subprocess.run(command, shell=True, check=check, capture_output=True, text=True, cwd=cwd) + except subprocess.CalledProcessError as e: - logger.error(f"Command failed: {e.stderr.strip() if e.stderr else str(e)}") + logger.error(f"Command failed with return value: {e.returncode}") + logger.error(f"stderr: {e.stderr.strip() if e.stderr else str(e)}") + logger.error(f"stdout: {e.stdout.strip()}") raise Exception(e) - if result.stderr: - logger.error(f"Error: {result.stderr.strip()}") - return result.stdout.strip() + stderr = result.stderr.strip() + if stderr: + if result.returncode == 0: + logger.debug(f"Successful return value, yet there is content in stderr: {stderr}") + else: + logger.error(f"Error: {stderr}") + return result.stdout.strip() def run_command_for_result(command): """ @@ -142,7 +144,7 @@ def run_command_for_result(command): - "returncode" (int): The return code of the command. """ command = command.strip() - logger.info(f'Running for result: {command}') + logger.debug(f'Running for result: {command}') try: result = subprocess.check_output(command, shell=True, stderr=subprocess.sys.stdout) return {"output": result.decode("utf-8").strip(), "returncode": 0} @@ -191,6 +193,9 @@ def cleanup_file(file_path): ------- - Exception: If an error occurs while trying to delete the file. """ + + logger.debug(f"Cleaning file {file_path}") + try: if os.path.exists(file_path): os.remove(file_path) @@ -272,21 +277,29 @@ def umount_dir(MOUNT_DIR, UMOUNT_HOST_FS=False): """ Unmounts a specified directory and optionally unmounts host filesystem mounts. + If the directory is not mounted, (ie, return code 32 from umount) then it is + silently ignored. + Args: ----- - MOUNT_DIR (str): The directory to unmount. - UMOUNT_HOST_FS (bool): If True, unmounts the host filesystem directories. """ + + logger.debug(f"umount dir {MOUNT_DIR}") + if UMOUNT_HOST_FS: for direc in HOST_FS_MOUNT: - try: - run_command(f"umount -l {MOUNT_DIR}/{direc}") - except: - logger.warning(f"Failed to unmount {MOUNT_DIR}/{direc}. Not mounted or busy. Ignoring.") - try: - run_command(f"umount -l {MOUNT_DIR}") - except: - logger.warning(f"Failed to unmount {MOUNT_DIR}. Not mounted or busy. Ignoring.") + result = subprocess.run(f"umount -l {MOUNT_DIR}/{direc}", + shell=True, capture_output=True, text=True) + + if result.returncode != 0 and result.returncode != 32: + logger.error(f"Failed to unmount {MOUNT_DIR}/{direc}: {result.stderr}") + + result = subprocess.run(f"umount -l {MOUNT_DIR}", + shell=True, capture_output=True, text=True) + if result.returncode != 0 and result.returncode != 32: + logger.error(f"Failed to unmount {MOUNT_DIR}: {result.stderr}") def change_folder_perm_read_write(DIR): """ @@ -365,29 +378,31 @@ def print_build_logs(directory): logger.error(content) logger.info("===== Build Logs End ======") -def start_local_apt_server(direc): +def start_local_apt_server(dir): """ Starts a local APT server in the specified directory and returns the APT repository line. Args: ----- - - direc (str): The directory to serve as the APT repository. + - dir (str): The directory to serve as the APT repository. Returns: -------- - str: The APT repository line to add to sources.list. """ - server = AptServer(directory=direc, port=random.randint(7500, 8500)) + + server = AptServer(directory=dir, port=random.randint(7500, 8500)) server.start() + return f"deb [trusted=yes arch=arm64] http://localhost:{server.port} stable main" -def build_deb_package_gz(direc, start_server=True) -> str: +def build_deb_package_gz(dir, start_server=True) -> str: """ Builds a Debian package and creates a compressed Packages file, optionally starting a local APT server. Args: ----- - - direc (str): The directory where the package is built. + - dir (str): The directory where the package is built. - start_server (bool): If True, starts a local APT server after building the package. Returns: @@ -398,21 +413,99 @@ def build_deb_package_gz(direc, start_server=True) -> str: ------- - Exception: If an error occurs while creating the Packages file. """ - global servers + + packages_dir = os.path.join(dir, 'dists', 'stable', 'main', 'binary-arm64') + packages_path = os.path.join(packages_dir, "Packages") + try: - packages_dir = os.path.join(direc, 'dists', 'stable', 'main', 'binary-arm64') os.makedirs(packages_dir, exist_ok=True) - cmd = f'dpkg-scanpackages -m . /dev/null > {os.path.join(packages_dir, "Packages")}' - run_command(cmd, cwd=direc) + cmd = f'dpkg-scanpackages -m . > {packages_path}' + + result = subprocess.run(cmd, shell=True, cwd=dir, check=False, capture_output=True, text=True) + + if result.returncode != 0: + logger.error(f"Error running : {cmd}") + logger.error(f"stdout : {result.stdout}") + logger.error(f"stderr : {result.stderr}") + + raise Exception(result.stderr) + + # Even with a successful exit code, dpkg-scanpackages still outputs the number of entries written to stderr logger.debug(result.stderr.strip()) + - packages_path = os.path.join(packages_dir, "Packages") - run_command(f"gzip -k -f {packages_path}") + cmd = f"gzip -k -f {packages_path}" + result = subprocess.run(cmd, shell=True, cwd=dir, check=False, capture_output=True, text=True) + + if result.returncode != 0: + logger.error(f"Error running : {cmd}") + logger.error(f"stdout : {result.stdout}") + logger.error(f"stderr : {result.stderr}") + + raise Exception(result.stderr) + + logger.debug(f"Packages file created at {packages_path}.gz") - logger.info(f"Packages file created in {direc}") except Exception as e: - logger.error(f"Error creating Packages file in {direc}, Ignoring.") + logger.error(f"Error creating Packages file in {dir} : {e}") + raise Exception(e) if start_server: - return start_local_apt_server(direc) + return start_local_apt_server(dir) return None + + +def pull_debs_wget(manifest_file_path, out_dir,DEBS_to_download_list,base_url): + """ + Downloads Debian packages from a remote repository using wget. + + Args: + ----- + - manifest_file_path (str): Path to the manifest file containing package versions. + - out_dir (str): Directory where downloaded packages will be saved. + - DEBS_to_download_list (list): List of package name prefixes to download. + - base_url (str): Base URL of the repository to download packages from. + + Returns: + -------- + - int: Number of packages successfully downloaded. + + Raises: + ------- + - Exception: If an error occurs while downloading packages. + """ + # Read manifest file + # Parse manifest into a dictionary + with open(manifest_file_path, 'r') as f: + manifest_text = f.read() + + # Parse manifest into a dictionary + version_map = {} + for line in manifest_text.strip().splitlines(): + if not line.strip(): + continue + parts = line.split() + if len(parts) >= 2: + name, version = parts[0], parts[1] + version_map[name] = version + + + # Generate wget links and download + os.makedirs(out_dir, exist_ok=True) + for module in DEBS_to_download_list: + for name, version in version_map.items(): + if name.startswith(module): + first_letter = name[0] + deb_name = f"{name}_{version}_arm64.deb" + url = f"{base_url}/{first_letter}/{name}/{deb_name}" + output_path = os.path.join(out_dir,name,deb_name) + create_new_directory(os.path.join(out_dir,name)) + # Construct wget command + wget_cmd = ["wget", "--no-check-certificate", url, "-O", output_path] + try: + logger.info(f"Downloading {url}...") + subprocess.run(wget_cmd, check=True) + logger.info(f"Saved to {output_path}") + except subprocess.CalledProcessError as e: + logger.error(f"error: Failed to download {url}: {e}") + break # Stop after first match diff --git a/ubuntu/pack_deb.py b/ubuntu/pack_deb.py index c8361fe..54907dd 100644 --- a/ubuntu/pack_deb.py +++ b/ubuntu/pack_deb.py @@ -21,8 +21,9 @@ from queue import Queue from collections import defaultdict, deque from constants import * -from helpers import create_new_file, check_if_root, logger, run_command, create_new_directory, run_command_for_result, mount_img, umount_dir, cleanup_file, build_deb_package_gz +from helpers import create_new_file, check_if_root, run_command, create_new_directory, run_command_for_result, mount_img, umount_dir, cleanup_file, build_deb_package_gz, parse_debs_manifest from deb_organize import search_manifest_map_for_path +from color_logger import logger class PackagePacker: def __init__(self, MOUNT_DIR, IMAGE_TYPE, VARIANT, OUT_DIR, OUT_SYSTEM_IMG, APT_SERVER_CONFIG, TEMP_DIR, DEB_OUT_DIR, DEBIAN_INSTALL_DIR, IS_CLEANUP_ENABLED, PACKAGES_MANIFEST_PATH=None): @@ -100,7 +101,7 @@ def set_efi_bin(self): create_new_directory(self.EFI_MOUNT_PATH) run_command(f"mount -o loop {self.EFI_BIN_PATH} {self.EFI_MOUNT_PATH}") - grub_update_cmd = f"""echo 'GRUB_CMDLINE_LINUX="ro console=ttyMSM0,115200n8 pcie_pme=nomsi earlycon qcom_scm.download_mode=1 panic=reboot_warm" + grub_update_cmd = f"""echo 'GRUB_CMDLINE_LINUX="ro console=ttyMSM0,115200n8 pcie_pme=nomsi earlycon qcom_scm.download_mode=1 reboot=panic_warm" GRUB_DEVICE="/dev/disk/by-partlabel/system" GRUB_TERMINAL="console" GRUB_DISABLE_LINUX_UUID="true" @@ -114,49 +115,23 @@ def parse_manifests(self): self.QCOM_MANIFEST = None # 1. User-provided manifest if self.PACKAGES_MANIFEST_PATH: - user_manifest = Path(self.PACKAGES_MANIFEST_PATH) - if not user_manifest.is_file() or not user_manifest.name.endswith('.manifest'): - raise ValueError(f"Provided manifest path '{user_manifest}' is not a valid '.manifest' file.") - self.BASE_MANIFEST = str(user_manifest) - logger.info(f"Packages manifest path: {self.BASE_MANIFEST}") + logger.info(f"Packages manifest path: {self.PACKAGES_MANIFEST_PATH}") # Load packages from user manifest - with open(self.BASE_MANIFEST, 'r') as f: - for line in f: - line = line.strip() - if line and not line.startswith('#'): - parts = list(line.split('\t')) - self.DEBS.append({ - 'package': parts[0], - 'version': parts[1] if len(parts) > 1 else None, - }) + self.DEBS = parse_debs_manifest(self.PACKAGES_MANIFEST_PATH) return # Done if user manifest is found and valid + # 2. Default manifest(s) from packages/base and/or packages/qcom base_path = os.path.join(self.cur_file, "packages", "base", f"{self.IMAGE_TYPE}.manifest") if os.path.isfile(base_path): self.BASE_MANIFEST = base_path - with open(self.BASE_MANIFEST, 'r') as f: - for line in f: - line = line.strip() - if line and not line.startswith('#'): - parts = list(line.split('\t')) - self.DEBS.append({ - 'package': parts[0], - 'version': parts[1] if len(parts) > 1 else None, - }) + logger.debug(f"Using base manifest: {self.BASE_MANIFEST}") + self.DEBS = parse_debs_manifest(self.BASE_MANIFEST) # Also include qcom manifest if variant == qcom if self.VARIANT == "qcom": qcom_path = os.path.join(self.cur_file, "packages", "qcom", f"{self.IMAGE_TYPE}.manifest") - if os.path.isfile(qcom_path): - self.QCOM_MANIFEST = qcom_path - with open(self.QCOM_MANIFEST, 'r') as f: - for line in f: - line = line.strip() - if line and not line.startswith('#'): - parts = list(line.split('\t')) - self.DEBS.append({ - 'package': parts[0], - 'version': parts[1] if len(parts) > 1 else None, - }) + self.QCOM_MANIFEST = qcom_path + logger.debug(f"Using QCOM manifest: {self.QCOM_MANIFEST}") + self.DEBS.extend(parse_debs_manifest(self.QCOM_MANIFEST)) return # 3. No manifest found: print message and exit logger.error("No manifest found. Please provide a valid .manifest file via PACKAGES_MANIFEST_PATH or ensure default manifests exist.") @@ -191,6 +166,7 @@ def build_image(self): bash_command = f""" sudo mmdebstrap --verbose --logfile={log_file} \ --customize-hook='echo root:password | chroot "$1" chpasswd' \ +--customize-hook='cp {self.cur_file}/99-network-manager.cfg "$1/etc/cloud/cloud.cfg.d/99-network-manager.cfg"' \ --customize-hook='echo "PermitRootLogin yes" >> "$1/etc/ssh/sshd_config"' \ --setup-hook='echo /dev/disk/by-partlabel/system / ext4 defaults 0 1 > "$1/etc/fstab"' \ --arch=arm64 \ diff --git a/ubuntu/read_dsc.py b/ubuntu/read_dsc.py index a5c288f..88e7a91 100644 --- a/ubuntu/read_dsc.py +++ b/ubuntu/read_dsc.py @@ -5,8 +5,8 @@ """ read_dsc.py -This script provides a function to extract MD5 checksums, sizes, and filenames from a Debian source control (DSC) file. -It allows for filtering of the extracted files based on an optional filename pattern, making it useful for +This script provides a function to extract MD5 checksums, sizes, and filenames from a Debian source control (DSC) file. +It allows for filtering of the extracted files based on an optional filename pattern, making it useful for analyzing package files in Debian packaging workflows. """ @@ -48,10 +48,10 @@ def extract_md5sum_from_files(dsc_path, filename_pattern=None): if line.startswith('Files:'): in_files_section = True continue - + if in_files_section and not line.startswith(' '): break - + if in_files_section and line.startswith(' '): parts = line.strip().split(maxsplit=2) if len(parts) == 3: @@ -62,5 +62,5 @@ def extract_md5sum_from_files(dsc_path, filename_pattern=None): 'size': size, 'filename': filename }) - + return entries diff --git a/ubuntu/requirements.txt b/ubuntu/requirements.txt index 0e4489e..db4e9a0 100644 --- a/ubuntu/requirements.txt +++ b/ubuntu/requirements.txt @@ -11,4 +11,4 @@ lxml==5.3.1 pymssql==2.3.2 requests==2.32.3 smmap==5.0.2 -urllib3==2.3.0 +urllib3==2.3.0 \ No newline at end of file