From 00a465df97c1254b132bcaba3d5f9214126af511 Mon Sep 17 00:00:00 2001 From: Pablo Cholaky Date: Sun, 16 Jan 2022 13:52:07 -0500 Subject: [PATCH 01/13] extract_utils: Ensure comm uses LC_COLLATE=C comm behavior will fail if does not receive a properly lexicographic sort. Not using appropiated LC_COLLATE leads to error. This is matching current LC_ALL=C already used at sort commands. Change-Id: I77f8acb419545a74812d3b36ba608719c2aa89e3 --- extract_utils.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/extract_utils.sh b/extract_utils.sh index 80094c5..a84aaf0 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -685,9 +685,9 @@ function write_product_packages() { # I really should not be doing this in bash due to shitty array passing :( local T_LIB32=( $(prefix_match "lib/") ) local T_LIB64=( $(prefix_match "lib64/") ) - local MULTILIBS=( $(comm -12 <(printf '%s\n' "${T_LIB32[@]}") <(printf '%s\n' "${T_LIB64[@]}")) ) - local LIB32=( $(comm -23 <(printf '%s\n' "${T_LIB32[@]}") <(printf '%s\n' "${MULTILIBS[@]}")) ) - local LIB64=( $(comm -23 <(printf '%s\n' "${T_LIB64[@]}") <(printf '%s\n' "${MULTILIBS[@]}")) ) + local MULTILIBS=( $(LC_ALL=C comm -12 <(printf '%s\n' "${T_LIB32[@]}") <(printf '%s\n' "${T_LIB64[@]}")) ) + local LIB32=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_LIB32[@]}") <(printf '%s\n' "${MULTILIBS[@]}")) ) + local LIB64=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_LIB64[@]}") <(printf '%s\n' "${MULTILIBS[@]}")) ) if [ "${#MULTILIBS[@]}" -gt "0" ]; then write_blueprint_packages "SHARED_LIBRARIES" "" "both" "MULTILIBS" >> "$ANDROIDBP" @@ -701,9 +701,9 @@ function write_product_packages() { local T_S_LIB32=( $(prefix_match "system/lib/") ) local T_S_LIB64=( $(prefix_match "system/lib64/") ) - local S_MULTILIBS=( $(comm -12 <(printf '%s\n' "${T_S_LIB32[@]}") <(printf '%s\n' "${T_S_LIB64[@]}")) ) - local S_LIB32=( $(comm -23 <(printf '%s\n' "${T_S_LIB32[@]}") <(printf '%s\n' "${S_MULTILIBS[@]}")) ) - local S_LIB64=( $(comm -23 <(printf '%s\n' "${T_S_LIB64[@]}") <(printf '%s\n' "${S_MULTILIBS[@]}")) ) + local S_MULTILIBS=( $(LC_ALL=C comm -12 <(printf '%s\n' "${T_S_LIB32[@]}") <(printf '%s\n' "${T_S_LIB64[@]}")) ) + local S_LIB32=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_S_LIB32[@]}") <(printf '%s\n' "${S_MULTILIBS[@]}")) ) + local S_LIB64=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_S_LIB64[@]}") <(printf '%s\n' "${S_MULTILIBS[@]}")) ) if [ "${#S_MULTILIBS[@]}" -gt "0" ]; then write_blueprint_packages "SHARED_LIBRARIES" "system" "both" "S_MULTILIBS" >> "$ANDROIDBP" @@ -717,9 +717,9 @@ function write_product_packages() { local T_V_LIB32=( $(prefix_match "vendor/lib/") ) local T_V_LIB64=( $(prefix_match "vendor/lib64/") ) - local V_MULTILIBS=( $(comm -12 <(printf '%s\n' "${T_V_LIB32[@]}") <(printf '%s\n' "${T_V_LIB64[@]}")) ) - local V_LIB32=( $(comm -23 <(printf '%s\n' "${T_V_LIB32[@]}") <(printf '%s\n' "${V_MULTILIBS[@]}")) ) - local V_LIB64=( $(comm -23 <(printf '%s\n' "${T_V_LIB64[@]}") <(printf '%s\n' "${V_MULTILIBS[@]}")) ) + local V_MULTILIBS=( $(LC_ALL=C comm -12 <(printf '%s\n' "${T_V_LIB32[@]}") <(printf '%s\n' "${T_V_LIB64[@]}")) ) + local V_LIB32=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_V_LIB32[@]}") <(printf '%s\n' "${V_MULTILIBS[@]}")) ) + local V_LIB64=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_V_LIB64[@]}") <(printf '%s\n' "${V_MULTILIBS[@]}")) ) if [ "${#V_MULTILIBS[@]}" -gt "0" ]; then write_blueprint_packages "SHARED_LIBRARIES" "vendor" "both" "V_MULTILIBS" >> "$ANDROIDBP" @@ -733,9 +733,9 @@ function write_product_packages() { local T_P_LIB32=( $(prefix_match "product/lib/") ) local T_P_LIB64=( $(prefix_match "product/lib64/") ) - local P_MULTILIBS=( $(comm -12 <(printf '%s\n' "${T_P_LIB32[@]}") <(printf '%s\n' "${T_P_LIB64[@]}")) ) - local P_LIB32=( $(comm -23 <(printf '%s\n' "${T_P_LIB32[@]}") <(printf '%s\n' "${P_MULTILIBS[@]}")) ) - local P_LIB64=( $(comm -23 <(printf '%s\n' "${T_P_LIB64[@]}") <(printf '%s\n' "${P_MULTILIBS[@]}")) ) + local P_MULTILIBS=( $(LC_ALL=C comm -12 <(printf '%s\n' "${T_P_LIB32[@]}") <(printf '%s\n' "${T_P_LIB64[@]}")) ) + local P_LIB32=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_P_LIB32[@]}") <(printf '%s\n' "${P_MULTILIBS[@]}")) ) + local P_LIB64=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_P_LIB64[@]}") <(printf '%s\n' "${P_MULTILIBS[@]}")) ) if [ "${#P_MULTILIBS[@]}" -gt "0" ]; then write_blueprint_packages "SHARED_LIBRARIES" "product" "both" "P_MULTILIBS" >> "$ANDROIDBP" @@ -749,9 +749,9 @@ function write_product_packages() { local T_SE_LIB32=( $(prefix_match "system_ext/lib/") ) local T_SE_LIB64=( $(prefix_match "system_ext/lib64/") ) - local SE_MULTILIBS=( $(comm -12 <(printf '%s\n' "${T_SE_LIB32[@]}") <(printf '%s\n' "${T_SE_LIB64[@]}")) ) - local SE_LIB32=( $(comm -23 <(printf '%s\n' "${T_SE_LIB32[@]}") <(printf '%s\n' "${SE_MULTILIBS[@]}")) ) - local SE_LIB64=( $(comm -23 <(printf '%s\n' "${T_SE_LIB64[@]}") <(printf '%s\n' "${SE_MULTILIBS[@]}")) ) + local SE_MULTILIBS=( $(LC_ALL=C comm -12 <(printf '%s\n' "${T_SE_LIB32[@]}") <(printf '%s\n' "${T_SE_LIB64[@]}")) ) + local SE_LIB32=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_SE_LIB32[@]}") <(printf '%s\n' "${SE_MULTILIBS[@]}")) ) + local SE_LIB64=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_SE_LIB64[@]}") <(printf '%s\n' "${SE_MULTILIBS[@]}")) ) if [ "${#SE_MULTILIBS[@]}" -gt "0" ]; then write_blueprint_packages "SHARED_LIBRARIES" "system_ext" "both" "SE_MULTILIBS" >> "$ANDROIDBP" @@ -765,9 +765,9 @@ function write_product_packages() { local T_O_LIB32=( $(prefix_match "odm/lib/") ) local T_O_LIB64=( $(prefix_match "odm/lib64/") ) - local O_MULTILIBS=( $(comm -12 <(printf '%s\n' "${T_O_LIB32[@]}") <(printf '%s\n' "${T_O_LIB64[@]}")) ) - local O_LIB32=( $(comm -23 <(printf '%s\n' "${T_O_LIB32[@]}") <(printf '%s\n' "${O_MULTILIBS[@]}")) ) - local O_LIB64=( $(comm -23 <(printf '%s\n' "${T_O_LIB64[@]}") <(printf '%s\n' "${O_MULTILIBS[@]}")) ) + local O_MULTILIBS=( $(LC_ALL=C comm -12 <(printf '%s\n' "${T_O_LIB32[@]}") <(printf '%s\n' "${T_O_LIB64[@]}")) ) + local O_LIB32=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_O_LIB32[@]}") <(printf '%s\n' "${O_MULTILIBS[@]}")) ) + local O_LIB64=( $(LC_ALL=C comm -23 <(printf '%s\n' "${T_O_LIB64[@]}") <(printf '%s\n' "${O_MULTILIBS[@]}")) ) if [ "${#O_MULTILIBS[@]}" -gt "0" ]; then write_blueprint_packages "SHARED_LIBRARIES" "odm" "both" "O_MULTILIBS" >> "$ANDROIDBP" From 9df0d3ac1aaf4a7c350ab4bbf3f4b3de604a8950 Mon Sep 17 00:00:00 2001 From: Alexander Koskovich Date: Sat, 22 Jan 2022 22:27:29 -0700 Subject: [PATCH 02/13] extract_utils: Support copying out to /vendor_dlkm. Change-Id: I600c9d9eb8295e92da2016093462c6d658ca4062 --- extract_utils.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extract_utils.sh b/extract_utils.sh index a84aaf0..883986b 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -303,6 +303,10 @@ function write_product_copy_files() { local OUTTARGET=$(truncate_file $TARGET) printf ' %s/proprietary/%s:$(TARGET_COPY_OUT_VENDOR)/%s%s\n' \ "$OUTDIR" "$TARGET" "$OUTTARGET" "$LINEEND" >> "$PRODUCTMK" + elif prefix_match_file "vendor_dlkm/" $TARGET ; then + local OUTTARGET=$(truncate_file $TARGET) + printf ' %s/proprietary/%s:$(TARGET_COPY_OUT_VENDOR_DLKM)/%s%s\n' \ + "$OUTDIR" "$TARGET" "$OUTTARGET" "$LINEEND" >> "$PRODUCTMK" elif prefix_match_file "system/vendor/" $TARGET ; then local OUTTARGET=$(truncate_file $TARGET) printf ' %s/proprietary/%s:$(TARGET_COPY_OUT_VENDOR)/%s%s\n' \ From b223e7a8143bac08b67350aaa53d70885d06681f Mon Sep 17 00:00:00 2001 From: Tim Zimmermann Date: Thu, 23 Dec 2021 07:06:17 +0100 Subject: [PATCH 03/13] extract_utils: allow overriding package name * Useful for BUILD_BROKEN_ELF_PREBUILT_PRODUCT_COPY_FILES if package name conflicts with something source-built but the output location differs Change-Id: I3af45eae6b2d2cfc583128628edfef9e37ea3976 --- extract_utils.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/extract_utils.sh b/extract_utils.sh index 883986b..6be4e85 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -351,17 +351,28 @@ function write_blueprint_packages() { local EXTENSION= local PKGNAME= local SRC= + local STEM= local OVERRIDEPKG= for P in "${FILELIST[@]}"; do FILE=$(target_file "$P") ARGS=$(target_args "$P") + ARGS=(${ARGS//;/ }) BASENAME=$(basename "$FILE") DIRNAME=$(dirname "$FILE") EXTENSION=${BASENAME##*.} PKGNAME=${BASENAME%.*} + # Allow overriding module name + STEM= + for ARG in "${ARGS[@]}"; do + if [[ "$ARG" =~ "MODULE" ]]; then + STEM="$PKGNAME" + PKGNAME=${ARG#*=} + fi + done + # Add to final package list PACKAGE_LIST+=("$PKGNAME") @@ -381,6 +392,9 @@ function write_blueprint_packages() { if [ "$CLASS" = "SHARED_LIBRARIES" ]; then printf 'cc_prebuilt_library_shared {\n' printf '\tname: "%s",\n' "$PKGNAME" + if [ ! -z "$STEM" ]; then + printf '\tstem: "%s",\n' "$STEM" + fi printf '\towner: "%s",\n' "$VENDOR" printf '\tstrip: {\n' printf '\t\tnone: true,\n' @@ -425,7 +439,6 @@ function write_blueprint_packages() { SRC="$SRC/app" fi printf '\tapk: "%s/%s",\n' "$SRC" "$FILE" - ARGS=(${ARGS//;/ }) USE_PLATFORM_CERTIFICATE="true" for ARG in "${ARGS[@]}"; do if [ "$ARG" = "PRESIGNED" ]; then From 6c20accc160a4b6f560ffeb9bf8fd9561692514b Mon Sep 17 00:00:00 2001 From: Tim Zimmermann Date: Thu, 23 Dec 2021 09:26:54 +0100 Subject: [PATCH 04/13] extract_utils: disable strip for ELF binaries as well * Just like we do for libraries * Without this some binaries might break Change-Id: Ida82f5ee35f02112eb3a35be66c2f7e1642d56b1 --- extract_utils.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extract_utils.sh b/extract_utils.sh index 6be4e85..b0c4380 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -494,9 +494,9 @@ function write_blueprint_packages() { if [ "$EXTENSION" != "sh" ]; then printf '\tsrcs: ["%s/%s"],\n' "$SRC" "$FILE" printf '\tcheck_elf_files: false,\n' - printf '\tprefer: true,\n' - else - printf '\tsrc: "%s/%s",\n' "$SRC" "$FILE" + printf '\tstrip: {\n' + printf '\t\tnone: true,\n' + printf '\t},\n' fi unset EXTENSION else From 495d2b06205c7559a9eebe6caf66b07c4235d83d Mon Sep 17 00:00:00 2001 From: Tim Zimmermann Date: Thu, 23 Dec 2021 10:15:26 +0100 Subject: [PATCH 05/13] extract_utils: do not cut off extension for cc_binary_prebuilt * Otherwise extracting HAL services breaks Change-Id: Ic6302df5c9b512adf4ca81c49cfb1d472aa15c84 --- extract_utils.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extract_utils.sh b/extract_utils.sh index b0c4380..7cbd701 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -364,6 +364,11 @@ function write_blueprint_packages() { EXTENSION=${BASENAME##*.} PKGNAME=${BASENAME%.*} + if [ "$CLASS" = "EXECUTABLES" ] && [ "$EXTENSION" != "sh" ]; then + PKGNAME="$BASENAME" + EXTENSION="" + fi + # Allow overriding module name STEM= for ARG in "${ARGS[@]}"; do From bbba5383df125560795fa4557a56a0e6f62d63da Mon Sep 17 00:00:00 2001 From: Michael Bestas Date: Mon, 28 Dec 2020 04:44:52 +0200 Subject: [PATCH 06/13] extract_utils: Remove support for /sbin Android 11 has removed /sbin support completely Remove rootfs handling as that was only added for /sbin Remove write_makefile_packages since it's no longer used Change-Id: Ie44656d5e30da9637a850df53c4bf3fa1ab0fb2b --- extract_utils.sh | 184 +++-------------------------------------------- 1 file changed, 8 insertions(+), 176 deletions(-) diff --git a/extract_utils.sh b/extract_utils.sh index 7cbd701..64e7683 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -485,17 +485,9 @@ function write_blueprint_packages() { printf '\tname: "%s",\n' "$BASENAME" printf '\towner: "%s",\n' "$VENDOR" fi - if [ "$ARGS" = "rootfs" ]; then - SRC="$SRC/rootfs" - if [ "$EXTRA" = "sbin" ]; then - SRC="$SRC/sbin" - printf '\tdist {\n' - printf '\t\tdest: "%s",\n' "root/sbin" - printf '\t},' - fi - else - SRC="$SRC/bin" - fi + printf '\tname: "%s",\n' "$PKGNAME" + printf '\towner: "%s",\n' "$VENDOR" + printf '\tsrcs: ["%s/bin/%s"],\n' "$SRC" "$FILE" if [ "$EXTENSION" != "sh" ]; then printf '\tsrcs: ["%s/%s"],\n' "$SRC" "$FILE" printf '\tcheck_elf_files: false,\n' @@ -538,154 +530,6 @@ function write_blueprint_packages() { done } -# -# write_makefile_packages: -# -# $1: The LOCAL_MODULE_CLASS for the given module list -# $2: /odm, /product, /system_ext, or /vendor partition -# $3: type-specific extra flags -# $4: Name of the array holding the target list -# -# Internal function which writes out the BUILD_PREBUILT stanzas -# for all modules in the list. This is called by write_product_packages -# after the modules are categorized. -# -function write_makefile_packages() { - - local CLASS="$1" - local PARTITION="$2" - local EXTRA="$3" - - # Yes, this is a horrible hack - we create a new array using indirection - local ARR_NAME="$4[@]" - local FILELIST=("${!ARR_NAME}") - - local FILE= - local ARGS= - local BASENAME= - local EXTENSION= - local PKGNAME= - local SRC= - - for P in "${FILELIST[@]}"; do - FILE=$(target_file "$P") - ARGS=$(target_args "$P") - - BASENAME=$(basename "$FILE") - DIRNAME=$(dirname "$FILE") - EXTENSION=${BASENAME##*.} - EXTENSION="."$EXTENSION - if [ "$EXTENSION" = ".jar" ]; then - EXTENSION="\$(COMMON_JAVA_PACKAGE_SUFFIX)" - elif [ "$EXTENSION" = ".apk" ]; then - EXTENSION="\$(COMMON_ANDROID_PACKAGE_SUFFIX)" - fi - PKGNAME=${BASENAME%.*} - - # Add to final package list - PACKAGE_LIST+=("$PKGNAME") - - SRC="proprietary" - if [ "$PARTITION" = "system" ]; then - SRC+="/system" - elif [ "$PARTITION" = "vendor" ]; then - SRC+="/vendor" - elif [ "$PARTITION" = "product" ]; then - SRC+="/product" - elif [ "$PARTITION" = "system_ext" ]; then - SRC+="/system_ext" - elif [ "$PARTITION" = "odm" ]; then - SRC+="/odm" - fi - - printf 'include $(CLEAR_VARS)\n' - printf 'LOCAL_MODULE := %s\n' "$PKGNAME" - printf 'LOCAL_MODULE_OWNER := %s\n' "$VENDOR" - if [ "$CLASS" = "SHARED_LIBRARIES" ]; then - if [ "$EXTRA" = "both" ]; then - printf 'LOCAL_SRC_FILES_64 := %s/lib64/%s\n' "$SRC" "$FILE" - printf 'LOCAL_SRC_FILES_32 := %s/lib/%s\n' "$SRC" "$FILE" - #if [ "$VENDOR_PKG" = "true" ]; then - # echo "LOCAL_MODULE_PATH_64 := \$(TARGET_OUT_VENDOR_SHARED_LIBRARIES)" - # echo "LOCAL_MODULE_PATH_32 := \$(2ND_TARGET_OUT_VENDOR_SHARED_LIBRARIES)" - #else - # echo "LOCAL_MODULE_PATH_64 := \$(TARGET_OUT_SHARED_LIBRARIES)" - # echo "LOCAL_MODULE_PATH_32 := \$(2ND_TARGET_OUT_SHARED_LIBRARIES)" - #fi - elif [ "$EXTRA" = "64" ]; then - printf 'LOCAL_SRC_FILES := %s/lib64/%s\n' "$SRC" "$FILE" - else - printf 'LOCAL_SRC_FILES := %s/lib/%s\n' "$SRC" "$FILE" - fi - if [ "$EXTRA" != "none" ]; then - printf 'LOCAL_MULTILIB := %s\n' "$EXTRA" - fi - elif [ "$CLASS" = "APPS" ]; then - if [ "$EXTRA" = "priv-app" ]; then - SRC="$SRC/priv-app" - else - SRC="$SRC/app" - fi - printf 'LOCAL_SRC_FILES := %s/%s\n' "$SRC" "$FILE" - local CERT=platform - if [ ! -z "$ARGS" ]; then - CERT="$ARGS" - fi - printf 'LOCAL_CERTIFICATE := %s\n' "$CERT" - elif [ "$CLASS" = "JAVA_LIBRARIES" ]; then - printf 'LOCAL_SRC_FILES := %s/framework/%s\n' "$SRC" "$FILE" - local CERT=platform - if [ ! -z "$ARGS" ]; then - CERT="$ARGS" - fi - printf 'LOCAL_CERTIFICATE := %s\n' "$CERT" - elif [ "$CLASS" = "ETC" ]; then - printf 'LOCAL_SRC_FILES := %s/etc/%s\n' "$SRC" "$FILE" - elif [ "$CLASS" = "EXECUTABLES" ]; then - if [ "$ARGS" = "rootfs" ]; then - SRC="$SRC/rootfs" - if [ "$EXTRA" = "sbin" ]; then - SRC="$SRC/sbin" - printf '%s\n' "LOCAL_MODULE_PATH := \$(TARGET_ROOT_OUT_SBIN)" - printf '%s\n' "LOCAL_UNSTRIPPED_PATH := \$(TARGET_ROOT_OUT_SBIN_UNSTRIPPED)" - fi - else - SRC="$SRC/bin" - fi - printf 'LOCAL_SRC_FILES := %s/%s\n' "$SRC" "$FILE" - unset EXTENSION - else - printf 'LOCAL_SRC_FILES := %s/%s\n' "$SRC" "$FILE" - fi - printf 'LOCAL_MODULE_TAGS := optional\n' - printf 'LOCAL_MODULE_CLASS := %s\n' "$CLASS" - if [ "$CLASS" = "APPS" ]; then - printf 'LOCAL_DEX_PREOPT := false\n' - fi - if [ ! -z "$EXTENSION" ]; then - printf 'LOCAL_MODULE_SUFFIX := %s\n' "$EXTENSION" - fi - if [ "$CLASS" = "SHARED_LIBRARIES" ] || [ "$CLASS" = "EXECUTABLES" ]; then - if [ "$DIRNAME" != "." ]; then - printf 'LOCAL_MODULE_RELATIVE_PATH := %s\n' "$DIRNAME" - fi - fi - if [ "$EXTRA" = "priv-app" ]; then - printf 'LOCAL_PRIVILEGED_MODULE := true\n' - fi - if [ "$PARTITION" = "vendor" ]; then - printf 'LOCAL_VENDOR_MODULE := true\n' - elif [ "$PARTITION" = "product" ]; then - printf 'LOCAL_PRODUCT_MODULE := true\n' - elif [ "$PARTITION" = "system_ext" ]; then - printf 'LOCAL_SYSTEM_EXT_MODULE := true\n' - elif [ "$PARTITION" = "odm" ]; then - printf 'LOCAL_ODM_MODULE := true\n' - fi - printf 'include $(BUILD_PREBUILT)\n\n' - done -} - # # write_product_packages: # @@ -946,11 +790,6 @@ function write_product_packages() { if [ "${#O_BIN[@]}" -gt "0" ]; then write_blueprint_packages "EXECUTABLES" "odm" "" "O_BIN" >> "$ANDROIDBP" fi - local SBIN=( $(prefix_match "sbin/") ) - if [ "${#SBIN[@]}" -gt "0" ]; then - write_makefile_packages "EXECUTABLES" "" "sbin" "SBIN" >> "$ANDROIDMK" - fi - # Actually write out the final PRODUCT_PACKAGES list local PACKAGE_COUNT=${#PACKAGE_LIST[@]} @@ -1614,17 +1453,10 @@ function extract() { IS_PRODUCT_PACKAGE=true fi - if [ "${SPEC_ARGS}" = "rootfs" ]; then - OUTPUT_DIR="${OUTPUT_ROOT}/rootfs" - TMP_DIR="${OUTPUT_TMP}/rootfs" - SRC_FILE="/${SPEC_SRC_FILE}" - DST_FILE="/${SPEC_DST_FILE}" - else - OUTPUT_DIR="${OUTPUT_ROOT}" - TMP_DIR="${OUTPUT_TMP}" - SRC_FILE="/system/${SPEC_SRC_FILE}" - DST_FILE="/system/${SPEC_DST_FILE}" - fi + OUTPUT_DIR="${OUTPUT_ROOT}" + TMP_DIR="${OUTPUT_TMP}" + SRC_FILE="/system/${SPEC_SRC_FILE}" + DST_FILE="/system/${SPEC_DST_FILE}" # Strip the file path in the vendor repo of "system", if present local BLOB_DISPLAY_NAME="${DST_FILE#/system/}" @@ -1702,7 +1534,7 @@ function extract() { if [ -f "${VENDOR_REPO_FILE}" ]; then local DIR=$(dirname "${VENDOR_REPO_FILE}") local TYPE="${DIR##*/}" - if [ "$TYPE" = "bin" -o "$TYPE" = "sbin" ]; then + if [ "$TYPE" = "bin" ]; then chmod 755 "${VENDOR_REPO_FILE}" else chmod 644 "${VENDOR_REPO_FILE}" From 5413cb75db01f2cb7e44a27e292fe2df89b810a9 Mon Sep 17 00:00:00 2001 From: Chenyang Zhong Date: Thu, 10 Feb 2022 21:40:41 -0500 Subject: [PATCH 07/13] extract_utils: sort the package list before writing blueprints "comm" requires both input to be sorted. The T_X_LIB32 and T_X_LIB64 arrays from prefix_match-ing PRODUCT_PACKAGES_LIST may not always be sorted, which causes certain multilib to be mis-categorized as 32-bit and 64-bit separately. The final blueprint would then have duplicated targets. Therefore, sort the package list first so that arrays generated from the package list are also sorted. Set LC_ALL=C to avoid "sort" using system locale. Change-Id: Ia2bc9733d89de66eda52059237a54d624ba4b0ad Signed-off-by: Chenyang Zhong --- extract_utils.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extract_utils.sh b/extract_utils.sh index 64e7683..1613a8e 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -541,6 +541,9 @@ function write_blueprint_packages() { function write_product_packages() { PACKAGE_LIST=() + # Sort the package list for comm + PRODUCT_PACKAGES_LIST=($( printf '%s\n' "${PRODUCT_PACKAGES_LIST[@]}" | LC_ALL=C sort)) + local COUNT=${#PRODUCT_PACKAGES_LIST[@]} if [ "$COUNT" = "0" ]; then From cdc487c39f7da44ede06ee6e88605ca0949eed93 Mon Sep 17 00:00:00 2001 From: Chirayu Desai Date: Thu, 17 Mar 2022 06:27:33 +0530 Subject: [PATCH 08/13] Use simg2img from prebuilts/extract-tools Change-Id: I15772394a24c2ff1fdd6eece86548a72a5d4d748 --- extract_utils.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/extract_utils.sh b/extract_utils.sh index 1613a8e..0d6541c 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -94,6 +94,18 @@ function setup_vendor() { VENDOR_RADIO_STATE=0 fi + export BINARIES_LOCATION="$ANDROID_ROOT"/prebuilts/extract-tools/${HOST}-x86/bin + + export SIMG2IMG="$BINARIES_LOCATION"/simg2img + + for version in 0_8 0_9; do + export PATCHELF_${version}="$BINARIES_LOCATION"/patchelf-"${version}" + done + + if [ -z "$PATCHELF_VERSION" ]; then + export PATCHELF_VERSION=0_9 + fi + if [ -z "$PATCHELF" ]; then export PATCHELF="$ANDROID_ROOT"/prebuilts/extract-tools/${HOST}-x86/bin/patchelf fi @@ -1416,7 +1428,7 @@ function extract() { if [[ $(file -b "$IMAGE") == Linux* ]]; then extract_img_data "$IMAGE" "$DUMPDIR"/"$PARTITION" elif [[ $(file -b "$IMAGE") == Android* ]]; then - simg2img "$IMAGE" "$DUMPDIR"/"$PARTITION".raw + "$SIMG2IMG" "$IMAGE" "$DUMPDIR"/"$PARTITION".raw extract_img_data "$DUMPDIR"/"$PARTITION".raw "$DUMPDIR"/"$PARTITION"/ else echo "Unsupported "$IMAGE"" @@ -1660,7 +1672,7 @@ function generate_prop_list_from_image() { if [[ $(file -b "$image_file") == Linux* ]]; then extract_img_data "$image_file" "$image_dir" elif [[ $(file -b "$image_file") == Android* ]]; then - simg2img "$image_file" "$image_dir"/"$(basename "$image_file").raw" + "$SIMG2IMG" "$image_file" "$image_dir"/"$(basename "$image_file").raw" extract_img_data "$image_dir"/"$(basename "$image_file").raw" "$image_dir" rm "$image_dir"/"$(basename "$image_file").raw" else From d71b47469050867314e59e1d91deb1e87f64cdcd Mon Sep 17 00:00:00 2001 From: Michael Bestas Date: Fri, 18 Feb 2022 23:45:09 +0200 Subject: [PATCH 09/13] extract_utils: Support extracting from super.img Change-Id: I2a30da25f2d55df1080b1795e08a14195c1e3ec0 --- extract_utils.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/extract_utils.sh b/extract_utils.sh index 0d6541c..3e0eb12 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -97,6 +97,7 @@ function setup_vendor() { export BINARIES_LOCATION="$ANDROID_ROOT"/prebuilts/extract-tools/${HOST}-x86/bin export SIMG2IMG="$BINARIES_LOCATION"/simg2img + export LPUNPACK="$BINARIES_LOCATION"/lpunpack for version in 0_8 0_9; do export PATCHELF_${version}="$BINARIES_LOCATION"/patchelf-"${version}" @@ -1416,6 +1417,26 @@ function extract() { SRC="$DUMPDIR" fi + if [ -d "$SRC" ] && [ -f "$SRC"/super.img ]; then + DUMPDIR="$TMPDIR"/super_dump + mkdir -p "$DUMPDIR" + + echo "Unpacking super.img" + "$SIMG2IMG" "$SRC"/super.img "$DUMPDIR"/super.raw + + for PARTITION in "system" "odm" "product" "system_ext" "vendor" + do + echo "Preparing "$PARTITION"" + if "$LPUNPACK" -p "$PARTITION"_a "$DUMPDIR"/super.raw "$DUMPDIR" ; then + mv "$DUMPDIR"/"$PARTITION"_a.img "$DUMPDIR"/"$PARTITION".img + else + "$LPUNPACK" -p "$PARTITION" "$DUMPDIR"/super.raw "$DUMPDIR" + fi + done + + SRC="$DUMPDIR" + fi + if [ -d "$SRC" ] && [ -f "$SRC"/system.img ]; then DUMPDIR="$TMPDIR"/system_dump mkdir -p "$DUMPDIR" From ebf49e042933ebd2fa3ec645f5f8859aac729444 Mon Sep 17 00:00:00 2001 From: Michael Bestas Date: Wed, 23 Mar 2022 23:15:23 +0200 Subject: [PATCH 10/13] extract_utils: export SIGSCAN For use in device scripts Change-Id: I7a8c9abd0b585a67963e65d5ad042da79d4d3c77 --- extract_utils.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/extract_utils.sh b/extract_utils.sh index 3e0eb12..3b7d478 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -98,6 +98,7 @@ function setup_vendor() { export SIMG2IMG="$BINARIES_LOCATION"/simg2img export LPUNPACK="$BINARIES_LOCATION"/lpunpack + export SIGSCAN="$BINARIES_LOCATION"/SigScan for version in 0_8 0_9; do export PATCHELF_${version}="$BINARIES_LOCATION"/patchelf-"${version}" From 320f943de32e0e69c5304be5d9d8a06a6697365f Mon Sep 17 00:00:00 2001 From: Chirayu Desai Date: Tue, 5 Apr 2022 03:32:25 +0530 Subject: [PATCH 11/13] Split up setup_vendor * For usage by regen-vendor.sh Change-Id: Ideb3d4ea68e7aa6ef4a5a73c15d507d47fda4b56 --- extract_utils.sh | 51 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/extract_utils.sh b/extract_utils.sh index 3b7d478..86acf62 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -33,6 +33,39 @@ function cleanup() { trap cleanup 0 +# +# setup_vendor_deps +# +# $1: Android root directory +# Sets up common dependencies for extraction +# +function setup_vendor_deps() { + export ANDROID_ROOT="$1" + if [ ! -d "$ANDROID_ROOT" ]; then + echo "\$ANDROID_ROOT must be set and valid before including this script!" + exit 1 + fi + + export BINARIES_LOCATION="$ANDROID_ROOT"/prebuilts/extract-tools/${HOST}-x86/bin + + export SIMG2IMG="$BINARIES_LOCATION"/simg2img + export LPUNPACK="$BINARIES_LOCATION"/lpunpack + export SIGSCAN="$BINARIES_LOCATION"/SigScan + + for version in 0_8 0_9; do + export PATCHELF_${version}="$BINARIES_LOCATION"/patchelf-"${version}" + done + + if [ -z "$PATCHELF_VERSION" ]; then + export PATCHELF_VERSION=0_9 + fi + + if [ -z "$PATCHELF" ]; then + local patchelf_variable="PATCHELF_${PATCHELF_VERSION}" + export PATCHELF=${!patchelf_variable} + fi +} + # # setup_vendor # @@ -94,23 +127,7 @@ function setup_vendor() { VENDOR_RADIO_STATE=0 fi - export BINARIES_LOCATION="$ANDROID_ROOT"/prebuilts/extract-tools/${HOST}-x86/bin - - export SIMG2IMG="$BINARIES_LOCATION"/simg2img - export LPUNPACK="$BINARIES_LOCATION"/lpunpack - export SIGSCAN="$BINARIES_LOCATION"/SigScan - - for version in 0_8 0_9; do - export PATCHELF_${version}="$BINARIES_LOCATION"/patchelf-"${version}" - done - - if [ -z "$PATCHELF_VERSION" ]; then - export PATCHELF_VERSION=0_9 - fi - - if [ -z "$PATCHELF" ]; then - export PATCHELF="$ANDROID_ROOT"/prebuilts/extract-tools/${HOST}-x86/bin/patchelf - fi + setup_vendor_deps "$ANDROID_ROOT" } # Helper functions for parsing a spec. From 89a9ca673cc77a4508d4fcef07af636db97840e8 Mon Sep 17 00:00:00 2001 From: Sebastiano Barezzi Date: Sat, 28 May 2022 15:13:49 +0200 Subject: [PATCH 12/13] extract_utils: Add a script to reorder blobs list Change-Id: I76e85e07f5a562b7e2a3683da264ccebff5c3613 --- sort-blobs-list.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100755 sort-blobs-list.py diff --git a/sort-blobs-list.py b/sort-blobs-list.py new file mode 100755 index 0000000..a0581d4 --- /dev/null +++ b/sort-blobs-list.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021-2022 The LineageOS Project +# +# SPDX-License-Identifier: Apache-2.0 +# + +import re, sys +from functools import cmp_to_key +from locale import LC_ALL, setlocale, strcoll +from pathlib import Path + + +def strcoll_extract_utils(string1: str, string2: str) -> int: + # Skip logic if one of the string if empty + if not string1 or not string2: + return strcoll(string1, string2) + + # Remove '-' from strings if there, + # it is used to indicate a build target + string1 = re.sub("^-", "", string1) + string2 = re.sub("^-", "", string2) + + # If no directories, compare normally + if "/" not in string1 and "/" not in string2: + return strcoll(string1, string2) + + string1_dir = string1.rsplit("/", 1)[0] + "/" + string2_dir = string2.rsplit("/", 1)[0] + "/" + if string1_dir == string2_dir: + # Same directory, compare normally + return strcoll(string1, string2) + + if string1_dir.startswith(string2_dir): + # First string dir is a subdirectory of the second one, + # return string1 > string2 + return -1 + + if string2_dir.startswith(string1_dir): + # Second string dir is a subdirectory of the first one, + # return string2 > string1 + return 1 + + # Compare normally + return strcoll(string1, string2) + + +if __name__ == "__main__": + setlocale(LC_ALL, "C") + + for file in sys.argv[1:] or ["proprietary-files.txt"]: + if not Path(file).is_file(): + print(f"File {file} not found") + continue + + with open(file, "r") as f: + sections = f.read().split("\n\n") + + ordered_sections = [] + for section in sections: + section_list = [line.strip() for line in section.splitlines()] + section_list.sort(key=cmp_to_key(strcoll_extract_utils)) + ordered_sections.append("\n".join(section_list)) + + with open(file, "w") as f: + f.write("\n\n".join(ordered_sections).strip() + "\n") From abbe0a187612d2b2c58a8c1ca8e9fc48d7198ec3 Mon Sep 17 00:00:00 2001 From: Chirayu Desai Date: Thu, 26 May 2022 14:50:35 +0530 Subject: [PATCH 13/13] Allow re-using extracted content Usage: export TMPDIR=/some/dir/$device export SKIP_CLEANUP=true Change-Id: I41b992708ca5fc7bc377b22a418f5fd1395cd08a --- extract_utils.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/extract_utils.sh b/extract_utils.sh index 86acf62..96db86a 100644 --- a/extract_utils.sh +++ b/extract_utils.sh @@ -19,7 +19,8 @@ COMMON=-1 ARCHES= FULLY_DEODEXED=-1 -TMPDIR=$(mktemp -d) +SKIP_CLEANUP=${SKIP_CLEANUP:-0} +TMPDIR=${TMPDIR:-$(mktemp -d)} HOST="$(uname | tr '[:upper:]' '[:lower:]')" # @@ -28,7 +29,11 @@ HOST="$(uname | tr '[:upper:]' '[:lower:]')" # kill our tmpfiles with fire on exit # function cleanup() { - rm -rf "${TMPDIR:?}" + if [ "$SKIP_CLEANUP" == "true" ] || [ "$SKIP_CLEANUP" == "1" ]; then + echo "Skipping cleanup of $TMPDIR" + else + rm -rf "${TMPDIR:?}" + fi } trap cleanup 0