[Build] allow to generate symbols for a subset of binaries (swiftlang#37120)

This would be needed to reduce overall build times in scenarios when
generating symbols for all binaries is too expensive and/or not needed.

At the same time, introduce tests around the logic that handles symbols.

Addresses rdar://76865276

Author: edymtt

utils/build-script

@@ -814,6 +814,12 @@ class BuildScriptInvocation(object):
                 ' '.join(args.llvm_ninja_targets_for_cross_compile_hosts)
             ]
 
+        if args.darwin_symroot_path_filters:
+            impl_args += [
+                "--darwin_symroot_path_filters=%s" %
+                ' '.join(args.darwin_symroot_path_filters)
+            ]
+
         # Compute the set of host-specific variables, which we pass through to
         # the build script via environment variables.
         host_specific_variables = self.compute_host_specific_variables()

utils/build-script-impl

@@ -103,6 +103,7 @@ KNOWN_SETTINGS=(
     darwin-deployment-version-watchos             "2.0"             "minimum deployment target version for watchOS"
     darwin-install-extract-symbols                ""                "whether to extract symbols with dsymutil during installations"
     darwin-install-extract-symbols-use-just-built-dsymutil "1"       "whether we should extract symbols using the just built dsymutil"
+    darwin-symroot-path-filters                   ""                "space-separated list of path patterns to consider for symbol generation"
     darwin-overlay-target                         ""                "single overlay target to build, dependencies are computed later"
     darwin-sdk-deployment-targets                 "xctest-ios-8.0"  "semicolon-separated list of triples like 'fookit-ios-9.0;barkit-watchos-9.0'"
     darwin-stdlib-install-name-dir                ""                "the directory of the install_name for standard library dylibs"
@@ -1015,11 +1016,19 @@ if [ -z "${CMAKE}" ] ; then
 fi
 
 function xcrun_find_tool() {
-  xcrun --sdk macosx --toolchain "${DARWIN_XCRUN_TOOLCHAIN}" --find "$@"
+  if [[ "${DRY_RUN}" ]]; then
+    echo echo "$@"
+  else
+    xcrun --sdk macosx --toolchain "${DARWIN_XCRUN_TOOLCHAIN}" --find "$@"
+  fi
 }
 
 function find_just_built_local_host_llvm_tool() {
-  find $(build_directory "${LOCAL_HOST}" llvm) -name "$1" -type f -print
+  if [[ "${DRY_RUN}" ]]; then
+    echo echo "$1"
+  else
+    find $(build_directory "${LOCAL_HOST}" llvm) -name "$1" -type f -print
+  fi
 }
 
 function not() {
@@ -3104,6 +3113,26 @@ function printJSONEndTimestamp() {
     printJSONTimestamp ${command} "end"
 }
 
+function grep_that_allows_no_matches() {
+    # This will not cause the script to fail
+    # if no line in the input matches the pattern
+    grep "$@" || test $? = 1
+}
+
+function filter_paths() {
+    if [[ -n "$1" ]]; then
+        local -a filters
+        filters=($1)
+        local -a grep_arguments
+        for cfilter in "${filters[@]}"; do
+            grep_arguments+=('-e' "$cfilter")
+        done
+        grep_that_allows_no_matches "${grep_arguments[@]}"
+    else
+        cat
+    fi
+}
+
 for host in "${ALL_HOSTS[@]}"; do
     # Check if we should perform this action.
     if ! [[ $(should_execute_action "${host}-extractsymbols") ]]; then
@@ -3131,15 +3160,10 @@ for host in "${ALL_HOSTS[@]}"; do
         # descibes
         host_symroot="${INSTALL_SYMROOT}/${host}"
 
-        # FIXME: Since it's hard to trace output pipe call,
-        #        For now, We don't support dry-run trace for this block
-        #        Instead, just echo we do "darwin_intall_extract_symbols".
-        if [[ "${DRY_RUN}" ]]; then
-            call darwin_install_extract_symbols
-            printJSONStartTimestamp dsymutil
-            echo xargs -n 1 -P ${DSYMUTIL_JOBS} dsymutil
-            printJSONEndTimestamp dsymutil
-        else
+        # FIXME this if statement is a trick to have a more readable
+        # diff for the PR that made the following code
+        # amenable to dry run testing
+        if [ -n "${DRY_RUN}" -o -z "${DRY_RUN}" ]; then
             set -x
 
             CURRENT_INSTALL_DIR=${host_install_destdir}
@@ -3148,7 +3172,9 @@ for host in "${ALL_HOSTS[@]}"; do
             # Copy executables and shared libraries from the `host_install_destdir` to
             # INSTALL_SYMROOT and run dsymutil on them.
             (cd "${CURRENT_INSTALL_DIR}" &&
-             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | cpio --insecure -pdm "${host_symroot}")
+             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | \
+             filter_paths "${DARWIN_SYMROOT_PATH_FILTERS}" | \
+             cpio --insecure -pdm -v "${host_symroot}")
 
             dsymutil_path=
             if [[ -n "${DARWIN_INSTALL_EXTRACT_SYMBOLS_USE_JUST_BUILT_DSYMUTIL}" ]]; then
@@ -3160,15 +3186,11 @@ for host in "${ALL_HOSTS[@]}"; do
             # Run dsymutil on executables and shared libraries.
             #
             # Exclude shell scripts and static archives.
-            # Exclude swift-api-digester dSYM to reduce debug toolchain size.
             # Tweak carefully the amount of parallelism -- dsymutil can be memory intensive and
             # as such too many instance can exhaust the memory and slow down/panic the machine
             printJSONStartTimestamp dsymutil
             (cd "${host_symroot}" &&
-             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | \
-               grep -v '.py$' | \
-               grep -v '.a$' | \
-               grep -v 'swift-api-digester' | \
+             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -not -name "*.a" -not -name "*.py" -print | \
                xargs -n 1 -P ${DSYMUTIL_JOBS} ${dsymutil_path})
             printJSONEndTimestamp dsymutil
 

utils/build_swift/build_swift/driver_arguments.py

@@ -693,6 +693,12 @@ def create_argument_parser():
     option('--symbols-package', store_path,
            help='if provided, an archive of the symbols directory will be '
                 'generated at this path')
+    option('--darwin-symroot-path-filters', append,
+           type=argparse.ShellSplitType(),
+           help='Space separated list of patterns used to match '
+                'a subset of files to generate symbols for. '
+                'Only supported on Darwin. Can be called multiple times '
+                'to add multiple such options.')
 
     # -------------------------------------------------------------------------
     in_group('Build variant')

utils/build_swift/tests/expected_options.py

@@ -137,6 +137,7 @@
         defaults.DARWIN_DEPLOYMENT_VERSION_TVOS,
     'darwin_deployment_version_watchos':
         defaults.DARWIN_DEPLOYMENT_VERSION_WATCHOS,
+    'darwin_symroot_path_filters': [],
     'darwin_xcrun_toolchain': None,
     'distcc': False,
     'sccache': False,
@@ -695,6 +696,7 @@ class BuildScriptImplOption(_BaseOption):
     AppendOption('--test-paths'),
     AppendOption('--llvm-ninja-targets'),
     AppendOption('--llvm-ninja-targets-for-cross-compile-hosts'),
+    AppendOption('--darwin-symroot-path-filters'),
 
     UnsupportedOption('--build-jobs'),
     UnsupportedOption('--common-cmake-options'),

validation-test/BuildSystem/dsymutil_jobs.test

@@ -1,10 +1,12 @@
+# REQUIRES: standalone_build,OS=macosx
+
 # RUN: %empty-directory(%t)
 # RUN: mkdir -p %t
-# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --darwin-install-extract-symbols --dsymutil-jobs 5 --cmake %cmake 2>&1 | %FileCheck %s
-
-# REQUIRES: standalone_build,OS=macosx
+# RUN: mkdir -p %t/destdir
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --darwin-install-extract-symbols --dsymutil-jobs 5 --cmake %cmake --install-symroot=%t/symroot --install-destdir=%t/destdir --toolchain-prefix= 2>&1 | %FileCheck %s
 
 # CHECK: --- Extracting symbols ---
 # CHECK: { "command": "dsymutil", "start": "
-# CHECK-NEXT: xargs -n 1 -P 5 dsymutil
-# CHECK-NEXT: { "command": "dsymutil", "end": "
+# CHECK: xargs -n 1 -P 5 {{.*}}dsymutil
+# CHECK: { "command": "dsymutil", "end": "

validation-test/BuildSystem/extractsymbols-darwin-symroot-path-filters.test

@@ -0,0 +1,111 @@
+# REQUIRES: standalone_build
+# REQUIRES: OS=macosx
+
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: split-file %s %t
+
+# Even though we are running build-script with dry-run,
+# symbol extraction runs real commands against the file system.
+# Thus we generate a series of files
+# to exercise the filtering logic
+# RUN: mkdir -p %t/destdir/bin
+# RUN: mkdir -p %t/destdir/lib
+# RUN: %swiftc_driver %t/hello.swift -o %t/destdir/bin/swift-demangle
+# RUN: %swiftc_driver %t/hello.swift -o %t/destdir/bin/swift-def-to-yaml-converter
+# RUN: ln -s %t/destdir/swift-demangle %t/destdir/bin/swift-api-digester
+# RUN: cp %t/swift-util.py %t/destdir/bin
+# RUN: chmod a+x %t/destdir/bin/swift-util.py
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -o %t/destdir/lib/libswiftDemangle.dylib
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -o %t/destdir/lib/lib_InternalSwiftScan.dylib
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -static -o %t/destdir/lib/libswiftASTSectionImporter.a 
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+
+# test build-script-impl on its own
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 --darwin-symroot-path-filters="/lib/ /swift-demangle"  2>&1 | tee %t/build-script-impl-output.txt
+# RUN: %FileCheck --input-file=%t/build-script-impl-output.txt %s
+# RUN: %FileCheck --input-file=%t/build-script-impl-output.txt --check-prefixes CHECK-SKIPPED %s
+
+# ensure build-script pass the argument to build-script-impl
+# RUN: %empty-directory(%t/symroot)
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --cmake %cmake --darwin-install-extract-symbols=1 --install-destdir=%t/destdir --toolchain-prefix="" --install-symroot=%t/symroot --darwin-symroot-path-filters="/lib/ /swift-demangle" --jobs=1  2>&1 | tee %t/build-script-output.txt
+# RUN: %FileCheck --input-file=%t/build-script-output.txt %s
+# RUN: %FileCheck --input-file=%t/build-script-output.txt --check-prefixes CHECK-SKIPPED %s
+
+# ensure we get all the values if we specify the flag multiple times
+# RUN: %empty-directory(%t/symroot)
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --cmake %cmake --darwin-install-extract-symbols=1 --install-destdir=%t/destdir --toolchain-prefix="" --install-symroot=%t/symroot --darwin-symroot-path-filters="/lib/" --darwin-symroot-path-filters="/swift-demangle" --jobs=1  2>&1 | tee %t/build-script-output.txt
+# RUN: %FileCheck --input-file=%t/build-script-output.txt %s
+# RUN: %FileCheck --input-file=%t/build-script-output.txt --check-prefixes CHECK-SKIPPED %s
+
+
+# CHECK: --- Extracting symbols ---
+
+# Ensure we copy all the files in lib and the swift-demangle executable
+# CHECK-LABEL: cpio
+# CHECK-DAG: swift-demangle
+# CHECK-DAG: libswiftDemangle.dylib
+# CHECK-DAG: lib_InternalSwiftScan.dylib
+
+# Ensure we generate symbols for the file in the symroot
+# CHECK-LABEL: "command": "dsymutil", "start"
+# CHECK-DAG: dsymutil {{.*}}swift-demangle
+# CHECK-DAG: dsymutil {{.*}}libswiftDemangle.dylib
+# CHECK-DAG: dsymutil {{.*}}lib_InternalSwiftScan.dylib
+
+# Ensure we strip the files in the installation directory
+# (which are not subject to the filters)
+# CHECK-LABEL: xcrun_find_tool strip
+# CHECK-DAG: strip {{.*}}swift-demangle
+# CHECK-DAG: strip {{.*}}swift-def-to-yaml-converter
+# CHECK-DAG: strip {{.*}}libswiftDemangle.dylib
+# CHECK-DAG: strip {{.*}}lib_InternalSwiftScan.dylib
+# CHECK-DAG: strip {{.*}}libswiftASTSectionImporter.a
+# CHECK-DAG: strip {{.*}}swift-util.py
+
+# Ensure we codesign dylibs
+# CHECK-LABEL: xcrun_find_tool codesign
+# CHECK-DAG: codesign {{.*}}libswiftDemangle.dylib
+# CHECK-DAG: codesign {{.*}}lib_InternalSwiftScan.dylib
+
+
+# CHECK-SKIPPED: --- Extracting symbols ---
+
+# Ensure we don't copy files not included by the filters
+# CHECK-SKIPPED-LABEL: cpio
+# CHECK-SKIPPED-NOT: swift-util.py
+# CHECK-SKIPPED-NOT: swift-def-to-yaml-converter
+# CHECK-SKIPPED-NOT: libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: swift-api-digester
+
+# Ensure we don't generate symbols for files we did not copy
+# CHECK-SKIPPED-LABEL: "command": "dsymutil", "start"
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-def-to-yaml-converter
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-util.py
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-api-digester
+
+# Ensure we don't strip symlinks
+# CHECK-SKIPPED-LABEL: xcrun_find_tool strip
+# CHECK-SKIPPED-NOT: strip {{.*}}swift-api-digester
+
+# Ensure we don't codesign executables, symlinks,
+# static archives and python scripts
+# CHECK-SKIPPED-LABEL: xcrun_find_tool codesign
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-demangle
+# CHECK-SKIPPED-NOT: codesign {{.*}}libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-util.py
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-api-digester
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-def-to-yaml-converter
+
+#--- hello.swift
+print("hello")
+
+#--- dylib.swift
+func greet(person: String) -> String {
+    return "Hello \(person)"
+}
+#--- swift-util.py
+print("hello")

validation-test/BuildSystem/extractsymbols-default-behaviour.test

@@ -0,0 +1,93 @@
+# REQUIRES: standalone_build
+# REQUIRES: OS=macosx
+
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: split-file %s %t
+
+# Even though we are running build-script with dry-run,
+# symbol extraction runs real commands against the file system.
+# Thus we generate a series of files
+# to target each of the cases handled by the code
+# RUN: mkdir -p %t/destdir
+# RUN: %swiftc_driver %t/hello.swift -o %t/destdir/swift-demangle
+# RUN: ln -s %t/destdir/swift-demangle %t/destdir/swift-api-digester
+# RUN: cp %t/swift-util.py %t/destdir/
+# RUN: chmod a+x %t/destdir/swift-util.py
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -o %t/destdir/libswiftDemangle.dylib
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -static -o %t/destdir/libswiftASTSectionImporter.a 
+# Targets marked with INSTALL_WITH_SHARED are executable (e.g. compatibility libraries)
+# RUN: cp %t/destdir/libswiftASTSectionImporter.a %t/destdir/libswiftCompatibility51.a
+# RUN: chmod a+x %t/destdir/libswiftCompatibility51.a
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 > %t/build-script-impl-output.txt 2>&1
+# RUN: %FileCheck --input-file=%t/build-script-impl-output.txt %s
+# RUN: %FileCheck --input-file=%t/build-script-impl-output.txt --check-prefixes CHECK-SKIPPED %s
+
+# CHECK: --- Extracting symbols ---
+
+# Ensure we copy executable regular files to the symroot
+# CHECK-LABEL: cpio
+# CHECK-DAG: swift-demangle
+# CHECK-DAG: swift-util.py
+# CHECK-DAG: libswiftDemangle.dylib
+# CHECK-DAG: libswiftCompatibility51.a
+
+# Ensure we extract symbols only for executables and
+# and dylibs
+# CHECK-LABEL: command": "dsymutil", "start"
+# CHECK-DAG: dsymutil {{.*}}swift-demangle
+# CHECK-DAG: dsymutil {{.*}}libswiftDemangle.dylib
+
+# Ensure we strip executables, shared libraries and static
+# libraries
+# CHECK-LABEL: xcrun_find_tool strip
+# CHECK-DAG: strip {{.*}}swift-demangle
+# CHECK-DAG: strip {{.*}}libswiftDemangle.dylib
+# CHECK-DAG: strip {{.*}}libswiftASTSectionImporter.a
+# CHECK-DAG: strip {{.*}}libswiftCompatibility51.a
+# CHECK-DAG: strip {{.*}}swift-util.py
+
+# Ensure we codesign dylibds
+# CHECK-LABEL: xcrun_find_tool codesign
+# CHECK: codesign {{.*}}libswiftDemangle.dylib
+
+# CHECK-SKIPPED: --- Extracting symbols ---
+
+# Ensure symroot does not contain symlinks and static archives
+# that are not executable
+# CHECK-SKIPPED-LABEL: cpio
+# CHECK-SKIPPED-NOT: swift-api-digester
+# CHECK-SKIPPED-NOT: libswiftASTSectionImporter.a
+
+# Ensure we don't extract symbols for static archives, symlinks
+# and Python scripts
+# CHECK-SKIPPED-LABEL: command": "dsymutil", "start"
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}libswiftCompatibility51.a
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-util.py
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-api-digester
+
+# Ensure we don't strip symlinks
+# CHECK-SKIPPED-LABEL: xcrun_find_tool strip
+# CHECK-SKIPPED-NOT: strip {{.*}}swift-api-digester
+
+# Ensure we don't codesign executables, symlinks,
+# static archives and python scripts
+# CHECK-SKIPPED-LABEL: xcrun_find_tool codesign
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-demangle
+# CHECK-SKIPPED-NOT: codesign {{.*}}libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: codesign {{.*}}libswiftCompatibility51.a
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-util.py
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-api-digester
+
+#--- hello.swift
+print("hello")
+
+#--- dylib.swift
+func greet(person: String) -> String {
+    return "Hello \(person)"
+}
+#--- swift-util.py
+print("hello")

validation-test/BuildSystem/extractsymbols-dry-run-does-not-fail.test

@@ -0,0 +1,24 @@
+# REQUIRES: standalone_build
+# REQUIRES: OS=macosx
+
+# As a result of incremental changes,
+# in dry-run mode the symbol extraction
+# still runs some real commands against the installation
+# and symbol directories (e.g. find and cpio)
+# This test explictly checks that such commands
+# do not cause build-script to fail when run
+# against empty directories (which is the typical
+# scenario in dry-run)
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: mkdir -p %t/destdir
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 2>&1 | %FileCheck %s
+
+# CHECK: --- Extracting symbols ---
+
+# CHECK-NOT: {{^}}echo dsymutil
+
+# CHECK-NOT: {{^[^\+].*}}strip
+
+# CHECK-NOT: {{^[^\+].*}}codesign