Skip to content

Commit 4803ec1

Browse files
akyrtziakyrtzi
authored
Merge pull request swiftlang#22827 from akyrtzi/utils-build-parser-lib
[utils] Add a utility script for building only the syntax parser library as fast as possible
2 parents 972a86b + 363da61 commit 4803ec1

File tree

3 files changed

+2768
-0
lines changed

3 files changed

+2768
-0
lines changed

‎utils/build-parser-lib

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
#!/usr/bin/env python
2+
# utils/build-parser-lib - Helper tool for building the parser library -*- python -*-
3+
#
4+
# This source file is part of the Swift.org open source project
5+
#
6+
# Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
7+
# Licensed under Apache License v2.0 with Runtime Library Exception
8+
#
9+
# See https://swift.org/LICENSE.txt for license information
10+
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11+
12+
# This is a utility for building only the syntax parser library as fast as possible
13+
# by only building the necessary libraries for the parser library and nothing
14+
# extraneous. To achieve this it does a single unified CMake configuration for
15+
# llvm/clang/swift and builds only the parser library target.
16+
# This mechanism is fundamentally different from build-script, which builds llvm/clang
17+
# in a separate build directory than swift.
18+
#
19+
# Even though this bypasses build-script, it does share some underlying helper
20+
# utilities from the python infrastructure.
21+
#
22+
# This utility also provides capability to gather profile data and build the parser
23+
# library with PGO optimization enabled.
24+
25+
from __future__ import print_function
26+
27+
import multiprocessing
28+
import os
29+
import platform
30+
import sys
31+
32+
from build_swift import argparse, defaults
33+
from swift_build_support.swift_build_support import (
34+
shell,
35+
)
36+
from swift_build_support.swift_build_support.SwiftBuildSupport import (
37+
HOME,
38+
SWIFT_BUILD_ROOT,
39+
SWIFT_SOURCE_ROOT,
40+
)
41+
from swift_build_support.swift_build_support.toolchain import host_toolchain
42+
43+
isMac = platform.system() == 'Darwin'
44+
45+
def call_without_sleeping(command, env=None, dry_run=False, echo=False):
46+
"""
47+
Execute a command during which system sleep is disabled.
48+
49+
By default, this ignores the state of the `shell.dry_run` flag.
50+
"""
51+
52+
# Disable system sleep, if possible.
53+
if platform.system() == 'Darwin':
54+
# Don't mutate the caller's copy of the arguments.
55+
command = ["caffeinate"] + list(command)
56+
57+
shell.call(command, env=env, dry_run=dry_run, echo=echo)
58+
59+
class Builder(object):
60+
def __init__(self, toolchain, args):
61+
self.toolchain = toolchain
62+
self.build_release = args.release
63+
self.enable_assertions = not args.no_assertions
64+
self.lto_type = args.lto_type
65+
self.pgo_type = args.pgo_type
66+
self.profile_input = args.profile_input
67+
self.dry_run = args.dry_run
68+
self.jobs = args.build_jobs
69+
self.verbose = args.verbose
70+
self.build_dir = args.build_dir
71+
self.install_symroot = args.install_symroot
72+
self.install_destdir = args.install_destdir
73+
self.install_prefix = args.install_prefix
74+
self.version = args.version
75+
76+
def call(self, command, env=None, without_sleeping=False):
77+
if without_sleeping:
78+
call_without_sleeping(command, env=env, dry_run=self.dry_run, echo=self.verbose)
79+
else:
80+
shell.call(command, env=env, dry_run=self.dry_run, echo=self.verbose)
81+
82+
def configure(self, enable_debuginfo, instrumentation=None, profile_data=None):
83+
cmake_args = [self.toolchain.cmake, '-G', 'Ninja']
84+
if isMac:
85+
cmake_args += ['-DCMAKE_OSX_DEPLOYMENT_TARGET=10.12', '-DSWIFT_DARWIN_DEPLOYMENT_VERSION_OSX=10.12']
86+
else:
87+
dispatch_source_path = os.path.join(SWIFT_SOURCE_ROOT, 'swift-corelibs-libdispatch')
88+
cmake_args += ['-DSWIFT_HOST_VARIANT=linux', '-DSWIFT_HOST_VARIANT_SDK=LINUX', '-DSWIFT_HOST_VARIANT_ARCH=x86_64',
89+
'-DSWIFT_PATH_TO_LIBDISPATCH_SOURCE:PATH='+dispatch_source_path,
90+
'-DLLVM_ENABLE_LLD=ON']
91+
cmake_args += ['-DLLVM_TARGETS_TO_BUILD=X86']
92+
if self.build_release:
93+
if enable_debuginfo:
94+
cmake_args += ['-DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo']
95+
else:
96+
cmake_args += ['-DCMAKE_BUILD_TYPE:STRING=Release']
97+
else:
98+
cmake_args += ['-DCMAKE_BUILD_TYPE:STRING=Debug']
99+
if self.enable_assertions:
100+
cmake_args += ['-DLLVM_ENABLE_ASSERTIONS:BOOL=ON']
101+
if instrumentation:
102+
cmake_args += ['-DLLVM_BUILD_INSTRUMENTED='+instrumentation]
103+
if profile_data:
104+
cmake_args += ['-DLLVM_PROFDATA_FILE='+profile_data]
105+
if self.lto_type and not instrumentation:
106+
cmake_args += ['-DLLVM_ENABLE_LTO='+self.lto_type.upper()]
107+
if self.install_prefix:
108+
cmake_args += ['-DCMAKE_INSTALL_PREFIX:PATH='+self.install_prefix]
109+
if self.version:
110+
cmake_args += ['-DSWIFT_LIBPARSER_VER:STRING='+self.version]
111+
cmake_args += ['-DLLVM_ENABLE_PROJECTS=clang;cmark;swift', '-DLLVM_EXTERNAL_PROJECTS=cmark;swift']
112+
cmake_args += ['-DSWIFT_BUILD_ONLY_SYNTAXPARSERLIB=TRUE']
113+
cmake_args += ['-DSWIFT_BUILD_PERF_TESTSUITE=NO', '-DSWIFT_INCLUDE_DOCS=NO']
114+
cmake_args += ['-DSWIFT_BUILD_REMOTE_MIRROR=FALSE', '-DSWIFT_BUILD_DYNAMIC_STDLIB=FALSE',
115+
'-DSWIFT_BUILD_STATIC_STDLIB=FALSE', '-DSWIFT_BUILD_DYNAMIC_SDK_OVERLAY=FALSE',
116+
'-DSWIFT_BUILD_STATIC_SDK_OVERLAY=FALSE']
117+
cmake_args += [os.path.join(SWIFT_SOURCE_ROOT, 'llvm')]
118+
self.call(cmake_args)
119+
120+
def build_target(self, build_dir, target, env=None):
121+
invocation = [self.toolchain.cmake, '--build', build_dir]
122+
invocation += ['--', '-j%d'%self.jobs]
123+
if self.verbose:
124+
invocation += ['-v']
125+
invocation += [target]
126+
self.call(invocation, env=env, without_sleeping=True)
127+
128+
def install(self):
129+
print("--- Installing ---", file=sys.stderr)
130+
env = None
131+
if self.install_destdir:
132+
env = {'DESTDIR': self.install_destdir}
133+
self.build_target(self.build_dir, 'tools/swift/tools/libSwiftSyntaxParser/install', env=env)
134+
135+
def extract_symbols(self):
136+
if not isMac:
137+
return
138+
extract_script = os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "parser-lib", "darwin-extract-symbols")
139+
print("--- Extracting symbols ---", file=sys.stderr)
140+
env = {'INSTALL_DIR': self.install_destdir,
141+
'INSTALL_PREFIX': self.install_prefix,
142+
'INSTALL_SYMROOT': self.install_symroot,
143+
'BUILD_JOBS': str(self.jobs)}
144+
self.call([extract_script], env=env)
145+
146+
def get_profile_data(self):
147+
profile_dir = os.path.join(self.build_dir, 'profiling')
148+
shell.makedirs(profile_dir, dry_run=self.dry_run)
149+
instrumentation = 'IR' if self.pgo_type == 'ir' else 'Frontend'
150+
with shell.pushd(profile_dir, dry_run=self.dry_run):
151+
self.configure(enable_debuginfo=False, instrumentation=instrumentation)
152+
self.build_target(profile_dir, 'swift-syntax-parser-test')
153+
# Delete existing profile data that were generated during building from running tablegen.
154+
shell.rmtree("profiles", dry_run=self.dry_run)
155+
self.call([os.path.join("bin", "swift-syntax-parser-test"), self.profile_input, '-time'])
156+
self.call([self.toolchain.llvm_profdata, "merge", "-output=profdata.prof", "profiles"])
157+
return os.path.join(profile_dir, "profdata.prof")
158+
159+
def run(self):
160+
shell.makedirs(self.build_dir, dry_run=self.dry_run)
161+
162+
profile_data = None
163+
if self.pgo_type:
164+
profile_data = self.get_profile_data()
165+
166+
with shell.pushd(self.build_dir, dry_run=self.dry_run):
167+
self.configure(enable_debuginfo=True, profile_data=profile_data)
168+
169+
self.build_target(self.build_dir, 'swift-syntax-parser-test')
170+
171+
if self.install_destdir:
172+
self.install()
173+
if self.install_symroot:
174+
self.extract_symbols()
175+
176+
177+
def main():
178+
parser = argparse.ArgumentParser(
179+
formatter_class=argparse.RawDescriptionHelpFormatter,
180+
description="""Builds Swift Syntax Parser library.""")
181+
optbuilder = parser.to_builder()
182+
option = optbuilder.add_option
183+
store = optbuilder.actions.store
184+
store_true = optbuilder.actions.store_true
185+
store_int = optbuilder.actions.store_int
186+
store_path = optbuilder.actions.store_path
187+
188+
option('--release', store_true,
189+
help='build in release mode')
190+
option('--lto', store('lto_type'),
191+
choices=['thin', 'full'],
192+
const='full',
193+
metavar='LTO_TYPE',
194+
help='use lto optimization.'
195+
'Options: thin, full. If no optional arg is provided, full is '
196+
'chosen by default')
197+
option('--pgo', store('pgo_type'),
198+
choices=['frontend', 'ir'],
199+
const='ir',
200+
metavar='PGO_TYPE',
201+
help='use pgo optimization.'
202+
'Options: frontend, ir. If no optional arg is provided, ir is '
203+
'chosen by default')
204+
option('--profile-input', store_path,
205+
default=os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "parser-lib", "profile-input.swift"),
206+
help='the source file to use for PGO profiling input')
207+
option('--no-assertions', store_true,
208+
help='disable assertions')
209+
option(['-v', '--verbose'], store_true,
210+
help='print the commands executed during the build')
211+
option('--dry-run', store_true,
212+
help='print the commands to execute but not actually execute them')
213+
option(['-j', '--jobs'], store_int('build_jobs'),
214+
default=multiprocessing.cpu_count(),
215+
help='the number of parallel build jobs to use')
216+
option('--build-dir', store_path,
217+
default=os.path.join(SWIFT_BUILD_ROOT, 'parser-lib'),
218+
help='the path where the build products will be placed. '
219+
'If none is specified it will use <workspace>/build/parser-lib')
220+
option('--install-symroot', store_path,
221+
help='the path to install debug symbols into')
222+
option('--install-destdir', store_path,
223+
help='the path to use as the filesystem root for the installation')
224+
option('--install-prefix', store,
225+
default = defaults.DARWIN_INSTALL_PREFIX if isMac else UNIX_INSTALL_PREFIX,
226+
help='the install prefix to use')
227+
option('--version', store,
228+
help='version string to use for the parser library')
229+
230+
parser = optbuilder.build()
231+
args = parser.parse_args()
232+
233+
toolchain = host_toolchain(xcrun_toolchain='default')
234+
builder = Builder(toolchain, args)
235+
builder.run()
236+
return 0
237+
238+
if __name__ == "__main__":
239+
try:
240+
sys.exit(main())
241+
except KeyboardInterrupt:
242+
sys.exit(1)

‎utils/parser-lib/darwin-extract-symbols

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/usr/bin/env bash
2+
#===--- darwin-extract-symbols - Extract symbols part of build-parser-lib --===#
3+
#
4+
## This source file is part of the Swift.org open source project
5+
##
6+
## Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
7+
## Licensed under Apache License v2.0 with Runtime Library Exception
8+
##
9+
## See https://swift.org/LICENSE.txt for license information
10+
## See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11+
#
12+
#===------------------------------------------------------------------------===#
13+
14+
set -x
15+
16+
if [ -z "$INSTALL_DIR" ]
17+
then
18+
echo "\$INSTALL_DIR is not set"
19+
exit 1
20+
fi
21+
if [ -z "$INSTALL_PREFIX" ]
22+
then
23+
echo "\$INSTALL_PREFIX is not set"
24+
exit 1
25+
fi
26+
if [ -z "$INSTALL_SYMROOT" ]
27+
then
28+
echo "\$INSTALL_SYMROOT is not set"
29+
exit 1
30+
fi
31+
if [ -z "$BUILD_JOBS" ]
32+
then
33+
echo "\$BUILD_JOBS is not set"
34+
exit 1
35+
fi
36+
37+
function xcrun_find_tool() {
38+
xcrun --sdk macosx --toolchain default --find "$@"
39+
}
40+
41+
# Copy executables and shared libraries from the INSTALL_DIR to
42+
# INSTALL_SYMROOT and run dsymutil on them.
43+
(cd "${INSTALL_DIR}" &&
44+
find ./"${INSTALL_PREFIX}" -perm -0111 -type f -print | cpio --insecure -pdm "${INSTALL_SYMROOT}")
45+
46+
# Run dsymutil on executables and shared libraries.
47+
#
48+
# Exclude shell scripts.
49+
(cd "${INSTALL_SYMROOT}" &&
50+
find ./"${INSTALL_PREFIX}" -perm -0111 -type f -print | \
51+
grep -v crashlog.py | \
52+
grep -v symbolication.py | \
53+
xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool dsymutil))
54+
55+
# Strip executables, shared libraries and static libraries in INSTALL_DIR.
56+
find "${INSTALL_DIR}${INSTALL_PREFIX}/" \
57+
'(' -perm -0111 -or -name "*.a" ')' -type f -print | \
58+
xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool strip) -S
59+
60+
# Codesign dylibs after strip tool
61+
# rdar://45388785
62+
find "${INSTALL_DIR}${INSTALL_PREFIX}/" \
63+
'(' -name "*.dylib" ')' -type f -print | \
64+
xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool codesign) -f -s -

0 commit comments

Comments
 (0)