Skip to content

Commit bae185e

Browse files
chronos: add initial python helper for management (#13650)
The bash scripts are quickly getting annoying to work with for larger scale. It will be beneficial moving forward having something more sophisticated and that is easier to extend. This adds support for a Python-based workflow for managing Chronos artifacts. Sample commands: ``` python3 infra/experimental/chronos/manager.py check-replay-script valijson python3 infra/experimental/chronos/manager.py check-test json-c ``` Signed-off-by: David Korczynski <david@adalogics.com>
1 parent c5566bb commit bae185e

File tree

1 file changed

+188
-0
lines changed

1 file changed

+188
-0
lines changed
Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
"""Module for managing Chronos cached builds."""
15+
16+
import os
17+
import sys
18+
import logging
19+
import argparse
20+
import time
21+
import subprocess
22+
23+
logger = logging.getLogger(__name__)
24+
25+
26+
def _get_project_cached_named(project, sanitizer='address'):
27+
"""Gets the name of the cached project image."""
28+
return f'us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/{project}-ofg-cached-{sanitizer}'
29+
30+
31+
def _get_project_cached_named_local(project, sanitizer='address'):
32+
return f'{project}-origin-{sanitizer}'
33+
34+
35+
def build_project_image(project):
36+
"""Build OSS-Fuzz base image for a project."""
37+
cmd = ['docker', 'build', '-t', 'gcr.io/oss-fuzz/' + project, '.']
38+
subprocess.check_call(' '.join(cmd),
39+
shell=True,
40+
cwd=os.path.join('projects', project))
41+
42+
43+
def build_cached_project(project, cleanup=True, sanitizer='address'):
44+
"""Build cached image for a project."""
45+
container_name = _get_project_cached_named_local(project, sanitizer)
46+
47+
# Clean up the container if it exists.
48+
if cleanup:
49+
try:
50+
subprocess.check_call(['docker', 'container', 'rm', '-f', container_name])
51+
except subprocess.CalledProcessError:
52+
pass
53+
54+
project_language = 'c++'
55+
cwd = os.getcwd()
56+
# Build the cached image.
57+
cmd = [
58+
'docker', 'run', '--env=SANITIZER=' + sanitizer,
59+
'--env=CCACHE_DIR=/workspace/ccache',
60+
f'--env=FUZZING_LANGUAGE={project_language}',
61+
'--env=CAPTURE_REPLAY_SCRIPT=1', f'--name={container_name}',
62+
f'-v={cwd}/ccaches/{project}/ccache:/workspace/ccache',
63+
f'-v={cwd}/build/out/{project}/:/out/', f'gcr.io/oss-fuzz/{project}',
64+
'bash', '-c',
65+
'"export PATH=/ccache/bin:\$PATH && compile && cp -n /usr/local/bin/replay_build.sh \$SRC/"'
66+
]
67+
68+
logger.info('Running: [%s]', ' '.join(cmd))
69+
subprocess.check_call(' '.join(cmd), shell=True)
70+
71+
# Save the container.
72+
cmd = [
73+
'docker', 'container', 'commit', '-c', '"ENV REPLAY_ENABLED=1"', '-c',
74+
'"ENV CAPTURE_REPLAY_SCRIPT=1"', container_name,
75+
_get_project_cached_named(project, sanitizer)
76+
]
77+
logger.info('Saving image: [%s]', ' '.join(cmd))
78+
subprocess.check_call(' '.join(cmd), shell=True)
79+
80+
81+
def check_cached_replay(project, sanitizer='address'):
82+
"""Checks if a cache build succeeds and times is."""
83+
build_project_image(project)
84+
build_cached_project(project, sanitizer=sanitizer)
85+
86+
# Run the cached replay script.
87+
cmd = [
88+
'docker', 'run', '--rm', '--env=SANITIZER=' + sanitizer,
89+
'--env=FUZZING_LANGUAGE=c++',
90+
'-v=' + os.getcwd() + '/build/out/' + project + '/:/out/',
91+
'--name=' + project + '-origin-' + sanitizer + '-replay-recached',
92+
_get_project_cached_named(project, sanitizer), '/bin/bash', '-c',
93+
'"export PATH=/ccache/bin:$PATH && rm -rf /out/* && compile"'
94+
]
95+
start = time.time()
96+
subprocess.check_call(' '.join(cmd), shell=True)
97+
end = time.time()
98+
logger.info('Cached build completion time: %.2f seconds', (end - start))
99+
100+
101+
def check_test(args):
102+
"""Run the `run_tests.sh` script for a specific project. Will
103+
build a cached container first."""
104+
project = args.project
105+
script_path = os.path.join('projects', project, 'run_tests.sh')
106+
107+
if not os.path.exists(script_path):
108+
logger.info('Error: The script for project "%s" does not exist at %s',
109+
project, script_path)
110+
sys.exit(1)
111+
112+
# Build an OSS-Fuzz image of the project
113+
build_project_image(project)
114+
115+
# build a cached version of the project
116+
build_cached_project(project, args.sanitizer)
117+
118+
# Run the test script
119+
cmd = [
120+
'docker', 'run', '--rm', '-ti',
121+
_get_project_cached_named(project, args.sanitizer), '/bin/bash', '-c',
122+
'"chmod +x /src/run_tests.sh && /src/run_tests.sh"'
123+
]
124+
start = time.time()
125+
subprocess.check_call(' '.join(cmd), shell=True)
126+
end = time.time()
127+
logger.info('Test completion time: %.2f seconds', (end - start))
128+
129+
130+
def parse_args():
131+
"""Parses command line arguments for the manager script."""
132+
parser = argparse.ArgumentParser(
133+
'manager.py',
134+
description='Chronos Mnaager: a tool for managing cached OSS-Fuzz builds.'
135+
)
136+
subparsers = parser.add_subparsers(dest='command')
137+
138+
check_test_parser = subparsers.add_parser(
139+
'check-test', help='Checks run_test.sh for specific project.')
140+
check_test_parser.add_argument(
141+
'project',
142+
help='The name of the project to check (e.g., "libpng").',
143+
)
144+
check_test_parser.add_argument(
145+
'--sanitizer',
146+
default='address',
147+
help='The sanitizer to use (default: address).')
148+
149+
check_replay_script_parser = subparsers.add_parser(
150+
'check-replay-script',
151+
help='Checks if the replay script works for a specific project.')
152+
153+
check_replay_script_parser.add_argument(
154+
'project', help='The name of the project to check.')
155+
check_replay_script_parser.add_argument(
156+
'--sanitizer',
157+
default='address',
158+
help='The sanitizer to use for the cached build (default: address).')
159+
160+
build_cached_image_parser = subparsers.add_parser(
161+
'build-cached-image',
162+
help='Builds a cached image for a specific project.')
163+
build_cached_image_parser.add_argument(
164+
'project', help='The name of the project to build.')
165+
build_cached_image_parser.add_argument(
166+
'--sanitizer',
167+
default='address',
168+
help='The sanitizer to use for the cached build (default: address).')
169+
170+
return parser.parse_args()
171+
172+
173+
def main():
174+
"""Main"""
175+
logging.basicConfig(level=logging.INFO)
176+
177+
args = parse_args()
178+
179+
if args.command == 'check-test':
180+
check_test(args)
181+
if args.command == 'check-replay-script':
182+
check_cached_replay(args.project, args.sanitizer)
183+
if args.command == 'build-cached-image':
184+
build_cached_project(args.project, sanitizer=args.sanitizer)
185+
186+
187+
if __name__ == '__main__':
188+
main()

0 commit comments

Comments
 (0)