Skip to content

Commit a428179

Browse files
ret2libcret2libc
authored
Merge branch 'master' into fix-reproduce
2 parents 6c289ee + ce8bd43 commit a428179

File tree

112 files changed

+5168
-805
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

112 files changed

+5168
-805
lines changed

‎.github/workflows/ubuntu_version_sync.yml‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,19 @@
1414
#
1515
################################################################################
1616

17-
name: 'Ubuntu Version Sync'
17+
name: 'Ubuntu Version Sync Check'
1818

1919
on:
2020
pull_request:
2121
types: [opened, synchronize, reopened]
2222

2323
jobs:
2424
check-sync:
25+
name: Ubuntu File Synchronization Check
2526
runs-on: ubuntu-latest
27+
env:
28+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
29+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
2630
steps:
2731
- name: 'Checkout code'
2832
uses: actions/checkout@v4
@@ -34,7 +38,7 @@ jobs:
3438
run: |
3539
set -e
3640
37-
MODIFIED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
41+
MODIFIED_FILES=$(git diff --name-only $BASE_SHA...$HEAD_SHA)
3842
echo "Checking for synchronized file changes..."
3943
echo "Modified files in this PR:"
4044
echo "$MODIFIED_FILES"
@@ -63,27 +67,27 @@ jobs:
6367
VERSIONS=("ubuntu-20-04" "ubuntu-24-04")
6468
6569
# Check Dockerfiles
66-
for legacy_file in "${{!LEGACY_DOCKERFILES[@]}}"; do
67-
if echo "$MODIFIED_FILES" | grep -q "^${legacy_file}$"; then
70+
for legacy_file in "${!LEGACY_DOCKERFILES[@]}"; do
71+
if [[ "${legacy_file}" == infra/* ]] && echo "$MODIFIED_FILES" | grep -q "^${legacy_file}$"; then
6872
echo "Legacy file changed: $legacy_file. Verifying counterparts..."
69-
for version in "${{VERSIONS[@]}}"; do
70-
pattern=${{LEGACY_DOCKERFILES[$legacy_file]}}
71-
versioned_file="${{pattern/{{version}}/$version}}"
72-
if ! echo "$MODIFIED_FILES" | grep -q "^${{versioned_file}}$"; then
73+
for version in "${VERSIONS[@]}"; do
74+
pattern="${LEGACY_DOCKERFILES[$legacy_file]}"
75+
versioned_file="${pattern/\{version\}/$version}"
76+
if ! echo "$MODIFIED_FILES" | grep -q "^${versioned_file}$"; then
7377
ERRORS+="\n- Legacy file '${legacy_file}' was changed, but its counterpart '${versioned_file}' was not."
7478
fi
7579
done
7680
fi
7781
done
7882
7983
# Check Scripts
80-
for legacy_file in "${{!LEGACY_SCRIPTS[@]}}"; do
84+
for legacy_file in "${!LEGACY_SCRIPTS[@]}"; do
8185
if echo "$MODIFIED_FILES" | grep -q "^${legacy_file}$"; then
8286
echo "Legacy script changed: $legacy_file. Verifying counterparts..."
83-
for version in "${{VERSIONS[@]}}"; do
84-
pattern=${{LEGACY_SCRIPTS[$legacy_file]}}
85-
versioned_file="${{pattern/{{version}}/$version}}"
86-
if ! echo "$MODIFIED_FILES" | grep -q "^${{versioned_file}}$"; then
87+
for version in "${VERSIONS[@]}"; do
88+
pattern="${LEGACY_SCRIPTS[$legacy_file]}"
89+
versioned_file="${pattern/\{version\}/$version}"
90+
if ! echo "$MODIFIED_FILES" | grep -q "^${versioned_file}$"; then
8791
ERRORS+="\n- Legacy script '${legacy_file}' was changed, but its counterpart '${versioned_file}' was not."
8892
fi
8993
done

‎infra/base-images/base-builder-ruby/Dockerfile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,8 @@ FROM gcr.io/oss-fuzz-base/base-builder
1818

1919
RUN git clone https://github.com/trailofbits/ruzzy.git $SRC/ruzzy
2020

21-
RUN install_ruby.sh
22-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
23-
24-
RUN gem update --system 3.5.11
21+
RUN /usr/local/bin/install_ruby.sh
22+
RUN /usr/local/bin/gem update --system 3.5.11
2523

2624
# Install ruzzy
2725
WORKDIR $SRC/ruzzy

‎infra/base-images/base-builder-ruby/ubuntu-20-04.Dockerfile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,8 @@ FROM gcr.io/oss-fuzz-base/base-builder:ubuntu-20-04
1818

1919
RUN git clone https://github.com/trailofbits/ruzzy.git $SRC/ruzzy
2020

21-
RUN install_ruby.sh
22-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
23-
24-
RUN gem update --system 3.5.11
21+
RUN /usr/local/bin/install_ruby.sh
22+
RUN /usr/local/bin/gem update --system 3.5.11
2523

2624
# Install ruzzy
2725
WORKDIR $SRC/ruzzy

‎infra/base-images/base-builder-ruby/ubuntu-24-04.Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ FROM gcr.io/oss-fuzz-base/base-builder:ubuntu-24-04
1818

1919
RUN git clone https://github.com/trailofbits/ruzzy.git $SRC/ruzzy
2020

21-
RUN install_ruby.sh
22-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
21+
RUN /usr/local/bin/install_ruby.sh
22+
RUN /usr/local/bin/gem update --system 3.5.11
2323

2424
RUN gem update --system 3.5.11
2525

‎infra/base-images/base-builder/indexer/utils.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@
2222
import subprocess
2323
from typing import Final, Sequence
2424

25+
from absl import logging
26+
27+
from google3.pyglib import gfile
28+
import pathlib
29+
30+
2531
LD_BINARY_NAME: Final[str] = "ld-linux-x86-64.so.2"
2632
_LD_BINARY_PATH: Final[pathlib.Path] = pathlib.Path("/lib64") / LD_BINARY_NAME
2733

@@ -79,3 +85,52 @@ def get_shared_libraries(
7985
)
8086

8187
return _parse_ld_trace_output(result.stdout.decode())
88+
89+
90+
def copy_shared_libraries(
91+
libraries: Sequence[SharedLibrary], dst_path: pathlib.Path
92+
) -> None:
93+
"""Copies the shared libraries to the shared directory."""
94+
for lib in libraries:
95+
try:
96+
logging.info("Copying %s => %s", lib.name, lib.path)
97+
gfile.Copy(lib.path, dst_path / lib.path.name, overwrite=True, mode=0o755)
98+
except gfile.GOSError:
99+
logging.exception("Could not copy %s to %s", lib.path, dst_path)
100+
raise
101+
102+
103+
def patch_binary_rpath_and_interpreter(
104+
binary_path: os.PathLike[str],
105+
lib_mount_path: pathlib.Path,
106+
):
107+
"""Patches the binary rpath and interpreter."""
108+
subprocess.run(
109+
[
110+
"patchelf",
111+
"--set-rpath",
112+
lib_mount_path.as_posix(),
113+
"--force-rpath",
114+
binary_path,
115+
],
116+
check=True,
117+
)
118+
119+
subprocess.run(
120+
[
121+
"patchelf",
122+
"--set-interpreter",
123+
(lib_mount_path / LD_BINARY_NAME).as_posix(),
124+
binary_path,
125+
],
126+
check=True,
127+
)
128+
129+
130+
def get_library_mount_path(binary_id: str) -> pathlib.Path:
131+
return pathlib.Path("/tmp") / (binary_id + "_lib")
132+
133+
134+
def report_progress(stage: str, is_done: bool = False) -> None:
135+
"""Reports progress of a stage of the snapshotting process."""
136+
logging.info("%s%s", stage, "..." if not is_done else "")

‎infra/base-images/base-builder/install_ruby.sh‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/bin/bash -eux
22
# Copyright 2024 Google LLC
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,14 +15,19 @@
1515
#
1616
################################################################################
1717

18-
apt update
19-
apt install -y lsb-release software-properties-common gnupg2 binutils xz-utils libyaml-dev
20-
gpg2 --keyserver keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
18+
echo "Starting ruby installation"
19+
RUBY_VERSION=3.3.1
20+
RUBY_DEPS="binutils xz-utils libyaml-dev libffi-dev zlib1g-dev"
21+
apt update && apt install -y $RUBY_DEPS
22+
curl -O https://cache.ruby-lang.org/pub/ruby/3.3/ruby-$RUBY_VERSION.tar.gz
23+
tar -xvf ruby-$RUBY_VERSION.tar.gz
24+
cd ruby-$RUBY_VERSION
25+
./configure
26+
make -j$(nproc)
27+
make install
28+
cd ../
2129

22-
curl -sSL https://get.rvm.io > ruby_installation.sh
23-
chmod +x ruby_installation.sh
24-
bash ruby_installation.sh stable
30+
# Clean up the sources.
31+
rm -rf ./ruby-$RUBY_VERSION ruby-$RUBY_VERSION.tar.gz
2532

26-
. /etc/profile.d/rvm.sh
27-
28-
rvm install ruby-3.3.1
33+
echo "Finished installing ruby"

‎infra/base-images/base-runner/Dockerfile‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,14 +109,14 @@ RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jaco
109109
COPY install_javascript.sh /
110110
RUN /install_javascript.sh && rm /install_javascript.sh
111111

112-
# Copy built ruby and ruzzy from builder
113-
COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
114-
COPY --from=base-ruby /install/ruzzy /install/ruzzy
115-
COPY ruzzy /usr/local/bin/ruzzy
116-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
117-
# RubyGems installation directory
118-
ENV GEM_HOME="$OUT/fuzz-gem"
119-
ENV GEM_PATH="/install/ruzzy"
112+
# Copy built ruby. It is up to the fuzzing harnesses
113+
# themselves to set GEM_HOME and GEM_PATH appropriately, as this depends
114+
# on how the harnesses are packaged.
115+
COPY --from=base-ruby /usr/local/bin/ruby /usr/local/bin/ruby
116+
COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem
117+
COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby
118+
COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0
119+
120120

121121
# Do this last to make developing these files easier/faster due to caching.
122122
COPY bad_build_check \

‎infra/base-images/base-runner/ubuntu-20-04.Dockerfile‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,14 +109,13 @@ RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jaco
109109
COPY install_javascript.sh /
110110
RUN /install_javascript.sh && rm /install_javascript.sh
111111

112-
# Copy built ruby and ruzzy from builder
113-
COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
114-
COPY --from=base-ruby /install/ruzzy /install/ruzzy
115-
COPY ruzzy /usr/bin/ruzzy
116-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
117-
# RubyGems installation directory
118-
ENV GEM_HOME="$OUT/fuzz-gem"
119-
ENV GEM_PATH="/install/ruzzy"
112+
# Copy built ruby. It is up to the fuzzing harnesses
113+
# themselves to set GEM_HOME and GEM_PATH appropriately, as this depends
114+
# on how the harnesses are packaged.
115+
COPY --from=base-ruby /usr/local/bin/ruby /usr/local/bin/ruby
116+
COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem
117+
COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby
118+
COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0
120119

121120
# Do this last to make developing these files easier/faster due to caching.
122121
COPY bad_build_check \

‎infra/base-images/base-runner/ubuntu-24-04.Dockerfile‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,14 +109,13 @@ RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jaco
109109
COPY install_javascript.sh /
110110
RUN /install_javascript.sh && rm /install_javascript.sh
111111

112-
# Copy built ruby and ruzzy from builder
113-
COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
114-
COPY --from=base-ruby /install/ruzzy /install/ruzzy
115-
COPY ruzzy /usr/bin/ruzzy
116-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
117-
# RubyGems installation directory
118-
ENV GEM_HOME="$OUT/fuzz-gem"
119-
ENV GEM_PATH="/install/ruzzy"
112+
# Copy built ruby. It is up to the fuzzing harnesses
113+
# themselves to set GEM_HOME and GEM_PATH appropriately, as this depends
114+
# on how the harnesses are packaged.
115+
COPY --from=base-ruby /usr/local/bin/ruby /usr/local/bin/ruby
116+
COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem
117+
COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby
118+
COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0
120119

121120
# Do this last to make developing these files easier/faster due to caching.
122121
COPY bad_build_check \

‎infra/build/functions/base_images.py‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,23 @@
4747
# This version will receive the ':v1' tag.
4848
DEFAULT_VERSION = 'legacy'
4949

50+
# Defines the dependency graph for base images.
51+
IMAGE_DEPENDENCIES = {
52+
'base-clang': ['base-image'],
53+
'base-clang-full': ['base-clang'],
54+
'base-builder': ['base-clang'],
55+
'base-builder-go': ['base-builder'],
56+
'base-builder-javascript': ['base-builder'],
57+
'base-builder-jvm': ['base-builder'],
58+
'base-builder-python': ['base-builder'],
59+
'base-builder-ruby': ['base-builder'],
60+
'base-builder-rust': ['base-builder'],
61+
'base-builder-swift': ['base-builder'],
62+
'base-runner': ['base-image', 'base-builder'],
63+
'base-runner-debug': ['base-runner'],
64+
'indexer': ['base-clang-full'],
65+
}
66+
5067

5168
class ImageConfig:
5269
"""Configuration for a specific base image version."""
@@ -85,6 +102,8 @@ def _resolve_dockerfile(self) -> str:
85102
if os.path.exists(versioned_dockerfile):
86103
logging.info('Using versioned Dockerfile: %s', versioned_dockerfile)
87104
return versioned_dockerfile
105+
raise FileNotFoundError(
106+
f'Versioned Dockerfile not found for {self.name}:{self.version}')
88107

89108
legacy_dockerfile = os.path.join(self.path, 'Dockerfile')
90109
logging.info('Using legacy Dockerfile: %s', legacy_dockerfile)
@@ -156,6 +175,8 @@ def full_image_name_with_tag(self) -> str:
156175
def get_base_image_steps(images: Sequence[ImageConfig]) -> list[dict]:
157176
"""Returns build steps for a given list of image configurations."""
158177
steps = [build_lib.get_git_clone_step()]
178+
build_ids = {}
179+
159180
for image_config in images:
160181
# The final tag is ':v1' for the default version, or the version name
161182
# (e.g., ':ubuntu-24-04') for others.
@@ -167,11 +188,20 @@ def get_base_image_steps(images: Sequence[ImageConfig]) -> list[dict]:
167188
tags.append(f'{IMAGE_NAME_PREFIX}{image_config.name}:latest')
168189

169190
dockerfile_path = os.path.join('oss-fuzz', image_config.dockerfile_path)
170-
steps.append(
171-
build_lib.get_docker_build_step(tags,
172-
image_config.path,
173-
dockerfile_path=dockerfile_path,
174-
build_args=image_config.build_args))
191+
step = build_lib.get_docker_build_step(tags,
192+
image_config.path,
193+
dockerfile_path=dockerfile_path,
194+
build_args=image_config.build_args)
195+
196+
# Check for dependencies and add 'waitFor' if necessary.
197+
dependencies = IMAGE_DEPENDENCIES.get(image_config.name, [])
198+
wait_for = [build_ids[dep] for dep in dependencies if dep in build_ids]
199+
if wait_for:
200+
step['waitFor'] = wait_for
201+
202+
build_ids[image_config.name] = step['id']
203+
steps.append(step)
204+
175205
return steps
176206

177207

0 commit comments

Comments
 (0)