Skip to content

Commit 3a7a888

Browse files
hunschehunsche
authored
Merge branch 'master' into feat/abseil-ubuntu-migration
2 parents 1d45388 + e89ca7a commit 3a7a888

File tree

29 files changed

+1801
-338
lines changed

29 files changed

+1801
-338
lines changed

‎.github/workflows/ubuntu_version_sync.yml‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,19 @@
1414
#
1515
################################################################################
1616

17-
name: 'Ubuntu Version Sync'
17+
name: 'Ubuntu Version Sync Check'
1818

1919
on:
2020
pull_request:
2121
types: [opened, synchronize, reopened]
2222

2323
jobs:
2424
check-sync:
25+
name: Ubuntu File Synchronization Check
2526
runs-on: ubuntu-latest
27+
env:
28+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
29+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
2630
steps:
2731
- name: 'Checkout code'
2832
uses: actions/checkout@v4
@@ -34,7 +38,7 @@ jobs:
3438
run: |
3539
set -e
3640
37-
MODIFIED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
41+
MODIFIED_FILES=$(git diff --name-only $BASE_SHA...$HEAD_SHA)
3842
echo "Checking for synchronized file changes..."
3943
echo "Modified files in this PR:"
4044
echo "$MODIFIED_FILES"
@@ -63,27 +67,27 @@ jobs:
6367
VERSIONS=("ubuntu-20-04" "ubuntu-24-04")
6468
6569
# Check Dockerfiles
66-
for legacy_file in "${{!LEGACY_DOCKERFILES[@]}}"; do
67-
if echo "$MODIFIED_FILES" | grep -q "^${legacy_file}$"; then
70+
for legacy_file in "${!LEGACY_DOCKERFILES[@]}"; do
71+
if [[ "${legacy_file}" == infra/* ]] && echo "$MODIFIED_FILES" | grep -q "^${legacy_file}$"; then
6872
echo "Legacy file changed: $legacy_file. Verifying counterparts..."
69-
for version in "${{VERSIONS[@]}}"; do
70-
pattern=${{LEGACY_DOCKERFILES[$legacy_file]}}
71-
versioned_file="${{pattern/{{version}}/$version}}"
72-
if ! echo "$MODIFIED_FILES" | grep -q "^${{versioned_file}}$"; then
73+
for version in "${VERSIONS[@]}"; do
74+
pattern="${LEGACY_DOCKERFILES[$legacy_file]}"
75+
versioned_file="${pattern/\{version\}/$version}"
76+
if ! echo "$MODIFIED_FILES" | grep -q "^${versioned_file}$"; then
7377
ERRORS+="\n- Legacy file '${legacy_file}' was changed, but its counterpart '${versioned_file}' was not."
7478
fi
7579
done
7680
fi
7781
done
7882
7983
# Check Scripts
80-
for legacy_file in "${{!LEGACY_SCRIPTS[@]}}"; do
84+
for legacy_file in "${!LEGACY_SCRIPTS[@]}"; do
8185
if echo "$MODIFIED_FILES" | grep -q "^${legacy_file}$"; then
8286
echo "Legacy script changed: $legacy_file. Verifying counterparts..."
83-
for version in "${{VERSIONS[@]}}"; do
84-
pattern=${{LEGACY_SCRIPTS[$legacy_file]}}
85-
versioned_file="${{pattern/{{version}}/$version}}"
86-
if ! echo "$MODIFIED_FILES" | grep -q "^${{versioned_file}}$"; then
87+
for version in "${VERSIONS[@]}"; do
88+
pattern="${LEGACY_SCRIPTS[$legacy_file]}"
89+
versioned_file="${pattern/\{version\}/$version}"
90+
if ! echo "$MODIFIED_FILES" | grep -q "^${versioned_file}$"; then
8791
ERRORS+="\n- Legacy script '${legacy_file}' was changed, but its counterpart '${versioned_file}' was not."
8892
fi
8993
done

‎infra/base-images/base-builder-ruby/Dockerfile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,8 @@ FROM gcr.io/oss-fuzz-base/base-builder
1818

1919
RUN git clone https://github.com/trailofbits/ruzzy.git $SRC/ruzzy
2020

21-
RUN install_ruby.sh
22-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
23-
24-
RUN gem update --system 3.5.11
21+
RUN /usr/local/bin/install_ruby.sh
22+
RUN /usr/local/bin/gem update --system 3.5.11
2523

2624
# Install ruzzy
2725
WORKDIR $SRC/ruzzy

‎infra/base-images/base-builder-ruby/ubuntu-20-04.Dockerfile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,8 @@ FROM gcr.io/oss-fuzz-base/base-builder:ubuntu-20-04
1818

1919
RUN git clone https://github.com/trailofbits/ruzzy.git $SRC/ruzzy
2020

21-
RUN install_ruby.sh
22-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
23-
24-
RUN gem update --system 3.5.11
21+
RUN /usr/local/bin/install_ruby.sh
22+
RUN /usr/local/bin/gem update --system 3.5.11
2523

2624
# Install ruzzy
2725
WORKDIR $SRC/ruzzy

‎infra/base-images/base-builder-ruby/ubuntu-24-04.Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ FROM gcr.io/oss-fuzz-base/base-builder:ubuntu-24-04
1818

1919
RUN git clone https://github.com/trailofbits/ruzzy.git $SRC/ruzzy
2020

21-
RUN install_ruby.sh
22-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
21+
RUN /usr/local/bin/install_ruby.sh
22+
RUN /usr/local/bin/gem update --system 3.5.11
2323

2424
RUN gem update --system 3.5.11
2525

‎infra/base-images/base-builder/indexer/utils.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@
2222
import subprocess
2323
from typing import Final, Sequence
2424

25+
from absl import logging
26+
27+
from google3.pyglib import gfile
28+
import pathlib
29+
30+
2531
LD_BINARY_NAME: Final[str] = "ld-linux-x86-64.so.2"
2632
_LD_BINARY_PATH: Final[pathlib.Path] = pathlib.Path("/lib64") / LD_BINARY_NAME
2733

@@ -79,3 +85,52 @@ def get_shared_libraries(
7985
)
8086

8187
return _parse_ld_trace_output(result.stdout.decode())
88+
89+
90+
def copy_shared_libraries(
91+
libraries: Sequence[SharedLibrary], dst_path: pathlib.Path
92+
) -> None:
93+
"""Copies the shared libraries to the shared directory."""
94+
for lib in libraries:
95+
try:
96+
logging.info("Copying %s => %s", lib.name, lib.path)
97+
gfile.Copy(lib.path, dst_path / lib.path.name, overwrite=True, mode=0o755)
98+
except gfile.GOSError:
99+
logging.exception("Could not copy %s to %s", lib.path, dst_path)
100+
raise
101+
102+
103+
def patch_binary_rpath_and_interpreter(
104+
binary_path: os.PathLike[str],
105+
lib_mount_path: pathlib.Path,
106+
):
107+
"""Patches the binary rpath and interpreter."""
108+
subprocess.run(
109+
[
110+
"patchelf",
111+
"--set-rpath",
112+
lib_mount_path.as_posix(),
113+
"--force-rpath",
114+
binary_path,
115+
],
116+
check=True,
117+
)
118+
119+
subprocess.run(
120+
[
121+
"patchelf",
122+
"--set-interpreter",
123+
(lib_mount_path / LD_BINARY_NAME).as_posix(),
124+
binary_path,
125+
],
126+
check=True,
127+
)
128+
129+
130+
def get_library_mount_path(binary_id: str) -> pathlib.Path:
131+
return pathlib.Path("/tmp") / (binary_id + "_lib")
132+
133+
134+
def report_progress(stage: str, is_done: bool = False) -> None:
135+
"""Reports progress of a stage of the snapshotting process."""
136+
logging.info("%s%s", stage, "..." if not is_done else "")

‎infra/base-images/base-builder/install_ruby.sh‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/bin/bash -eux
22
# Copyright 2024 Google LLC
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,14 +15,19 @@
1515
#
1616
################################################################################
1717

18-
apt update
19-
apt install -y lsb-release software-properties-common gnupg2 binutils xz-utils libyaml-dev
20-
gpg2 --keyserver keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
18+
echo "Starting ruby installation"
19+
RUBY_VERSION=3.3.1
20+
RUBY_DEPS="binutils xz-utils libyaml-dev libffi-dev zlib1g-dev"
21+
apt update && apt install -y $RUBY_DEPS
22+
curl -O https://cache.ruby-lang.org/pub/ruby/3.3/ruby-$RUBY_VERSION.tar.gz
23+
tar -xvf ruby-$RUBY_VERSION.tar.gz
24+
cd ruby-$RUBY_VERSION
25+
./configure
26+
make -j$(nproc)
27+
make install
28+
cd ../
2129

22-
curl -sSL https://get.rvm.io > ruby_installation.sh
23-
chmod +x ruby_installation.sh
24-
bash ruby_installation.sh stable
30+
# Clean up the sources.
31+
rm -rf ./ruby-$RUBY_VERSION ruby-$RUBY_VERSION.tar.gz
2532

26-
. /etc/profile.d/rvm.sh
27-
28-
rvm install ruby-3.3.1
33+
echo "Finished installing ruby"

‎infra/base-images/base-runner/Dockerfile‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,14 +109,14 @@ RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jaco
109109
COPY install_javascript.sh /
110110
RUN /install_javascript.sh && rm /install_javascript.sh
111111

112-
# Copy built ruby and ruzzy from builder
113-
COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
114-
COPY --from=base-ruby /install/ruzzy /install/ruzzy
115-
COPY ruzzy /usr/local/bin/ruzzy
116-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
117-
# RubyGems installation directory
118-
ENV GEM_HOME="$OUT/fuzz-gem"
119-
ENV GEM_PATH="/install/ruzzy"
112+
# Copy built ruby. It is up to the fuzzing harnesses
113+
# themselves to set GEM_HOME and GEM_PATH appropriately, as this depends
114+
# on how the harnesses are packaged.
115+
COPY --from=base-ruby /usr/local/bin/ruby /usr/local/bin/ruby
116+
COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem
117+
COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby
118+
COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0
119+
120120

121121
# Do this last to make developing these files easier/faster due to caching.
122122
COPY bad_build_check \

‎infra/base-images/base-runner/ubuntu-20-04.Dockerfile‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,14 +109,13 @@ RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jaco
109109
COPY install_javascript.sh /
110110
RUN /install_javascript.sh && rm /install_javascript.sh
111111

112-
# Copy built ruby and ruzzy from builder
113-
COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
114-
COPY --from=base-ruby /install/ruzzy /install/ruzzy
115-
COPY ruzzy /usr/bin/ruzzy
116-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
117-
# RubyGems installation directory
118-
ENV GEM_HOME="$OUT/fuzz-gem"
119-
ENV GEM_PATH="/install/ruzzy"
112+
# Copy built ruby. It is up to the fuzzing harnesses
113+
# themselves to set GEM_HOME and GEM_PATH appropriately, as this depends
114+
# on how the harnesses are packaged.
115+
COPY --from=base-ruby /usr/local/bin/ruby /usr/local/bin/ruby
116+
COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem
117+
COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby
118+
COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0
120119

121120
# Do this last to make developing these files easier/faster due to caching.
122121
COPY bad_build_check \

‎infra/base-images/base-runner/ubuntu-24-04.Dockerfile‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,14 +109,13 @@ RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jaco
109109
COPY install_javascript.sh /
110110
RUN /install_javascript.sh && rm /install_javascript.sh
111111

112-
# Copy built ruby and ruzzy from builder
113-
COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
114-
COPY --from=base-ruby /install/ruzzy /install/ruzzy
115-
COPY ruzzy /usr/bin/ruzzy
116-
ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
117-
# RubyGems installation directory
118-
ENV GEM_HOME="$OUT/fuzz-gem"
119-
ENV GEM_PATH="/install/ruzzy"
112+
# Copy built ruby. It is up to the fuzzing harnesses
113+
# themselves to set GEM_HOME and GEM_PATH appropriately, as this depends
114+
# on how the harnesses are packaged.
115+
COPY --from=base-ruby /usr/local/bin/ruby /usr/local/bin/ruby
116+
COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem
117+
COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby
118+
COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0
120119

121120
# Do this last to make developing these files easier/faster due to caching.
122121
COPY bad_build_check \

‎infra/build/functions/build_lib.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -707,7 +707,7 @@ def get_runner_image_name(test_image_suffix, base_image_tag=None):
707707
# Only add a tag if it's specified and not 'legacy', as 'legacy' implies
708708
# 'latest', which is the default behavior.
709709
if base_image_tag and base_image_tag != 'legacy':
710-
image += ':' + base_image_tag
710+
image += f":{base_image_tag}"
711711

712712
return image
713713

0 commit comments

Comments
 (0)