Skip to content

Commit d1fdc21

Browse files
ebyerlyphlax
ebyerly
and
phlax
authored
Envoy commit integration (#1467)
Move Envoy commit integration up to envoyproxy/envoy@e593c78 phlax@ has been making significant contributions to Envoy's CI process. What did we need to do to update Nighthawk's CI? Explicitly set config=gcc or config=clang for all tasks Bind mount spare disks - the runners have ~100GB hard drives but default to storing Bazel caches in /tmp with caps Update Bazel config name references - remote-envoy-engflow and bes-envoy-engflow no longer exist Incorporate RBE - a huge win! Nighthawk's CI process is down to ~30 minutes for build and test. Update our Envoy integration tooling to also copy over .github/config.yml and ci/envoy_build_sha.sh Statically link libatomic instead of relying on dynamic linking. Use Bazel's .stripped output instead of stripping inside the Docker container build process. Add the Docker container builder user to the docker group. Replace rsync with cp and find|rm. --------- Signed-off-by: Elizabeth Byerly <[email protected]> Co-authored-by: Ryan Northey <[email protected]>
1 parent 407510a commit d1fdc21

File tree

13 files changed

+840
-413
lines changed

13 files changed

+840
-413
lines changed

‎.bazelrc‎

Lines changed: 271 additions & 372 deletions
Large diffs are not rendered by default.

‎.github/config.yml‎

Lines changed: 440 additions & 0 deletions
Large diffs are not rendered by default.

‎.github/workflows/_ci.yml‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ on:
1515
type: string
1616
default: >-
1717
--config=remote-ci
18+
--config=rbe
1819
docker-in-docker:
1920
description: "Enable Docker in Docker"
2021
required: false
@@ -37,17 +38,34 @@ jobs:
3738
runs-on: ubuntu-22.04
3839
steps:
3940
- uses: actions/[email protected]
41+
- name: Set gcc
42+
run: |
43+
if [[ "${{ inputs.task }}" == *"gcc"* ]]; then
44+
echo "build --config=gcc" >> repo.bazelrc
45+
else
46+
echo "common --config=clang" >> repo.bazelrc
47+
fi
4048
- name: Configure repo Bazel settings
4149
run: |
42-
echo "build:remote-envoy-engflow --config=bes-envoy-engflow" > repo.bazelrc
50+
echo "common --jobs=200" >> repo.bazelrc
51+
echo "build --config=bes" >> repo.bazelrc
52+
- name: Bind mount spare disk for build artifacts
53+
uses: envoyproxy/toolshed/gh-actions/[email protected]
54+
with:
55+
mounts: |
56+
- src: /mnt/envoy-docker-build
57+
target: /tmp/envoy-docker-build
58+
chown: runner:runner
59+
- src: /mnt/bazel-shared
60+
target: /tmp/bazel-shared
61+
chown: runner:runner
4362
- name: Run CI script
4463
run: |
4564
./ci/run_envoy_docker.sh './ci/do_ci.sh ${{ inputs.task }}'
4665
env:
4766
GITHUB_TOKEN: ${{ github.token }}
4867
BAZEL_BUILD_EXTRA_OPTIONS: >-
4968
${{ inputs.bazel-extra }}
50-
--config=remote-envoy-engflow
5169
GH_BRANCH: ${{ github.ref }}
5270
GH_SHA1: ${{ github.sha }}
5371
ENVOY_DOCKER_IN_DOCKER: ${{ inputs.docker-in-docker && '1' || '' }}

‎BUILD‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,18 @@ envoy_package()
1313
filegroup(
1414
name = "nighthawk",
1515
srcs = [
16-
":nighthawk_adaptive_load_client",
17-
":nighthawk_client",
18-
":nighthawk_output_transform",
19-
":nighthawk_service",
20-
":nighthawk_test_server",
16+
":nighthawk_adaptive_load_client.stripped",
17+
":nighthawk_client.stripped",
18+
":nighthawk_output_transform.stripped",
19+
":nighthawk_service.stripped",
20+
":nighthawk_test_server.stripped",
2121
],
2222
)
2323

2424
envoy_cc_binary(
2525
name = "nighthawk_adaptive_load_client",
2626
linkopts = [
27-
"-latomic",
27+
"-l:libatomic.a",
2828
"-lrt",
2929
],
3030
repository = "@envoy",
@@ -36,7 +36,7 @@ envoy_cc_binary(
3636
envoy_cc_binary(
3737
name = "nighthawk_client",
3838
linkopts = [
39-
"-latomic",
39+
"-l:libatomic.a",
4040
"-lrt",
4141
],
4242
repository = "@envoy",
@@ -50,7 +50,7 @@ envoy_cc_binary(
5050
envoy_cc_binary(
5151
name = "nighthawk_client_testonly",
5252
linkopts = [
53-
"-latomic",
53+
"-l:libatomic.a",
5454
"-lrt",
5555
],
5656
repository = "@envoy",
@@ -64,7 +64,7 @@ envoy_cc_binary(
6464
envoy_cc_binary(
6565
name = "nighthawk_test_server",
6666
linkopts = [
67-
"-latomic",
67+
"-l:libatomic.a",
6868
"-lrt",
6969
],
7070
repository = "@envoy",
@@ -79,7 +79,7 @@ envoy_cc_binary(
7979
envoy_cc_binary(
8080
name = "nighthawk_service",
8181
linkopts = [
82-
"-latomic",
82+
"-l:libatomic.a",
8383
"-lrt",
8484
],
8585
repository = "@envoy",
@@ -91,7 +91,7 @@ envoy_cc_binary(
9191
envoy_cc_binary(
9292
name = "nighthawk_output_transform",
9393
linkopts = [
94-
"-latomic",
94+
"-l:libatomic.a",
9595
"-lrt",
9696
],
9797
repository = "@envoy",

‎WORKSPACE‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,6 @@ load("@envoy//bazel:api_repositories.bzl", "envoy_api_dependencies")
1717

1818
envoy_api_dependencies()
1919

20-
load("@envoy//bazel:repo.bzl", "envoy_repo")
21-
22-
envoy_repo()
23-
2420
load("@envoy//bazel:repositories.bzl", "envoy_dependencies")
2521

2622
envoy_dependencies()
@@ -41,6 +37,18 @@ load("@envoy//bazel:dependency_imports.bzl", "envoy_dependency_imports")
4137

4238
envoy_dependency_imports()
4339

40+
load("@envoy//bazel:repo.bzl", "envoy_repo")
41+
42+
envoy_repo()
43+
44+
load("@envoy//bazel:toolchains.bzl", "envoy_toolchains")
45+
46+
envoy_toolchains()
47+
48+
load("@envoy//bazel:dependency_imports_extra.bzl", "envoy_dependency_imports_extra")
49+
50+
envoy_dependency_imports_extra()
51+
4452
load("//bazel:python_dependencies.bzl", "nighthawk_python_dependencies")
4553

4654
nighthawk_python_dependencies()

‎bazel/repositories.bzl‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
22

3-
ENVOY_COMMIT = "ee9df7b308de9f7e998619d1c86bc87b9058d0c9"
4-
ENVOY_SHA = "583d91fb5c87e1c92b1a75ad1e259fe9e1164515cd714e887eed01c6cb5ec953"
3+
ENVOY_COMMIT = "e593c7820b15aab315ddc2b615bd639012288469"
4+
ENVOY_SHA = "6f5614477880fc392844e2b63efb9b55774e908e1bc5f18e6ee57068f76addf8"
55

66
HDR_HISTOGRAM_C_VERSION = "0.11.8" # June 18th, 2025
77
HDR_HISTOGRAM_C_SHA = "bb95351a6a8b242dc9be1f28562761a84d4cf0a874ffc90a9b630770a6468e94"
@@ -48,7 +48,7 @@ cc_library(
4848
"-Wno-implicit-function-declaration",
4949
"-Wno-error",
5050
],
51-
deps = ["@envoy//bazel/foreign_cc:zlib",],
51+
deps = ["@zlib",],
5252
visibility = ["//visibility:public"],
5353
)
5454
""",

‎ci/do_ci.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,8 @@ export BAZEL_TEST_OPTIONS="${BAZEL_BUILD_OPTIONS} \
310310
--test_env=UBSAN_OPTIONS=print_stacktrace=1 \
311311
--cache_test_results=no --test_output=errors ${BAZEL_EXTRA_TEST_OPTIONS}"
312312

313+
export CARGO_BAZEL_REPIN=true
314+
313315
case "$1" in
314316
build)
315317
setup_clang_toolchain

‎ci/docker-compose.yml‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
x-envoy-build-base: &envoy-build-base
22
image: >-
3-
${ENVOY_BUILD_IMAGE:-docker.io/envoyproxy/envoy-build-ubuntu:4ce0cb04f941fb89d475597a640176ae64070bb1@sha256:de54e91a8d27623ffbd3fb9bd1aba8241567e41c0fb82b167128e3629c2ede18}
3+
${ENVOY_BUILD_IMAGE:?ERROR: ENVOY_BUILD_IMAGE is not set! either set it in your environment or use ci/run_envoy_docker.sh to set it}
44
cpus: ${ENVOY_DOCKER_CPUS:-0}
55
user: root:root
66
working_dir: ${ENVOY_DOCKER_SOURCE_DIR:-/source}
@@ -81,26 +81,40 @@ x-envoy-build-base: &envoy-build-base
8181
- "/bin/bash"
8282
- "-c"
8383
- |
84-
groupadd --gid ${DOCKER_GID:-${USER_GID:-$(id -g)}} -f envoygroup
85-
useradd -o \
86-
--uid ${USER_UID:-$(id -u)} \
87-
--gid ${DOCKER_GID:-${USER_GID:-$(id -g)}} \
88-
--no-create-home \
89-
-s /bin/bash \
90-
--home-dir /build envoybuild
91-
usermod -a -G pcap envoybuild
92-
chown envoybuild:envoygroup /build
84+
set -e
85+
86+
if [[ -n "${USER_GID:-}" ]]; then
87+
groupmod -g "$USER_GID" envoybuild >/dev/null
88+
fi
89+
if [[ -n "${USER_UID:-}" ]]; then
90+
usermod -u "$USER_UID" envoybuild >/dev/null
91+
fi
92+
usermod -d /build envoybuild >/dev/null
93+
if [[ -f /.build-id ]]; then
94+
build_id="$(cat /.build-id)"
95+
echo "Envoy build image: $${build_id}"
96+
fi
97+
chown envoybuild:envoybuild /build
9398
chown envoybuild /proc/self/fd/2 2>/dev/null || true
9499
[[ -e /entrypoint-extra.sh ]] && /entrypoint-extra.sh
100+
101+
if [ -S /var/run/docker.sock ]; then
102+
DOCKER_GID=$(stat -c '%g' /var/run/docker.sock)
103+
if ! getent group "$DOCKER_GID" >/dev/null; then
104+
groupadd -g "$DOCKER_GID" docker_sock
105+
fi
106+
usermod -a -G "$(getent group "$DOCKER_GID" | cut -d: -f1)" envoybuild
107+
fi
108+
95109
sudo -EHs -u envoybuild bash -c 'cd ${ENVOY_DOCKER_SOURCE_DIR:-/source} && exec ${DOCKER_COMMAND:-bash}'
96110
97111
services:
98112
envoy-build:
99113
<<: *envoy-build-base
100114
volumes:
101-
- ${ENVOY_DOCKER_BUILD_DIR:-/tmp/envoy-docker-build}:/build
102-
- ${SOURCE_DIR:-..}:/source
103-
- ${SHARED_TMP_DIR:-/tmp/bazel-shared}:${SHARED_TMP_DIR:-/tmp/bazel-shared}
115+
- ${ENVOY_DOCKER_BUILD_DIR:-/tmp/envoy-docker-build}:/build${MOUNT_ACCESS_MODE:-}
116+
- ${SOURCE_DIR:-..}:/source${MOUNT_ACCESS_MODE:-}
117+
- ${SHARED_TMP_DIR:-/tmp/bazel-shared}:${SHARED_TMP_DIR:-/tmp/bazel-shared}${MOUNT_ACCESS_MODE:-}
104118

105119
envoy-build-gpg:
106120
<<: *envoy-build-base
@@ -117,4 +131,4 @@ services:
117131
- ${ENVOY_DOCKER_BUILD_DIR:-/tmp/envoy-docker-build}:/build
118132
- ${SOURCE_DIR:-..}:/source
119133
- /var/run/docker.sock:/var/run/docker.sock
120-
- ${SHARED_TMP_DIR:-/tmp/bazel-shared}:${SHARED_TMP_DIR:-/tmp/bazel-shared}
134+
- ${SHARED_TMP_DIR:-/tmp/bazel-shared}:${SHARED_TMP_DIR:-/tmp/bazel-shared} - ${ENVOY_ENTRYPOINT_EXTRA:-./docker-entrypoint-extra.sh}:/entrypoint-extra.sh

‎ci/docker/benchmark_build.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ TMP_DIR="$(mktemp -d ${WORKSPACE}/tmp-docker-build-context-XXXXXXXX)"
2222
echo "Preparing docker build context in ${TMP_DIR}"
2323
cp -r "${WORKSPACE}/ci/docker/" "${TMP_DIR}/"
2424
# Exclude any venv files since they aren't necessary and docker can't handle symlinks.
25-
rsync -a --exclude='*.venv/' "${BAZEL_BIN}/benchmarks" "${TMP_DIR}"
25+
cp -r "${BAZEL_BIN}/benchmarks" "${TMP_DIR}"
26+
find "${TMP_DIR}/benchmarks" -name "*.venv" -type d -exec rm -rf {} +
2627

2728
cd "${TMP_DIR}"
2829
echo "running docker build ... "

‎ci/docker/docker_build.sh‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,11 @@ echo "Preparing docker build context in ${TMP_DIR}"
2323
cp -r "${WORKSPACE}/ci/docker/" "${TMP_DIR}/"
2424

2525
for BINARY in "${BINARIES[@]}"; do
26-
echo "Copy and strip ${BINARY}"
26+
echo "Copy ${BINARY}"
2727
TARGET="${TMP_DIR}/${BINARY}"
2828
# Docker won't follow symlinks
29-
cp "${BAZEL_BIN}/${BINARY}" "${TARGET}"
29+
cp "${BAZEL_BIN}/${BINARY}.stripped" "${TARGET}"
3030
chmod +w "${TARGET}"
31-
strip --strip-debug "${TARGET}"
3231
done
3332

3433
cd "${TMP_DIR}"

0 commit comments

Comments
 (0)