Skip to content

Commit f134b62

Browse files
sandeepgupta12sandeepgupta12
committed
self-hosted runner changes
1 parent c1005d1 commit f134b62

File tree

10 files changed

+730
-4
lines changed

10 files changed

+730
-4
lines changed

‎.github/scripts/ppc64le-ci/README.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Configuring the builder.
2+
3+
## Install prerequisites.
4+
5+
```
6+
Install Docker
7+
```
8+
## Clone pytorch repository
9+
10+
## Add services.
11+
12+
```
13+
$ sudo cp self-hosted-builder/*.service /etc/systemd/system/
14+
$ sudo systemctl daemon-reload
15+
```
16+
Next step is to build `actions-runner` image using:
17+
18+
```
19+
$ cd self-hosted-builder
20+
$ sudo docker build \
21+
--pull \
22+
-f actions-runner.Dockerfile \
23+
--build-arg RUNNERPATCH="runner-sdk-8.ppc64le.patch" \
24+
-t iiilinuxibmcom/actions-runner.<name> \
25+
.
26+
```
27+
28+
Now prepare all necessary files for runner registration:
29+
30+
```
31+
$ sudo mkdir -p /etc/actions-runner/<name>
32+
$ sudo chmod 700 /etc/actions-runner/<name>
33+
$ sudo /bin/cp <github_app_private_key_file> /etc/actions-runner/<name>/key_private.pem
34+
$ sudo echo <github_app_id> | sudo tee /etc/actions-runner/<name>/appid.env
35+
$ sudo echo <github_app_install_id> | sudo tee /etc/actions-runner/<name>/installid.env
36+
$ sudo echo NAME=<worker_name> | sudo tee /etc/actions-runner/<name>/env
37+
$ sudo echo ORG=<github_org> | sudo tee -a /etc/actions-runner/<name>/env
38+
$ cd self-hosted-builder
39+
$ sudo /bin/cp helpers/*.sh /local/bin/
40+
$ sudo chmod 755 /usr/local/bin/app_token.sh /usr/local/bin/gh_token_generator.sh
41+
```
42+
43+
## Autostart the runner.
44+
45+
```
46+
$ sudo systemctl enable --now actions-runner@$NAME
47+
```

‎.github/scripts/ppc64le-ci/self-hosted-builder/actions-runner.Dockerfile‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# Self-Hosted IBM Power Github Actions Runner.
2+
FROM ubuntu:22.04
3+
4+
# Set non-interactive mode for apt
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
7+
# Fix sources to point to ports.ubuntu.com for ppc64le
8+
RUN echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy main restricted universe multiverse" > /etc/apt/sources.list && \
9+
echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy-updates main restricted universe multiverse" >> /etc/apt/sources.list && \
10+
echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy-backports main restricted universe multiverse" >> /etc/apt/sources.list && \
11+
echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy-security main restricted universe multiverse" >> /etc/apt/sources.list
12+
13+
# Fix sources for ppc64le and update system
14+
RUN apt-get update -o Acquire::Retries=5 -o Acquire::http::Timeout="10" && \
15+
apt-get -y install --no-install-recommends \
16+
build-essential \
17+
curl \
18+
sudo \
19+
jq \
20+
gnupg-agent \
21+
iptables \
22+
ca-certificates \
23+
software-properties-common \
24+
vim \
25+
zip \
26+
python3 \
27+
python3-pip && \
28+
apt-get clean && rm -rf /var/lib/apt/lists/*
29+
30+
# Switch to iptables-legacy
31+
RUN update-alternatives --set iptables /usr/sbin/iptables-legacy && \
32+
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
33+
34+
35+
# Add Docker GPG key and repository
36+
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
37+
echo "deb [arch=ppc64el signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list && \
38+
apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io && \
39+
apt-get clean && rm -rf /var/lib/apt/lists/*
40+
41+
# Install dotnet SDK and other dependencies
42+
RUN apt-get update && apt-get install -y --no-install-recommends \
43+
wget \
44+
git \
45+
dotnet-sdk-8.0 \
46+
cmake \
47+
make \
48+
automake \
49+
autoconf \
50+
m4 \
51+
libtool && \
52+
apt-get clean && rm -rf /var/lib/apt/lists/*
53+
54+
55+
# Setup user and permissions
56+
RUN useradd -c "Action Runner" -m runner && \
57+
usermod -L runner && \
58+
echo "runner ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/runner && \
59+
groupadd docker || true && \
60+
usermod -aG docker runner && \
61+
(test -S /var/run/docker.sock && chmod 660 /var/run/docker.sock && chgrp docker /var/run/docker.sock || true)
62+
63+
64+
# Add and configure GitHub Actions runner
65+
ARG RUNNERREPO="https://github.com/actions/runner"
66+
ARG RUNNERPATCH
67+
68+
ADD ${RUNNERPATCH} /tmp/runner.patch
69+
70+
RUN git clone -q ${RUNNERREPO} /tmp/runner && \
71+
cd /tmp/runner && \
72+
git checkout main -b build && \
73+
git apply /tmp/runner.patch && \
74+
sed -i'' -e /version/s/8......\"$/${SDK}.0.100\"/ src/global.json
75+
76+
RUN cd /tmp/runner/src && \
77+
./dev.sh layout && \
78+
./dev.sh package && \
79+
./dev.sh test && \
80+
rm -rf /root/.dotnet /root/.nuget
81+
82+
RUN mkdir -p /opt/runner && \
83+
tar -xf /tmp/runner/_package/*.tar.gz -C /opt/runner && \
84+
chown -R runner:runner /opt/runner && \
85+
su - runner -c "/opt/runner/config.sh --version"
86+
87+
RUN rm -rf /tmp/runner /tmp/runner.patch
88+
89+
# Copy custom scripts and set permissions
90+
COPY fs/ /
91+
RUN chmod 777 /usr/bin/actions-runner /usr/bin/entrypoint
92+
93+
# Switch to the runner user
94+
USER runner
95+
96+
# Set working directory
97+
WORKDIR /opt/runner
98+
99+
# Define entry point and command
100+
ENTRYPOINT ["/usr/bin/entrypoint"]
101+
CMD ["/usr/bin/actions-runner"]
102+

‎.github/scripts/ppc64le-ci/self-hosted-builder/[email protected]‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
[Unit]
2+
Description=Self-Hosted IBM power Github Actions Runner
3+
StartLimitIntervalSec=0
4+
5+
[Service]
6+
Type=simple
7+
Restart=always
8+
9+
# Cleanup stale containers
10+
ExecStartPre=-/usr/bin/docker rm --force actions-runner.%i
11+
ExecStartPre=-/usr/local/bin/gh_token_generator.sh /etc/actions-runner/%i/appid.env /etc/actions-runner/%i/installid.env /etc/actions-runner/%i/key_private.pem /etc/actions-runner/%i/ghtoken.env
12+
ExecStartPre=-/usr/local/bin/gh_cat_token.sh /etc/actions-runner/%i/ghtoken.env /etc/actions-runner/%i/ghtoken.txt
13+
14+
ExecStart=/usr/bin/docker run \
15+
--env-file=/etc/actions-runner/%i/env \
16+
--volume /etc/actions-runner/%i/ghtoken.txt:/run/runner_secret \
17+
--volume /var/run/docker.sock:/var/run/docker.sock \
18+
--init \
19+
--interactive \
20+
--name=actions-runner.%i \
21+
--rm \
22+
--privileged \
23+
--log-driver=journald \
24+
iiilinuxibmcom/actions-runner.%i
25+
ExecStop=/bin/sh -c "docker exec actions-runner.%i kill -INT -- -1"
26+
ExecStop=/bin/sh -c "docker wait actions-runner.%i"
27+
ExecStop=/bin/sh -c "docker rm actions-runner.%i"
28+
29+
ExecStop=/usr/bin/env rm -f /etc/actions-runner/%i/ghtoken.env /etc/actions-runner/%i/ghtoken.txt
30+
31+
[Install]
32+
WantedBy=multi-user.target

‎.github/scripts/ppc64le-ci/self-hosted-builder/fs/usr/bin/actions-runner‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/env bash
2+
3+
set -e -u
4+
5+
trap cleanup EXIT
6+
7+
token_file=registration-token.json
8+
9+
# Function to clean up and unregister the runner
10+
cleanup() {
11+
echo "Cleaning up temporary files..."
12+
[ -f "$token_file" ] && rm -f "$token_file"
13+
[ -f "runner-id.json" ] && rm -f "runner-id.json"
14+
15+
echo "Unregistering the runner from GitHub..."
16+
ACCESS_TOKEN="$(cat /run/runner_secret)"
17+
runner_id=$(curl -s \
18+
-H "Accept: application/vnd.github.v3+json" \
19+
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
20+
"https://api.github.com/repos/${OWNER}/${REPO}/actions/runners" | \
21+
jq --raw-output '.runners[] | select(.name=="'"${NAME}"'") | .id')
22+
23+
if [ -n "$runner_id" ]; then
24+
curl -s \
25+
-X DELETE \
26+
-H "Accept: application/vnd.github.v3+json" \
27+
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
28+
"https://api.github.com/repos/${OWNER}/${REPO}/actions/runners/$runner_id"
29+
echo "Runner unregistered successfully."
30+
else
31+
echo "Warning: Runner ID for ${NAME} not found. It may already be removed."
32+
fi
33+
34+
unset ACCESS_TOKEN runner_id
35+
}
36+
37+
# Fetch GitHub access token
38+
if [ ! -f /run/runner_secret ]; then
39+
echo "Error: Access token file not found at /run/runner_secret."
40+
exit 1
41+
fi
42+
43+
44+
ACCESS_TOKEN="$(cat /run/runner_secret)"
45+
46+
# Generate registration token
47+
curl \
48+
-X POST \
49+
-H "Accept: application/vnd.github.v3+json" \
50+
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
51+
"https://api.github.com/repos/${OWNER}/${REPO}/actions/runners/registration-token" \
52+
-o "$token_file"
53+
54+
unset ACCESS_TOKEN
55+
56+
# register runner as ephemeral runner
57+
# it does one job, stops and unregisters
58+
registration_token=$(jq --raw-output .token "$token_file")
59+
60+
./config.sh \
61+
--unattended \
62+
--ephemeral \
63+
--url "https://github.com/${OWNER}/${REPO}" \
64+
--token "${registration_token}" \
65+
--name "${NAME}" \
66+
--no-default-labels \
67+
--labels self-hosted,linux.ppc64le
68+
69+
unset registration_token
70+
rm -f "$token_file"
71+
72+
# Run one job.
73+
./run.sh
74+
75+
echo "Ephemeral runner workflow completed."

‎.github/scripts/ppc64le-ci/self-hosted-builder/fs/usr/bin/entrypoint‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Container entrypoint that waits for all spawned processes.
5+
#
6+
7+
set -e -u
8+
9+
# Create a FIFO and start reading from its read end.
10+
tempdir=$(mktemp -d "/tmp/done.XXXXXXXXXX")
11+
trap 'rm -r "$tempdir"' EXIT
12+
done="$tempdir/pipe"
13+
mkfifo "$done"
14+
cat "$done" & waiter=$!
15+
16+
# Start the workload. Its descendants will inherit the FIFO's write end.
17+
status=0
18+
if [ "$#" -eq 0 ]; then
19+
bash 9>"$done" || status=$?
20+
else
21+
"$@" 9>"$done" || status=$?
22+
fi
23+
24+
# When the workload and all of its descendants exit, the FIFO's write end will
25+
# be closed and `cat "$done"` will exit. Wait until it happens. This is needed
26+
# in order to handle SelfUpdater, which the workload may start in background
27+
# before exiting.
28+
wait "$waiter"
29+
30+
exit "$status"

‎.github/scripts/ppc64le-ci/self-hosted-builder/helpers/app_token.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Request an ACCESS_TOKEN to be used by a GitHub APP
4+
# Environment variable that need to be set up:
5+
# * APP_ID, the GitHub's app ID
6+
# * INSTALL_ID, the Github's app's installation ID
7+
# * APP_PRIVATE_KEY, the content of GitHub app's private key in PEM format.
8+
#
9+
# https://github.com/orgs/community/discussions/24743#discussioncomment-3245300
10+
#
11+
12+
set -o pipefail
13+
14+
set -e # Exit on error
15+
16+
# Generate JWT
17+
header='{"alg":"RS256","typ":"JWT"}'
18+
payload="{\"iat\":$(date +%s),\"exp\":$(( $(date +%s) + 600 )),\"iss\":${APP_ID}}"
19+
20+
header_base64=$(echo -n "$header" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
21+
payload_base64=$(echo -n "$payload" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
22+
23+
signature=$(echo -n "${header_base64}.${payload_base64}" | \
24+
openssl dgst -sha256 -sign "${APP_PRIVATE_KEY}" | \
25+
openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
26+
27+
generated_jwt="${header_base64}.${payload_base64}.${signature}"
28+
29+
API_VERSION=v3
30+
API_HEADER="Accept: application/vnd.github+json"
31+
32+
auth_header="Authorization: Bearer ${generated_jwt}"
33+
34+
app_installations_response=$(curl -sX POST \
35+
-H "${auth_header}" \
36+
-H "${API_HEADER}" \
37+
--url "https://api.github.com/app/installations/${INSTALL_ID}/access_tokens" \
38+
)
39+
40+
echo "$app_installations_response" | jq --raw-output '.token'

‎.github/scripts/ppc64le-ci/self-hosted-builder/helpers/gh_cat_token.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
3+
TOKEN_FILE=$1
4+
OUTPUT_FILE=$2
5+
6+
echo "Starting gh_cat_token.sh with TOKEN_FILE=${TOKEN_FILE}, OUTPUT_FILE=${OUTPUT_FILE}"
7+
8+
# Validate inputs
9+
if [[ ! -r "${TOKEN_FILE}" ]]; then
10+
echo "Error: Token file '${TOKEN_FILE}' does not exist or is not readable."
11+
exit 1
12+
fi
13+
14+
# Write the token to the output file
15+
cat "${TOKEN_FILE}" > "${OUTPUT_FILE}"
16+
echo "Token written to ${OUTPUT_FILE}"

‎.github/scripts/ppc64le-ci/self-hosted-builder/helpers/gh_token_generator.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/usr/bin/env bash
2+
3+
SCRIPT_DIR=$(dirname "$0")
4+
APP_ID=$1
5+
INSTALL_ID=$2
6+
APP_PRIVATE_KEY=$3
7+
DST_FILE="$4"
8+
9+
ACCESS_TOKEN="$(APP_ID="$(<"${APP_ID}")" INSTALL_ID="$(<"${INSTALL_ID}")" APP_PRIVATE_KEY="${APP_PRIVATE_KEY}" "${SCRIPT_DIR}/app_token.sh")"
10+
echo "${ACCESS_TOKEN}" > "${DST_FILE}"

0 commit comments

Comments
 (0)