Skip to content

Commit 2f07b4d

Browse files
sandeepgupta12sandeepgupta12
committed
added file for ppc arch to run on osu vm
1 parent 92634a3 commit 2f07b4d

File tree

10 files changed

+464
-0
lines changed

10 files changed

+464
-0
lines changed

‎.github/scripts/ppc64le/README.md‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Configuring the builder.
2+
3+
## Install prerequisites.
4+
5+
```
6+
$ sudo dnf install podman podman-docker jq
7+
```
8+
9+
## Add services.
10+
11+
```
12+
$ sudo cp self-hosted-builder/*.service /etc/systemd/system/
13+
$ sudo systemctl daemon-reload
14+
```
15+
16+
## Download qemu-user-static image
17+
18+
```
19+
# sudo docker pull docker.io/iiilinuxibmcom/qemu-user-static:6.1.0-1
20+
```
21+
22+
## Autostart the x86_64 emulation support.
23+
24+
```
25+
$ sudo systemctl enable --now qemu-user-static
26+
```
27+
28+
## Rebuild the image
29+
30+
First build s390x builder image `docker.io/pytorch/manylinuxs390x-builder`,
31+
using following commands:
32+
33+
```
34+
$ cd ~
35+
$ git clone https://github.com/pytorch/pytorch
36+
$ cd pytorch
37+
$ git submodule update --init --recursive
38+
$ GPU_ARCH_TYPE=cpu-s390x "$(pwd)/.ci/docker/manywheel/build.sh" manylinuxs390x-builder
39+
$ docker image tag localhost/pytorch/manylinuxs390x-builder docker.io/pytorch/manylinuxs390x-builder:cpu-s390x
40+
$ docker image save -o ~/manywheel-s390x.tar docker.io/pytorch/manylinuxs390x-builder:cpu-s390x
41+
```
42+
43+
Next step is to build `actions-runner` image using:
44+
45+
```
46+
$ cd self-hosted-builder
47+
$ sudo docker build \
48+
--pull \
49+
-f actions-runner.Dockerfile \
50+
-t iiilinuxibmcom/actions-runner.<name> \
51+
.
52+
```
53+
54+
If there are failures, ensure that selinux doesn't prevent it from working.
55+
In worst case, selinux can be disabled with `setenforce 0`.
56+
57+
Now prepare all necessary files for runner registration:
58+
59+
```
60+
$ sudo mkdir -p /etc/actions-runner/<name>
61+
$ sudo chmod 700 /etc/actions-runner/<name>
62+
$ sudo /bin/cp <github_app_private_key_file> /etc/actions-runner/<name>/key_private.pem
63+
$ sudo echo <github_app_id> | sudo tee /etc/actions-runner/<name>/appid.env
64+
$ sudo echo <github_app_install_id> | sudo tee /etc/actions-runner/<name>/installid.env
65+
$ sudo echo NAME=<worker_name> | sudo tee /etc/actions-runner/<name>/env
66+
$ sudo echo ORG=<github_org> | sudo tee -a /etc/actions-runner/<name>/env
67+
$ cd self-hosted-builder
68+
$ sudo /bin/cp helpers/*.sh /usr/local/bin/
69+
$ sudo chmod 755 /usr/local/bin/app_token.sh /usr/local/bin/gh_token_generator.sh
70+
```
71+
72+
## Autostart the runner.
73+
74+
```
75+
$ sudo systemctl enable --now actions-runner@$NAME
76+
```

‎.github/scripts/ppc64le/self-hosted-builder/actions-runner copy.Dockerfile‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Stage 1: Main image for ppc64le Ubuntu
2+
FROM --platform=linux/ppc64le ubuntu:22.04
3+
4+
# Set non-interactive mode for apt
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
7+
# Fix sources to point to ports.ubuntu.com for ppc64le
8+
RUN echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy main restricted universe multiverse" > /etc/apt/sources.list && \
9+
echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy-updates main restricted universe multiverse" >> /etc/apt/sources.list && \
10+
echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy-backports main restricted universe multiverse" >> /etc/apt/sources.list && \
11+
echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy-security main restricted universe multiverse" >> /etc/apt/sources.list
12+
13+
# Update and install basic tools
14+
RUN apt-get clean && rm -rf /var/lib/apt/lists/* && \
15+
apt-get update -o Acquire::Retries=5 -o Acquire::http::Timeout="10" && \
16+
apt-get -y install --no-install-recommends \
17+
build-essential \
18+
curl \
19+
sudo \
20+
gnupg-agent \
21+
iptables iptables-legacy \
22+
ca-certificates \
23+
software-properties-common && \
24+
apt-get clean && rm -rf /var/lib/apt/lists/*
25+
26+
# Switch to iptables-legacy
27+
RUN update-alternatives --set iptables /usr/sbin/iptables-legacy && \
28+
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
29+
30+
# Add Docker GPG key and repository
31+
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
32+
echo "deb [arch=ppc64el signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list && \
33+
apt-get update && apt-get install -y \
34+
docker-ce \
35+
docker-ce-cli \
36+
containerd.io && \
37+
apt-get clean && rm -rf /var/lib/apt/lists/*
38+
39+
# Replace apt sources for ppc64el
40+
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://ports.ubuntu.com/ubuntu-ports|g' /etc/apt/sources.list && \
41+
sed -i 's|http://security.ubuntu.com/ubuntu|http://ports.ubuntu.com/ubuntu-ports|g' /etc/apt/sources.list
42+
43+
# Install additional dependencies
44+
RUN apt-get update && apt-get install -y \
45+
vim \
46+
python3 \
47+
python3-dev \
48+
python3-pip \
49+
virtualenv && \
50+
apt-get clean && rm -rf /var/lib/apt/lists/*
51+
52+
# Set up Python virtual environment
53+
RUN virtualenv --system-site-packages venv
54+
55+
# Copy custom scripts
56+
COPY fs/ /
57+
RUN chmod 777 /usr/bin/actions-runner /usr/bin/entrypoint
58+
59+
# Download and extract GitHub Actions Runner
60+
RUN curl -L https://github.com/actions/runner/releases/download/v2.317.0/actions-runner-linux-x64-2.317.0.tar.gz | tar -xz

‎.github/scripts/ppc64le/self-hosted-builder/actions-runner.Dockerfile‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Self-Hosted IBM Power Github Actions Runner.
2+
3+
4+
# Stage 1: Main image for ppc64le Ubuntu
5+
FROM ubuntu:22.04
6+
7+
# Set non-interactive mode for apt
8+
ENV DEBIAN_FRONTEND=noninteractive
9+
10+
# Fix sources to point to ports.ubuntu.com for ppc64le
11+
RUN echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy main restricted universe multiverse" > /etc/apt/sources.list && \
12+
echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy-updates main restricted universe multiverse" >> /etc/apt/sources.list && \
13+
echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy-backports main restricted universe multiverse" >> /etc/apt/sources.list && \
14+
echo "deb [arch=ppc64el] http://ports.ubuntu.com/ubuntu-ports jammy-security main restricted universe multiverse" >> /etc/apt/sources.list
15+
16+
# Update and install basic tools
17+
RUN apt-get clean && rm -rf /var/lib/apt/lists/* && \
18+
apt-get update -o Acquire::Retries=5 -o Acquire::http::Timeout="10" && \
19+
apt-get -y install --no-install-recommends \
20+
build-essential \
21+
curl \
22+
sudo \
23+
jq \
24+
gnupg-agent \
25+
iptables \
26+
ca-certificates \
27+
software-properties-common && \
28+
apt-get clean && rm -rf /var/lib/apt/lists/*
29+
30+
# Switch to iptables-legacy
31+
RUN update-alternatives --set iptables /usr/sbin/iptables-legacy && \
32+
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
33+
34+
# Add Docker GPG key and repository
35+
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
36+
echo "deb [arch=ppc64el signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list && \
37+
apt-get update && apt-get install -y \
38+
docker-ce \
39+
docker-ce-cli \
40+
containerd.io && \
41+
apt-get clean && rm -rf /var/lib/apt/lists/*
42+
43+
# Replace apt sources for ppc64el
44+
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://ports.ubuntu.com/ubuntu-ports|g' /etc/apt/sources.list && \
45+
sed -i 's|http://security.ubuntu.com/ubuntu|http://ports.ubuntu.com/ubuntu-ports|g' /etc/apt/sources.list
46+
47+
# Install additional dependencies
48+
RUN apt-get update && apt-get install -y \
49+
vim \
50+
python3 \
51+
python3-dev \
52+
python3-pip \
53+
virtualenv && \
54+
apt-get clean && rm -rf /var/lib/apt/lists/*
55+
56+
# Set up Python virtual environment
57+
RUN virtualenv --system-site-packages venv
58+
59+
# Copy custom scripts
60+
COPY fs/ /
61+
RUN chmod 777 /usr/bin/actions-runner /usr/bin/entrypoint
62+
63+
# Download and extract GitHub Actions Runner
64+
#RUN curl -L https://github.com/actions/runner/releases/download/v2.317.0/actions-runner-linux-x64-2.317.0.tar.gz | tar -xz

‎.github/scripts/ppc64le/self-hosted-builder/[email protected]‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
[Unit]
2+
Description=Self-Hosted IBM power Github Actions Runner
3+
4+
StartLimitIntervalSec=0
5+
6+
[Service]
7+
Type=simple
8+
Restart=always
9+
ExecStartPre=-/usr/bin/docker rm --force actions-runner.%i
10+
ExecStartPre=-/usr/local/bin/gh_token_generator.sh /etc/actions-runner/%i/appid.env /etc/actions-runner/%i/installid.env /etc/actions-runner/%i/key_private.pem /etc/actions-runner/%i/ghtoken.env
11+
ExecStart=/usr/bin/docker run \
12+
--env-file=/etc/actions-runner/%i/env \
13+
--env-file=/etc/actions-runner/%i/ghtoken.env \
14+
--init \
15+
--interactive \
16+
--name=actions-runner.%i \
17+
--rm \
18+
--privileged \
19+
iiilinuxibmcom/actions-runner.%i
20+
ExecStop=/bin/sh -c "docker exec actions-runner.%i kill -INT -- -1"
21+
ExecStop=/bin/sh -c "docker wait actions-runner.%i"
22+
ExecStop=/bin/sh -c "docker rm actions-runner.%i"
23+
24+
[Install]
25+
WantedBy=multi-user.target

‎.github/scripts/ppc64le/self-hosted-builder/fs/usr/bin/actions-runner‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/usr/bin/env bash
2+
3+
set -e -u
4+
5+
# first import docker image
6+
if [ -f ./manywheel-ppc64le.tar ] ; then
7+
docker image load --input manywheel-ppc64le.tar
8+
docker image tag docker.io/pytorch/manylinuxppc64le-builder:cpu-ppc64le docker.io/pytorch/manylinuxsppc64le-builder:cpu-ppc64le-main
9+
rm -f manywheel-ppc64le.tar
10+
fi
11+
12+
token_file=registration-token.json
13+
14+
ACCESS_TOKEN="$(cat /run/runner_secret)"
15+
16+
# Generate registration token
17+
curl \
18+
-X POST \
19+
-H "Accept: application/vnd.github.v3+json" \
20+
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
21+
"https://api.github.com/orgs/${ORG}/actions/runners/registration-token" \
22+
-o "$token_file"
23+
24+
unset ACCESS_TOKEN
25+
26+
# register runner as ephemeral runner
27+
# it does one job, stops and unregisters
28+
registration_token=$(jq --raw-output .token "$token_file")
29+
30+
./config.sh \
31+
--unattended \
32+
--ephemeral \
33+
--url "https://github.com/${ORG}" \
34+
--token "${registration_token}" \
35+
--name "${NAME}" \
36+
--no-default-labels \
37+
--labels self-hosted,linux.ppc64le
38+
39+
unset registration_token
40+
rm -f "$token_file"
41+
42+
# enter into python virtual environment.
43+
# build workflows use "python -m pip install ...",
44+
# and it doesn't work for non-root user
45+
source venv/bin/activate
46+
47+
# Run one job.
48+
./run.sh

‎.github/scripts/ppc64le/self-hosted-builder/fs/usr/bin/entrypoint‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Container entrypoint that waits for all spawned processes.
5+
#
6+
7+
set -e -u
8+
9+
# Create a FIFO and start reading from its read end.
10+
tempdir=$(mktemp -d "/tmp/done.XXXXXXXXXX")
11+
trap 'rm -r "$tempdir"' EXIT
12+
done="$tempdir/pipe"
13+
mkfifo "$done"
14+
cat "$done" & waiter=$!
15+
16+
# Start the workload. Its descendants will inherit the FIFO's write end.
17+
status=0
18+
if [ "$#" -eq 0 ]; then
19+
bash 9>"$done" || status=$?
20+
else
21+
"$@" 9>"$done" || status=$?
22+
fi
23+
24+
# When the workload and all of its descendants exit, the FIFO's write end will
25+
# be closed and `cat "$done"` will exit. Wait until it happens. This is needed
26+
# in order to handle SelfUpdater, which the workload may start in background
27+
# before exiting.
28+
wait "$waiter"
29+
30+
exit "$status"

‎.github/scripts/ppc64le/self-hosted-builder/helpers/app_token.sh‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Request an ACCESS_TOKEN to be used by a GitHub APP
4+
# Environment variable that need to be set up:
5+
# * APP_ID, the GitHub's app ID
6+
# * INSTALL_ID, the Github's app's installation ID
7+
# * APP_PRIVATE_KEY, the content of GitHub app's private key in PEM format.
8+
#
9+
# https://github.com/orgs/community/discussions/24743#discussioncomment-3245300
10+
#
11+
12+
set -o pipefail
13+
14+
_GITHUB_HOST=${GITHUB_HOST:="github.com"}
15+
16+
# If URL is not github.com then use the enterprise api endpoint
17+
if [[ ${GITHUB_HOST} = "github.com" ]]; then
18+
URI="https://api.${_GITHUB_HOST}"
19+
else
20+
URI="https://${_GITHUB_HOST}/api/v3"
21+
fi
22+
23+
API_VERSION=v3
24+
API_HEADER="Accept: application/vnd.github.${API_VERSION}+json"
25+
CONTENT_LENGTH_HEADER="Content-Length: 0"
26+
APP_INSTALLATIONS_URI="${URI}/app/installations"
27+
28+
29+
# JWT parameters based off
30+
# https://docs.github.com/en/developers/apps/building-github-apps/authenticating-with-github-apps#authenticating-as-a-github-app
31+
#
32+
# JWT token issuance and expiration parameters
33+
JWT_IAT_DRIFT=60
34+
JWT_EXP_DELTA=600
35+
36+
JWT_JOSE_HEADER='{
37+
"alg": "RS256",
38+
"typ": "JWT"
39+
}'
40+
41+
42+
build_jwt_payload() {
43+
now=$(date +%s)
44+
iat=$((now - JWT_IAT_DRIFT))
45+
jq -c \
46+
--arg iat_str "${iat}" \
47+
--arg exp_delta_str "${JWT_EXP_DELTA}" \
48+
--arg app_id_str "${APP_ID}" \
49+
'
50+
($iat_str | tonumber) as $iat
51+
| ($exp_delta_str | tonumber) as $exp_delta
52+
| ($app_id_str | tonumber) as $app_id
53+
| .iat = $iat
54+
| .exp = ($iat + $exp_delta)
55+
| .iss = $app_id
56+
' <<< "{}" | tr -d '\n'
57+
}
58+
59+
base64url() {
60+
base64 | tr '+/' '-_' | tr -d '=\n'
61+
}
62+
63+
rs256_sign() {
64+
openssl dgst -binary -sha256 -sign <(echo "$1")
65+
}
66+
67+
request_access_token() {
68+
jwt_payload=$(build_jwt_payload)
69+
encoded_jwt_parts=$(base64url <<<"${JWT_JOSE_HEADER}").$(base64url <<<"${jwt_payload}")
70+
encoded_mac=$(echo -n "$encoded_jwt_parts" | rs256_sign "${APP_PRIVATE_KEY}" | base64url)
71+
generated_jwt="${encoded_jwt_parts}.${encoded_mac}"
72+
73+
auth_header="Authorization: Bearer ${generated_jwt}"
74+
75+
app_installations_response=$(curl -sX POST \
76+
-H "${auth_header}" \
77+
-H "${API_HEADER}" \
78+
--header "X-GitHub-Api-Version: 2022-11-28" \
79+
--url "https://api.github.com/app/installations/${INSTALL_ID}/access_tokens" \
80+
)
81+
echo "$app_installations_response" | jq --raw-output '.token'
82+
}
83+
84+
request_access_token

0 commit comments

Comments
 (0)