Skip to content

Commit b4dd7b7

Browse files
haampiehaampie
authored
Add patchelf to the list of binaries provided (#13)
1 parent 111ee5c commit b4dd7b7

File tree

4 files changed

+236
-0
lines changed

4 files changed

+236
-0
lines changed

‎.github/workflows/patchelf.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: "Patchelf"
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
tags:
8+
- v0.**
9+
pull_request:
10+
branches:
11+
- main
12+
workflow_dispatch:
13+
14+
concurrency:
15+
group: patchelf-${{ github.ref }}
16+
cancel-in-progress: true
17+
18+
jobs:
19+
20+
manylinux2014:
21+
runs-on: ubuntu-20.04
22+
outputs:
23+
spack_manylinux_tag: ${{ fromJSON(steps.docker_meta.outputs.json).tags[0] }}
24+
25+
steps:
26+
- uses: actions/checkout@v3
27+
28+
# Setup tags to be used for docker images
29+
- uses: docker/metadata-action@v4
30+
id: docker_meta
31+
with:
32+
images: ghcr.io/${{ github.repository_owner }}/patchelf_manylinux2014
33+
34+
# Login to Github Packages
35+
- uses: docker/login-action@v2
36+
with:
37+
registry: ghcr.io
38+
username: ${{ github.repository_owner }}
39+
password: ${{ secrets.GITHUB_TOKEN }}
40+
41+
- uses: docker/setup-qemu-action@v2
42+
id: qemu
43+
with:
44+
platforms: linux/ppc64le,linux/arm64,linux/amd64
45+
46+
- uses: docker/setup-buildx-action@v2
47+
48+
# Build and eventually push to registry
49+
- uses: docker/build-push-action@v3
50+
with:
51+
file: ./patchelf/Dockerfile.manylinux2014
52+
platforms: linux/arm64,linux/ppc64le,linux/amd64
53+
pull: ${{ github.event_name == 'pull_request' }}
54+
# Self-hosted runners use the local cache
55+
# cache-from: type=gha
56+
# cache-to: type=gha,mode=max
57+
cache-from: type=local,src=/tmp/.buildx-cache-patchelf-manylinux2014
58+
cache-to: type=local,dest=/tmp/.buildx-cache-patchelf-manylinux2014,mode=max
59+
push: true
60+
tags: ${{ steps.docker_meta.outputs.tags }}
61+
labels: ${{ steps.docker_meta.outputs.labels }}
62+
63+
upload-manylinux2014:
64+
runs-on: ubuntu-latest
65+
needs: [ manylinux2014 ]
66+
env:
67+
SPACK_MANYLINUX2014_TAG: ${{ needs.manylinux2014.outputs.spack_manylinux_tag }}
68+
69+
steps:
70+
- uses: actions/checkout@v3
71+
- run: |
72+
. ./patchelf/scripts/copy_mirror_manylinux2014.sh
73+
- uses: actions/upload-artifact@v3
74+
with:
75+
name: patchelf_binary_mirror
76+
path: |
77+
binary-mirror
78+
79+
patchelf_json:
80+
runs-on: ubuntu-latest
81+
needs: [ upload-manylinux2014 ]
82+
steps:
83+
- uses: actions/checkout@v3
84+
- uses: actions/setup-python@v3
85+
with:
86+
python-version: 3.9
87+
- uses: actions/download-artifact@v3
88+
with:
89+
name: patchelf_binary_mirror
90+
- run: |
91+
pip install ruamel.yaml
92+
python patchelf/scripts/patchelf_json.py
93+
- uses: actions/upload-artifact@v3
94+
with:
95+
name: patchelf_manifest
96+
path: |
97+
patchelf.json

‎patchelf/Dockerfile.manylinux2014‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM ghcr.io/spack/manylinux2014:latest
2+
3+
RUN adduser -m spack
4+
5+
# Switch to a non-root user
6+
USER spack
7+
ENV SPACK_CMD="/opt/python/cp39-cp39/bin/python /home/spack/spack/bin/spack"
8+
WORKDIR /home/spack
9+
10+
# Clone the repo and install Spack
11+
RUN git clone https://www.github.com/spack/spack.git
12+
13+
# Set externals, locate compilers
14+
RUN ${SPACK_CMD} external find --not-buildable cmake
15+
RUN ${SPACK_CMD} compiler find
16+
17+
RUN ${SPACK_CMD} python -c "import archspec.cpu;print(archspec.cpu.host().family)" > target.txt
18+
RUN ${SPACK_CMD} install patchelf 'ldflags="-static-libstdc++ -static-libgcc"' target=$(cat target.txt)
19+
20+
RUN mkdir -p /home/spack/binary-mirror && \
21+
${SPACK_CMD} buildcache create -d /home/spack/binary-mirror -a -u -f patchelf

‎patchelf/scripts/copy_mirror_manylinux2014.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env sh
2+
3+
id=$(docker create --platform linux/amd64 ${SPACK_MANYLINUX2014_TAG})
4+
mkdir amd64
5+
docker cp $id:/home/spack/binary-mirror amd64/binary-mirror
6+
docker rm -v $id
7+
8+
id=$(docker create --platform linux/arm64 ${SPACK_MANYLINUX2014_TAG})
9+
mkdir arm64
10+
docker cp $id:/home/spack/binary-mirror arm64/binary-mirror
11+
docker rm -v $id
12+
13+
id=$(docker create --platform linux/ppc64le ${SPACK_MANYLINUX2014_TAG})
14+
mkdir ppc64le
15+
docker cp $id:/home/spack/binary-mirror ppc64le/binary-mirror
16+
docker rm -v $id
17+
18+
# Unify the two mirrors. This is required since the upload-artifacts action
19+
# computes the common ancestore among different paths and starts from there
20+
# instead of "merging" the copies
21+
mkdir binary-mirror
22+
rsync -a ./amd64/binary-mirror/ ./binary-mirror
23+
rsync -a ./arm64/binary-mirror/ ./binary-mirror
24+
rsync -a ./ppc64le/binary-mirror/ ./binary-mirror

‎patchelf/scripts/patchelf_json.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
"""Produce the clingo.json file associated with the mirror"""
2+
import json
3+
import glob
4+
import hashlib
5+
import json
6+
import os.path
7+
8+
9+
# Each entry in patchelf.json has the following keys:
10+
#
11+
# "spec": root spec to be matched
12+
# "binaries": list of objects, to be installed in order (each one has a "hash", "sha256" and "name" attribute)
13+
# "python": constraints on the python interpreter
14+
# "compiler": compiler entry used to build the spec
15+
#
16+
17+
# Dictionary that maps (OS, TARGET) to info for the spec
18+
SPEC_INFO = {
19+
('centos7', 'x86_64'): {
20+
'spec': '[email protected]: %gcc platform=linux target=x86_64',
21+
},
22+
('centos7', 'aarch64'): {
23+
'spec': '[email protected]: %gcc platform=linux target=aarch64',
24+
},
25+
('centos7', 'ppc64le'): {
26+
'spec': '[email protected]: %gcc platform=linux target=ppc64le',
27+
}
28+
}
29+
30+
def compiler_entry(name, version, os, target):
31+
return {
32+
"spec": "{0}@{1}".format(name, version),
33+
"paths": {
34+
"cc": "/dev/null",
35+
"cxx": "/dev/null",
36+
"f77": "/dev/null",
37+
"fc": "/dev/null"
38+
},
39+
"operating_system": "{0}".format(os),
40+
"target": "{0}".format(target),
41+
"modules": []
42+
}
43+
44+
def sha256(path):
45+
fn = hashlib.sha256()
46+
with open(path, "rb") as f:
47+
fn.update(f.read())
48+
return fn.hexdigest()
49+
50+
def tarball_hash(path):
51+
filename = os.path.basename(path)
52+
filename = filename.replace('.spack', '')
53+
return filename.split('-')[-1]
54+
55+
tarballs = glob.glob( './build_cache/**/*.spack', recursive=True)
56+
shas = {tarball_hash(tarball): sha256(tarball) for tarball in tarballs}
57+
spec_yaml_files = glob.glob('./build_cache/*.json')
58+
59+
mirror_info = []
60+
spec_yaml_dict = {}
61+
for spec_yaml in spec_yaml_files:
62+
if 'patchelf' not in spec_yaml:
63+
continue
64+
print(spec_yaml)
65+
# Get the raw data from spec.yaml
66+
with open(spec_yaml) as f:
67+
spec_yaml_data = json.load(f)['spec']['nodes']
68+
69+
# Find the patchelf entry and store it somewhere
70+
binary_data = {}
71+
for entry in spec_yaml_data:
72+
if 'patchelf' == entry['name']:
73+
binary_data['patchelf'] = entry
74+
assert 'patchelf' in binary_data
75+
76+
current_os = binary_data['patchelf']['arch']['platform_os']
77+
current_target = binary_data['patchelf']['arch']['target']
78+
79+
compiler_name = binary_data['patchelf']['compiler']['name']
80+
compiler_version = str(binary_data['patchelf']['compiler']['version'])
81+
82+
current_hash = binary_data['patchelf']['hash']
83+
binaries = [
84+
('patchelf', current_hash, shas[current_hash])
85+
]
86+
mirror_entry = {
87+
"spec": SPEC_INFO[(current_os, current_target)]['spec'],
88+
"binaries": binaries,
89+
}
90+
mirror_info.append(mirror_entry)
91+
92+
mirror_info = sorted(mirror_info, key=lambda x: x['spec'])
93+
with open('./patchelf.json', 'w') as f:
94+
json.dump({'verified': mirror_info}, f, sort_keys=True, indent=2)

0 commit comments

Comments
 (0)