Skip to content

Commit 7b49fa6

Browse files
vsochvsoch
authored
adding shpc hosted action to update from listing and cache (#616)
* adding shpc hosted action to update from listing and cache Signed-off-by: vsoch <[email protected]>
1 parent a3858f0 commit 7b49fa6

File tree

7 files changed

+459
-1
lines changed

7 files changed

+459
-1
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
1414
The versions coincide with releases on pip. Only major versions will be released as tags on Github.
1515

1616
## [0.0.x](https://github.com/singularityhub/singularity-hpc/tree/main) (0.0.x)
17+
- GitHub action to update a registry from a cache or listing (0.1.17)
1718
- Fix bugs uninstalling all tags of a module (0.1.16)
1819
- support for install using registry recipe and local image (0.1.15)
1920
- fix views .view_module modulefile and loading (0.1.14)

‎actions/cache-update/action.yaml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
name: "Update an shpc registry from a cache"
2+
description: "Install the container_discovery module for your own usage."
3+
inputs:
4+
token:
5+
description: Token (required if pull_request is true)
6+
required: true
7+
namespace:
8+
description: Add a namespace prefix to containers in listing (if listing defined)
9+
required: false
10+
cache:
11+
description: GitHub repository that has cache to use
12+
required: true
13+
registry:
14+
description: path to root of cloned registry
15+
required: false
16+
min-count-inclusion:
17+
description: Minimum alias count for inclusion
18+
required: false
19+
default: 10
20+
max-count-inclusion:
21+
description: Do not include counts over this value
22+
required: false
23+
default: 1000
24+
additional-count-inclusion:
25+
description: Additional container binaries to include after initial set derived.
26+
required: false
27+
default: 25
28+
branch:
29+
description: Branch to open pull request against (defaults to main)
30+
required: false
31+
default: "main"
32+
pull_request:
33+
description: Open a pull request with changes
34+
required: false
35+
default: true
36+
maintainer:
37+
description: GitHub maintainer of recipe
38+
required: false
39+
default: "@vsoch"
40+
url_format_string:
41+
required: false
42+
listing:
43+
description: Containers listing to update from (to supplement cache).
44+
required: false
45+
46+
runs:
47+
using: "composite"
48+
steps:
49+
# Install container_discovery module
50+
- name: Install Container Discovery and Dependencies
51+
uses: singularityhub/container-executable-discovery/install@main
52+
53+
- name: Install Additional Dependencies
54+
run: pip install git+https://github.com/singularityhub/singularity-hpc@main
55+
shell: bash
56+
57+
- name: Set root url and namespace
58+
env:
59+
namespace: ${{ inputs.namespace }}
60+
url: ${{ inputs.url_format_string }}
61+
maintainer: ${{ inputs.maintainer }}
62+
root: ${{ inputs.registry }}
63+
run: |
64+
if [ "${root}" == "" ]; then
65+
root=$(pwd)
66+
fi
67+
if [ "${url}" != "" ]; then
68+
url="--url ${url}"
69+
fi
70+
if [ "${namespace}" != "" ]; then
71+
namespace="--namespace ${namespace}"
72+
fi
73+
echo "registry=${root}" >> $GITHUB_ENV
74+
echo "url=${url}" >> $GITHUB_ENV
75+
echo "namespace=${namespace}" >> $GITHUB_ENV
76+
shell: bash
77+
78+
- name: Clone Cache
79+
env:
80+
cache: ${{ inputs.cache }}
81+
run: git clone ${cache} /tmp/cache
82+
shell: bash
83+
84+
- name: Update Containers
85+
env:
86+
registry: ${{ env.registry }}
87+
add_count: ${{ inputs.additional-count-inclusion }}
88+
min_count: ${{ inputs.min-count-inclusion }}
89+
max_count: ${{ inputs.max-count-inclusion }}
90+
maintainer: ${{ inputs.maintainer }}
91+
url: ${{ env.url }}
92+
run: |
93+
cmd="python ${{ github.action_path }}/scripts/update-from-cache.py --cache /tmp/cache --registry ${registry} --min-count-inclusion ${min_count} --additional-count-inclusion ${add_count} --max-count-inclusion ${max_count} --maintainer ${maintainer} ${url}"
94+
echo "${cmd}"
95+
${cmd}
96+
shell: bash
97+
98+
- name: Update from Listing
99+
if: (inputs.listing != '')
100+
env:
101+
listing: ${{ inputs.listing }}
102+
registry: ${{ env.registry }}
103+
maintainer: ${{ inputs.maintainer }}
104+
namespace: ${{ env.namespace }}
105+
url: ${{ env.url }}
106+
run: |
107+
cmd="python ${{ github.action_path }}/scripts/update-from-listing.py --listing ${listing} --registry ${registry} --maintainer ${maintainer} ${url} ${namespace}"
108+
echo "${cmd}"
109+
${cmd}
110+
shell: bash
111+
112+
- name: Checkout Update branch
113+
if: (inputs.pull_request == 'true')
114+
env:
115+
GITHUB_TOKEN: ${{ inputs.token }}
116+
BRANCH_AGAINST: ${{ inputs.branch }}
117+
run: |
118+
printf "GitHub Actor: ${GITHUB_ACTOR}\n"
119+
export BRANCH_FROM="update/containers-$(date '+%Y-%m-%d')"
120+
git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
121+
BRANCH_EXISTS=$(git ls-remote --heads origin ${BRANCH_FROM})
122+
if [[ -z ${BRANCH_EXISTS} ]]; then
123+
printf "Branch does not exist in remote.\n"
124+
else
125+
printf "Branch already exists in remote.\n"
126+
exit 1
127+
fi
128+
git branch
129+
git checkout -b "${BRANCH_FROM}" || git checkout "${BRANCH_FROM}"
130+
git branch
131+
git config --global user.name "github-actions"
132+
git config --global user.email "[email protected]"
133+
git add ./*
134+
git status
135+
if git diff-index --quiet HEAD --; then
136+
export OPEN_PULL_REQUEST=0
137+
printf "No changes\n"
138+
else
139+
export OPEN_PULL_REQUEST=1
140+
printf "Changes\n"
141+
git commit -a -m "Automated deployment to update containers $(date '+%Y-%m-%d')"
142+
git push origin "${BRANCH_FROM}"
143+
fi
144+
echo "OPEN_PULL_REQUEST=${OPEN_PULL_REQUEST}" >> $GITHUB_ENV
145+
echo "PULL_REQUEST_FROM_BRANCH=${BRANCH_FROM}" >> $GITHUB_ENV
146+
echo "PULL_REQUEST_TITLE=[bot] ${BRANCH_FROM}" >> $GITHUB_ENV
147+
echo "PULL_REQUEST_BODY=Update container modules pull request." >> $GITHUB_ENV
148+
shell: bash
149+
150+
- name: Open Pull Request
151+
uses: vsoch/pull-request-action@50f22f6d146226ee6b73b7a001f26a3d4579f360 # 1.0.22
152+
if: ${{ env.OPEN_PULL_REQUEST == '1' }}
153+
env:
154+
GITHUB_TOKEN: ${{ inputs.token }}
155+
PULL_REQUEST_BRANCH: ${{ inputs.branch }}

‎actions/cache-update/scripts/helpers.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import shpc.utils
2+
3+
4+
def add_container(cli, container, maintainer, entry_name, aliases, url=None):
5+
"""
6+
Add a container from a unique resource identifier.
7+
"""
8+
print(f"Generating container entry for {container}")
9+
module_name = container.split(":")[0]
10+
container_yaml = cli.add(image="docker://" + container, module_name=module_name)
11+
12+
# First clean up commented lines - not intended for automation
13+
toclean = shpc.utils.read_file(container_yaml).split("\n")
14+
lines = [x for x in toclean if not (x.strip().startswith("#") or not x.strip())]
15+
shpc.utils.write_file(container_yaml, "\n".join(lines))
16+
17+
# Now read in as yaml (without comments)
18+
result = shpc.utils.read_yaml(container_yaml)
19+
result["maintainer"] = maintainer
20+
result[
21+
"description"
22+
] = f"singularity registry hpc automated addition for {entry_name}"
23+
24+
if not url:
25+
url = "https://singularity-hpc.readthedocs.io/"
26+
try:
27+
result["url"] = url % entry_name
28+
except Exception:
29+
result["url"] = url
30+
31+
if aliases:
32+
result["aliases"] = aliases
33+
shpc.utils.write_yaml(result, container_yaml)

‎actions/cache-update/scripts/update-from-cache.py‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#!/usr/bin/env python3
2+
3+
# This script uses count data from https://github.com/singularityhub/shpc-registry-cache
4+
# to:
5+
6+
# - Subsetting to those in a container, including counts
7+
# - Rank ordering from least to greatest (lower frequency is a more unique command)
8+
# - Including any counts with a frequency < 10
9+
# - Above that, including the next N, but don't go over max_count (1000)
10+
# - Use these to generate a new container.yaml for the file (if it does not exist yet)
11+
12+
# It requires a local clone of shpc-registry-cache with a counts file available,
13+
# along with a json file with commands found there.
14+
# Usage:
15+
# python update.py --cache ../shpc-registry-cache --registry $PWD
16+
17+
import argparse
18+
import os
19+
import sys
20+
21+
from container_discovery.cache import iter_new_cache
22+
23+
from shpc.main import get_client
24+
25+
here = os.path.dirname(os.path.abspath(__file__))
26+
sys.path.insert(0, here)
27+
28+
from helpers import add_container # noqa
29+
30+
31+
def get_parser():
32+
parser = argparse.ArgumentParser(
33+
description="SHPC Container Adder",
34+
formatter_class=argparse.RawTextHelpFormatter,
35+
)
36+
parser.add_argument("--cache", help="Path to cache with counts.json.")
37+
parser.add_argument(
38+
"--registry", help="Path to registry root.", default=os.getcwd()
39+
)
40+
parser.add_argument(
41+
"--url", help="URL optionally with format string %s for entry_name"
42+
)
43+
parser.add_argument("--maintainer", help="Author", default="@vsoch")
44+
parser.add_argument(
45+
"--min-count-inclusion",
46+
help="Include all executables under this count",
47+
default=10,
48+
type=int,
49+
dest="min_count",
50+
)
51+
parser.add_argument(
52+
"--max-count-inclusion",
53+
help="Do not include counts over this value",
54+
default=1000,
55+
type=int,
56+
dest="max_count",
57+
)
58+
parser.add_argument(
59+
"--additional-count-inclusion",
60+
help="Author",
61+
default=25,
62+
type=int,
63+
dest="add_count",
64+
)
65+
return parser
66+
67+
68+
def main():
69+
70+
parser = get_parser()
71+
72+
# If an error occurs while parsing the arguments, the interpreter will exit with value 2
73+
args, extra = parser.parse_known_args()
74+
75+
# Show args to the user
76+
print(" maintainer: %s" % args.maintainer)
77+
print(" registry: %s" % args.registry)
78+
print(" cache: %s" % args.cache)
79+
80+
cli = get_client()
81+
cli.settings.registry = [args.registry]
82+
cli.reload_registry()
83+
84+
for path in args.registry, args.cache:
85+
if not os.path.exists(path):
86+
sys.exit(f"{path} does not exist")
87+
88+
for entry in iter_new_cache(args.cache, registry=args.registry):
89+
aliases = entry.filter_aliases(
90+
add_count=args.add_count, min_count=args.min_count, max_count=args.max_count
91+
)
92+
93+
# Now add the container, and use the tag
94+
container = f"{entry.image}:{entry.tag}"
95+
entry_name = entry.image.split(os.sep)[-1]
96+
97+
# First derive and save the aliases - we can filter based
98+
try:
99+
add_container(
100+
cli,
101+
container,
102+
args.maintainer,
103+
entry_name,
104+
aliases=aliases,
105+
url=args.url,
106+
)
107+
except Exception as e:
108+
print(f"Issue adding container {container}: {e}")
109+
110+
111+
if __name__ == "__main__":
112+
main()

0 commit comments

Comments
 (0)