Skip to content

Commit 1849784

Browse files
vsochvsoch
committed
refactoring organization to better support different container arhitectures
Signed-off-by: vsoch <[email protected]>
1 parent 0b83f88 commit 1849784

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+2173
-1309
lines changed

‎.circleci/config.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,16 @@ build_singularity: &install_singularity
5656
./mconfig -p /usr/local && \
5757
make -C builddir && \
5858
sudo make -C builddir install
59-
59+
60+
install_podman: &install_podman
61+
name: Install Podman
62+
command: |-
63+
. /etc/os-release
64+
echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/testing/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:testing.list
65+
curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/testing/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add -
66+
sudo apt-get update -qq
67+
sudo apt-get -qq -y install podman
68+
6069
install_dependencies: &install_dependencies
6170
name: install dependencies
6271
command: |-
@@ -109,6 +118,7 @@ jobs:
109118
keys: v1-dependencies
110119
- run: *install_dependencies
111120
- run: *setup_environment
121+
- run: *install_podman
112122
- run: *update_go
113123
- run: *fetch_deb_deps
114124
- run: *install_singularity

‎.github/workflows/main.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ jobs:
1818
- name: Setup black linter
1919
run: conda create --quiet --name black pyflakes
2020

21+
- name: Check Spelling
22+
uses: crate-ci/typos@592b36d23c62cb378f6097a292bc902ee73f93ef # version 1.0.4
23+
with:
24+
files: ./docs/getting_started/ ./docs/index.rst
25+
2126
- name: Lint python code
2227
run: |
2328
export PATH="/usr/share/miniconda/bin:$PATH"
@@ -29,4 +34,8 @@ jobs:
2934
run: |
3035
export PATH="/usr/share/miniconda/bin:$PATH"
3136
source activate black
32-
pyflakes shpc/main
37+
pyflakes shpc/main/*.py
38+
pyflakes shpc/main/modules
39+
pyflakes shpc/main/container/base.py
40+
pyflakes shpc/main/container/podman.py
41+
pyflakes shpc/main/container/singularity.py

‎.github/workflows/test-container.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: Test Registry Module
2+
3+
on:
4+
pull_request:
5+
types: [labeled]
6+
7+
jobs:
8+
test-registry:
9+
if: ${{ github.event.label.name == 'container-recipe' }}
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Install Dependencies
13+
run: |
14+
sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
15+
16+
- name: Install LMOD
17+
run: |
18+
PKG_VERSION=8.4.27
19+
PKG_URL="https://github.com/TACC/Lmod/archive/${PKG_VERSION}.tar.gz"
20+
wget $PKG_URL
21+
tar xfz ${PKG_VERSION}.tar.gz
22+
cd Lmod-${PKG_VERSION}
23+
./configure --prefix=/usr/share && make && make install
24+
25+
- uses: eWaterCycle/setup-singularity@v6
26+
name: Install Singularity
27+
with:
28+
singularity-version: 3.6.4
29+
30+
- name: Derive number of commits
31+
run: |
32+
cd /opt
33+
branch="${{ github.head_ref }}"
34+
git config --global user.email "github-actions"
35+
git config --global user.name "[email protected]"
36+
git clone -b main https://github.com/singularityhub/singularity-hpc
37+
cd singularity-hpc
38+
git checkout -b $branch
39+
git pull origin $branch
40+
commits=$(git log --oneline "$branch" ^main | wc -l)
41+
echo "Found $commits since main"
42+
cd registry
43+
touch changes.txt
44+
# Note that you can also do: git diff --name-only $branch..main .
45+
# We show the count of commits as a sanity check to the developer
46+
for container_yaml in $(git diff --name-only HEAD~$commits..HEAD .); do
47+
echo $container_yaml >> changes.txt
48+
done
49+
cat changes.txt
50+
51+
- name: Save changelist
52+
uses: actions/upload-artifact@v2
53+
with:
54+
name: changes.txt
55+
path: /opt/singularity-hpc/registry/changes.txt
56+
57+
- name: Create conda environment
58+
run: conda create --quiet -c conda-forge --name shpc spython
59+
60+
- uses: actions/checkout@v2
61+
- name: Install shpc
62+
run: |
63+
export PATH="/usr/share/miniconda/bin:$PATH"
64+
source activate shpc
65+
pip install -e .
66+
67+
- name: Download changelist
68+
uses: actions/download-artifact@v2
69+
with:
70+
name: changes.txt
71+
72+
- name: Run module tests
73+
run: |
74+
export PATH="/usr/share/miniconda/bin:$PATH"
75+
76+
source activate shpc
77+
cd registry
78+
for container_yaml in $(cat ../changes.txt); do
79+
module=$(dirname $container_yaml)
80+
module=$(echo "${module/registry\//}")
81+
echo "Testing $module"
82+
shpc test --template ../shpc/tests/test-registry-module.sh --commands $module
83+
done

‎.github/workflows/test.yml‎

Lines changed: 70 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,35 @@
1-
name: Test Registry Module
1+
name: Test Modules
22

33
on:
4-
pull_request:
5-
types: [labeled]
4+
pull_request: []
65

76
jobs:
8-
test-registry:
9-
if: ${{ github.event.label.name == 'container-recipe' }}
7+
test:
108
runs-on: ubuntu-latest
9+
strategy:
10+
fail-fast: false
11+
matrix:
12+
module: ["lmod", "tcl"]
13+
container_tech: ["podman", "singularity"]
1114
steps:
15+
1216
- name: Install Dependencies
1317
run: |
1418
sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
1519
20+
- name: Install environment modules
21+
if: ${{ matrix.module == 'tcl' }}
22+
run: |
23+
curl -LJO https://github.com/cea-hpc/modules/releases/download/v4.7.1/modules-4.7.1.tar.gz
24+
tar xfz modules-4.7.1.tar.gz
25+
ls
26+
cd modules-4.7.1
27+
./configure
28+
make
29+
sudo make install
30+
1631
- name: Install LMOD
32+
if: ${{ matrix.module == 'lmod' }}
1733
run: |
1834
PKG_VERSION=8.4.27
1935
PKG_URL="https://github.com/TACC/Lmod/archive/${PKG_VERSION}.tar.gz"
@@ -23,37 +39,11 @@ jobs:
2339
./configure --prefix=/usr/share && make && make install
2440
2541
- uses: eWaterCycle/setup-singularity@v6
42+
if: ${{ matrix.container_tech == 'singularity' }}
2643
name: Install Singularity
2744
with:
2845
singularity-version: 3.6.4
2946

30-
- name: Derive number of commits
31-
run: |
32-
cd /opt
33-
branch="${{ github.head_ref }}"
34-
git config --global user.email "github-actions"
35-
git config --global user.name "[email protected]"
36-
git clone -b main https://github.com/singularityhub/singularity-hpc
37-
cd singularity-hpc
38-
git checkout -b $branch
39-
git pull origin $branch
40-
commits=$(git log --oneline "$branch" ^main | wc -l)
41-
echo "Found $commits since main"
42-
cd registry
43-
touch changes.txt
44-
# Note that you can also do: git diff --name-only $branch..main .
45-
# We show the count of commits as a sanity check to the developer
46-
for container_yaml in $(git diff --name-only HEAD~$commits..HEAD .); do
47-
echo $container_yaml >> changes.txt
48-
done
49-
cat changes.txt
50-
51-
- name: Save changelist
52-
uses: actions/upload-artifact@v2
53-
with:
54-
name: changes.txt
55-
path: /opt/singularity-hpc/registry/changes.txt
56-
5747
- name: Create conda environment
5848
run: conda create --quiet -c conda-forge --name shpc spython
5949

@@ -64,20 +54,55 @@ jobs:
6454
source activate shpc
6555
pip install -e .
6656
67-
- name: Download changelist
68-
uses: actions/download-artifact@v2
69-
with:
70-
name: changes.txt
71-
72-
- name: Run module tests
57+
- name: Run python module tests
58+
if: ${{ matrix.module == 'lmod' }}
7359
run: |
7460
export PATH="/usr/share/miniconda/bin:$PATH"
61+
. /usr/share/lmod/lmod/init/sh
62+
source activate shpc
63+
shpc config set container_tech:${{ matrix.container_tech }}
64+
shpc config set module_sys:${{ matrix.module }}
65+
shpc install python:3.9.5-alpine
66+
module use ./modules
67+
module load python/3.9.5-alpine
68+
alias
69+
70+
printf "\n\nmodule show ============================================\n"
71+
module show python/3.9.5-alpine
7572
73+
printf "\n\nmodule whatis ==========================================\n"
74+
module whatis python/3.9.5-alpine
75+
76+
printf "\n\nmodule help ============================================\n"
77+
module help python/3.9.5-alpine
78+
python-exec echo donuts
79+
python-run python --version
80+
81+
- name: Run python module tests
82+
if: ${{ matrix.module == 'tcl' }}
83+
shell: bash
84+
run: |
85+
export PATH="/usr/share/miniconda/bin:$PATH"
86+
. /usr/local/Modules/init/sh
7687
source activate shpc
77-
cd registry
78-
for container_yaml in $(cat ../changes.txt); do
79-
module=$(dirname $container_yaml)
80-
module=$(echo "${module/registry\//}")
81-
echo "Testing $module"
82-
shpc test --template ../shpc/tests/test-registry-module.sh --commands $module
83-
done
88+
shpc config set container_tech:${{ matrix.container_tech }}
89+
shpc config set module_sys:${{ matrix.module }}
90+
shpc install python:3.9.5-alpine
91+
92+
shopt expand_aliases || true
93+
shopt -s expand_aliases
94+
95+
module use ./modules
96+
module load python/3.9.5-alpine
97+
alias
98+
99+
printf "\n\nmodule show ============================================\n"
100+
module show python/3.9.5-alpine
101+
102+
printf "\n\nmodule whatis ==========================================\n"
103+
module whatis python/3.9.5-alpine
104+
105+
printf "\n\nmodule help ============================================\n"
106+
module help python/3.9.5-alpine
107+
python-exec echo donuts
108+
python-run python --version

‎CHANGELOG.md‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,28 @@ and **Merged pull requests**. Critical items to know are:
1414
The versions coincide with releases on pip. Only major versions will be released as tags on Github.
1515

1616
## [0.0.x](https://github.scom/singularityhub/singularity-hpc/tree/master) (0.0.x)
17-
- Podman support (0.0.25)
18-
- adding namespaces to make module install, show, inspect easier (0.0.24)
19-
- added support for env section to render to bound environment file
20-
- added support for container features like gpu
21-
- allowing for a `container:tag` convention to be used for commands.
22-
- list defaults to showing modules/tags one per line, unless --short used
17+
- Podman support (0.0.24)
18+
- container_tech is now a settings.yml variable
19+
- allow for custom test interpreter "test_shell"
20+
- adding config edit for interactive edit
21+
- adding show --filter to filter list of modules shown
22+
- split documentation into user and developer guide
23+
- added automated spell checking with crate-ci/typos
24+
- support for container features
25+
- shpc get -e to show path to an envrironment file
26+
- adding namespaces to make module install, show, inspect easier
27+
- added support for env section to render to bound environment file
28+
- added support for container features like gpu
29+
- allowing for a `container:tag` convention to be used for commands.
30+
- list defaults to showing modules/tags one per line, unless --short used
2331
- container url does not render in docs (0.0.23)
2432
- addition of tcl modules, removal of un-needed database (0.0.22)
2533
- first update of all containers, and bugfix to pull (0.0.21)
2634
- allowing for a custom container base, container_base (0.0.2)
27-
- adding more nvidia containers, rapidsai and paraview
35+
- adding more nvidia containers, rapidsai and paraview
2836
- more packages and shell for a container (0.0.19)
2937
- adding description to docgen, and descriptions for all containers (0.0.18)
30-
- descriptions are now required
38+
- descriptions are now required
3139
- adding docgen command (0.0.17)
3240
- additional recipes and bug fix for uninstall (0.0.16)
3341
- updating testing to be with ubuntu (on native) (0.0.15)

‎Dockerfile.tcl‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,13 @@ RUN apt-get update && \
1717
python3-dev \
1818
python3-pip \
1919
curl \
20-
less
20+
less \
21+
wget
22+
23+
RUN wget -O- http://neuro.debian.net/lists/xenial.us-ca.full | tee /etc/apt/sources.list.d/neurodebian.sources.list && \
24+
apt-key adv --recv-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 && \
25+
apt-get update && \
26+
apt-get install -y singularity-container
2127

2228
RUN curl -LJO https://github.com/cea-hpc/modules/releases/download/v4.7.0/modules-4.7.0.tar.gz && \
2329
tar xfz modules-4.7.0.tar.gz && \

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ And container technologies:
1515
- [Singularity](https://github.com/sylabs/singularity)
1616
- [Podman](https://podman.io)
1717

18+
Coming soon:
19+
20+
- [Shifter](https://github.com/NERSC/shifter)
21+
1822
You can use shpc if you are:
1923

2024
1. a linux administrator wanting to manage containers as modules for your cluster

‎docs/getting_started/installation.rst‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,16 @@ Virtual Environment
1313
===================
1414

1515
The recommended approach is to install from the repository directly, whether
16-
you use pip or another setup approach. Here is how to clone the repository
17-
and do a local install.
16+
you use pip or another setup approach, and to install a `known release <https://github.com/singularityhub/singularity-hpc/releases/>`_. Here is how to clone the repository and do a local install.
1817

1918
.. code:: console
2019
21-
$ git clone [email protected]:singularityhub/singularity-hpc
20+
# Install release 0.0.24
21+
$ git clone -b 0.0.24 [email protected]:singularityhub/singularity-hpc
2222
$ cd singularity-hpc
2323
$ pip install -e .[all]
2424
25-
or
25+
or (an example with python setuptools and installing from the main branch)
2626

2727
.. code:: console
2828
@@ -40,7 +40,7 @@ if you install to a system python, meaning either of these commands:
4040
4141
You will need to put the registry files elsewhere (update the ``registry`` config argument to the path), as they will not be installed
4242
alongside the package. The same is the case for modules - if you install to system
43-
python it's recomended to define ``lmod_base`` as something else, unless you
43+
python it's recommended to define ``lmod_base`` as something else, unless you
4444
can write to your install location. Installing locally ensures that you
4545
can easily store your module files along with the install (the default until you
4646
change it). Installation of singularity-hpc adds an executable, `shpc` to your path.
@@ -91,11 +91,23 @@ and interacting with your modules.
9191
to install container modules.
9292

9393

94+
Environment Modules
95+
-------------------
96+
97+
If you are using `environment modules (tcl) <http://modules.sourceforge.net/>`_
98+
and you find that your aliases do not expand, you can use `shopt <https://www.gnu.org/software/bash/manual/html_node/The-Shopt-Builtin.html>`_ to fix this issue:
99+
100+
.. code-block:: console
101+
102+
$ shopt expand_aliases || true
103+
$ shopt -s expand_aliases
104+
105+
94106
Pypi
95107
====
96108

97-
The module is avaiable in pypi as `singularity-hpc <https://pypi.org/project/singularity-hpc/>`_,
98-
and this is primairly to have a consistent means for release, and an interface to show the package. Since the registry
109+
The module is available in pypi as `singularity-hpc <https://pypi.org/project/singularity-hpc/>`_,
110+
and this is primarily to have a consistent means for release, and an interface to show the package. Since the registry
99111
files will not install and you would need to change the registry path
100112
and module base (making it hard to update from the git remote) we do not
101113
encourage you to install from pip unless you know exactly what you are doing.

0 commit comments

Comments
 (0)