Skip to content

Commit 3b164e0

Browse files
vsochvsoch
authored
Shell script wrappers (#586)
* proof of concept for shell wrapper! I still need to do the container command, exec, run, and the two inspects, and then the same for docker. This is a WIP - the proof of concept seems to be okay so far! * adding reminder of singularity wrappers * adding docker wrapper scripts and tests * singularity exec/run should not have -s shell * ensure template is fully loaded * fixing bug that filesystem wrapper returns the filename instead of the content! * update test: load_wrapper_script should have equivalent behavior regardless! * Update shpc/main/modules/templates/singularity.tcl Signed-off-by: vsoch <[email protected]>
1 parent e4c8650 commit 3b164e0

File tree

22 files changed

+285
-75
lines changed

22 files changed

+285
-75
lines changed

‎shpc/client/sync.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@
22
__copyright__ = "Copyright 2021-2022, Vanessa Sochat"
33
__license__ = "MPL 2.0"
44

5+
import os
6+
57
import shpc.logger as logger
68
import shpc.utils
7-
import os
89

910

1011
def sync_registry(args, parser, extra, subparser):

‎shpc/main/modules/templates/docker.lua‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,17 +53,13 @@ local containerPath = '{{ image }}'
5353

5454
-- service environment variable to access docker URI
5555
setenv("PODMAN_CONTAINER", containerPath)
56-
set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
5756

5857
local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath
5958
-- execCmd needs entrypoint to be the executor
6059
local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "
6160
local runCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath
6261
local inspectCmd = "{{ command }} ${PODMAN_OPTS} inspect ${PODMAN_COMMAND_OPTS} " .. containerPath
6362

64-
-- set_shell_function takes bashStr and cshStr
65-
set_shell_function("{|module_name|}-shell", shellCmd, shellCmd)
66-
6763
-- conflict with modules with the same name
6864
conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %})
6965

@@ -80,14 +76,19 @@ if (myShellName() == "bash") then
8076
{% endfor %}
8177
end{% endif %}
8278

79+
{% if wrapper_scripts %}{% else %}set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
80+
81+
-- set_shell_function takes bashStr and cshStr
82+
set_shell_function("{|module_name|}-shell", shellCmd, shellCmd)
83+
8384
-- A customizable exec function
8485
set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. containerPath .. " \"$@\"", execCmd .. " --entrypoint \"\" " .. containerPath)
8586

8687
-- Always provide a container run
8788
set_shell_function("{|module_name|}-run", runCmd .. " \"$@\"", runCmd)
8889

8990
-- Inspect runscript or deffile easily!
90-
set_shell_function("{|module_name|}-inspect", inspectCmd, inspectCmd)
91+
set_shell_function("{|module_name|}-inspect", inspectCmd, inspectCmd){% endif %}
9192

9293
whatis("Name : " .. myModuleName())
9394
whatis("Version : " .. myModuleVersion())

‎shpc/main/modules/templates/docker.tcl‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ conflict {{ parsed_name.tool }}
7070

7171
# service environment variable to access full SIF image path
7272
setenv PODMAN_CONTAINER "${containerPath}"
73-
set-alias {|module_name|}-container "echo ${containerPath}"
7473

7574
# interactive shell to any container, plus exec for aliases
7675
set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if settings.enable_tty %}t{% endif %} --entrypoint {{ shell }} {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}"
@@ -80,9 +79,6 @@ set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{%
8079
set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}"
8180
set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}"
8281

83-
# set_shell_function takes bashStr and cshStr
84-
set-alias {|module_name|}-shell "${shellCmd}"
85-
8682
# wrapper scripts? Add bin to path
8783
{% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %}
8884

@@ -101,6 +97,11 @@ set-alias {|module_name|}-shell "${shellCmd}"
10197
{% endfor %}
10298
}{% endif %}
10399

100+
{% if wrapper_scripts %}{% else %}
101+
set-alias {|module_name|}-container "echo ${containerPath}"
102+
103+
set-alias {|module_name|}-shell "${shellCmd}"
104+
104105
# A customizable exec function
105106
if { [ module-info shell bash ] } {
106107
set-alias {|module_name|}-exec "${execCmd} --entrypoint \"\" ${containerPath} \"\$@\""
@@ -116,7 +117,7 @@ if { [ module-info shell bash ] } {
116117
}
117118

118119
# Inspect runscript or deffile easily!
119-
set-alias {|module_name|}-inspect "${inspectCmd} ${containerPath}"
120+
set-alias {|module_name|}-inspect "${inspectCmd} ${containerPath}"{% endif %}
120121

121122
#=====
122123
# Module options

‎shpc/main/modules/templates/singularity.lua‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,17 +56,13 @@ if not os.getenv("SINGULARITY_COMMAND_OPTS") then setenv ("SINGULARITY_COMMAND_O
5656
local containerPath = '{{ container_sif }}'
5757
-- service environment variable to access full SIF image path
5858
setenv("SINGULARITY_CONTAINER", containerPath)
59-
set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
6059

6160
-- interactive shell to any container, plus exec for aliases
6261
local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath
6362
local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} "
6463
local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath
6564
local inspectCmd = "singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} "
6665

67-
-- set_shell_function takes bashStr and cshStr
68-
set_shell_function("{|module_name|}-shell", shellCmd, shellCmd)
69-
7066
-- conflict with modules with the same name
7167
conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %})
7268

@@ -83,6 +79,12 @@ if (myShellName() == "bash") then
8379
{% endfor %}
8480
end{% endif %}
8581

82+
-- Only set shell functions if we don't use wrapper scripts
83+
{% if wrapper_scripts %}{% else %}set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
84+
85+
-- set_shell_function takes bashStr and cshStr
86+
set_shell_function("{|module_name|}-shell", shellCmd, shellCmd)
87+
8688
-- A customizable exec function
8789
set_shell_function("{|module_name|}-exec", execCmd .. containerPath .. " \"$@\"", execCmd .. containerPath)
8890

@@ -91,7 +93,8 @@ set_shell_function("{|module_name|}-run", runCmd .. " \"$@\"", runCmd)
9193

9294
-- Inspect runscript or deffile easily!
9395
set_shell_function("{|module_name|}-inspect-runscript", inspectCmd .. " -r " .. containerPath, inspectCmd .. containerPath)
94-
set_shell_function("{|module_name|}-inspect-deffile", inspectCmd .. " -d " .. containerPath, inspectCmd .. containerPath)
96+
set_shell_function("{|module_name|}-inspect-deffile", inspectCmd .. " -d " .. containerPath, inspectCmd .. containerPath){% endif %}
97+
9598

9699
whatis("Name : " .. myModuleName())
97100
whatis("Version : " .. myModuleVersion())

‎shpc/main/modules/templates/singularity.tcl‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,18 +78,13 @@ setenv SINGULARITY_SHELL {{ settings.singularity_shell }}
7878

7979
# service environment variable to access full SIF image path
8080
setenv SINGULARITY_CONTAINER "${containerPath}"
81-
set-alias {|module_name|}-container "echo ${containerPath}"
8281

8382
# interactive shell to any container, plus exec for aliases
8483
set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}"
8584
set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} "
8685
set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}"
8786
set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} "
8887

89-
# set_shell_function takes bashStr and cshStr
90-
set-alias {|module_name|}-shell "${shellCmd}"
91-
92-
9388
# if we have any wrapper scripts, add bin to path
9489
{% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %}
9590

@@ -108,7 +103,11 @@ set-alias {|module_name|}-shell "${shellCmd}"
108103
{% endfor %}
109104
}{% endif %}
110105

111-
# A customizable exec function
106+
{% if wrapper_scripts %}{% else %}
107+
set-alias {|module_name|}-shell "${shellCmd}"
108+
set-alias {|module_name|}-container "echo ${containerPath}"
109+
110+
112111
if { [ module-info shell bash ] } {
113112
set-alias {|module_name|}-exec "${execCmd} ${containerPath} \"\$@\""
114113
} else {
@@ -124,8 +123,7 @@ if { [ module-info shell bash ] } {
124123

125124
# Inspect runscript or deffile easily!
126125
set-alias {|module_name|}-inspect-runscript "${inspectCmd} -r ${containerPath}"
127-
set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}"
128-
126+
set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}"{% endif %}
129127

130128
#=====
131129
# Module options

‎shpc/main/registry/filesystem.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ def load_wrapper_script(self, container_tech, script):
5858
"""
5959
wrapper_script = self.find_wrapper_script(container_tech, script)
6060
if wrapper_script:
61-
return os.path.join(self.dirname, wrapper_script)
61+
return shpc.utils.read_file(os.path.join(self.dirname, wrapper_script))
6262

6363
def override_exists(self, tag):
6464
"""

‎shpc/main/wrappers/__init__.py‎

Lines changed: 24 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,18 @@
22
__copyright__ = "Copyright 2022, Vanessa Sochat"
33
__license__ = "MPL 2.0"
44

5-
import os
6-
7-
from shpc.logger import logger
85

6+
from . import generators as gen
97
from .base import WrapperScript
108

11-
here = os.path.abspath(os.path.dirname(__file__))
12-
139

1410
def generate(image, container, config, **kwargs):
1511
"""
1612
Generate one or more wrapper scripts for a container.
1713
1814
Required arguments for all include container (class), image (path), settings
19-
All kwargs go in optional. And this can be extended to include custom arguments.
15+
All kwargs go in optional. The core set of constructor kwargs are provided
16+
to each wrapper generator. This can be extended to include custom arguments.
2017
"""
2118
# Return list of generated templates
2219
generated = []
@@ -33,46 +30,30 @@ def generate(image, container, config, **kwargs):
3330
}
3431

3532
# Default wrapper for container technology, used for aliases unless overridden
33+
default_wrapper = load_default_wrapper(constructor_kwargs)
34+
35+
# Generate wrappers for command aliases
36+
generated += gen.alias_wrappers(aliases, default_wrapper, constructor_kwargs)
37+
38+
# Generate wrappers for container interactions
39+
generated += gen.container_wrappers(constructor_kwargs)
40+
41+
# Container level wrapper scripts (allow eventually supporting custom podman)
42+
generated += gen.custom_container_wrappers(constructor_kwargs)
43+
return list(set(generated))
44+
45+
46+
def load_default_wrapper(constructor_kwargs):
47+
"""
48+
Given a container and user settings, load a default wrapper.
49+
"""
3650
default_wrapper = None
51+
container = constructor_kwargs["container"]
52+
settings = constructor_kwargs["settings"]
53+
3754
default_template_name = settings.wrapper_scripts.get(container.command)
3855
if default_template_name:
3956
default_wrapper = WrapperScript(default_template_name, **constructor_kwargs)
4057
# include_container_dir not set -> only look in the global locations
4158
default_wrapper.load_template()
42-
43-
# Command aliases
44-
custom_wrapper_option_name = "%s_script" % container.templatefile
45-
for alias in aliases:
46-
# Allow overriding the template name in the script option
47-
if custom_wrapper_option_name in alias:
48-
wrapper = WrapperScript(
49-
alias[custom_wrapper_option_name], **constructor_kwargs
50-
)
51-
wrapper.load_template(include_container_dir=True)
52-
elif default_wrapper:
53-
wrapper = default_wrapper
54-
else:
55-
logger.exit(
56-
"Can't generate a wrapper script for '%s' as there is no template defined for %s"
57-
% (alias["name"], container.templatefile)
58-
)
59-
60-
# NB: alias is a dictionary
61-
generated += wrapper.generate(alias["name"], alias)
62-
63-
# Container level wrapper scripts (allow eventually supporting custom podman)
64-
scripts = {
65-
"singularity": config.singularity_scripts,
66-
"docker": config.docker_scripts,
67-
"podman": config.docker_scripts,
68-
}
69-
# Additional commands defined in the custom container.yaml script section
70-
listing = scripts.get(container.templatefile) or {}
71-
for alias, template_name in listing.items():
72-
wrapper = WrapperScript(template_name, **constructor_kwargs)
73-
# Template wrapper scripts may live alongside container.yaml
74-
wrapper.load_template(include_container_dir=True)
75-
# NB: alias is a string
76-
generated += wrapper.generate(alias, alias)
77-
78-
return list(set(generated))
59+
return default_wrapper

‎shpc/main/wrappers/base.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,16 +104,21 @@ def load_template(self, include_container_dir=False):
104104
result = self.find_wrapper_script(template_paths, include_container_dir)
105105
loader = FileSystemLoader(template_paths)
106106
env = Environment(loader=loader)
107+
108+
# Do we have a filesystem path to load directly?
107109
if "path" in result:
108110
self.template = env.get_template(self.wrapper_template)
111+
112+
# Or string content to load?
109113
else:
110114
self.template = env.from_string(result["content"])
111115

112-
def generate(self, wrapper_name, alias_definition):
116+
def generate(self, wrapper_name, alias_definition=None):
113117
"""
114118
Template generation function.
115119
NB: alias_definition is a dictionary for command aliases, and a string
116-
for additional arbitrary commands
120+
for additional arbitrary commands. It is not required for container
121+
interaction wrappers (e.g., exec, shell, etc.)
117122
"""
118123

119124
# Write scripts into container directory

0 commit comments

Comments
 (0)