Skip to content

Commit a5a0e0a

Browse files
authored
Merge pull request #384 from singularityhub/add/docker
adding support for Docker as a new container technology
2 parents 975d4c9 + 08ac2d6 commit a5a0e0a

File tree

19 files changed

+333
-238
lines changed

19 files changed

+333
-238
lines changed

.github/workflows/test.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010
fail-fast: false
1111
matrix:
1212
module: ["lmod", "tcl"]
13-
container_tech: ["podman", "singularity"]
13+
container_tech: ["podman", "singularity", "docker"]
1414
steps:
1515

1616
- name: Install Dependencies
@@ -62,6 +62,7 @@ jobs:
6262
source activate shpc
6363
shpc config set container_tech:${{ matrix.container_tech }}
6464
shpc config set module_sys:${{ matrix.module }}
65+
shpc config set enable_tty:false
6566
shpc install python:3.9.5-alpine
6667
module use ./modules
6768
module load python/3.9.5-alpine
@@ -87,6 +88,7 @@ jobs:
8788
source activate shpc
8889
shpc config set container_tech:${{ matrix.container_tech }}
8990
shpc config set module_sys:${{ matrix.module }}
91+
shpc config set enable_tty:false
9092
shpc install python:3.9.5-alpine
9193
9294
shopt expand_aliases || true

CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ and **Merged pull requests**. Critical items to know are:
1414
The versions coincide with releases on pip. Only major versions will be released as tags on Github.
1515

1616
## [0.0.x](https://github.scom/singularityhub/singularity-hpc/tree/master) (0.0.x)
17+
- Docker support (0.0.26)
18+
- added an enable_tty setting to allow disabling add -t in recipes
19+
- enforced not adding extra settings to setting.yml
20+
- settings are validated when an update is attempted
1721
- Podman support (0.0.25)
1822
- container_tech is now a settings.yml variable
1923
- allow for custom test interpreter "test_shell"

docs/getting_started/user-guide.rst

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -146,10 +146,13 @@ A summary table of variables is included below, and then further discussed in de
146146
- null
147147
* - singularity_shell
148148
- exported to SINGULARITY_SHELL, defaults to /bin/bash.
149-
- /bin/bash
149+
- /bin/sh
150150
* - podman_shell
151151
- The shell used for podman
152-
- /bin/bash
152+
- /bin/sh
153+
* - docker_shell
154+
- The shell used for docker
155+
- /bin/sh
153156
* - test_shell
154157
- The shell used for the test.sh file
155158
- /bin/bash
@@ -159,6 +162,9 @@ A summary table of variables is included below, and then further discussed in de
159162
* - environment_file
160163
- The name of the environment file to generate and bind to the container.
161164
- 99-shpc.sh
165+
* - enable_tty
166+
- For container technologies that require -t for tty, enable (add) or disable (do not add)
167+
- true
162168
* - config_editor
163169
- The editor to use for your config editing
164170
- vim
@@ -356,7 +362,7 @@ Container Technology
356362
====================
357363

358364
The default container technology to pull and then provide to users is Singularity,
359-
and we have also recently added Podman, and will add support for Shifter and Sarus soon.
365+
and we have also recently added Podman and Docker, and will add support for Shifter and Sarus soon.
360366
Akin to module software, you can specify the container technology to use on a global
361367
setting, or via a one-off command:
362368

shpc/client/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ def get_parser():
230230
"--container-tech",
231231
dest="container_tech",
232232
help="container technology to use to override settings.yaml",
233-
choices=["singularity", "podman"],
233+
choices=["singularity", "podman", "docker"],
234234
default=None,
235235
)
236236

shpc/client/config.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ def main(args, parser, extra, subparser):
3232
)
3333
continue
3434
key, value = param.split(":", 1)
35-
logger.info("Updating %s to be %s" % (key, value))
3635
cli.settings.set(key, value)
36+
logger.info("Updated %s to be %s" % (key, value))
3737

3838
# Save settings
3939
cli.settings.save()

shpc/main/__init__.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ def get_client(quiet=False, **kwargs):
4848

4949
Client.container = PodmanContainer()
5050

51+
elif container == "docker":
52+
from .container import DockerContainer
53+
54+
Client.container = DockerContainer()
55+
5156
# The containe should have access to settings too
5257
if hasattr(Client, "container"):
5358
Client.container.settings = settings

shpc/main/container/__init__.py

Lines changed: 3 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
from .singularity import SingularityContainer
77
from .podman import PodmanContainer
8-
import shpc.main.templates as templates
8+
from .docker import DockerContainer
99

1010
from shpc.logger import logger
1111
import shpc.main.schemas as schemas
@@ -17,7 +17,6 @@
1717
from ruamel.yaml import YAML
1818

1919
import os
20-
import re
2120
import jsonschema
2221
import sys
2322

@@ -63,32 +62,6 @@ def __repr__(self):
6362
return str(self)
6463

6564

66-
class ContainerName:
67-
"""
68-
Parse a container name into named parts
69-
"""
70-
71-
def __init__(self, raw):
72-
self.raw = raw
73-
self.registry = None
74-
self.namespace = None
75-
self.tool = None
76-
self.version = None
77-
self.digest = None
78-
self.parse(raw)
79-
80-
def parse(self, raw):
81-
"""
82-
Parse a name into known pieces
83-
"""
84-
match = re.search(templates.docker_regex, raw)
85-
if not match:
86-
logger.exit("%s does not match a known identifier pattern." % raw)
87-
for key, value in match.groupdict().items():
88-
value = value.strip("/") if value else None
89-
setattr(self, key, value)
90-
91-
9265
class ContainerConfig:
9366
"""A ContainerConfig wraps a container.yaml file, intended for install."""
9467

@@ -127,6 +100,8 @@ def name(self):
127100
"""
128101
Return the name, whether it's docker or GitHub
129102
"""
103+
from .base import ContainerName
104+
130105
if self.docker:
131106
return ContainerName(self.docker)
132107
elif self.gh:

shpc/main/container/base.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,33 @@
99

1010
from jinja2 import Template
1111
import os
12+
import re
13+
14+
15+
class ContainerName:
16+
"""
17+
Parse a container name into named parts
18+
"""
19+
20+
def __init__(self, raw):
21+
self.raw = raw
22+
self.registry = None
23+
self.namespace = None
24+
self.tool = None
25+
self.version = None
26+
self.digest = None
27+
self.parse(raw)
28+
29+
def parse(self, raw):
30+
"""
31+
Parse a name into known pieces
32+
"""
33+
match = re.search(shpc.main.templates.docker_regex, raw)
34+
if not match:
35+
logger.exit("%s does not match a known identifier pattern." % raw)
36+
for key, value in match.groupdict().items():
37+
value = value.strip("/") if value else None
38+
setattr(self, key, value)
1239

1340

1441
class ContainerTechnology:

shpc/main/container/docker.py

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
__author__ = "Vanessa Sochat"
2+
__copyright__ = "Copyright 2021, Vanessa Sochat"
3+
__license__ = "MPL 2.0"
4+
5+
6+
from shpc.logger import logger
7+
from .base import ContainerTechnology
8+
import shpc.main.templates
9+
import shpc.utils
10+
11+
from datetime import datetime
12+
import json
13+
import os
14+
import sys
15+
16+
17+
class DockerContainer(ContainerTechnology):
18+
"""
19+
A Docker container controller.
20+
"""
21+
22+
# The module technology adds extensions here
23+
templatefile = "docker"
24+
command = "docker"
25+
features = {}
26+
27+
def __init__(self):
28+
if shpc.utils.which("docker")["return_code"] != 0:
29+
logger.exit("Docker is required to use the 'docker' base.")
30+
super(DockerContainer, self).__init__()
31+
32+
def shell(self, image):
33+
"""
34+
Interactive shell into a container image.
35+
"""
36+
os.system(
37+
"docker run -it --rm --entrypoint %s %s"
38+
% (self.settings.docker_shell, image)
39+
)
40+
41+
def add_registry(self, uri):
42+
"""
43+
Given a "naked" name, add the registry if it's Docker Hub
44+
"""
45+
# Is this a core library container, or Docker Hub without prefix?
46+
if uri.count("/") == 0:
47+
uri = "docker.io/library/%s" % uri
48+
elif uri.count("/") == 1:
49+
uri = "docker.io/%s" % uri
50+
return uri
51+
52+
def registry_pull(self, module_dir, container_dir, config, tag):
53+
"""
54+
Pull a container to the library.
55+
"""
56+
pull_type = "docker" if getattr(config, "docker") else "gh"
57+
if pull_type != "docker":
58+
logger.exit("%s only supports Docker (oci registry) pulls." % self.command)
59+
60+
# Podman doesn't keep a record of digest->tag, so we use tag
61+
uri = "%s:%s" % (self.add_registry(config.docker), tag.name)
62+
return self.pull(uri)
63+
64+
def pull(self, uri):
65+
"""
66+
Pull a unique resource identifier.
67+
"""
68+
res = shpc.utils.run_command([self.command, "pull", uri], stream=True)
69+
if res["return_code"] != 0:
70+
logger.exit("There was an issue pulling %s" % uri)
71+
return uri
72+
73+
def inspect(self, image):
74+
"""
75+
Inspect an image
76+
"""
77+
res = shpc.utils.run_command([self.command, "inspect", image])
78+
if res["return_code"] != 0:
79+
logger.exit("There was an issue getting the manifest for %s" % image)
80+
raw = res["message"]
81+
return json.loads(raw)
82+
83+
def exists(self, image):
84+
"""
85+
Exists is a derivative of inspect that just determines existence.
86+
"""
87+
if not image:
88+
return False
89+
res = shpc.utils.run_command([self.command, "inspect", image])
90+
if res["return_code"] != 0:
91+
return False
92+
return True
93+
94+
def get(self, module_name):
95+
"""
96+
Determine if a container uri exists.
97+
"""
98+
# If no module tag provided, try to deduce from install tree
99+
module_name = self.guess_tag(module_name)
100+
101+
uri = self.add_registry(module_name)
102+
# If there isn't a tag in the name, add it back
103+
if ":" not in uri:
104+
uri = ":".join(uri.rsplit("/", 1))
105+
if uri and self.exists(uri):
106+
return uri
107+
108+
def delete(self, image):
109+
"""
110+
Delete a container when a module is deleted.
111+
"""
112+
image = self.get(image)
113+
if self.exists(image):
114+
shpc.utils.run_command([self.command, "rmi", image])
115+
116+
def check(self, module_name, config):
117+
"""
118+
Given a module name, check if it's the latest version.
119+
"""
120+
# Case 1: a specific tag is selected
121+
image = self.get(module_name)
122+
if self.exists(image):
123+
tag = image.split(":")[-1]
124+
if tag == config.latest.name:
125+
logger.info("⭐️ tag %s is up to date. ⭐️" % config.tag.name)
126+
else:
127+
logger.exit(
128+
"👉️ tag %s can be updated to %s! 👈️"
129+
% (module_name, config.latest.name)
130+
)
131+
132+
def install(
133+
self,
134+
module_path,
135+
container_path,
136+
name,
137+
template,
138+
parsed_name,
139+
aliases=None,
140+
url=None,
141+
description=None,
142+
version=None,
143+
config_features=None,
144+
features=None,
145+
):
146+
"""Install a general container path to a module
147+
148+
The module_dir should be created by the calling function, and
149+
the container should already be added to the module directory. In
150+
the case of install this means we did a pull from a registry,
151+
and for add we moved the container there explicitly.
152+
"""
153+
# Container features are defined in container.yaml and the settings
154+
# and specific values are determined by the container technology
155+
features = self.get_features(
156+
config_features, self.settings.container_features, features
157+
)
158+
159+
# Ensure that the container exists
160+
# Do we want to clean up other versions here too?
161+
manifest = self.inspect(container_path)
162+
if not manifest:
163+
sys.exit("Container %s was not found. Was it pulled?" % container_path)
164+
165+
labels = manifest[0].get("Labels", {})
166+
167+
# If there's a tag in the name, don't use it
168+
name = name.split(":", 1)[0]
169+
170+
# Make sure to render all values!
171+
out = template.render(
172+
podman_module=self.settings.podman_module,
173+
bindpaths=self.settings.bindpaths,
174+
shell=self.settings.podman_shell
175+
if self.command == "podman"
176+
else self.settings.docker_shell,
177+
image=container_path,
178+
description=description,
179+
module_dir=os.path.dirname(module_path),
180+
aliases=aliases,
181+
url=url,
182+
features=features,
183+
version=version,
184+
labels=labels,
185+
prefix=self.settings.module_exc_prefix,
186+
creation_date=datetime.now(),
187+
name=name,
188+
tool=parsed_name.tool,
189+
registry=parsed_name.registry,
190+
namespace=parsed_name.namespace,
191+
envfile=self.settings.environment_file,
192+
command=self.command,
193+
tty=self.settings.enable_tty,
194+
)
195+
shpc.utils.write_file(module_path, out)

0 commit comments

Comments
 (0)