Skip to content

Commit f90c565

Browse files
jiridanekjiridanek
committed
NO-JIRA: add "Subscribe with subscription manager" dockerfile fragment
1 parent a772fc7 commit f90c565

File tree

1 file changed

+95
-151
lines changed

1 file changed

+95
-151
lines changed

‎scripts/dockerfile_fragments.py‎

Lines changed: 95 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env python3
2-
from __future__ import annotations
32

43
"""
54
This script is inspired by the AIPCC `replace-markers.sh` script, invoked from `make regen`
@@ -13,170 +12,115 @@
1312
run Python function `funcname` and paste in the return value.
1413
"""
1514

16-
import os
15+
from __future__ import annotations
16+
1717
import textwrap
18-
import pathlib
1918
from typing import TYPE_CHECKING
2019

20+
import ntb
21+
2122
if TYPE_CHECKING:
23+
import pathlib
24+
2225
from pyfakefs.fake_filesystem import FakeFilesystem
2326

24-
ROOT_DIR = pathlib.Path(__file__).parent.parent
27+
# restricting to the relevant directories significantly speeds up the processing
28+
docker_directories = (
29+
ntb.ROOT_DIR / "jupyter",
30+
ntb.ROOT_DIR / "codeserver",
31+
ntb.ROOT_DIR / "rstudio",
32+
ntb.ROOT_DIR / "runtimes",
33+
)
34+
35+
36+
def sanity_check(dockerfile: pathlib.Path, replacements: dict[str, str]):
37+
"""Sanity check that we don't have any unexpected `### BEGIN`s and `### END`s"""
38+
begin = "#" * 3 + " BEGIN"
39+
end = "#" * 3 + " END"
40+
with open(dockerfile, "rt") as fp:
41+
for line_no, line in enumerate(fp, start=1):
42+
for prefix in (begin, end):
43+
if line.rstrip().startswith(prefix):
44+
suffix = line[len(prefix) + 1:].rstrip()
45+
if suffix not in replacements:
46+
raise ValueError(
47+
f"Expected replacement for '{prefix} {suffix}' "
48+
f"not found in {dockerfile}:{line_no}"
49+
)
2550

2651

2752
def main():
28-
for dockerfile in ROOT_DIR.glob("**/Dockerfile*"):
29-
if not dockerfile.is_file():
30-
continue
31-
if dockerfile.is_relative_to(ROOT_DIR / "base-images"):
32-
continue
33-
if dockerfile.is_relative_to(ROOT_DIR / "examples"):
34-
continue
35-
36-
replacements = {
37-
"upgrade first to avoid fixable vulnerabilities": textwrap.dedent(r"""
38-
# If we have a Red Hat subscription prepared, refresh it
39-
RUN /bin/bash <<'EOF'
40-
set -Eeuxo pipefail
41-
if command -v subscription-manager &> /dev/null; then
42-
subscription-manager identity &>/dev/null && subscription-manager refresh || echo "No identity, skipping refresh."
43-
fi
44-
EOF
45-
46-
# Problem: The operation would result in removing the following protected packages: systemd
47-
# (try to add '--allowerasing' to command line to replace conflicting packages or '--skip-broken' to skip uninstallable packages)
48-
# Solution: --best --skip-broken does not work either, so use --nobest
49-
RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked,id=notebooks-dnf /bin/bash <<'EOF'
50-
set -Eeuxo pipefail
51-
dnf -y upgrade --refresh --nobest --skip-broken --nodocs --noplugins --setopt=install_weak_deps=0 --setopt=keepcache=1
52-
EOF
53-
54-
"""),
55-
"Install micropipenv and uv to deploy packages from requirements.txt": '''RUN pip install --no-cache-dir --extra-index-url https://pypi.org/simple -U "micropipenv[toml]==1.9.0" "uv==0.9.6"''',
56-
"Install the oc client": textwrap.dedent(r"""
57-
RUN /bin/bash <<'EOF'
58-
set -Eeuxo pipefail
59-
curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/ocp/stable/openshift-client-linux.tar.gz \
60-
-o /tmp/openshift-client-linux.tar.gz
61-
tar -xzvf /tmp/openshift-client-linux.tar.gz oc
62-
rm -f /tmp/openshift-client-linux.tar.gz
63-
EOF
64-
65-
"""),
66-
"Dependencies for PDF export": textwrap.dedent(r"""
67-
RUN ./utils/install_pdf_deps.sh
68-
ENV PATH="/usr/local/texlive/bin/linux:/usr/local/pandoc/bin:$PATH"
69-
"""),
70-
"Download Elyra Bootstrapper": textwrap.dedent(r"""
71-
RUN curl -fL https://raw.githubusercontent.com/opendatahub-io/elyra/refs/tags/v4.3.1/elyra/kfp/bootstrapper.py \
72-
-o ./utils/bootstrapper.py
73-
# Prevent Elyra from re-installing the dependencies
74-
ENV ELYRA_INSTALL_PACKAGES="false"
75-
"""),
76-
}
77-
78-
# sanity check that we don't have any unexpected `### BEGIN`s and `### END`s
79-
begin = "#" * 3 + " BEGIN"
80-
end = "#" * 3 + " END"
81-
with open(dockerfile, "rt") as fp:
82-
for line_no, line in enumerate(fp, start=1):
83-
for prefix in (begin, end):
84-
if line.rstrip().startswith(prefix):
85-
suffix = line[len(prefix) + 1:].rstrip()
86-
if suffix not in replacements:
87-
raise ValueError(
88-
f"Expected replacement for '{prefix} {suffix}' "
89-
f"not found in {dockerfile}:{line_no}"
90-
)
91-
92-
for prefix, contents in replacements.items():
93-
blockinfile(
94-
filename=dockerfile,
95-
contents=contents,
96-
prefix=prefix,
97-
)
98-
99-
100-
def blockinfile(
101-
filename: str | os.PathLike,
102-
contents: str, prefix: str | None = None,
103-
*,
104-
comment: str = "#",
105-
) -> None:
106-
"""This is similar to the functions in
107-
* https://homely.readthedocs.io/en/latest/ref/files.html#homely-files-blockinfile-1
108-
* ansible.modules.lineinfile
109-
"""
110-
begin_marker = f"{comment * 3} BEGIN{" " + prefix if prefix else ""}"
111-
end_marker = f"{comment * 3} END{" " + prefix if prefix else ""}"
112-
113-
begin = end = -1
114-
try:
115-
with open(filename, "rt") as fp:
116-
original_lines = fp.readlines()
117-
except (IOError, OSError) as e:
118-
raise RuntimeError(f"Failed to read {filename}: {e}") from e
119-
for line_no, line in enumerate(original_lines):
120-
if line.rstrip() == begin_marker:
121-
begin = line_no
122-
elif line.rstrip() == end_marker:
123-
end = line_no
124-
125-
if begin != -1 and end == -1:
126-
raise ValueError(f"Found begin marker but no matching end marker in {filename}")
127-
if begin == -1 and end != -1:
128-
raise ValueError(f"Found end marker but no matching begin marker in {filename}")
129-
if begin > end:
130-
raise ValueError(f"Begin marker appears after end marker in {filename}")
131-
132-
lines = original_lines[:]
133-
# NOTE: textwrap.dedent() with raw strings leaves leading and trailing newline
134-
# we want to preserve the trailing one because HEREDOC has to have an empty trailing line for hadolint
135-
new_contents = contents.lstrip("\n").splitlines(keepends=True)
136-
if new_contents and new_contents[-1] == "\n":
137-
new_contents = new_contents[:-1]
138-
if begin == end == -1:
139-
# no markers found
140-
return
141-
else:
142-
lines[begin: end + 1] = [f"{begin_marker}\n", *new_contents, f"\n{end_marker}\n"]
143-
144-
if lines == original_lines:
145-
return
146-
with open(filename, "wt") as fp:
147-
fp.writelines(lines)
53+
subscription_manager_register_refresh = textwrap.dedent(r"""
54+
# If we have a Red Hat subscription prepared, refresh it
55+
RUN /bin/bash <<'EOF'
56+
set -Eeuxo pipefail
57+
if command -v subscription-manager &> /dev/null; then
58+
subscription-manager identity &>/dev/null && subscription-manager refresh || echo "No identity, skipping refresh."
59+
fi
60+
EOF
61+
""")
62+
63+
replacements = {
64+
"Subscribe with subscription manager": textwrap.dedent(subscription_manager_register_refresh),
65+
"upgrade first to avoid fixable vulnerabilities": textwrap.dedent(ntb.process_template_with_indents(rt"""
66+
{subscription_manager_register_refresh}
67+
# Problem: The operation would result in removing the following protected packages: systemd
68+
# (try to add '--allowerasing' to command line to replace conflicting packages or '--skip-broken' to skip uninstallable packages)
69+
# Solution: --best --skip-broken does not work either, so use --nobest
70+
RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked,id=notebooks-dnf /bin/bash <<'EOF'
71+
set -Eeuxo pipefail
72+
dnf -y upgrade --refresh --nobest --skip-broken --nodocs --noplugins --setopt=install_weak_deps=0 --setopt=keepcache=1
73+
EOF
74+
75+
""")),
76+
"Install micropipenv and uv to deploy packages from requirements.txt": '''RUN pip install --no-cache-dir --extra-index-url https://pypi.org/simple -U "micropipenv[toml]==1.9.0" "uv==0.9.6"''',
77+
"Install the oc client": textwrap.dedent(r"""
78+
RUN /bin/bash <<'EOF'
79+
set -Eeuxo pipefail
80+
curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/ocp/stable/openshift-client-linux.tar.gz \
81+
-o /tmp/openshift-client-linux.tar.gz
82+
tar -xzvf /tmp/openshift-client-linux.tar.gz oc
83+
rm -f /tmp/openshift-client-linux.tar.gz
84+
EOF
85+
86+
"""),
87+
"Dependencies for PDF export": textwrap.dedent(r"""
88+
RUN ./utils/install_pdf_deps.sh
89+
ENV PATH="/usr/local/texlive/bin/linux:/usr/local/pandoc/bin:$PATH"
90+
"""),
91+
"Download Elyra Bootstrapper": textwrap.dedent(r"""
92+
RUN curl -fL https://raw.githubusercontent.com/opendatahub-io/elyra/refs/tags/v4.3.1/elyra/kfp/bootstrapper.py \
93+
-o ./utils/bootstrapper.py
94+
# Prevent Elyra from re-installing the dependencies
95+
ENV ELYRA_INSTALL_PACKAGES="false"
96+
"""),
97+
}
98+
99+
for docker_dir in docker_directories:
100+
for dockerfile in docker_dir.glob("**/Dockerfile*"):
101+
if not dockerfile.is_file():
102+
continue
103+
if dockerfile.is_relative_to(ntb.ROOT_DIR / "base-images"):
104+
continue
105+
if dockerfile.is_relative_to(ntb.ROOT_DIR / "examples"):
106+
continue
107+
108+
sanity_check(dockerfile, replacements)
109+
110+
for prefix, contents in replacements.items():
111+
ntb.blockinfile(
112+
filename=dockerfile,
113+
contents=contents,
114+
prefix=prefix,
115+
)
148116

149117

150118
if __name__ == "__main__":
151119
main()
152120

153121

154-
class TestBlockinfile:
155-
def test_adding_new_block(self, fs: FakeFilesystem):
156-
"""the file should not be modified if there is no block already"""
157-
fs.create_file("/config.txt", contents="hello\nworld")
158-
159-
blockinfile("/config.txt", "key=value")
160-
161-
assert fs.get_object("/config.txt").contents == "hello\nworld"
162-
163-
def test_updating_value_in_block(self, fs: FakeFilesystem):
164-
fs.create_file("/config.txt", contents="hello\nworld\n### BEGIN\nkey=value1\n### END\n")
165-
166-
blockinfile("/config.txt", "key=value2")
167-
168-
assert fs.get_object("/config.txt").contents == "hello\nworld\n### BEGIN\nkey=value2\n### END\n"
169-
170-
def test_lastnewline_removal(self, fs: FakeFilesystem):
171-
fs.create_file("/config.txt", contents="hello\nworld\n### BEGIN\n### END\n")
172-
173-
blockinfile("/config.txt", "key=value\n\n")
174-
175-
assert fs.get_object("/config.txt").contents == "hello\nworld\n### BEGIN\nkey=value\n\n### END\n"
176-
122+
class TestMain:
177123
def test_dry_run(self, fs: FakeFilesystem):
178-
fs.add_real_directory(source_path=ROOT_DIR / "jupyter", read_only=False)
179-
fs.add_real_directory(source_path=ROOT_DIR / "codeserver", read_only=False)
180-
fs.add_real_directory(source_path=ROOT_DIR / "rstudio", read_only=False)
181-
fs.add_real_directory(source_path=ROOT_DIR / "runtimes", read_only=False)
124+
for docker_dir in docker_directories:
125+
fs.add_real_directory(source_path=docker_dir, read_only=False)
182126
main()

0 commit comments

Comments
 (0)