Skip to content

Commit 0b37ac2

Browse files
yarikopticyarikoptic
committed
Perspective neurodocker invocation + produced Dockerfile (broken)
It is based on WIP done in ReproNim/neurodocker#611 to add support for Gentoo. But it is just a dirty draft so produced docker file is not good.
1 parent 85b0be6 commit 0b37ac2

File tree

7 files changed

+222
-8
lines changed

7 files changed

+222
-8
lines changed

‎Dockerfile

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
# Generated by Neurodocker and Reproenv.
2+
3+
FROM docker.io/gentoo/portage:20240324 as portage
4+
FROM docker.io/gentoo/stage3:20240318
5+
RUN COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo \
6+
&& RUN emerge -v --noreplace dev-vcs/git \
7+
&& RUN emerge -v1u portage \
8+
# Pinned commits for the dependency tree state
9+
&& ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6 \
10+
&& ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448 \
11+
&& ARG FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" \
12+
# This will be bound, and contents available outside of container
13+
&& RUN mkdir /outputs \
14+
&& COPY gentoo-portage/ /etc/portage/ \
15+
# Moving gentoo repo from default rsync to git
16+
&& RUN rm /var/db/repos/gentoo -rf \
17+
# Cloning manually to prevent vdb update, pinning state via git
18+
# Allegedly it's better to chain everything in one command, something with container layers 🤔
19+
&& RUN \
20+
REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/gentoo | sed -e "s/sync-uri *= *//g") && \
21+
mkdir -p /var/db/repos/gentoo && pushd /var/db/repos/gentoo && git init . && \
22+
git remote add origin ${REPO_URL} && \
23+
git fetch --filter="blob:none" origin $gentoo_hash && \
24+
git reset --hard $gentoo_hash && rm .git -rf && popd && \
25+
REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/science | sed -e "s/sync-uri *= *//g") && \
26+
mkdir -p /var/db/repos/science && pushd /var/db/repos/science && git init . && \
27+
git remote add origin ${REPO_URL} && \
28+
git fetch --filter="blob:none" origin $science_hash && \
29+
git reset --hard $science_hash && rm .git -rf && popd \
30+
# Old Christian: Remove sync-uri to not accidentally re-sync if we work with the package management interactively
31+
# Christian from the future: Maybe we want the option to re-sync if we're debugging it interactively...
32+
#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "s/sync-type *= *git/sync-type =/g"
33+
#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-uri/d"
34+
#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-git-verify-commit-signature/d"
35+
# Make sure all CPU flags supported by the hardware are whitelisted
36+
# This only affects packages with handwritten assembly language optimizations, e.g. ffmpeg.
37+
# Removing it is safe, software will just not take full advantage of processor capabilities.
38+
#RUN emerge cpuid2cpuflags
39+
#RUN echo "*/* $(cpuid2cpuflags)" > /etc/portage/package.use/00cpu-flags
40+
### Emerge cool stuff here
41+
### Autounmask-continue enables all features on dependencies which the top level packages require
42+
### By default this needs user confirmation which would interrupt the build.
43+
RUN emerge --autounmask-continue \
44+
afni \
45+
fsl \
46+
&& rm -rf /var/tmp/portage/*
47+
COPY ["environment.yml", \
48+
"/opt/environment.yml"]
49+
COPY ["src", \
50+
"/opt/dsst-defacing-pipeline"]
51+
ENV CONDA_DIR="/opt/miniconda-latest" \
52+
PATH="/opt/miniconda-latest/bin:$PATH"
53+
RUN \
54+
# Install dependencies.
55+
&& export PATH="/opt/miniconda-latest/bin:$PATH" \
56+
&& echo "Downloading Miniconda installer ..." \
57+
&& conda_installer="/tmp/miniconda.sh" \
58+
&& curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
59+
&& bash "$conda_installer" -b -p /opt/miniconda-latest \
60+
&& rm -f "$conda_installer" \
61+
&& conda update -yq -nbase conda \
62+
# Prefer packages in conda-forge
63+
&& conda config --system --prepend channels conda-forge \
64+
# Packages in lower-priority channels not considered if a package with the same
65+
# name exists in a higher priority channel. Can dramatically speed up installations.
66+
# Conda recommends this as a default
67+
# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html
68+
&& conda config --set channel_priority strict \
69+
&& conda config --system --set auto_update_conda false \
70+
&& conda config --system --set show_channel_urls true \
71+
# Enable `conda activate`
72+
&& conda init bash \
73+
&& conda env create --name dsstdeface --file /opt/environment.yml \
74+
# Clean up
75+
&& sync && conda clean --all --yes && sync \
76+
&& rm -rf ~/.cache/pip/*
77+
RUN test "$(getent passwd dsst)" \
78+
|| useradd --no-user-group --create-home --shell /bin/bash dsst
79+
USER dsst
80+
ENTRYPOINT ["/opt/dsst-defacing-pipeline/run.py"]
81+
82+
# Save specification to JSON.
83+
USER root
84+
RUN printf '{ \
85+
"pkg_manager": "portage", \
86+
"existing_users": [ \
87+
"root" \
88+
], \
89+
"instructions": [ \
90+
{ \
91+
"name": "from_", \
92+
"kwds": { \
93+
"base_image": "docker.io/gentoo/portage:20240324 as portage" \
94+
} \
95+
}, \
96+
{ \
97+
"name": "from_", \
98+
"kwds": { \
99+
"base_image": "docker.io/gentoo/stage3:20240318" \
100+
} \
101+
}, \
102+
{ \
103+
"name": "run", \
104+
"kwds": { \
105+
"command": "COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo\\nRUN emerge -v --noreplace dev-vcs/git\\nRUN emerge -v1u portage\\n# Pinned commits for the dependency tree state\\nARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\\nARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448\\nARG FEATURES=\\"-ipc-sandbox -network-sandbox -pid-sandbox\\"\\n# This will be bound, and contents available outside of container\\nRUN mkdir /outputs\\nCOPY gentoo-portage/ /etc/portage/\\n# Moving gentoo repo from default rsync to git\\nRUN rm /var/db/repos/gentoo -rf\\n# Cloning manually to prevent vdb update, pinning state via git\\n# Allegedly it'"'"'s better to chain everything in one command, something with container layers \\ud83e\\udd14\\nRUN \\\\\\n REPO_URL=$\(grep \\"^sync-uri\\" /etc/portage/repos.conf/gentoo | sed -e \\"s/sync-uri *= *//g\\"\) && \\\\\\n mkdir -p /var/db/repos/gentoo && pushd /var/db/repos/gentoo && git init . && \\\\\\n git remote add origin ${REPO_URL} && \\\\\\n git fetch --filter=\\"blob:none\\" origin $gentoo_hash && \\\\\\n git reset --hard $gentoo_hash && rm .git -rf && popd && \\\\\\n REPO_URL=$\(grep \\"^sync-uri\\" /etc/portage/repos.conf/science | sed -e \\"s/sync-uri *= *//g\\"\) && \\\\\\n mkdir -p /var/db/repos/science && pushd /var/db/repos/science && git init . && \\\\\\n git remote add origin ${REPO_URL} && \\\\\\n git fetch --filter=\\"blob:none\\" origin $science_hash && \\\\\\n git reset --hard $science_hash && rm .git -rf && popd\\n# Old Christian: Remove sync-uri to not accidentally re-sync if we work with the package management interactively\\n# Christian from the future: Maybe we want the option to re-sync if we'"'"'re debugging it interactively...\\n#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e \\"s/sync-type *= *git/sync-type =/g\\"\\n#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e \\"/sync-uri/d\\"\\n#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e \\"/sync-git-verify-commit-signature/d\\"\\n# Make sure all CPU flags supported by the hardware are whitelisted\\n# This only affects packages with handwritten assembly language optimizations, e.g. ffmpeg.\\n# Removing it is safe, software will just not take full advantage of processor capabilities.\\n#RUN emerge cpuid2cpuflags\\n#RUN echo \\"*/* $\(cpuid2cpuflags\)\\" > /etc/portage/package.use/00cpu-flags\\n### Emerge cool stuff here\\n### Autounmask-continue enables all features on dependencies which the top level packages require\\n### By default this needs user confirmation which would interrupt the build." \
106+
} \
107+
}, \
108+
{ \
109+
"name": "install", \
110+
"kwds": { \
111+
"pkgs": [ \
112+
"afni", \
113+
"fsl" \
114+
], \
115+
"opts": null \
116+
} \
117+
}, \
118+
{ \
119+
"name": "run", \
120+
"kwds": { \
121+
"command": "emerge --autounmask-continue \\\\\\n afni \\\\\\n fsl \\\\\\n && rm -rf /var/tmp/portage/*" \
122+
} \
123+
}, \
124+
{ \
125+
"name": "copy", \
126+
"kwds": { \
127+
"source": [ \
128+
"environment.yml", \
129+
"/opt/environment.yml" \
130+
], \
131+
"destination": "/opt/environment.yml" \
132+
} \
133+
}, \
134+
{ \
135+
"name": "copy", \
136+
"kwds": { \
137+
"source": [ \
138+
"src", \
139+
"/opt/dsst-defacing-pipeline" \
140+
], \
141+
"destination": "/opt/dsst-defacing-pipeline" \
142+
} \
143+
}, \
144+
{ \
145+
"name": "env", \
146+
"kwds": { \
147+
"CONDA_DIR": "/opt/miniconda-latest", \
148+
"PATH": "/opt/miniconda-latest/bin:$PATH" \
149+
} \
150+
}, \
151+
{ \
152+
"name": "run", \
153+
"kwds": { \
154+
"command": "\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-latest/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-latest\\nrm -f \\"$conda_installer\\"\\nconda update -yq -nbase conda\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda env create --name dsstdeface --file /opt/environment.yml\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \
155+
} \
156+
}, \
157+
{ \
158+
"name": "user", \
159+
"kwds": { \
160+
"user": "dsst" \
161+
} \
162+
}, \
163+
{ \
164+
"name": "entrypoint", \
165+
"kwds": { \
166+
"args": [ \
167+
"/opt/dsst-defacing-pipeline/run.py" \
168+
] \
169+
} \
170+
} \
171+
] \
172+
}' > /.reproenv.json
173+
USER dsst
174+
# End saving to specification to JSON.

‎generate_container.sh

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,29 @@ generate() {
66
# more details might come on https://github.com/ReproNim/neurodocker/issues/330
77
[ "$1" == singularity ] && add_entry=' "$@"' || add_entry=''
88
#neurodocker generate "$1" \
9-
ndversion=0.9.5
9+
#ndversion=0.9.5
1010
#ndversion=master
1111
#docker run --rm repronim/neurodocker:$ndversion \
1212
# ATM needs devel version of neurodocker for a fix to AFNI recipe
13+
#--base-image neurodebian:bookworm \
14+
#--ndfreeze date=20240320 \
15+
dest=/opt/dsst-defacing-pipeline
1316
neurodocker \
1417
generate "$1" \
15-
--base-image neurodebian:bookworm \
16-
--ndfreeze date=20240320 \
18+
--pkg-manager portage \
19+
--base-image "docker.io/gentoo/portage:20240324 as portage" \
20+
--base-image "docker.io/gentoo/stage3:20240318" \
21+
--gentoo gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6 \
22+
--install afni fsl \
1723
--copy environment.yml /opt/environment.yml \
24+
--copy src "$dest" \
1825
--miniconda \
1926
version=latest \
2027
env_name=dsstdeface \
2128
env_exists=false \
2229
yaml_file=/opt/environment.yml \
23-
--pkg-manager=apt \
24-
--install vim wget strace time ncdu gnupg curl procps pigz less tree \
25-
--run "apt-get update && apt-get -y dist-upgrade" \
26-
--afni method=binaries version=latest \
2730
--user=dsst \
28-
--entrypoint "bash"
31+
--entrypoint "$dest/run.py"
2932
#--run "curl -sL https://deb.nodesource.com/setup_16.x | bash - " \
3033
#--install nodejs npm \
3134
#--run "npm install -g [email protected]" \

‎gentoo-portage/make.conf

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
### This file contains system-wide build variables, including Gentoo variables such as USE, which enable/disable optional package features.
2+
3+
COMMON_FLAGS="-O2 -pipe -march=native"
4+
# Comment the following out on systems with less than 8 threads
5+
MAKEOPTS="--jobs 8 --load-average 9"
6+
CFLAGS="${COMMON_FLAGS}"
7+
CXXFLAGS="${COMMON_FLAGS}"
8+
FCFLAGS="${COMMON_FLAGS}"
9+
FFLAGS="${COMMON_FLAGS}"
10+
11+
# NOTE: This stage was built with the bindist Use flag enabled
12+
13+
# This sets the language of build output to English.
14+
# Please keep this setting intact when reporting bugs.
15+
LC_MESSAGES=C
16+
17+
USE="${USE} science"
18+
ACCEPT_LICENSE="*"
19+
20+
# Needed in the container environment
21+
#FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox"

‎gentoo-portage/package.accept_keywords/gen

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
### This is needed because ::science packages are generally not marked as stable
2+
*/* ~amd64

‎gentoo-portage/package.mask/bugs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
### This is empty, thankfully.
2+
### If we find bugs in some version of some package we can blacklist the package, version, or feature that causes it here.

‎gentoo-portage/repos.conf/gentoo

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[gentoo]
2+
location = /var/db/repos/gentoo
3+
# We sync manually, but we need sync-uri to be written down somewhere to do so
4+
sync-type = git
5+
sync-uri = https://anongit.gentoo.org/git/repo/gentoo.git
6+
sync-git-verify-commit-signature = yes

‎gentoo-portage/repos.conf/science

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[science]
2+
location = /var/db/repos/science
3+
# We sync manually, but we need sync-uri to be written down somewhere to do so
4+
sync-type = git
5+
sync-uri = https://anongit.gentoo.org/git/proj/sci.git
6+
priority = 7777

0 commit comments

Comments
 (0)