Skip to content

Commit 1025f0b

Browse files
vsochvsoch
committed
more testing
Signed-off-by: Vanessa Sochat <[email protected]>
1 parent 5f3fea1 commit 1025f0b

File tree

4 files changed

+68
-36
lines changed

4 files changed

+68
-36
lines changed

‎singularity/build/scripts/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Changes
2+
3+
- [build.go](build.go) is from [here](https://github.com/sylabs/singularity/blob/release-3.4/internal/pkg/build/build.go)
4+
- [bundle.go](bundle.go) is from [here](https://github.com/sylabs/singularity/blob/release-3.4/pkg/build/types/bundle.go)
5+
6+
For bundle.go, we accept an extra argument for the stage, and the directory
7+
created is named based on that. Build.go then provides this stage. If no stage
8+
is provided, it works as normal.

‎singularity/build/scripts/bundle.go‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
package types
77

88
import (
9-
"io/ioutil"
109
"os"
1110
"path/filepath"
1211

@@ -86,11 +85,10 @@ func bundleCommon(bundleDir, bundlePrefix string, keyInfo *crypt.KeyInfo, stageN
8685
bundlePrefix = "sbuild-"
8786
}
8887

89-
// Bundle path must be predictable
90-
// TODO if this isn't defined just do sbuild
91-
sylog.Debugf("Stage name is %v\n", stageName)
88+
// Bundle path must be predictable (/tmp/sbuild if not defined)
89+
sylog.Infof("Stage name is %v\n", stageName)
9290
b.Path = "/tmp/sbuild" + stageName
93-
sylog.Debugf("Will use path for bundle %v\n", b.Path)
91+
sylog.Infof("Will use path for bundle %v\n", b.Path)
9492

9593
err = os.Mkdir(b.Path, 0755)
9694
if err != nil {
@@ -120,7 +118,7 @@ func NewEncryptedBundle(bundleDir, bundlePrefix string, keyInfo *crypt.KeyInfo,
120118
}
121119

122120
// NewBundle creates a Bundle environment
123-
func NewBundle(bundleDir, bundlePrefix string) (b *Bundle, err error, stageName string) {
121+
func NewBundle(bundleDir, bundlePrefix string, stageName string) (b *Bundle, err error) {
124122
return bundleCommon(bundleDir, bundlePrefix, nil, stageName)
125123
}
126124

‎singularity/build/scripts/secure-build.sh‎

Lines changed: 51 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,13 @@ fi
1515
# Get all stages
1616
CONTENDER_STAGES=$(cat $SINGULARITY_BUILDDEF | grep ^[[:blank:]]*Stage:)
1717

18-
stages="build"
18+
# Case 1: We have stages
1919
for stage in $CONTENDER_STAGES; do
2020
if [ "$stage" != "Stage" ]; then
2121
stages="$stage $stages"
2222
fi
2323
done
2424

25-
echo "Found stages $stages"
26-
2725
SINGULARITY_confdir="/usr/local/etc/singularity"
2826
SINGULARITY_bindir="/usr/local/bin"
2927
SINGULARITY_libexecdir="/usr/local/libexec/singularity"
@@ -63,18 +61,9 @@ cat > "$SINGULARITY_WORKDIR/root/.rpmmacros" << RPMMAC
6361
%_dbpath %{_var}/lib/rpm
6462
RPMMAC
6563

66-
# Staged build image is the main / final build
64+
# Location for the repository
6765
REPO_DIR="/root/repo"
68-
STAGED_BUILD_IMAGE="/root/build"
69-
7066
mkdir -p ${SINGULARITY_WORKDIR}${REPO_DIR}
71-
mkdir -p ${SINGULARITY_WORKDIR}${STAGED_BUILD_IMAGE}
72-
73-
# Make directory for each stage (build included)
74-
#for stage in $stages; do
75-
# echo "Creating ${SINGULARITY_WORKDIR}/root/$stage"
76-
# mkdir -p "${SINGULARITY_WORKDIR}/root/$stage"
77-
#done
7867

7968
# Move the repo to be the REPO_DIR
8069
cp -R $BUILDDEF_DIR/* ${SINGULARITY_WORKDIR}$REPO_DIR
@@ -88,10 +77,7 @@ HOSTS_FILE="$SINGULARITY_WORKDIR/hosts"
8877
cp /etc/resolv.conf $RESOLV_CONF
8978
cp /etc/hosts $HOSTS_FILE
9079

91-
cat > "$FSTAB_FILE" << FSTAB
92-
none $STAGED_BUILD_IMAGE bind dev 0 0
93-
FSTAB
94-
80+
# Create base temporary config file
9581
cat > "$TMP_CONF_FILE" << CONF
9682
config passwd = yes
9783
config group = no
@@ -109,7 +95,6 @@ user bind control = no
10995
bind path = $SINGULARITY_WORKDIR/root:/root
11096
bind path = $SINGULARITY_WORKDIR/tmp:/tmp
11197
bind path = $SINGULARITY_WORKDIR/var_tmp:/var/tmp
112-
bind path = /tmp/sbuild/fs:$STAGED_BUILD_IMAGE
11398
bind path = $FSTAB_FILE:/etc/fstab
11499
bind path = $RESOLV_CONF:/etc/resolv.conf
115100
bind path = $HOSTS_FILE:/etc/hosts
@@ -118,10 +103,7 @@ allow user capabilities = no
118103
allow setuid = yes
119104
CONF
120105

121-
# We only use the builder once, make default config
122-
cp "$TMP_CONF_FILE" "${SINGULARITY_confdir}/singularity.conf"
123-
124-
# here build pre-stage
106+
# Shared starting script (will have build command appended)
125107
cat > "$BUILD_SCRIPT" << SCRIPT
126108
#!/bin/sh
127109
mount -r --no-mtab -t proc proc /proc
@@ -135,10 +117,53 @@ if [ \$? != 0 ]; then
135117
exit 1
136118
fi
137119
cd $REPO_DIR
138-
singularity build $STAGED_BUILD_IMAGE/container.sif $BUILDDEF
139-
exit \$?
140120
SCRIPT
141121

122+
123+
# Proceed based on stages (or not)
124+
if [ "$stages" == "" ]; then
125+
echo "NO STAGES"
126+
127+
# Staged build image is the main / final build
128+
STAGED_BUILD_IMAGE="/root/build"
129+
mkdir -p ${SINGULARITY_WORKDIR}${STAGED_BUILD_IMAGE}
130+
131+
cat > "$FSTAB_FILE" << FSTAB
132+
none $STAGED_BUILD_IMAGE bind dev 0 0
133+
FSTAB
134+
135+
# Update config file with staged build image
136+
echo "bind path = /tmp/sbuild/fs:$STAGED_BUILD_IMAGE" >> "$TMP_CONF_FILE"
137+
138+
# We only use the builder once, make default config
139+
cp "$TMP_CONF_FILE" "${SINGULARITY_confdir}/singularity.conf"
140+
141+
# add final lines of build script
142+
echo "singularity build $STAGED_BUILD_IMAGE/container.sif $BUILDDEF" >> "${BUILD_SCRIPT}"
143+
echo "exit \$?" >> ${BUILD_SCRIPT}
144+
145+
else
146+
echo "Found stages $stages"
147+
148+
# Make directory for each stage (/root/<stage>
149+
for stage in $stages; do
150+
STAGED_BUILD_IMAGE="/root/$stage"
151+
echo "Creating ${SINGULARITY_WORKDIR}$STAGED_BUILD_IMAGE"
152+
mkdir -p "${SINGULARITY_WORKDIR}$STAGED_BUILD_IMAGE"
153+
echo "none ${STAGED_BUILD_IMAGE} bind dev 0 0" >> "$FSTAB_FILE"
154+
echo "bind path = /tmp/sbuild$stage/fs:$STAGED_BUILD_IMAGE" >> "$TMP_CONF_FILE"
155+
echo "bind path = /tmp/sbuild$stage/fs:$STAGED_BUILD_IMAGE" >> "$TMP_CONF_FILE"
156+
done
157+
158+
# The final stage is where we put the container
159+
echo "Final staged build image is $STAGED_BUILD_IMAGE"
160+
161+
# We only use the builder once, make default config
162+
cp "$TMP_CONF_FILE" "${SINGULARITY_confdir}/singularity.conf"
163+
164+
fi
165+
166+
# Shared final tweaks
142167
chmod +x $BUILD_SCRIPT
143168

144169
unset SINGULARITY_IMAGE
@@ -154,8 +179,8 @@ if [ $? != 0 ]; then
154179
fi
155180

156181
if [ ! -f "${SINGULARITY_WORKDIR}${STAGED_BUILD_IMAGE}/container.sif" ]; then
157-
echo "Container was not built.";
158-
exit 1;
182+
echo "Container was not built.";
183+
exit 1;
159184
fi
160185

161186
mv "${SINGULARITY_WORKDIR}${STAGED_BUILD_IMAGE}/container.sif" "$BUILDDEF_DIR/${SINGULARITY_FINAL}"

‎singularity/build/scripts/singularity-prepare-instance.sh‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,15 @@ sudo -H python3 -m pip install oauth2client==3.0.0
4747
sudo -H python3 -m pip install ipython
4848

4949
# Install GoLang
50-
export VERSION=1.12.6 OS=linux ARCH=amd64
50+
export VERSION=1.13 OS=linux ARCH=amd64
5151

5252
wget -O /tmp/go${VERSION}.${OS}-${ARCH}.tar.gz https://dl.google.com/go/go${VERSION}.${OS}-${ARCH}.tar.gz && \
5353
sudo tar -C /usr/local -xzf /tmp/go${VERSION}.${OS}-${ARCH}.tar.gz
5454

5555
# Install Singularity from Github
5656

5757
mkdir -p /tmp/go
58-
export GOPATH=/tmp/go SINGULARITY_VERSION=3.2.1
58+
export GOPATH=/tmp/go SINGULARITY_VERSION=3.4.2
5959
export PATH=/usr/local/go/bin:$PATH
6060
echo 'Defaults env_keep += "GOPATH"' | sudo tee --append /etc/sudoers.d/env_keep
6161

@@ -69,7 +69,8 @@ mkdir -p ${GOPATH}/src/github.com/sylabs && \
6969

7070
# https://github.com/sylabs/singularity/blob/release-3.4/internal/pkg/build/build.go is 3.4.2 at build time
7171
cd ${GOPATH}/src/github.com/sylabs/singularity && \
72-
wget https://raw.githubusercontent.com/singularityhub/singularity-python/v3.2.1/singularity/build/scripts/bundle.go && \
72+
wget https://raw.githubusercontent.com/singularityhub/singularity-python/v$SINGULARITY_VERSION/singularity/build/scripts/bundle.go && \
73+
wget https://raw.githubusercontent.com/singularityhub/singularity-python/v$SINGULARITY_VERSION/singularity/build/scripts/build.go && \
7374
mv bundle.go pkg/build/types/ && \
7475
mv build.go internal/pkg/build/build.go
7576
./mconfig && \
@@ -97,7 +98,7 @@ From: ubuntu:18.04
9798
export VERSION=1.12.6 OS=linux ARCH=amd64
9899
export SINGULARITY_VERSION=3.2.1
99100
apt-get update -y
100-
apt-get -y install git build-essential libssl-dev uuid-dev pkg-config curl gcc
101+
apt-get -y install git build-essential libssl-dev uuid-dev pkg-config curl gcc cryptsetup
101102
apt-get -y install libgpgme11-dev libseccomp-dev squashfs-tools libc6-dev-i386
102103
apt-get install -y debootstrap yum pacman
103104
apt-get clean

0 commit comments

Comments
 (0)