Skip to content

Commit 18d73e9

Browse files
committed
try adding bundle
Signed-off-by: Vanessa Sochat <[email protected]>
1 parent 38a451c commit 18d73e9

File tree

3 files changed

+121
-132
lines changed

3 files changed

+121
-132
lines changed
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved.
2+
// This software is licensed under a 3-clause BSD license. Please consult the
3+
// LICENSE.md file distributed with the sources of this project regarding your
4+
// rights to use or distribute this software.
5+
6+
package types
7+
8+
import (
9+
"io/ioutil"
10+
"os"
11+
"path/filepath"
12+
13+
ocitypes "github.com/containers/image/types"
14+
"github.com/sylabs/singularity/internal/pkg/sylog"
15+
)
16+
17+
// Bundle is the temporary build environment used during the image
18+
// building process. A Bundle is the programmatic representation of
19+
// the directory structure which will constitute this environmenb.
20+
// /tmp/...:
21+
// fs/ - A chroot filesystem
22+
// .singularity.d/ - Container metadata (from 2.x image format)
23+
// config.json (optional) - Contain information for OCI image bundle
24+
// etc... - The Bundle dir can theoretically contain arbitrary directories,
25+
// files, etc... which can be interpreted by the Chef
26+
type Bundle struct {
27+
// FSObjects is a map of the filesystem objects contained in the Bundle. An object
28+
// will be built as one section of a SIF file.
29+
//
30+
// Known FSObjects labels:
31+
// * rootfs -> root file system
32+
// * .singularity.d -> .singularity.d directory (includes image exec scripts)
33+
// * data -> directory containing data files
34+
FSObjects map[string]string `json:"fsObjects"`
35+
JSONObjects map[string][]byte `json:"jsonObjects"`
36+
Recipe Definition `json:"rawDeffile"`
37+
BindPath []string `json:"bindPath"`
38+
Path string `json:"bundlePath"`
39+
Opts Options `json:"opts"`
40+
}
41+
42+
// Options defines build time behavior to be executed on the bundle
43+
type Options struct {
44+
// sections are the parts of the definition to run during the build
45+
Sections []string `json:"sections"`
46+
// TmpDir specifies a non-standard temporary location to perform a build
47+
TmpDir string
48+
// LibraryURL contains URL to library where base images can be pulled
49+
LibraryURL string `json:"libraryURL"`
50+
// LibraryAuthToken contains authentication token to access specified library
51+
LibraryAuthToken string `json:"libraryAuthToken"`
52+
// contains docker credentials if specified
53+
DockerAuthConfig *ocitypes.DockerAuthConfig
54+
// noTest indicates if build should skip running the test script
55+
NoTest bool `json:"noTest"`
56+
// force automatically deletes an existing container at build destination while performing build
57+
Force bool `json:"force"`
58+
// update detects and builds using an existing sandbox container at build destination
59+
Update bool `json:"update"`
60+
// noHTTPS
61+
NoHTTPS bool `json:"noHTTPS"`
62+
// NoCleanUp allows a user to prevent a bundle from being cleaned up after a failed build
63+
// useful for debugging
64+
NoCleanUp bool `json:"noCleanUp"`
65+
}
66+
67+
// NewBundle creates a Bundle environment
68+
func NewBundle(bundleDir, bundlePrefix string) (b *Bundle, err error) {
69+
b = &Bundle{}
70+
b.JSONObjects = make(map[string][]byte)
71+
72+
if bundlePrefix == "" {
73+
bundlePrefix = "sbuild-"
74+
}
75+
76+
// Bundle path must be predictable
77+
b.Path = "/tmp/sbuild"
78+
err = os.MkDir(b.Path)
79+
if err != nil {
80+
return nil, err
81+
}
82+
sylog.Debugf("Created temporary directory for bundle %v\n", b.Path)
83+
84+
b.FSObjects = map[string]string{
85+
"rootfs": "fs",
86+
}
87+
88+
for _, fso := range b.FSObjects {
89+
if err = os.MkdirAll(filepath.Join(b.Path, fso), 0755); err != nil {
90+
return
91+
}
92+
}
93+
94+
return b, nil
95+
}
96+
97+
// Rootfs give the path to the root filesystem in the Bundle
98+
func (b *Bundle) Rootfs() string {
99+
return filepath.Join(b.Path, b.FSObjects["rootfs"])
100+
}
101+
102+
// RunSection iterates through the sections specified in a bundle
103+
// and returns true if the given string, s, is a section of the
104+
// definition that should be executed during the build process
105+
func (b Bundle) RunSection(s string) bool {
106+
for _, section := range b.Opts.Sections {
107+
if section == "none" {
108+
return false
109+
}
110+
if section == "all" || section == s {
111+
return true
112+
}
113+
}
114+
return false
115+
}

singularity/build/scripts/singularity-build-latest.sh

Lines changed: 0 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -24,133 +24,6 @@
2424
#
2525
################################################################################
2626

27-
SINGULARITY_libexecdir="/usr/local/libexec/singularity"
28-
SINGULARITY_PATH="/usr/local/bin"
29-
SECBUILD_IMAGE="$SINGULARITY_libexecdir/singularity/secure-build/secbuild.sif"
30-
31-
# Set the isolated root
32-
if [ -z "${SINGULARITY_ISOLATED_ROOT:-}" ]; then
33-
BUILDDEF_DIR_NAME=$(dirname ${SINGULARITY_BUILDDEF:-})
34-
else
35-
BUILDDEF_DIR_NAME=$(readlink -f ${SINGULARITY_ISOLATED_ROOT:-})
36-
fi
37-
BUILDDEF_DIR=$(readlink -f ${BUILDDEF_DIR_NAME:-})
38-
39-
if [ -z "${BUILDDEF_DIR:-}" ]; then
40-
message ERROR "Can't find parent directory of $SINGULARITY_BUILDDEF\n"
41-
exit 1
42-
fi
43-
44-
BUILDDEF=$(basename ${SINGULARITY_BUILDDEF:-})
45-
46-
# create a temporary dir per build instance
47-
export SINGULARITY_WORKDIR=$(mktemp -d)
48-
49-
# create /tmp and /var/tmp into WORKDIR
50-
mkdir -p $SINGULARITY_WORKDIR/tmp $SINGULARITY_WORKDIR/var_tmp
51-
52-
# set sticky bit for these directories
53-
chmod 1777 $SINGULARITY_WORKDIR/tmp
54-
chmod 1777 $SINGULARITY_WORKDIR/var_tmp
55-
56-
# setup a fake root directory
57-
cp -a /etc/skel $SINGULARITY_WORKDIR/root
58-
59-
cat > "$SINGULARITY_WORKDIR/root/.rpmmacros" << RPMMAC
60-
%_var /var
61-
%_dbpath %{_var}/lib/rpm
62-
RPMMAC
63-
64-
REPO_DIR="/root/repo"
65-
STAGED_BUILD_IMAGE="/root/build"
66-
67-
mkdir ${SINGULARITY_WORKDIR}${REPO_DIR}
68-
mkdir ${SINGULARITY_WORKDIR}${STAGED_BUILD_IMAGE}
69-
70-
BUILD_SCRIPT="$SINGULARITY_WORKDIR/tmp/build-script"
71-
TMP_CONF_FILE="$SINGULARITY_WORKDIR/tmp.conf"
72-
FSTAB_FILE="$SINGULARITY_WORKDIR/fstab"
73-
RESOLV_CONF="$SINGULARITY_WORKDIR/resolv.conf"
74-
HOSTS_FILE="$SINGULARITY_WORKDIR/hosts"
75-
76-
cp /etc/resolv.conf $RESOLV_CONF
77-
cp /etc/hosts $HOSTS_FILE
78-
79-
cat > "$FSTAB_FILE" << FSTAB
80-
none $STAGED_BUILD_IMAGE bind dev 0 0
81-
FSTAB
82-
83-
cat > "$TMP_CONF_FILE" << CONF
84-
config passwd = no
85-
config group = no
86-
config resolv_conf = no
87-
mount proc = no
88-
mount sys = no
89-
mount home = no
90-
mount dev = minimal
91-
mount devpts = no
92-
mount tmp = no
93-
enable overlay = no
94-
user bind control = no
95-
bind path = $SINGULARITY_WORKDIR/root:/root
96-
bind path = $SINGULARITY_WORKDIR/tmp:/tmp
97-
bind path = $SINGULARITY_WORKDIR/var_tmp:/var/tmp
98-
bind path = $SINGULARITY_ROOTFS:$STAGED_BUILD_IMAGE
99-
bind path = $BUILDDEF_DIR:$REPO_DIR
100-
bind path = $FSTAB_FILE:/etc/fstab
101-
bind path = $RESOLV_CONF:/etc/resolv.conf
102-
bind path = $HOSTS_FILE:/etc/hosts
103-
root default capabilities = default
104-
allow user capabilities = no
105-
CONF
106-
107-
# here build pre-stage
108-
cat > "$BUILD_SCRIPT" << SCRIPT
109-
#!/bin/sh
110-
mount -r --no-mtab -t proc proc /proc
111-
if [ \$? != 0 ]; then
112-
echo "Can't mount /proc directory"
113-
exit 1
114-
fi
115-
mount -r --no-mtab -t sysfs sysfs /sys
116-
if [ \$? != 0 ]; then
117-
echo "Can't mount /sys directory"
118-
exit 1
119-
fi
120-
mount -o remount,dev $STAGED_BUILD_IMAGE
121-
if [ \$? != 0 ]; then
122-
echo "Can't remount $STAGED_BUILD_IMAGE"
123-
exit 1
124-
fi
125-
cd $REPO_DIR
126-
singularity build --sandbox $STAGED_BUILD_IMAGE $BUILDDEF
127-
exit \$?
128-
SCRIPT
129-
130-
chmod +x $BUILD_SCRIPT
131-
132-
unset SINGULARITY_IMAGE
133-
unset SINGULARITY_NO_PRIVS
134-
unset SINGULARITY_KEEP_PRIVS
135-
unset SINGULARITY_ADD_CAPS
136-
unset SINGULARITY_DROP_CAPS
137-
138-
${SINGULARITY_bindir}/singularity -c $TMP_CONF_FILE exec -e -i -p $SECBUILD_IMAGE /tmp/build-script
139-
if [ $? != 0 ]; then
140-
rm -rf $SINGULARITY_WORKDIR
141-
exit 1
142-
fi
143-
144-
rm -rf $SINGULARITY_WORKDIR
145-
146-
147-
148-
149-
150-
151-
152-
153-
15427

15528
echo "Start Time: $(date)." > /tmp/.shub-log 2>&1
15629
timeout -s KILL 2h sudo python3 -c "from singularity.build.google import run_build; run_build()" >> /tmp/.shub-log 2>&1

singularity/build/scripts/singularity-prepare-instance.sh

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,11 @@ sudo apt-get -y install git \
3838
python3-pip
3939

4040
# Pip3 installs
41-
sudo pip3 install --upgrade pip &&
42-
sudo pip3 install pyasn1-modules -U &&
43-
sudo pip3 install --upgrade google-api-python-client &&
44-
sudo pip3 install --upgrade google &&
45-
sudo pip3 install oauth2client==3.0.0
41+
sudo -H pip3 install -H --upgrade pip
42+
sudo -H pip3 install pyasn1-modules -U
43+
sudo -H pip3 install --upgrade google-api-python-client
44+
sudo -H pip3 install --upgrade google
45+
sudo -H pip3 install oauth2client==3.0.0
4646

4747
# Install GoLang
4848
export VERSION=1.12.6 OS=linux ARCH=amd64
@@ -66,6 +66,7 @@ mkdir -p ${GOPATH}/src/github.com/sylabs && \
6666
echo "v${SINGULARITY_VERSION}" > VERSION
6767

6868
cd ${GOPATH}/src/github.com/sylabs/singularity && \
69+
wget
6970
./mconfig && \
7071
cd ./builddir && \
7172
make && \

0 commit comments

Comments
 (0)