Skip to content

Commit 97805d7

Browse files
Merge pull request #382 from ambiata/topic/fix-broken-test
s3: Fix chunkFilesBySize function
2 parents c0cb6eb + 7e7945f commit 97805d7

File tree

24 files changed

+240
-238
lines changed

24 files changed

+240
-238
lines changed

framework/mafia

Lines changed: 43 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -1,123 +1,63 @@
11
#!/bin/sh -eu
22

33
: ${MAFIA_HOME:=$HOME/.mafia}
4-
5-
fetch_latest () {
6-
if [ -z ${MAFIA_TEST_MODE+x} ]; then
7-
TZ=$(date +"%T")
8-
curl --silent "https://raw.githubusercontent.com/ambiata/mafia/master/script/mafia?$TZ"
9-
else
10-
cat ../script/mafia
11-
fi
12-
}
4+
: ${MAFIA_VERSIONS:=$MAFIA_HOME/versions}
135

146
latest_version () {
15-
git ls-remote https://github.com/ambiata/mafia | grep refs/heads/master | cut -f 1
7+
git ls-remote https://github.com/ambiata/mafia | grep refs/heads/master | cut -f 1
168
}
179

18-
local_version () {
19-
awk '/^# Version: / { print $3; exit 0; }' $0
10+
build_version() {
11+
MAFIA_VERSION="$1"
12+
MAFIA_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t 'exec_mafia')
13+
MAFIA_FILE=mafia-$MAFIA_VERSION
14+
MAFIA_PATH=$MAFIA_VERSIONS/$MAFIA_FILE
15+
mkdir -p $MAFIA_VERSIONS
16+
echo "Building $MAFIA_FILE in $MAFIA_TEMP"
17+
git clone https://github.com/ambiata/mafia $MAFIA_TEMP
18+
git --git-dir="$MAFIA_TEMP/.git" --work-tree="$MAFIA_TEMP" reset --hard $MAFIA_VERSION || {
19+
echo "mafia version ($MAFIA_VERSION) could not be found." >&2
20+
exit 1
21+
}
22+
(cd "$MAFIA_TEMP" && ./bin/bootstrap) || {
23+
got=$?
24+
echo "mafia version ($MAFIA_VERSION) could not be built." >&2
25+
exit "$got"
26+
}
27+
chmod +x "$MAFIA_TEMP/.cabal-sandbox/bin/mafia"
28+
# Ensure executable is on same file-system so final mv is atomic.
29+
mv -f "$MAFIA_TEMP/.cabal-sandbox/bin/mafia" "$MAFIA_PATH.$$"
30+
mv "$MAFIA_PATH.$$" "$MAFIA_PATH" || {
31+
rm -f "$MAFIA_PATH.$$"
32+
echo "INFO: mafia version ($MAFIA_VERSION) already exists not overiding," >&2
33+
echo "INFO: this is expected if parallel builds of the same version of" >&2
34+
echo "INFO: mafia occur, we are playing by first in, wins." >&2
35+
exit 0
36+
}
2037
}
2138

22-
run_upgrade () {
23-
MAFIA_TEMP=$(mktemp 2>/dev/null || mktemp -t 'upgrade_mafia')
24-
25-
clean_up () {
26-
rm -f "$MAFIA_TEMP"
27-
}
28-
29-
trap clean_up EXIT
30-
31-
MAFIA_CUR="$0"
32-
33-
if [ -L "$MAFIA_CUR" ]; then
34-
echo 'Refusing to overwrite a symlink; run `upgrade` from the canonical path.' >&2
35-
exit 1
36-
fi
37-
38-
echo "Checking for a new version of mafia ..."
39-
fetch_latest > $MAFIA_TEMP
40-
41-
LATEST_VERSION=$(latest_version)
42-
echo "# Version: $LATEST_VERSION" >> $MAFIA_TEMP
43-
44-
if ! cmp $MAFIA_CUR $MAFIA_TEMP >/dev/null 2>&1; then
45-
mv $MAFIA_TEMP $MAFIA_CUR
46-
chmod +x $MAFIA_CUR
47-
echo "New version found and upgraded. You can now commit it to your git repo."
48-
else
49-
echo "You have latest mafia."
50-
fi
39+
enable_version() {
40+
if [ $# -eq 0 ]; then
41+
MAFIA_VERSION="$(latest_version)"
42+
echo "INFO: No explicit mafia version requested installing latest ($MAFIA_VERSION)." >&2
43+
else
44+
MAFIA_VERSION="$1"
45+
fi
46+
[ -x "$MAFIA_HOME/versions/mafia-$MAFIA_VERSION" ] || build_version "$MAFIA_VERSION"
47+
ln -sf "$MAFIA_HOME/versions/mafia-$MAFIA_VERSION" "$MAFIA_HOME/versions/mafia"
5148
}
5249

5350
exec_mafia () {
54-
MAFIA_VERSION=$(local_version)
55-
56-
if [ "x$MAFIA_VERSION" = "x" ]; then
57-
# If we can't find the mafia version, then we need to upgrade the script.
58-
run_upgrade
59-
else
60-
MAFIA_BIN=$MAFIA_HOME/bin
61-
MAFIA_FILE=mafia-$MAFIA_VERSION
62-
MAFIA_PATH=$MAFIA_BIN/$MAFIA_FILE
63-
64-
[ -f "$MAFIA_PATH" ] || {
65-
# Create a temporary directory which will be deleted when the script
66-
# terminates. Unfortunately `mktemp` doesn't behave the same on
67-
# Linux and OS/X so we need to try two different approaches.
68-
MAFIA_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t 'exec_mafia')
69-
70-
# Create a temporary file in MAFIA_BIN so we can do an atomic copy/move dance.
71-
mkdir -p $MAFIA_BIN
72-
73-
clean_up () {
74-
rm -rf "$MAFIA_TEMP"
75-
}
76-
77-
trap clean_up EXIT
78-
79-
echo "Building $MAFIA_FILE in $MAFIA_TEMP"
80-
81-
( cd "$MAFIA_TEMP"
82-
83-
git clone https://github.com/ambiata/mafia
84-
cd mafia
85-
86-
git reset --hard $MAFIA_VERSION
87-
88-
bin/bootstrap ) || exit $?
89-
90-
MAFIA_PATH_TEMP=$(mktemp --tmpdir=$MAFIA_BIN $MAFIA_FILE-XXXXXX 2>/dev/null || TMPDIR=$MAFIA_BIN mktemp -t $MAFIA_FILE)
91-
92-
clean_up_temp () {
93-
clean_up
94-
rm -f "$MAFIA_PATH_TEMP"
95-
}
96-
trap clean_up_temp EXIT
97-
98-
cp "$MAFIA_TEMP/mafia/.cabal-sandbox/bin/mafia" "$MAFIA_PATH_TEMP"
99-
chmod 755 "$MAFIA_PATH_TEMP"
100-
mv "$MAFIA_PATH_TEMP" "$MAFIA_PATH"
101-
102-
clean_up_temp
103-
}
104-
105-
exec $MAFIA_PATH "$@"
106-
fi
51+
[ -x "$MAFIA_HOME/versions/mafia" ] || enable_version
52+
"$MAFIA_HOME/versions/mafia" "$@"
10753
}
10854

10955
#
11056
# The actual start of the script.....
11157
#
11258

113-
if [ $# -gt 0 ]; then
114-
MODE="$1"
115-
else
116-
MODE=""
117-
fi
118-
119-
case "$MODE" in
120-
upgrade) shift; run_upgrade "$@" ;;
59+
case "${1:-}" in
60+
upgrade) shift; enable_version "$@" ;;
12161
*) exec_mafia "$@"
12262
esac
123-
# Version: 7418efce0395b4dcb1898dcb5d99c21d8534c1f3
63+
# Version: 59b3c6aadb50447f8942b036f255325ded1b4649

mismi-autoscaling-core/master.toml

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,26 @@
11
[master]
2+
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20170724213527-c4b2de4/master-haskell-20170724213527-c4b2de4"
23
version = 1
3-
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20160623072912-ccea728/master-haskell-20160623072912-ccea728"
4-
sha1 = "e150d58a710c500eb908db492b3dcd0e532b2725"
4+
sha1 = "09aa2a97367596116e8d12112bf68464919d4815"
5+
6+
[global]
7+
CACHE = "true"
58

69
[build.dist]
710
GHC_VERSION="7.10.2"
8-
CABAL_VERSION = "1.22.4.0"
11+
CABAL_VERSION = "1.24.0.0"
912

1013
[build.dist-7-10]
1114
HADDOCK = "true"
1215
HADDOCK_S3 = "$AMBIATA_HADDOCK_MASTER"
1316
GHC_VERSION = "7.10.2"
14-
CABAL_VERSION = "1.22.4.0"
17+
CABAL_VERSION = "1.24.0.0"
1518

1619
[build.branches-7-10]
1720
HADDOCK = "true"
1821
HADDOCK_S3 = "$AMBIATA_HADDOCK_BRANCHES"
1922
GHC_VERSION = "7.10.2"
20-
CABAL_VERSION = "1.22.4.0"
23+
CABAL_VERSION = "1.24.0.0"
2124

2225
[build.dist-8-0]
2326
GHC_VERSION = "8.0.1"
@@ -29,12 +32,12 @@
2932

3033
[build.dist-7-10-cabal-test]
3134
GHC_VERSION = "7.10.2"
32-
CABAL_VERSION = "1.22.4.0"
35+
CABAL_VERSION = "1.24.0.0"
3336
TEST = "false"
3437
CABAL_TEST = "true"
3538

3639
[build.branches-7-10-cabal-test]
3740
GHC_VERSION = "7.10.2"
38-
CABAL_VERSION = "1.22.4.0"
41+
CABAL_VERSION = "1.24.0.0"
3942
TEST = "false"
4043
CABAL_TEST = "true"

mismi-autoscaling/master.toml

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,27 @@
11
[master]
2+
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20170724213527-c4b2de4/master-haskell-20170724213527-c4b2de4"
23
version = 1
3-
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20160623072912-ccea728/master-haskell-20160623072912-ccea728"
4-
sha1 = "e150d58a710c500eb908db492b3dcd0e532b2725"
4+
sha1 = "09aa2a97367596116e8d12112bf68464919d4815"
5+
6+
[global]
7+
CACHE = "true"
58

69
[build.dist]
710
GHC_VERSION="7.10.2"
8-
CABAL_VERSION = "1.22.4.0"
11+
CABAL_VERSION = "1.24.0.0"
912

1013
[build.dist-7-10]
1114
HADDOCK = "true"
1215
HADDOCK_S3 = "$AMBIATA_HADDOCK_MASTER"
1316
GHC_VERSION = "7.10.2"
14-
CABAL_VERSION = "1.22.4.0"
17+
CABAL_VERSION = "1.24.0.0"
1518

1619
[build.branches-7-10]
1720
# Change these once fully switched to boris
1821
HADDOCK = "true"
1922
HADDOCK_S3 = "$AMBIATA_HADDOCK_BRANCHES"
2023
GHC_VERSION = "7.10.2"
21-
CABAL_VERSION = "1.22.4.0"
24+
CABAL_VERSION = "1.24.0.0"
2225

2326
[build.dist-8-0]
2427
GHC_VERSION = "8.0.1"
@@ -30,12 +33,12 @@
3033

3134
[build.dist-7-10-cabal-test]
3235
GHC_VERSION = "7.10.2"
33-
CABAL_VERSION = "1.22.4.0"
36+
CABAL_VERSION = "1.24.0.0"
3437
TEST = "false"
3538
CABAL_TEST = "true"
3639

3740
[build.branches-7-10-cabal-test]
3841
GHC_VERSION = "7.10.2"
39-
CABAL_VERSION = "1.22.4.0"
42+
CABAL_VERSION = "1.24.0.0"
4043
TEST = "false"
4144
CABAL_TEST = "true"

mismi-cli/master.toml

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,21 @@
11
[master]
2+
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20170724213527-c4b2de4/master-haskell-20170724213527-c4b2de4"
23
version = 1
3-
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20160623072912-ccea728/master-haskell-20160623072912-ccea728"
4-
sha1 = "e150d58a710c500eb908db492b3dcd0e532b2725"
4+
sha1 = "09aa2a97367596116e8d12112bf68464919d4815"
5+
6+
[global]
7+
CACHE = "true"
58

69
[build.dist-7-10]
7-
HADDOCK = "true"
8-
HADDOCK_S3 = "$AMBIATA_HADDOCK_MASTER"
910
GHC_VERSION = "7.10.2"
10-
CABAL_VERSION = "1.22.4.0"
11+
CABAL_VERSION = "1.24.0.0"
1112
PUBLISH = "true"
1213
PUBLISH_S3 = "$AMBIATA_ARTEFACTS_MASTER"
1314
PUBLISH_EXECUTABLES = "s3"
1415

1516
[build.branches-7-10]
16-
HADDOCK = "true"
17-
HADDOCK_S3 = "$AMBIATA_HADDOCK_BRANCHES"
1817
GHC_VERSION = "7.10.2"
19-
CABAL_VERSION = "1.22.4.0"
18+
CABAL_VERSION = "1.24.0.0"
2019
PUBLISH = "true"
2120
PUBLISH_S3 = "$AMBIATA_ARTEFACTS_BRANCHES"
2221
PUBLISH_EXECUTABLES = "s3"

mismi-cloudwatch-logs/master.toml

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,26 @@
11
[master]
2+
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20170724213527-c4b2de4/master-haskell-20170724213527-c4b2de4"
23
version = 1
3-
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20160623072912-ccea728/master-haskell-20160623072912-ccea728"
4-
sha1 = "e150d58a710c500eb908db492b3dcd0e532b2725"
4+
sha1 = "09aa2a97367596116e8d12112bf68464919d4815"
5+
6+
[global]
7+
CACHE = "true"
58

69
[build.dist]
710
GHC_VERSION="7.10.2"
8-
CABAL_VERSION = "1.22.4.0"
11+
CABAL_VERSION = "1.24.0.0"
912

1013
[build.dist-7-10]
1114
HADDOCK = "true"
1215
HADDOCK_S3 = "$AMBIATA_HADDOCK_MASTER"
1316
GHC_VERSION = "7.10.2"
14-
CABAL_VERSION = "1.22.4.0"
17+
CABAL_VERSION = "1.24.0.0"
1518

1619
[build.branches-7-10]
1720
HADDOCK = "true"
1821
HADDOCK_S3 = "$AMBIATA_HADDOCK_BRANCHES"
1922
GHC_VERSION = "7.10.2"
20-
CABAL_VERSION = "1.22.4.0"
23+
CABAL_VERSION = "1.24.0.0"
2124

2225
[build.dist-8-0]
2326
GHC_VERSION = "8.0.1"

mismi-cloudwatch/master.toml

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,26 @@
11
[master]
2+
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20170724213527-c4b2de4/master-haskell-20170724213527-c4b2de4"
23
version = 1
3-
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20160623072912-ccea728/master-haskell-20160623072912-ccea728"
4-
sha1 = "e150d58a710c500eb908db492b3dcd0e532b2725"
4+
sha1 = "09aa2a97367596116e8d12112bf68464919d4815"
5+
6+
[global]
7+
CACHE = "true"
58

69
[build.dist]
710
GHC_VERSION="7.10.2"
8-
CABAL_VERSION = "1.22.4.0"
11+
CABAL_VERSION = "1.24.0.0"
912

1013
[build.dist-7-10]
1114
HADDOCK = "true"
1215
HADDOCK_S3 = "$AMBIATA_HADDOCK_MASTER"
1316
GHC_VERSION = "7.10.2"
14-
CABAL_VERSION = "1.22.4.0"
17+
CABAL_VERSION = "1.24.0.0"
1518

1619
[build.branches-7-10]
1720
HADDOCK = "true"
1821
HADDOCK_S3 = "$AMBIATA_HADDOCK_BRANCHES"
1922
GHC_VERSION = "7.10.2"
20-
CABAL_VERSION = "1.22.4.0"
23+
CABAL_VERSION = "1.24.0.0"
2124

2225
[build.dist-8-0]
2326
GHC_VERSION = "8.0.1"

0 commit comments

Comments
 (0)