Script for downloading snapshots using s5cmd and pigz (#120)

* add s5cmd to image
* add pigz to test faster decompress
* add downloading to script
* add concurrency parameter
* Add metrics to the download script

Author: wavetylor

Dockerfile

@@ -4,7 +4,10 @@ ARG VERSION
 
 RUN apk --update add gcc git curl alpine-sdk libc6-compat ca-certificates sqlite \
   && curl -SsL https://github.com/segmentio/chamber/releases/download/v2.13.2/chamber-v2.13.2-linux-amd64 -o /bin/chamber \
-  && chmod +x /bin/chamber
+  && curl -sL https://github.com/peak/s5cmd/releases/download/v2.1.0/s5cmd_2.1.0_Linux-64bit.tar.gz -o s5cmd.gz && tar -xzf s5cmd.gz -C /bin \
+  && chmod +x /bin/chamber \
+  && chmod +x /bin/s5cmd
+
 
 COPY . /go/src/${SRC}
 WORKDIR /go/src/${SRC}
@@ -15,11 +18,11 @@ RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/vers
 RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/version.version=$VERSION" ${SRC}/pkg/cmd/ctlstore-cli \
   && cp ${GOPATH}/bin/ctlstore-cli /usr/local/bin
 
-RUN apk del gcc git curl alpine-sdk libc6-compat
-
 FROM alpine
-RUN apk --no-cache add sqlite
+RUN apk --no-cache add sqlite pigz
 
+COPY --from=0 /go/src/github.com/segmentio/ctlstore/scripts/download.sh .
 COPY --from=0 /bin/chamber /bin/chamber
+COPY --from=0 /bin/s5cmd /bin/s5cmd
 COPY --from=0 /usr/local/bin/ctlstore /usr/local/bin/
 COPY --from=0 /usr/local/bin/ctlstore-cli /usr/local/bin/

scripts/download.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env sh
+
+set -eo pipefail
+
+CTLSTORE_BOOTSTRAP_URL=$1
+CONCURRENCY=${2:-20}
+STATS_IP=$3
+STATS_PORT=${4:-8125}
+
+TAGS="downloaded:false"
+START=$(date +%s)
+END=$(date +%s)
+if [ ! -f /var/spool/ctlstore/ldb.db ]; then
+  # busybox does not support sub-second resolution
+  START=$(date +%s)
+
+  mkdir -p /var/spool/ctlstore
+  cd /var/spool/ctlstore
+  s5cmd -r 0 --log debug cp --concurrency $CONCURRENCY $CTLSTORE_BOOTSTRAP_URL .
+
+  TAGS="downloaded:true"
+  if [[ ${CTLSTORE_BOOTSTRAP_URL: -2} == gz ]]; then
+    echo "Decompressing"
+    pigz -d snapshot.db.gz
+    TAGS="$TAGS,compressed:true"
+  fi
+
+  TAGS="$TAGS,concurrency:$CONCURRENCY"
+
+  mv snapshot.db ldb.db
+  END=$(date +%s)
+  echo "ldb.db ready in $(($END - $START)) seconds"
+else
+  echo "Snapshot already present"
+fi
+
+if [ ! -z "$STATS_IP" ]; then
+  echo -n "ctlstore.reflector.init_snapshot_download_time:$(($END - $START))|h|#$TAGS" | nc -u -w1 $NODE_IP $STATS_PORT
+fi