Alpine (#36)

* easy static builds for hts-nim projects

* curl almost working

* more curl stuff

* try for static build on alpine

* static builds with musl

* update gitignore

Author: brentp

.gitignore

@@ -4,3 +4,9 @@ hts/hts
 tests/htstest
 tests/vcftest
 tests/bgzftest
+tests/bamtest
+tests/all
+tests/run
+docker/nsb
+docker/hts_nim_static_builder
+

Dockerfile

@@ -1,11 +1,113 @@
-FROM ubuntu:16.04
+## this docker file is based on a relatively old setup so that libc dependencies
+## should not be a problem. It:
+# 1. builds htslib and all dependencies currently without libcurl
+# 2. installs nim
+# 3. sets up a nim binary (nsb) that is expected to be called from an external binary (static_builder)
+# These facilitate building static binaries for projects using hts-nim.
 
-RUN apt-get update \
-        && apt-get -qy install curl libssl-dev build-essential gcc \
-        && curl -sSfLo init.sh https://nim-lang.org/choosenim/init.sh \
-        && bash init.sh -y \ 
-        && rm init.sh \
-        && echo "export PATH=/root/.nimble/bin:$PATH" >> /etc/profile \
-        && echo "export PATH=/root/.nimble/bin:$PATH" >> /etc/bash.bashrc \
+# docker build -t brentp/hts-nim:latest -f Dockerfile .
+FROM centos:centos6
 
+RUN yum install -y git curl wget zlib-devel xz-devel bzip2-devel libcurl-devel && \
+    wget http://people.centos.org/tru/devtools-2/devtools-2.repo -O /etc/yum.repos.d/devtools-2.repo && \
+    yum install -y devtoolset-2-gcc devtoolset-2-binutils devtoolset-2-gcc-c++ lzma-devel glibc-static && \
+    source scl_source enable devtoolset-2 && \
+    echo "source scl_source enable devtoolset-2" >> ~/.bashrc && \
+    echo "source scl_source enable devtoolset-2" >> ~/.bash_profile && \
+    wget --quiet https://ftp.gnu.org/gnu/m4/m4-1.4.18.tar.gz && \
+    tar xzf m4-1.4.18.tar.gz && cd m4* && ./configure && make && make install && cd .. && \
+    rm -rf m4* && \
+    wget --quiet http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz && \
+    tar xzf autoconf-2.69.tar.gz && \
+    cd autoconf* && ./configure && make && make install && cd .. && rm -rf autoconf* && \
+    git clone --depth 1 https://github.com/ebiggers/libdeflate.git && \
+    cd libdeflate && make -j 2 CFLAGS='-fPIC -O3' libdeflate.a && \
+    cp libdeflate.a /usr/local/lib && cp libdeflate.h /usr/local/include && \
+    cd .. && rm -rf libdeflate && \
+    wget --quiet http://http.debian.net/debian/pool/main/b/bzip2/bzip2_1.0.6.orig.tar.bz2 && \
+    tar xjvf bzip2_1.0.6.orig.tar.bz2 && \
+    cd bzip2-1.0.6 && \
+    make -j2 install && \
+    cd ../ && \
+    rm -rf bzip2-* && \
+    wget --quiet https://www.zlib.net/zlib-1.2.11.tar.gz && \
+    tar xzf zlib-1.2.11.tar.gz && \
+    cd zlib-1.2.11 && \
+    ./configure && \
+    make -j4 install && \
+    cd .. && \
+    rm -rf zlib* && \
+    wget --quiet https://tukaani.org/xz/xz-5.2.4.tar.bz2 && \
+    tar xjf xz-5.2.4.tar.bz2 && \
+    cd xz-5.2.4 && \
+    ./configure && \
+    make -j4 install && \
+    cd .. && \
+    rm -r xz*
 
+
+RUN source scl_source enable devtoolset-2 && \
+    cd / && \
+    wget --quiet http://www.musl-libc.org/releases/musl-1.1.21.tar.gz && \
+    tar xvf musl-1.1.21.tar.gz && \ 
+    cd musl-1.1.21 && \ 
+    ./configure && \
+    make -j4 install && \
+    rm -rf musl-*
+
+
+RUN source scl_source enable devtoolset-2 && \
+    cd / && \
+    wget --quiet https://www.openssl.org/source/openssl-1.1.1b.tar.gz && \
+    tar xzvf openssl-1.1.1b.tar.gz && \
+    cd openssl-1.1.1b && \
+    ./config && \
+    make install && cd ../ && rm -rf openssl-1.1.1b
+
+
+RUN cd / && \
+    git clone -b devel --depth 10 git://github.com/nim-lang/nim nim && \
+    cd nim && \
+    chmod +x ./build_all.sh && \
+    scl enable devtoolset-2 ./build_all.sh && \
+    echo 'PATH=/nim/bin:$PATH' >> ~/.bashrc && \
+    echo 'PATH=/nim/bin:$PATH' >> ~/.bash_profile && \
+    echo 'PATH=/nim/bin:$PATH' >> /etc/environment
+
+RUN source scl_source enable devtoolset-2 && \
+    wget --quiet https://c-ares.haxx.se/download/c-ares-1.15.0.tar.gz && \
+    tar xzf c-ares-1.15.0.tar.gz && \
+    cd c-ares-1.15.0 && \
+    LIBS="-lrt"  LDFLAGS="-Wl,--no-as-needed -static" ./configure --enable-static && \
+    make LDFLAGS="-Wl,--no-as-needed -all-static -lrt -lssl -lcrypto -lc" -j4 install && \
+    cd .. && \
+    rm -rf c-ares-1.15.0* && \
+    wget --quiet https://curl.haxx.se/download/curl-7.64.0.tar.gz && \
+    tar xzf curl-7.64.0.tar.gz && \
+    cd curl-7.64.0 && \
+    LIBS="-ldl -lpthread -lrt -lssl -lcrypto -lcares -ldl -lc" LDFLAGS="-Wl,--no-as-needed -static" PKG_CONFIG="pkg-config --static" ./configure --disable-shared --enable-static --disable-ldap --with-ssl=/usr/local/ --disable-sspi --without-librtmp --disable-ftp --disable-file --disable-dict --disable-telnet --disable-tftp --disable-rtsp --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-smb --without-libidn --enable-ares && \
+    make curl_LDFLAGS=-all-static LDFLAGS="-Wl,--no-as-needed -all-static -lrt -lssl -lcrypto -lcares -ldl -lc" -j4 install && \
+    cd ../ && \
+    rm -rf curl-7.64.0*
+
+
+RUN source scl_source enable devtoolset-2 && \
+    git clone https://github.com/samtools/htslib && \
+    cd htslib && git checkout 1.9 && autoheader && autoconf && \
+    ./configure --enable-s3 --enable-libcurl --with-libdeflate && \
+    make LDFLAGS="-Wl,--no-as-needed -lrt -lssl -lcrypto -ldl -lcares -lc" -j4 CFLAGS="-fPIC -O3 -lcrypto" install && \
+    echo "/usr/local/lib" >> /etc/ld.so.conf && \
+    ldconfig && \
+    cd ../ && rm -rf htslib
+
+
+ENV PATH=:/root/.nimble/bin:/nim/bin/:$PATH:/opt/rh/devtoolset-2/root/usr/bin/
+
+ADD . /src/
+RUN cat /src/docker/docker.nim.cfg >> /nim/config/nim.cfg && \
+    echo "source scl_source enable devtoolset-2" >> /etc/environment && \
+    source ~/.bashrc && cd /src/ && nimble install -y && \
+    nimble install -y docopt && \
+    nimble install -y c2nim@#3ec45c24585ebaed && \
+    nim c -o:/usr/local/bin/nsb /src/docker/nsb.nim && \
+    rm -rf /src/

Dockerfile.musl-hts-nim

@@ -0,0 +1,38 @@
+## This dockerfile
+# 1. builds htslib and all dependencies currently without libcurl
+# 2. installs nim
+# 3. sets up a nim binary (nsb) that is expected to be called from an external binary (static_builder)
+# These facilitate building static binaries for projects using hts-nim.
+
+# docker build -t brentp/musl-hts-nim:latest -f Dockerfile.musl-hts-nim .
+FROM alpine:3.9
+
+ENV LDFLAGS=-static PKG_CONFIG='pkg-config --static'
+ENV curl_LDFLAGS=-all-static
+
+RUN apk add curl musl build-base git autoconf zlib-dev bzip2-dev xz-dev curl-dev
+
+RUN cd / && \
+    git clone -b devel --depth 10 git://github.com/nim-lang/nim nim && \
+    cd nim && sh ./build_all.sh
+
+RUN git clone --depth 1 https://github.com/ebiggers/libdeflate.git && \
+    cd libdeflate && make -j 2 CFLAGS='-fPIC -O3' libdeflate.a && \
+    cp libdeflate.a /usr/local/lib && cp libdeflate.h /usr/include && \
+    cd .. && rm -rf libdeflate && \
+    git clone https://github.com/samtools/htslib && \
+    cd htslib && git checkout 1.9 && autoheader && autoconf && \
+   ./configure --enable-plugins --disable-libcurl --with-libdeflate && \
+   make -j4 install && \
+   cd ../ && rm -rf htslib
+
+ADD . /src/
+
+ENV PATH=/root/.nimble/bin:/nim/bin/:$PATH
+
+RUN cat /src/docker/docker.nim.cfg >> /nim/config/nim.cfg && \
+    cd /src/ && nimble install -y && \
+    nimble install -y docopt && \
+    nimble install -y c2nim@#3ec45c24585ebaed && \
+    nim c -o:/usr/local/bin/nsb /src/docker/nsb.nim && \
+    rm -rf /src/

README.md

@@ -157,3 +157,40 @@ In all cases, it's recommended to use nim version 0.18.0 or more recent.
 
 Then, from this repo you can run `nimble test` and `nimble install` and then you can save the above snippets into `some.nim`
 and run them with `nim c -d:release -r some.nim`. This will run them and save an executable named `some`.
+
+## Static Builds
+
+`hts-nim` is meant to simplify and speed development and distribution. To that end, there is some machinery to help create
+truly static binaries for linux from nim-projects and for simple nim scripts. This means that there is no dependency on libhts.so. These builds only require docker and [this static binary XXX TODO](https://github.com/brentp/slivar/releases).
+
+For a single file application that does not have a nimble file we can specify the dependencies using `--deps`:
+
+```
+hts_nim_static_builder -s vcf_cleaner.nim --deps "hts@>=0.2.7" --deps "binaryheap"
+```
+
+This will create a static binary at `./vcf_cleaner`.
+
+
+
+Projects with `.nimble` files can use that directly to indicate dependencies.
+For example, to build [slivar](https://github.com/brentp/slivar), we can do:
+
+```
+hts_nim_static_builder -s ../slivar/src/slivar.nim -n ../slivar/slivar.nimble
+```
+
+After this finishes, a static `slivar` binary will appear in the current working directory.
+
+We can verify that it is static using:
+
+```
+$ file ./slivar 
+./slivar: ELF 64-bit LSB executable, x86-64, version 1 (GNU/Linux), statically linked, for GNU/Linux 2.6.18, BuildID[sha1]=c2b5b52cb7be7f81bf90355a4e44a08a08df91d8, not stripped
+```
+
+The [docker image](https://hub.docker.com/r/brentp/musl-hts-nim) is based on alpine linux and uses musl to create truly static binaries.
+At this time, libcurl is not supported so only binaries built using this method will only be able to access local files (no http/https/s3/gcs).
+
+The docker images does use [libdeflate](https://github.com/ebiggers/libdeflate) by default. That provides,
+for example, a 20% speed improvement when used to build [mosdepth](https://github.com/brentp/mosdepth).

docker/docker.nim.cfg

@@ -0,0 +1,23 @@
+# in htslib: ./configure --with-libdeflate --disable-libcurl
+# nim c -a -d:static -d:release mosdepth.nim
+@if nsb_static:
+  passC:"-static"
+  passl:"-static"
+
+  passl:"/usr/lib/libm.a"
+  passl:"/usr/local/lib/libhts.a"
+  #passl:"/usr/local/lib/libnghttp2.a"
+  passl:"/usr/local/lib/libdeflate.a"
+  passl:"/usr/lib/liblzma.a"
+  passl:"/lib/libz.a"
+  passl:"/usr/lib/libbz2.a"
+  passl:"/usr/lib/libpthread.a"
+  passl:"/usr/lib/libdl.a"
+  passl:"/usr/lib/libcurl.a"
+  passl:"/usr/lib/libc.a"
+  passl:"/usr/lib/libssl.a"
+  passl:"/usr/lib/libcrypto.a"
+  passl:"/usr/lib/libssh2.a"
+  passl:"/usr/lib/librt.a"
+  dynlibOverride:"hts"
+@end

docker/hts_nim_static_builder.nim

@@ -0,0 +1,72 @@
+# nim static builder
+# to be run outside the docker container
+import docopt
+import os
+import osproc
+import strformat
+import strutils
+
+let doc = """
+static-builder
+
+Usage: static-builder [options --deps <string>...] [--] [<nim_compiler_args>...]
+
+`nim_compiler_args` are passed on directly to the nim compiler, e.g. --excessiveStackTrace:on
+
+Options:
+
+  -n --nimble-file <string>            optional path to nimble file must be in the same or parent directory of the nim source file.
+  -s --nim-src-file <string>           required path to nim file to be compiled to binary
+  -x --debug                           debug build. default is to build in release mode.
+  -d --deps <string>...               any number of dependencies, e.g. --deps "hts@>=0.2.7" --deps "lapper"
+
+"""
+
+let args = docopt(doc)
+
+echo $args
+
+if $args["--nim-src-file"] == "nil":
+  echo doc
+  echo "source file required"
+  quit 1
+
+var source = expandFilename($args["--nim-src-file"])
+if not existsFile(source):
+  quit "couldn't find source file"
+
+var dir: string
+var nimblePath: string
+
+if $args["--nimble-file"] != "nil":
+  nimblePath = expandFileName($args["--nimble-file"])
+  var (d, _, _) = splitFile(nimblePath)
+  dir = d
+else:
+   var (d, name, _) = splitFile(source)
+   dir = d
+
+# file gets build and sent to /load so it appears in the users pwd
+var cmd = &"""docker run -v {dir}:{dir} -v {getCurrentDir()}:/load/ brentp/musl-hts-nim:latest /usr/local/bin/nsb """
+if $args["--nimble-file"] != "nil":
+  cmd &= &"""-n {nimblePath}"""
+
+cmd &= &""" -s {source}"""
+
+for d in @(args["--deps"]):
+  cmd &= &""" --deps "{d}" """
+
+var added_dash = false
+if len(@(args["<nim_compiler_args>"])) > 0:
+  cmd &= &""" -- {join(@(args["<nim_compiler_args>"]), " ")}"""
+  added_dash = true
+
+if not args["--debug"]:
+  if not added_dash:
+    cmd &= " -- "
+  cmd &= " -d:release"
+
+echo cmd
+
+if execCmd(cmd) != 0:
+  quit "failed"

docker/nsb.nim

@@ -0,0 +1,64 @@
+# nim static builder
+# to be run inside the docker container
+import docopt
+import os
+import osproc
+import strformat
+import strutils
+
+let doc = """
+nim-static-builder
+
+Usage: nsb [options --deps <string>...] [--] [<nim_compiler_args>...]
+
+Options:
+
+  -n --nimble-file <string>            optional path to nimble file
+  -s --nim-src-file <string>           required path to nim file to be compiled to binary
+  -d --deps <string>...                 any number of dependencies, e.g. --deps "hts@>=0.2.7" --deps "lapper"
+
+"""
+
+echo commandLineParams()
+
+let args = docopt(doc)
+
+echo $args
+
+if $args["--nim-src-file"] == "nil":
+  echo doc
+  echo "source file required"
+  quit 1
+
+var source = expandFilename($args["--nim-src-file"])
+if not existsFile(source):
+  quit "couldn't find source file"
+
+var path = getEnv("PATH")
+path &= ":/nim/bin"
+putEnv("PATH", path)
+
+for d in @(args["--deps"]):
+  if 0 != execCmd(&"""nimble install -y "{d}" """):
+    quit "failed on nimble install of " & d
+
+if $args["--nimble-file"] != "nil":
+  var (dir, _, _) = splitFile(expandFileName($args["--nimble-file"]))
+  dir.setCurrentDir
+
+  if execCmd(&"""sh -c "export PATH={path}; /nim/bin/nimble install -d -y " """) != 0:
+    quit "coudn't run nimble install"
+
+
+var (dir, name, _) = splitFile(source)
+dir.setCurrentDir
+
+removeFile("xx_exe_out")
+var cmd = &"""/nim/bin/nim c -d:nsb_static {join(@(args["<nim_compiler_args>"]), " ")} -o:xx_exe_out {name}"""
+if execCmd(&"""sh -c "{cmd}" """) != 0:
+  quit "error compiling code"
+
+copyFileWithPermissions("xx_exe_out", &"/load/{name}")
+removeFile("xx_exe_out")
+
+echo &"wrote executable: {name}"

src/hts/bam.nim

@@ -1,5 +1,6 @@
 import ./private/hts_concat
 include "./bam/enums"
+import strformat
 import strutils
 include "./bam/flag"
 include "./bam/cigar"
@@ -323,7 +324,7 @@ proc load_index*(b: Bam, path: string) =
   else:
     b.idx = sam_index_load2(b.hts, b.path, path.cstring)
   if b.idx == nil:
-    raise newException(IoError, "[bam] load_index error opening index %s for bam %s. %s" % [path, $b.path, $strerror(errno)])
+    raise newException(IoError, &"[bam] load_index error opening index {path} for bam {b.path}. {strerror(errno)}")
 
 proc hts_set_opt*(fp: ptr htsFile; opt: FormatOption): cint {.varargs, cdecl,
     importc: "hts_set_opt", dynlib: libname.}