Release 1.6: hfile_libcurl improvements; more checks on input files

Author: daviesrob

.appveyor.yml

@@ -0,0 +1,43 @@
+# version format.
+# you can use {branch} name in version format too
+# version: 1.0.{build}-{branch}
+version: 'vers.{build}'
+
+# branches to build
+branches:
+    # Whitelist
+    only:
+      - develop
+
+    # Blacklist
+    except:
+      - gh-pages
+
+# Do not build on tags (GitHub and BitBucket)
+skip_tags: true
+
+# Skipping commits affecting specific files (GitHub only). More details here: /docs/appveyor-yml
+#skip_commits:
+#  files:
+#    - docs/*
+#    - '**/*.html'
+
+# We use Mingw/Msys, so use pacman for installs
+install:
+  - set HOME=.
+  - set MSYSTEM=MINGW64
+  - set PATH=C:/msys64/usr/bin;C:/msys64/mingw64/bin;%PATH%
+  - set MINGWPREFIX=x86_64-w64-mingw32
+  - "sh -lc \"pacman -S --noconfirm --needed base-devel mingw-w64-x86_64-toolchain mingw-w64-x86_64-zlib mingw-w64-x86_64-bzip2 mingw-w64-x86_64-xz mingw-w64-x86_64-curl\""
+
+build_script:
+  - set HOME=.
+  - set MSYSTEM=MINGW64
+  - set PATH=C:/msys64/usr/bin;C:/msys64/mingw64/bin;%PATH%
+  - "sh -lc \"aclocal && autoheader && autoconf && ./configure && make -j2\""
+
+#build_script:
+#  - make
+
+test_script:
+  - "sh -lc \"make test\""

.gitattributes

@@ -10,3 +10,8 @@
 .git*           export-ignore
 /.travis.yml    export-ignore
 README.md       export-ignore
+
+# Remove the text attribute from reference files, so that git doesn't convert
+# line separators on Windows machines. It causes the index files to become out
+# of sync with the fasta files.
+*.fa* -text

.gitignore

@@ -1,7 +1,9 @@
 *.o
 *.pico
+*.obj
 *.dSYM
 *.exe
+*.dll
 *.pc.tmp
 *-uninstalled.pc
 /version.h

INSTALL

@@ -48,6 +48,12 @@ library is used.  Systems that do not have CChmac will get this from
 libcrypto.  libcrypto is part of OpenSSL or one of its derivatives (LibreSSL
 or BoringSSL).
 
+On Microsoft Windows we recommend use of Mingw64/Msys2.  Note that
+currently for the test harness to work you will need to override the
+test temporary directory with e.g.: make check TEST_OPTS="-t C:/msys64/tmp/_"
+Whilst the code may work on Windows with other environments, these have
+not be verified.
+
 Building Configure
 ==================
 

Makefile

@@ -76,7 +76,8 @@ BUILT_TEST_PROGRAMS = \
 	test/test_view \
 	test/test-vcf-api \
 	test/test-vcf-sweep \
-	test/test-bcf-sr
+	test/test-bcf-sr \
+	test/test-bcf-translate
 
 BUILT_THRASH_PROGRAMS = \
 	test/thrash_threads1 \
@@ -91,42 +92,27 @@ all: lib-static lib-shared $(BUILT_PROGRAMS) plugins $(BUILT_TEST_PROGRAMS)
 HTSPREFIX =
 include htslib_vars.mk
 
-
-PACKAGE_VERSION = 1.5
+# If not using GNU make, you need to copy the version number from version.sh
+# into here.
+PACKAGE_VERSION := $(shell ./version.sh)
 LIBHTS_SOVERSION = 2
 
-
 # $(NUMERIC_VERSION) is for items that must have a numeric X.Y.Z string
 # even if this is a dirty or untagged Git working tree.
-NUMERIC_VERSION = $(PACKAGE_VERSION)
-
-# If building from a Git repository, replace $(PACKAGE_VERSION) with the Git
-# description of the working tree: either a release tag with the same value
-# as $(PACKAGE_VERSION) above, or an exact description likely based on a tag.
-# Much of this is also GNU Make-specific.  If you don't have GNU Make and/or
-# are not building from a Git repository, comment out this conditional.
-ifneq "$(wildcard .git)" ""
-original_version := $(PACKAGE_VERSION)
-PACKAGE_VERSION := $(shell git describe --always --dirty)
-
-# Unless the Git description matches /\d*\.\d*(\.\d*)?/, i.e., is exactly a tag
-# with a numeric name, revert $(NUMERIC_VERSION) to the original version number
-# written above, but with the patchlevel field bumped to 255.
-ifneq "$(subst ..,.,$(subst 0,,$(subst 1,,$(subst 2,,$(subst 3,,$(subst 4,,$(subst 5,,$(subst 6,,$(subst 7,,$(subst 8,,$(subst 9,,$(PACKAGE_VERSION))))))))))))" "."
-empty :=
-NUMERIC_VERSION := $(subst $(empty) ,.,$(wordlist 1,2,$(subst ., ,$(original_version))) 255)
-endif
+NUMERIC_VERSION := $(shell ./version.sh numeric)
 
 # Force version.h to be remade if $(PACKAGE_VERSION) has changed.
 version.h: $(if $(wildcard version.h),$(if $(findstring "$(PACKAGE_VERSION)",$(shell cat version.h)),,force))
-endif
 
 version.h:
 	echo '#define HTS_VERSION "$(PACKAGE_VERSION)"' > $@
 
 print-version:
 	@echo $(PACKAGE_VERSION)
 
+show-version:
+	@echo PACKAGE_VERSION = $(PACKAGE_VERSION)
+	@echo NUMERIC_VERSION = $(NUMERIC_VERSION)
 
 .SUFFIXES: .bundle .c .cygdll .dll .o .pico .so
 
@@ -148,6 +134,7 @@ LIBHTS_OBJS = \
 	hfile.o \
 	hfile_net.o \
 	hts.o \
+	hts_os.o\
 	md5.o \
 	multipart.o \
 	probaln.o \
@@ -208,6 +195,8 @@ config.h:
 	echo '/* Default config.h generated by Makefile */' > $@
 	echo '#define HAVE_LIBBZ2 1' >> $@
 	echo '#define HAVE_LIBLZMA 1' >> $@
+	echo '#define HAVE_FSEEKO 1' >> $@
+	echo '#define HAVE_DRAND48 1' >> $@
 
 # And similarly for htslib.pc.tmp ("pkg-config template").  No dependency
 # on htslib.pc.in listed, as if that file is newer the usual way to regenerate
@@ -235,6 +224,9 @@ lib-shared: libhts.dylib
 else ifeq "$(findstring CYGWIN,$(PLATFORM))" "CYGWIN"
 SHLIB_FLAVOUR = cygdll
 lib-shared: cyghts-$(LIBHTS_SOVERSION).dll
+else ifeq "$(findstring MSYS,$(PLATFORM))" "MSYS"
+SHLIB_FLAVOUR = dll
+lib-shared: hts-$(LIBHTS_SOVERSION).dll
 else
 SHLIB_FLAVOUR = so
 lib-shared: libhts.so
@@ -276,6 +268,9 @@ libhts.dylib: $(LIBHTS_OBJS)
 cyghts-$(LIBHTS_SOVERSION).dll: $(LIBHTS_OBJS)
 	$(CC) -shared -Wl,--out-implib=libhts.dll.a -Wl,--export-all-symbols -Wl,--enable-auto-import $(LDFLAGS) -o $@ -Wl,--whole-archive $(LIBHTS_OBJS) -Wl,--no-whole-archive $(LIBS) -lpthread
 
+hts-$(LIBHTS_SOVERSION).dll: $(LIBHTS_OBJS)
+	$(CC) -shared -Wl,--out-implib=hts.dll.a -Wl,--export-all-symbols -Wl,--enable-auto-import $(LDFLAGS) -o $@ -Wl,--whole-archive $(LIBHTS_OBJS) -Wl,--no-whole-archive $(LIBS) -lpthread
+
 
 .pico.so:
 	$(CC) -shared -Wl,-E $(LDFLAGS) -o $@ $< $(LIBS) -lpthread
@@ -286,6 +281,9 @@ cyghts-$(LIBHTS_SOVERSION).dll: $(LIBHTS_OBJS)
 .o.cygdll:
 	$(CC) -shared $(LDFLAGS) -o $@ $< libhts.dll.a $(LIBS)
 
+.o.dll:
+	$(CC) -shared $(LDFLAGS) -o $@ $< hts.dll.a $(LIBS)
+
 
 bgzf.o bgzf.pico: bgzf.c config.h $(htslib_hts_h) $(htslib_bgzf_h) $(htslib_hfile_h) $(htslib_thread_pool_h) cram/pooled_alloc.h $(htslib_khash_h)
 errmod.o errmod.pico: errmod.c config.h $(htslib_hts_h) $(htslib_ksort_h)
@@ -348,6 +346,9 @@ tabix.o: tabix.c config.h $(htslib_tbx_h) $(htslib_sam_h) $(htslib_vcf_h) $(htsl
 
 # For tests that might use it, set $REF_PATH explicitly to use only reference
 # areas within the test suite (or set it to ':' to use no reference areas).
+#
+# If using MSYS, avoid poor shell expansion via:
+#    MSYS2_ARG_CONV_EXCL="*" make check
 check test: $(BUILT_PROGRAMS) $(BUILT_TEST_PROGRAMS)
 	test/hts_endian
 	test/fieldarith test/fieldarith.sam
@@ -356,7 +357,7 @@ check test: $(BUILT_PROGRAMS) $(BUILT_TEST_PROGRAMS)
 	cd test/tabix && ./test-tabix.sh tabix.tst
 	REF_PATH=: test/sam test/ce.fa test/faidx.fa
 	test/test-regidx
-	cd test && REF_PATH=: ./test.pl
+	cd test && REF_PATH=: ./test.pl $${TEST_OPTS:-}
 
 test/hts_endian: test/hts_endian.o
 	$(CC) $(LDFLAGS) -o $@ test/hts_endian.o $(LIBS)
@@ -388,6 +389,9 @@ test/test-vcf-sweep: test/test-vcf-sweep.o libhts.a
 test/test-bcf-sr: test/test-bcf-sr.o libhts.a
 	$(CC) $(LDFLAGS) -o $@ test/test-bcf-sr.o libhts.a -lz $(LIBS) -lpthread
 
+test/test-bcf-translate: test/test-bcf-translate.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test-bcf-translate.o libhts.a -lz $(LIBS) -lpthread
+
 test/hts_endian.o: test/hts_endian.c $(htslib_hts_endian_h)
 test/fieldarith.o: test/fieldarith.c config.h $(htslib_sam_h)
 test/hfile.o: test/hfile.c config.h $(htslib_hfile_h) $(htslib_hts_defs_h)
@@ -398,6 +402,7 @@ test/test_view.o: test/test_view.c config.h $(cram_h) $(htslib_sam_h)
 test/test-vcf-api.o: test/test-vcf-api.c config.h $(htslib_hts_h) $(htslib_vcf_h) $(htslib_kstring_h) $(htslib_kseq_h)
 test/test-vcf-sweep.o: test/test-vcf-sweep.c config.h $(htslib_vcf_sweep_h)
 test/test-bcf-sr.o: test/test-bcf-sr.c config.h $(htslib_vcf_sweep_h) bcf_sr_sort.h
+test/test-bcf-translate.o: test/test-bcf-translate.c config.h
 
 
 test/thrash_threads1: test/thrash_threads1.o libhts.a
@@ -446,6 +451,10 @@ install-cygdll: cyghts-$(LIBHTS_SOVERSION).dll installdirs
 	$(INSTALL_PROGRAM) cyghts-$(LIBHTS_SOVERSION).dll $(DESTDIR)$(bindir)/cyghts-$(LIBHTS_SOVERSION).dll
 	$(INSTALL_PROGRAM) libhts.dll.a $(DESTDIR)$(libdir)/libhts.dll.a
 
+install-dll: hts-$(LIBHTS_SOVERSION).dll installdirs
+	$(INSTALL_PROGRAM) hts-$(LIBHTS_SOVERSION).dll $(DESTDIR)$(bindir)/hts-$(LIBHTS_SOVERSION).dll
+	$(INSTALL_PROGRAM) hts.dll.a $(DESTDIR)$(libdir)/hts.dll.a
+
 install-dylib: libhts.dylib installdirs
 	$(INSTALL_PROGRAM) libhts.dylib $(DESTDIR)$(libdir)/libhts.$(PACKAGE_VERSION).dylib
 	ln -sf libhts.$(PACKAGE_VERSION).dylib $(DESTDIR)$(libdir)/libhts.dylib
@@ -483,6 +492,9 @@ clean-so:
 clean-cygdll:
 	-rm -f cyghts-*.dll libhts.dll.a
 
+clean-dll:
+	-rm -f hts-*.dll hts.dll.a
+
 clean-dylib:
 	-rm -f libhts.dylib libhts.*.dylib
 
@@ -498,7 +510,8 @@ tags TAGS:
 # (The wildcards attempt to omit non-exported files (.git*, README.md,
 # etc) and other detritus that might be in the top-level directory.)
 distdir:
-	tar -c *.[ch15] [ILMNRcht]*[ELSbcekmnt] | (cd $(distdir) && tar -x)
+	@if [ -z "$(distdir)" ]; then echo "Please supply a distdir=DIR argument."; false; fi
+	tar -c *.[ch15] [ILMNRchtv]*[ELSbcekmnth] | (cd $(distdir) && tar -x)
 	+cd $(distdir) && $(MAKE) distclean
 
 force:
@@ -507,7 +520,8 @@ force:
 .PHONY: all check clean distclean distdir force
 .PHONY: install install-pkgconfig installdirs lib-shared lib-static
 .PHONY: maintainer-clean mostlyclean plugins print-config print-version
-.PHONY: tags test testclean
+.PHONY: show-version tags test testclean
 .PHONY: clean-so install-so
 .PHONY: clean-cygdll install-cygdll
+.PHONY: clean-dll install-dll
 .PHONY: clean-dylib install-dylib

NEWS

@@ -1,3 +1,38 @@
+Noteworthy changes in release 1.6 (28th September 2017)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Fixed bug where iterators on CRAM files did not propagate error return
+  values to the caller correctly.  Thanks go to Chris Saunders.
+
+* Overhauled Windows builds.  Building with msys2/mingw64 now works
+  correctly and passes all tests.
+
+* More improvements to logging output (thanks again to Anders Kaplan).
+
+* Return codes from sam_read1() when reading cram have been made
+  consistent with those returned when reading sam/bam.  Thanks to
+  Chris Saunders (#575).
+
+* BGZF CRC32 checksums are now always verified.
+
+* It's now possible to set nthreads = 1 for cram files.
+
+* hfile_libcurl has been modified to make it thread-safe.  It's also
+  better at handling web servers that do not honour byte range requests
+  when attempting to seek - it now sets errno to ESPIPE and keeps
+  the existing connection open so callers can revert to streaming mode
+  it they want to.
+
+* hfile_s3 now recalculates access tokens if they have become stale.  This
+  fixes a reported problem where authentication failed after a file
+  had been in use for more than 15 minutes.
+
+* Fixed bug where remote index fetches would fail to notice errors when
+  writing files.
+
+* bam_read1() now checks that the query sequence length derived from the
+  CIGAR alignment matches the sequence length in the BAM record.
+
 Noteworthy changes in release 1.5 (21st June 2017)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -18,7 +53,7 @@ Noteworthy changes in release 1.4.1  (8th May 2017)
 
 This is primarily a security bug fix update.
 
-* Fixed SECURITY issue with buffer overruns with malicious data. (#514).
+* Fixed SECURITY (CVE-2017-1000206) issue with buffer overruns with malicious data. (#514).
 
 * S3 support for non Amazon AWS endpoints. (#506)