Merge branch 'develop' of github.com:samtools/bcftools into develop

Author: pd3

bcftools.h

@@ -50,7 +50,15 @@ void error(const char *format, ...) HTS_NORETURN HTS_FORMAT(HTS_PRINTF_FMT, 1, 2
 void error_errno(const char *format, ...) HTS_NORETURN HTS_FORMAT(HTS_PRINTF_FMT, 1, 2);
 
 // For on the fly index creation with --write-index
-int init_index(htsFile *fh, bcf_hdr_t *hdr, const char *fname, char **idx_fname);
+int init_index2(htsFile *fh, bcf_hdr_t *hdr, const char *fname,
+                char **idx_fname, int idx_fmt);
+int init_index(htsFile *fh, bcf_hdr_t *hdr, const char *fname,
+               char **idx_fname);
+
+// Used to set args->write_index in CLI.
+// It will be true if set correctly.
+// Note due to HTS_FMT_CSI being zero we have to use an additional bit.
+int write_index_parse(char *arg);
 
 void bcf_hdr_append_version(bcf_hdr_t *hdr, int argc, char **argv, const char *cmd);
 const char *hts_bcf_wmode(int file_type);

configure.ac

@@ -245,6 +245,11 @@ AS_IF([test "$enable_perl_filters" != "no" ], [dnl
         PERL_CCOPTS="`perl -MExtUtils::Embed -e ccopts | sed 's,-arch i386,,'`"
         PERL_LIBS="`perl -MExtUtils::Embed -e ldopts | sed 's,-arch i386,,'`"
     fi
+    AS_CASE([$PERL_LIBS],
+      [*redhat/redhat-hardened-ld*],[[
+        PERL_CCOPTS="`echo $PERL_CCOPTS | sed 's,-specs=[a-z/]*/redhat/redhat-hardened-[a-z0-9]* *,,g'`"
+        PERL_LIBS="`echo $PERL_LIBS | sed 's,-specs=[a-z/]*/redhat/redhat-hardened-[a-z0-9]* *,,g'`"]
+        AC_MSG_NOTICE([removing -specs=...redhat-hardened... from Perl options])])
     AC_SUBST([PERL_CCOPTS])
     AC_SUBST([PERL_LIBS])
 

csq.c

@@ -554,7 +554,9 @@ void init_data(args_t *args)
         if ( args->hdr_nsmpl )
             bcf_hdr_printf(args->hdr,"##FORMAT=<ID=%s,Number=.,Type=Integer,Description=\"Bitmask of indexes to INFO/BCSQ, with interleaved first/second haplotype. Use \\\"bcftools query -f'[%%CHROM\\t%%POS\\t%%SAMPLE\\t%%TBCSQ\\n]'\\\" to translate.\">",args->bcsq_tag);
         if ( bcf_hdr_write(args->out_fh, args->hdr)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname?args->output_fname:"standard output");
-        if ( args->write_index && init_index(args->out_fh,args->hdr,args->output_fname,&args->index_fn)<0 ) error("Error: failed to initialise index for %s\n",args->output_fname);
+        if ( init_index2(args->out_fh,args->hdr,args->output_fname,
+                         &args->index_fn, args->write_index) < 0 )
+            error("Error: failed to initialise index for %s\n",args->output_fname);
     }
     if ( args->verbosity > 0 ) fprintf(stderr,"Calling...\n");
 }
@@ -3328,7 +3330,7 @@ static const char *usage(void)
         "       --targets-overlap 0|1|2       Include if POS in the region (0), record overlaps (1), variant overlaps (2) [0]\n"
         "       --threads INT                 Use multithreading with <int> worker threads [0]\n"
         "   -v, --verbose INT                 Verbosity level 0-2 [1]\n"
-        "       --write-index                 Automatically index the output files [off]\n"
+        "   -W, --write-index[=FMT]           Automatically index the output files [off]\n"
         "\n"
         "Example:\n"
         "   bcftools csq -f hs37d5.fa -g Homo_sapiens.GRCh37.82.gff3.gz in.vcf\n"
@@ -3379,7 +3381,7 @@ int main_csq(int argc, char *argv[])
         {"targets-file",1,0,'T'},
         {"targets-overlap",required_argument,NULL,5},
         {"no-version",no_argument,NULL,3},
-        {"write-index",no_argument,NULL,6},
+        {"write-index",optional_argument,NULL,'W'},
         {"dump-gff",required_argument,NULL,7},
         {"unify-chr-names",required_argument,NULL,8},
         {0,0,0,0}
@@ -3388,7 +3390,7 @@ int main_csq(int argc, char *argv[])
     int regions_overlap = 1;
     int targets_overlap = 0;
     char *targets_list = NULL, *regions_list = NULL, *tmp;
-    while ((c = getopt_long(argc, argv, "?hr:R:t:T:i:e:f:o:O:g:s:S:p:qc:ln:bB:v:",loptions,NULL)) >= 0)
+    while ((c = getopt_long(argc, argv, "?hr:R:t:T:i:e:f:o:O:g:s:S:p:qc:ln:bB:v:W::",loptions,NULL)) >= 0)
     {
         switch (c)
         {
@@ -3470,7 +3472,10 @@ int main_csq(int argc, char *argv[])
                 targets_overlap = parse_overlap_option(optarg);
                 if ( targets_overlap < 0 ) error("Could not parse: --targets-overlap %s\n",optarg);
                 break;
-            case  6 : args->write_index = 1; break;
+            case 'W':
+                if (!(args->write_index = write_index_parse(optarg)))
+                    error("Unsupported index format '%s'\n", optarg);
+                break;
             case  7 : args->dump_gff = optarg; break;
             case  8 :
                 if ( !strcmp(optarg,"0") ) args->unify_chr_names = 0;
@@ -3490,7 +3495,7 @@ int main_csq(int argc, char *argv[])
     }
     else fname = argv[optind];
     if ( argc - optind>1 ) error("%s", usage());
-    if ( !args->fa_fname ) error("Missing the --fa-ref option\n");
+    if ( !args->fa_fname ) error("Missing the --fasta-ref option\n");
     if ( !args->gff_fname ) error("Missing the --gff option\n");
     args->sr = bcf_sr_init();
     if ( targets_list )

doc/bcftools.txt

@@ -303,8 +303,11 @@ Such a file can be easily created from a VCF using:
     Use multithreading with 'INT' worker threads. The option is currently used only for the compression of the
     output stream, only when '--output-type' is 'b' or 'z'. Default: 0.
 
-*--write-index*::
-    Automatically index the output files. Can be used only for compressed BCF and VCF output.
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output files. 'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format. Defaults to
+    CSI unless specified otherwise. Can be used only for compressed
+    BCF and VCF output.
 
 
 [[annotate]]
@@ -509,8 +512,9 @@ Add or remove annotations.
     "^INFO/FOO,INFO/BAR" (and similarly for FORMAT and FILTER).
     "INFO" can be abbreviated to "INF" and "FORMAT" to "FMT".
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 *Examples:*
 ----
@@ -615,8 +619,9 @@ demand. The original calling model can be invoked with the *-c* option.
 *--threads* 'INT'::
     see *<<common_options,Common Options>>*
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 ==== Input/output options:
 
@@ -895,8 +900,9 @@ are concatenated without being recompressed, which is very fast..
 *--threads* 'INT'::
     see *<<common_options,Common Options>>*
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 
 [[consensus]]
@@ -1051,8 +1057,9 @@ Note that the *-H, --haplotype* option requires the *-s, --samples* option, unle
 *--targets-overlap* '0'|'1'|'2'::
     see *<<common_options,Common Options>>*
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 ==== VCF output options:
 
@@ -1452,8 +1459,9 @@ output VCF and are ignored for the prediction analysis.
     and VCF, such as "chrX" vs "X". The chromosome names in the output VCF will match
     that of the input VCF. The default is to attempt the automatic translation.
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 *Examples:*
 ----
@@ -1622,8 +1630,9 @@ And similarly here, the second is filtered:
 *--threads* 'INT'::
     see *<<common_options,Common Options>>*
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 
 
@@ -1921,8 +1930,10 @@ in the other files.
     comma-separated list of input files to output given as 1-based indices. With *-p* and no
     *-w*, all files are written.
 
-*--write-index*::
-    Automatically index the output file. This is done automatically with the *-p* option.
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and defaults
+    to tbi for vcf.gz and csi for bcf.  This is done automatically
+    with the *-p* option if the output format is compressed.
 
 ==== Examples:
 
@@ -2030,7 +2041,7 @@ For "vertical" merge take a look at *<<concat,bcftools concat>>* or *<<norm,bcft
 
 *-m, --merge* 'snps'|'indels'|'both'|'snp-ins-del'|'all'|'none'|'id'[,'*']::
     The option controls what types of multiallelic records can be created. If single asterisk
-    '*' is appended, the unobserved allele '<*>' or '<NON_REF>' will be removed at variant sites;
+    '\*' is appended, the unobserved allele '<*>' or '<NON_REF>' will be removed at variant sites;
     if two asterisks '**' are appended, the unobserved allele will be removed all sites.
 ----
 -m none        ..  no new multiallelics, output multiple records instead
@@ -2079,8 +2090,9 @@ For "vertical" merge take a look at *<<concat,bcftools concat>>* or *<<norm,bcft
 *--threads* 'INT'::
     see *<<common_options,Common Options>>*
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 [[mpileup]]
 === bcftools mpileup ['OPTIONS'] *-f* 'ref.fa' 'in.bam' ['in2.bam' [...]]
@@ -2343,8 +2355,9 @@ INFO/DPR    .. Deprecated in favor of INFO/AD; Number of high-quality bases for
     used by the earlier Bcftools releases.  For excample BQBZ becomes
     BQB.
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 ==== Options for SNP/INDEL genotype likelihood computation
 
@@ -2632,8 +2645,9 @@ the *<<fasta_ref,--fasta-ref>>* option is supplied.
     maximum distance between two records to consider when locally
     sorting variants which changed position during the realignment
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 [[plugin]]
 
@@ -2691,8 +2705,9 @@ the usage examples that each plugin comes with.
 *--threads* 'INT'::
     see *<<common_options,Common Options>>*
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 ==== Plugin options:
 
@@ -3333,8 +3348,9 @@ Transition probabilities:
     Use this directory to store temporary files. If the last six characters of the string DIR are XXXXXX,
     then these are replaced with a string that makes the directory name unique.
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 
 
@@ -3483,8 +3499,9 @@ Convert between VCF and BCF. Former *bcftools subset*.
 *--threads* 'INT'::
     see *<<common_options,Common Options>>*
 
-*--write-index*::
-    Automatically index the output file
+*-W*['FMT']*, -W*[='FMT']*, --write-index*[='FMT']::
+    Automatically index the output file.  'FMT' is optional and can be
+    one of "tbi" or "csi" depending on output file format.
 
 
 ==== Subset options:

filter.c

@@ -3682,7 +3682,8 @@ static filter_t *filter_init_(bcf_hdr_t *hdr, const char *str, int exit_on_error
         if ( !out[i].tag ) continue;
         if ( out[i].setter==filters_set_type )
         {
-            if ( i+1==nout ) error("Could not parse the expression: %s\n", filter->str);
+            if ( i+1==nout || !out[i+1].key )
+                error("Could not parse the expression: %s\n", filter->str);
             int itok, ival;
             if ( out[i+1].tok_type==TOK_EQ || out[i+1].tok_type==TOK_NE ) ival = i - 1, itok = i + 1;
             else if ( out[i+1].tok_type==TOK_LIKE || out[i+1].tok_type==TOK_NLIKE ) ival = i - 1, itok = i + 1;

mpileup.c

@@ -863,7 +863,9 @@ static int mpileup(mplp_conf_t *conf)
     for (i=0; i<nsmpl; i++)
         bcf_hdr_add_sample(conf->bcf_hdr, smpl[i]);
     if ( bcf_hdr_write(conf->bcf_fp, conf->bcf_hdr)!=0 ) error("[%s] Error: failed to write the header to %s\n",__func__,conf->output_fname?conf->output_fname:"standard output");
-    if ( conf->write_index && init_index(conf->bcf_fp,conf->bcf_hdr,conf->output_fname,&conf->index_fn)<0 ) error("Error: failed to initialise index for %s\n",conf->output_fname);
+    if ( init_index2(conf->bcf_fp,conf->bcf_hdr,conf->output_fname,
+                     &conf->index_fn, conf->write_index) < 0 )
+        error("Error: failed to initialise index for %s\n",conf->output_fname);
 
     conf->bca = bcf_call_init(-1., conf->min_baseQ, conf->max_baseQ,
                               conf->delta_baseQ);
@@ -1256,7 +1258,7 @@ static void print_usage(FILE *fp, const mplp_conf_t *mplp)
         "  -O, --output-type TYPE  'b' compressed BCF; 'u' uncompressed BCF;\n"
         "                          'z' compressed VCF; 'v' uncompressed VCF; 0-9 compression level [v]\n"
         "      --threads INT       Use multithreading with INT worker threads [0]\n"
-        "      --write-index       Automatically index the output files [off]\n"
+        "  -W, --write-index[=FMT] Automatically index the output files [off]\n"
         "\n"
         "SNP/INDEL genotype likelihoods options:\n"
         "  -X, --config STR        Specify platform profile (use \"-X list\" for details)\n"
@@ -1445,15 +1447,15 @@ int main_mpileup(int argc, char *argv[])
         {"seed", required_argument, NULL, 13},
         {"ambig-reads", required_argument, NULL, 14},
         {"ar", required_argument, NULL, 14},
-        {"write-index",no_argument,NULL,21},
+        {"write-index",optional_argument,NULL,'W'},
         {"del-bias", required_argument, NULL, 23},
         {"poly-mqual", no_argument, NULL, 24},
         {"no-poly-mqual", no_argument, NULL, 26},
         {"score-vs-ref",required_argument, NULL, 27},
         {"seqq-offset", required_argument, NULL, 28},
         {NULL, 0, NULL, 0}
     };
-    while ((c = getopt_long(argc, argv, "Ag:f:r:R:q:Q:C:BDd:L:b:P:po:e:h:Im:F:EG:6O:xa:s:S:t:T:M:X:U",lopts,NULL)) >= 0) {
+    while ((c = getopt_long(argc, argv, "Ag:f:r:R:q:Q:C:BDd:L:b:P:po:e:h:Im:F:EG:6O:xa:s:S:t:T:M:X:UW::",lopts,NULL)) >= 0) {
         switch (c) {
         case 'x': mplp.flag &= ~MPLP_SMART_OVERLAPS; break;
         case  16 :
@@ -1594,7 +1596,10 @@ int main_mpileup(int argc, char *argv[])
             }
             break;
         case  20: mplp.indels_v20 = 1; mplp.edlib = 0; break;
-        case  21: mplp.write_index = 1; break;
+        case 'W':
+            if (!(mplp.write_index = write_index_parse(optarg)))
+                error("Unsupported index format '%s'\n", optarg);
+            break;
         case  22: mplp.edlib = 1; mplp.indels_v20 = 0; break;
         case  25: mplp.edlib = 0; break;
         case  28:

plugins/contrast.c

@@ -110,7 +110,7 @@ static const char *usage_text(void)
         "   -t, --targets REG                Similar to -r but streams rather than index-jumps\n"
         "   -T, --targets-file FILE          Similar to -R but streams rather than index-jumps\n"
         "       --targets-overlap 0|1|2      Include if POS in the region (0), record overlaps (1), variant overlaps (2) [0]\n"
-        "       --write-index                Automatically index the output files [off]\n"
+        "   -W, --write-index[=FMT]          Automatically index the output files [off]\n"
         "\n"
         "Example:\n"
         "   # Test if any of the samples a,b is different from the samples c,d,e\n"
@@ -236,7 +236,9 @@ static void init_data(args_t *args)
     args->out_fh = hts_open(args->output_fname ? args->output_fname : "-", wmode);
     if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
     if ( bcf_hdr_write(args->out_fh, args->hdr_out)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
-    if ( args->write_index && init_index(args->out_fh,args->hdr_out,args->output_fname,&args->index_fn)<0 ) error("Error: failed to initialise index for %s\n",args->output_fname);
+    if ( init_index2(args->out_fh,args->hdr_out,args->output_fname,
+                     &args->index_fn, args->write_index)<0 )
+        error("Error: failed to initialise index for %s\n",args->output_fname);
 
     if ( args->max_AC_str )
     {
@@ -485,12 +487,12 @@ int run(int argc, char **argv)
         {"targets",1,0,'t'},
         {"targets-file",1,0,'T'},
         {"targets-overlap",required_argument,NULL,4},
-        {"write-index",no_argument,NULL,5},
+        {"write-index",optional_argument,NULL,'W'},
         {NULL,0,NULL,0}
     };
     int c;
     char *tmp;
-    while ((c = getopt_long(argc, argv, "O:o:i:e:r:R:t:T:0:1:a:f:",loptions,NULL)) >= 0)
+    while ((c = getopt_long(argc, argv, "O:o:i:e:r:R:t:T:0:1:a:f:W::",loptions,NULL)) >= 0)
     {
         switch (c)
         {
@@ -536,7 +538,10 @@ int run(int argc, char **argv)
                 args->targets_overlap = parse_overlap_option(optarg);
                 if ( args->targets_overlap < 0 ) error("Could not parse: --targets-overlap %s\n",optarg);
                 break;
-            case  5 : args->write_index = 1; break;
+            case 'W':
+                if (!(args->write_index = write_index_parse(optarg)))
+                    error("Unsupported index format '%s'\n", optarg);
+                break;
             case 'h':
             case '?':
             default: error("%s", usage_text()); break;