Merge pull request #16 from eseiler/feature/file_input

[FEATURE] Accept file containing bin paths

Author: eseiler

CHANGELOG.md

@@ -1,5 +1,14 @@
+# 1.1.0
+
+## Features
+* Raptor accepts a text file containing the path to a bin on each line 
+  ([\#16](https://github.com/seqan/raptor/pull/16)).
+
+## Bug fixes
+* Threshold option not working ([\#14](https://github.com/seqan/raptor/pull/14)).
+
 # 1.0.1
 
 ## Bug fixes
 
-* Reduced the number of open file handles ([\#10](https://github.com/seqan/raptor/pull/10))
+* Reduced the number of open file handles ([\#10](https://github.com/seqan/raptor/pull/10)).

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required (VERSION 3.8)
 ## CUSTOMISE
 
 # Define the application name and version.
-project (raptor VERSION 1.0.2)
+project (raptor VERSION 1.1.0)
 set (CMAKE_CXX_STANDARD 17)
 set (CMAKE_CXX_STANDARD_REQUIRED ON)
 

README.md

@@ -99,6 +99,13 @@ raptor build --kmer 19 --window 23 --size 8m --output index.raptor $(seq -f "exa
 # You can replace `$(seq -f "example_data/64/bins/bin_%02g.fasta" 0 1 63)` by `example_data/64/bins/bin_{00..63}.fasta` if your shell supports this syntax.
 # The equivalent command for 1,024 bins is `$(seq -f "example_data/1024/bins/bin_%04g.fasta" 0 1 1023)`
 ```
+
+You can also prepare a file that contains one file path per line (a line corresponds to a bin) and use this file as input:
+```
+seq -f "example_data/64/bins/bin_%02g.fasta" 0 1 63 > all_bin_paths.txt
+raptor build --kmer 19 --window 23 --size 8m --output another_index.raptor all_bin_paths.txt
+```
+
 You may be prompted to enable or disable automatic update notifications. For questions, please consult [the SeqAn documentation](https://github.com/seqan/seqan3/wiki/Update-Notifications).
 
 Afterwards, we can search for all reads from bin 1:
@@ -143,7 +150,16 @@ raptor build --kmer 19 --window 23 --size 8m --compute-minimiser --output precom
 
 Then we run the build step again and use the computed minimisers as input:
 ```
-raptor build --kmer 19 --window 23 --size 8m --output index.raptor $(seq -f "precomputed_minimisers/bin_%02g.minimiser" 0 1 63)
+raptor build --kmer 19 --window 23 --size 8m --output minimiser_index.raptor $(seq -f "precomputed_minimisers/bin_%02g.minimiser" 0 1 63)
+```
+
+Alternatively, you can also prepare a file that contains one file path per line (a line corresponds to a bin)
+and use this file as input for both cases:
+```
+seq -f "example_data/64/bins/bin_%02g.fasta" 0 1 63 > all_bin_paths.txt
+raptor build --kmer 19 --window 23 --size 8m --compute-minimiser --output precomputed_minimisers/ all_bin_paths.txt
+seq -f "precomputed_minimisers/bin_%02g.minimiser" 0 1 63 > all_minimiser_paths.txt
+raptor build --kmer 19 --window 23 --size 8m --output another_minimiser_index.raptor all_minimiser_paths.txt
 ```
 
 ## Authorship and Copyright

src/raptor.cpp

@@ -117,6 +117,22 @@ class bin_validator
             {
                 if (value.extension() == ".minimiser")
                     minimiser_file_validator(value);
+                else if (values.size() == 1u)
+                {
+                    std::ifstream list_of_files{value};
+                    std::string line;
+                    while (std::getline(list_of_files, line))
+                    {
+                        if (!line.empty())
+                        {
+                            std::filesystem::path bin_path{line};
+                            if (bin_path.extension() == ".minimiser")
+                                minimiser_file_validator(bin_path);
+                            else
+                                sequence_file_validator(bin_path);
+                        }
+                    }
+                }
                 else
                     throw exception;
             }
@@ -131,9 +147,10 @@ class bin_validator
 
     std::string get_help_page_message() const
     {
-        return "The input file must exist and read permissions must be granted. Valid file extensions are: [minimiser],"
-               " or [embl, fasta, fa, fna, ffn, faa, frn, fas, fastq, fq, genbank, gb, gbk, sam] possibly followed by: "
-               "[gz, bgzf, bz2].";
+        return "The input file must exist and read permissions must be granted. Valid file extensions for bin files are"
+               " : [minimiser], or [embl, fasta, fa, fna, ffn, faa, frn, fas, fastq, fq, genbank, gb, gbk, sam] "
+               "possibly followed by: [gz, bgzf, bz2]. All other extensions will be assumed to contain one line per"
+               " path to a bin.";
     }
 
 private:
@@ -148,7 +165,7 @@ inline void init_shared_meta(seqan3::argument_parser & parser)
     parser.info.citation = "Seiler, E. et al. (2020). Raptor: A fast and space-efficient pre-filter for"
                            " querying very large collections of nucleotide sequences. bioRxiv 2020.10.08.330985. doi:"
                            " https://doi.org/10.1101/2020.10.08.330985";
-    parser.info.date = "12-10-2020";
+    parser.info.date = "16-12-2020";
     parser.info.email = "enrico.seiler@fu-berlin.de";
     parser.info.long_copyright = R"(BSD 3-Clause License
 
@@ -182,7 +199,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.)";
     parser.info.short_copyright = "BSD 3-Clause License";
     parser.info.short_description = "A fast and space-efficient pre-filter for querying very large collections of nucleotide sequences.";
     parser.info.url = "https://github.com/seqan/raptor";
-    parser.info.version = "1.0.2";
+    parser.info.version = "1.1.0";
 }
 
 void init_top_level_parser(seqan3::argument_parser & parser)
@@ -229,7 +246,8 @@ inline void init_build_parser(seqan3::argument_parser & parser, build_arguments
     init_shared_meta(parser);
     init_shared_options(parser, arguments);
     parser.add_positional_option(arguments.bin_path,
-                                 "Provide a list of input files. One file per bin. ",
+                                 "Provide a list of input files (one file per bin). Alternatively, provide a text file "
+                                 "containing the paths to the bins (one line per path to a bin). ",
                                  bin_validator{});
     parser.add_option(arguments.out_path,
                       '\0',
@@ -319,6 +337,34 @@ void run_build(seqan3::argument_parser & parser)
     init_build_parser(parser, arguments);
     try_parsing(parser);
 
+    // ==========================================
+    // Process bin_path
+    // ==========================================
+    if (arguments.bin_path.size() == 1u) // Either only one bin or a file containing bin paths
+    {
+        auto & file = arguments.bin_path[0];
+
+        if (file.extension() != ".minimiser")
+        {
+            try
+            {
+                seqan3::input_file_validator<seqan3::sequence_file_input<>> validator;
+                validator(file);
+            }
+            catch (seqan3::validation_error const & exception)
+            {
+                decltype(arguments.bin_path) new_values;
+                std::ifstream list_of_files{file};
+                std::string line;
+                while (std::getline(list_of_files, line))
+                    new_values.emplace_back(line);
+                while (new_values.back().empty())
+                    new_values.pop_back();
+                arguments.bin_path = std::move(new_values);
+            }
+        }
+    }
+
     // ==========================================
     // Various checks.
     // ==========================================

test/cli/raptor_test.cpp

@@ -19,7 +19,7 @@ inline std::string string_from_file(std::filesystem::path const & path, std::ios
 ///////////////////////////////////////////////// raptor build tests ///////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-TEST_P(raptor_build, build)
+TEST_P(raptor_build, build_with_list)
 {
     auto const [number_of_bins, run_parallel] = GetParam();
 
@@ -39,6 +39,38 @@ TEST_P(raptor_build, build)
     EXPECT_TRUE(expected == actual);
 }
 
+TEST_P(raptor_build, build_with_file)
+{
+    auto const [number_of_bins, run_parallel] = GetParam();
+
+    {
+        std::string const expanded_bins = expand_bins(number_of_bins);
+        std::ofstream file{"raptor_cli_test.txt"};
+        auto split_bins = expanded_bins | std::views::split(' ') | ranges::view::transform([](auto &&rng) {return std::string_view(&*rng.begin(), ranges::distance(rng));});
+        for (auto && file_path : split_bins)
+        {
+            file << file_path << '\n';
+        }
+        file << '\n';
+    }
+
+    cli_test_result result = execute_app("raptor", "build",
+                                                   "--kmer 19",
+                                                   "--window 23",
+                                                   "--size 8m",
+                                                   "--threads ", run_parallel ? "2" : "1",
+                                                   "--output index.ibf", 
+                                                   "raptor_cli_test.txt");
+    ASSERT_EQ(result.exit_code, 0);
+    ASSERT_EQ(result.out, std::string{});
+    ASSERT_EQ(result.err, std::string{});
+
+    std::string expected = string_from_file(data("expected_results/b" + std::to_string(number_of_bins) + "_k19_w23_s8m.ibf"), std::ios::binary);
+    std::string actual = string_from_file("index.ibf", std::ios::binary);
+
+    ASSERT_TRUE(expected == actual);
+}
+
 INSTANTIATE_TEST_SUITE_P(build_suite,
                          raptor_build,
                          testing::Combine(testing::Values(64, 1024), testing::Values(true, false)),