Merge pull request #82 from JesseMckinzie/array_input_v2

Add list of strings as an input to the FilePattern constructor

Author: sameeul

CMakeLists.txt

@@ -14,7 +14,7 @@ option(RUN_GTEST "Downloads google unit test API and runs google test scripts to
 
 
 if(BUILD_PYTHON_LIB)
-	find_package(pybind11 CONFIG REQUIRED)
+	find_package(pybind11 CONFIG REQUIRED) 
 	pybind11_add_module(backend
 						src/filepattern/cpp/bindings.cpp
 	)

src/filepattern/__init__.py

@@ -3,7 +3,7 @@
 
 from .filepattern import FilePattern
 
-from .functions import infer_pattern, get_regex
+from .functions import infer_pattern, get_regex, get_variables
 
 __all__ = ["FilePattern", "infer_pattern", "get_regex"]
 from . import _version

src/filepattern/cpp/bindings.cpp

@@ -3,6 +3,7 @@
 #include <pybind11/operators.h>
 #include <pybind11/complex.h>
 #include <pybind11/stl/filesystem.h>
+#include <string>
 
 #include "pattern_object.hpp"
 
@@ -17,7 +18,15 @@ namespace py = pybind11;
 PYBIND11_MODULE(backend, m){
 
     py::class_<FilePattern>(m, "FilePattern")
-        .def(py::init<const std::string &, const std::string &, const std::string&, bool, bool>())
+        .def(py::init<const std::string&,
+                      const std::string&, 
+                      const std::string&,  
+                      bool, 
+                      bool>())
+        .def(py::init<const std::vector<std::string>&,
+                      const std::string&, 
+                      bool, 
+                      bool>())
         .def("getMatching", &FilePattern::getMatching)
         .def("getOccurrences", &FilePattern::getOccurrences)
         .def("getUniqueValues", &FilePattern::getUniqueValues)
@@ -34,6 +43,7 @@ PYBIND11_MODULE(backend, m){
         .def("setGroupStr",   py::overload_cast<std::string&>(&FilePattern::setGroup))
         .def("length", &FilePattern::length)
         .def_static("getRegex", &FilePattern::getRegex)
+        .def_static("getVariablesFromPattern", &FilePattern::getVariablesFromPattern)
         .def_static("inferPattern", py::overload_cast<const std::string&, std::string&, const std::string&>(&FilePattern::inferPattern))
         .def_static("inferPattern", py::overload_cast<std::vector<std::string>&, std::string&>(&FilePattern::inferPattern))
         .def("isGrouped", [](FilePattern &v){

src/filepattern/cpp/include/filepattern.h

@@ -32,13 +32,16 @@ using Tuple = std::tuple<Map, std::vector<std::string>>;
 using Tuple = std::tuple<Map, std::vector<fs::path>>;
 #endif
 
+
 class PatternObject; // forward declaration
 class FILEPATTERN_EXPORT FilePattern {
 
     public:
 
         FilePattern(const std::string& path, const std::string& filePattern="", const std::string& block_size="", bool recursive=false, bool suppressWarnings=false);
 
+        FilePattern(const std::vector<std::string>& file_array, const std::string& filePattern, bool recursive=false, bool suppressWarnings=false);
+
         ~FilePattern();
 
         std::vector<Tuple> getMatchingByMap (Map& variables);
@@ -101,10 +104,12 @@ class FILEPATTERN_EXPORT FilePattern {
 
         static std::string getRegex(std::string filepattern, bool suppress_warnings);
 
+        static std::vector<std::string> getVariablesFromPattern(std::string& filepattern, bool supress_warnings);
+
         std::string getPattern();
         void setPattern(std::string& pattern);
         std::string getPath();
         const std::unique_ptr<PatternObject>& getPatternObject() const;
-    private:
+      private:
         std::unique_ptr<PatternObject> fp_;
 };

src/filepattern/cpp/interface/filepattern.cpp

@@ -1,6 +1,9 @@
 #include "../include/filepattern.h"
+#include "../pattern.hpp"
 #include "filepattern_factory.h"
 
+#include <tuple>
+
 FilePattern::FilePattern(const std::string& path, const std::string& filePattern, const std::string& block_size, bool recursive, bool suppressWarnings) {
 
     FilePatternFactory fpf = FilePatternFactory();
@@ -14,6 +17,17 @@ FilePattern::FilePattern(const std::string& path, const std::string& filePattern
     }
 
 }
+
+FilePattern::FilePattern(const std::vector<std::string>& file_array, const std::string& filePattern, bool recursive, bool suppressWarnings) {
+
+    FilePatternFactory fpf = FilePatternFactory();
+
+    this->fp_ = std::unique_ptr<PatternObject>(fpf.getObject(file_array, filePattern, suppressWarnings));
+
+    this->fp_->external = false;
+
+}
+
 FilePattern::~FilePattern() {
     this->fp_.reset();
 }
@@ -120,6 +134,7 @@ std::string FilePattern::inferPattern(const std::string& path, std::string& vari
 
     // create dummy object to avoid the need for static methods in virtual class
     std::unique_ptr<PatternObject> fp;
+
     if (block_size == "") {
         fp = std::unique_ptr<PatternObject>(fpf.getObject(path, "", block_size, false, true));
     } else {
@@ -189,3 +204,7 @@ std::pair<std::vector<std::pair<std::string, Types>> , std::vector<Tuple>> FileP
 std::string FilePattern::getRegex(std::string filepattern, bool suppress_warnings) {
     return std::get<0>(Pattern::getRegex(filepattern, suppress_warnings));
 }
+
+std::vector<std::string> FilePattern::getVariablesFromPattern(std::string& filepattern, bool supress_warnings) {
+    return std::get<1>(Pattern::getRegex(filepattern, supress_warnings));
+}

src/filepattern/cpp/interface/filepattern_factory.h

@@ -10,14 +10,28 @@
 #include "../util/vector_parser.hpp"
 
 
-
 class FilePatternFactory {
 
     public:
 
         FilePatternFactory() {}
 
-        std::unique_ptr<PatternObject> getObject(const std::string& path, const std::string& file_pattern, const std::string& block_size, bool recursive, bool suppressWarnings) {
+        std::unique_ptr<PatternObject> getObject(
+            const std::vector<std::string>& file_array, 
+            const std::string& file_pattern, 
+            bool suppressWarnings) {
+
+            return std::make_unique<ArrayPattern>(file_array, file_pattern, suppressWarnings);
+
+        }
+
+        std::unique_ptr<PatternObject> getObject(
+            const std::string& path, 
+            const std::string& file_pattern, 
+            const std::string& block_size, 
+            bool recursive, 
+            bool suppressWarnings) {
+
             if (block_size == "") {
                 if(fs::is_regular_file(path)) {
                     std::ifstream infile(path);

src/filepattern/cpp/internal/filepattern.cpp

@@ -143,3 +143,51 @@ void FilePatternObject::matchFiles() {
         this->matchFilesOneDir();
     }
 }
+
+ArrayPattern::ArrayPattern(const std::vector<std::string>& file_array, const std::string& pattern, bool suppress_warnings){
+
+    this->setSuppressWarnings(suppress_warnings);
+
+    this->setFilePattern(pattern);
+    this->setRegexFilePattern(""); // Regex version of pattern
+
+    this->matchFiles(file_array);
+
+    this->sortFiles();
+};
+
+void ArrayPattern::matchFiles(const std::vector<std::string>& file_array) {
+
+    filePatternToRegex(); // Get regex of filepattern
+
+    Map mapping;
+    std::vector<std::string> parsed_regex;
+
+    std::string s;
+    std::string file, file_path;
+    Tuple member;
+    // Iterate over every file in directory
+
+    // check if bracket expression was not properly parsed
+    if (this->getRegexFilePattern().find('{') != std::string::npos || this->getRegexFilePattern().find('}') != std::string::npos) {
+        auto start = this->getRegexFilePattern().find('{');
+        auto end = this->getRegexFilePattern().find('}');
+        auto length = end - start;
+        throw std::invalid_argument("Invalid pattern found in bracket expressions in filepattern: \"" + this->getRegexFilePattern().substr(start, length+1) + "\"");
+    }
+
+    std::regex pattern_regex = std::regex(this->getRegexFilePattern());
+    std::smatch sm;
+
+    // iterate over file array and add valid files
+    for (const auto& path : file_array) {
+        // Get the current file
+        auto file_path = path;
+        std::replace(file_path.begin(), file_path.end(), '\\', '/');
+        file = s::getBaseName(file_path);
+
+        if(regex_match(file, sm, pattern_regex)){
+            this->valid_files_.push_back(getVariableMap(file_path, sm)); // write to txt file
+        }
+    }
+};

src/filepattern/cpp/internal/filepattern.hpp

@@ -60,3 +60,10 @@ class FilePatternObject : public InternalPattern {
         void matchFilesMultDir();
 
 };
+
+class ArrayPattern : public InternalPattern {
+    public:
+        ArrayPattern(const std::vector<std::string>& file_array, const std::string& pattern, bool suppress_warnings);
+        
+        void matchFiles(const std::vector<std::string>& file_array);
+};

src/filepattern/cpp/util/sort.hpp

@@ -73,7 +73,7 @@ class ExternalMergeSort {
         std::string tmpdir_; // Create a directory to store temporary files
         std::string sort_variable_; // Variable to sort by if std_map
         std::string block_size_str_; // String input of blocksize
-        int map_size_; // Number of variables in the map
+        long unsigned int map_size_; // Number of variables in the map
         Map temp_map_;
 
         /**

src/filepattern/filepattern.py

@@ -3,6 +3,7 @@
 from typing import Dict, List, Tuple, Union, Set, Any
 import os
 from .pydantic_filepattern import create_pydantic_fp, get_pydantic_fp
+from pathlib import Path
 
 
 class PatternObject:
@@ -383,7 +384,7 @@ class FilePattern(PatternObject):
 
     def __init__(
         self,
-        path: str,
+        input: Union[str, Path, list] ="",
         pattern: str = "",
         block_size: str = "",
         recursive: bool = False,
@@ -400,16 +401,20 @@ def __init__(
         the names of the channel subdirectories will be captured for each file.
 
         Args:
-            path: Path to directory or text file
+            input: Path to directory or text file or a list of strings to be matched to the filepattern
             pattern: Pattern to compare each filename to
             block_size: Maximum amount of RAM to consume at once. Defaults to "".
             recursive: Iterate over subdirectories. Defaults to False.
             suppress_warnings: True to suppress warning printed to console. Defaults to False.
         """
 
-        path = str(path)  # change path type to string to support pathlib paths
-
-        self._file_pattern = backend.FilePattern(path, pattern, block_size, recursive, suppress_warnings)
+        if (isinstance(input, list)):
+            self._file_pattern = backend.FilePattern(input, pattern, recursive, suppress_warnings)
+        elif (isinstance(input, str) or isinstance(input, Path)):
+            input = str(input)  # change path type to string to support pathlib paths
+            self._file_pattern = backend.FilePattern(input, pattern, block_size, recursive, suppress_warnings)
+        else:
+            raise TypeError("Error: input type must either be a string/path to a file or directory or a list of strings")
 
         super().__init__(self._file_pattern, block_size)