Merge pull request #55 from JesseMckinzie/directory_capturing

Add directory capturing to filepattern

Author: sameeul

docs/source/Examples.rst

@@ -365,6 +365,45 @@ the output will be
 
     ['r', 'c']
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Capturing directory names
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Directory names can be also be captured when the the `recursive` option is set to `True`. In this case, 
+the directory name can be treated the same as a variable in the `filepattern`. For example, if a directory has the
+structure:
+
+.. code-block:: bash
+
+    data
+        DAPI
+            img_r001_c001.tif 
+        TXREAD
+            img_r001_c001.tif
+        GFP
+            img_r001_c001.tif
+
+Then the following `filepattern` will capture the directory names.
+
+.. code-block:: python 
+
+    path = '/path/to/data'
+
+    filepattern = '/{directory:c+}/img_r{r:ddd}_c{c:ddd}.tif'
+        
+    files = fp.FilePattern(path, filepattern, recursive=True)
+
+    for file in files():
+        print(file)
+
+The output will be:
+
+.. code-block:: bash
+
+    ({'c': 1, 'directory': 'DAPI', 'r': 1}, ['path/to/data/img_r001_c001.tif'])
+    ({'c': 1, 'directory': 'GFP', 'r': 1}, ['path/to/data/img_r001_c001.tif'])
+    ({'c': 1, 'directory': 'TXREAD', 'r': 1}, ['path/to/data/img_r001_c001.tif'])
+
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Pydantic models as return values
@@ -384,7 +423,7 @@ are dynamically created at runtime, allowing the fields of the model to be the v
 
     pattern = "img_r{r:ddd}_c{c:ddd}_{channel:c+}.tif"
 
-    files = fp.FilePattern(filepath, pattern)
+    files = fp.FilePattern(filepath, pattern, recursive=True)
 
     for file in files(pydantic_output=True):
         print(file.r)

src/filepattern/cpp/internal/filepattern.cpp

@@ -22,11 +22,21 @@ FilePatternObject::FilePatternObject(const string& path, const string& file_patt
 
     } else {
 
-        this->setJustPath(false);
-        this->setPath(path); // store path to target directory
-        this->setFilePattern(file_pattern); // cast input string to regex
         this->recursive_ = recursive; // Iterate over subdirectories
 
+        // check if filepattern contains directory capturing 
+        if (file_pattern.find('/') != std::string::npos || file_pattern.find('\\') != std::string::npos) {
+            this->setCaptureDirectoryNames(true);
+            this->recursive_ = true; // need to be recursive to capture directory names
+
+            this->setFilePattern(s::escapeForwardSlashes(file_pattern));
+        } else {
+            this->setFilePattern(file_pattern); // cast input string to regex
+        }
+
+        this->setJustPath(false);
+        this->setPath(path); // store path to target directory
+        
         try {
             if(recursive){
                 this->recursive_iterator_ = fs::recursive_directory_iterator(this->getPath());
@@ -86,29 +96,48 @@ void FilePatternObject::matchFilesOneDir(){
 }
 
 void FilePatternObject::matchFilesMultDir(){
-    // Iterate over every file in directory
+
     regex pattern_regex = regex(this->getRegexFilePattern());
+
     Tuple tup;
     smatch sm;
     string file, file_path;
+
+    bool is_pushed = false;
+
     // Iterate over directories and subdirectories
     for (const auto& entry : this->recursive_iterator_) {
+
         file_path = entry.path().string();
-        replace(file_path.begin(), file_path.end(), '\\', '/');
-        if(this->getJustPath()){
+
+        replace(file_path.begin(), file_path.end(), '\\', '/'); // escape slashes for regex
+
+        if(this->getJustPath() || this->captureDirectoryNames()){
             file = s::eraseSubStr(file_path, this->getPath());
         } else {
             file = s::getBaseName(file_path);
         }
 
         if(regex_match(file, sm, pattern_regex)){
-            if(this->getJustPath()) tup = getVariableMap(file_path, sm);
-            else tup = getVariableMapMultDir(file_path, sm);
+
+            if(this->getJustPath() || this->captureDirectoryNames()) {
+                tup = getVariableMap(file_path, sm);
+            } else  {
+                tup = getVariableMapMultDir(file_path, sm);
+            }
+
             if(get<0>(tup).size() > 0){
-            this->valid_files_.push_back(tup); // write to txt file
+                this->valid_files_.push_back(tup); 
+                is_pushed = true;
+            } else {
+                is_pushed = false;
             }
         }
     }
+
+    if (!is_pushed && get<1>(tup).size() > 0) {
+         this->valid_files_.push_back(tup); 
+    }
 }
 
 void FilePatternObject::matchFiles() {

src/filepattern/cpp/pattern.cpp

@@ -760,6 +760,14 @@ bool Pattern::getJustPath(){
     return this->just_path_;
 }
 
+bool Pattern::captureDirectoryNames() {
+    return this->capture_directory_names_;
+}
+
+void Pattern::setCaptureDirectoryNames(bool capture) {
+    this->capture_directory_names_ = capture;
+}
+
 bool Pattern::getSuppressWarnings(){
     return this->suppress_warnings_;
 }

src/filepattern/cpp/pattern.hpp

@@ -30,6 +30,7 @@ class Pattern : public PatternObject {
         //std::string path_;
         bool just_path_;
         bool suppress_warnings_;
+        bool capture_directory_names_;
 
         std::string VARIABLES_;
 
@@ -49,6 +50,7 @@ class Pattern : public PatternObject {
         std::string getPath();
         bool getJustPath();
         bool getSuppressWarnings();
+        void setCaptureDirectoryNames(bool capture);
 
 
         /**
@@ -236,4 +238,6 @@ class Pattern : public PatternObject {
          */
         std::vector<std::string> getTmpDirs();
 
+        bool captureDirectoryNames();
+
 };

src/filepattern/cpp/util/util.hpp

@@ -49,6 +49,7 @@ using Tuple = std::tuple<Map, std::vector<std::string>>;
 using Tuple = std::tuple<Map, std::vector<fs::path>>;
 #endif
 
+
 #if defined(WIN32) || defined(_WIN32) || defined(__WIN32) && !defined(__CYGWIN__)
 static const std::string SLASH = "\\";
 #else
@@ -62,6 +63,20 @@ static const std::string SLASH = "/";
  */
 namespace s {
 
+    inline std::string escapeForwardSlashes(const std::string& input) {
+        std::string result;
+        
+        for (char ch : input) {
+            if (ch == '/') {
+                result += "\\/";
+            } else {
+                result += ch;
+            }
+        }
+        
+        return result;
+    }
+
     inline std::string escape_regex_characters(const std::string& str) {
 
         const std::unordered_set<char> escape_chars = {'*', '?', '^', '$', '(', ')', '[', ']', '|', '\\'};

tests/test_filepattern.py

@@ -269,10 +269,81 @@ def test_named_group_direcotry(self):
                 basename = os.path.basename(mapping[1][0])
                 for filepath in mapping[1]:
                     assert basename == os.path.basename(filepath)
+    
+    def test_recursive_directory_fp(self):
+        
+        path = self.root_directory + '/test_data/recursive_data'
+   
+        filepattern = '/{directory:c+}/img_r{r:ddd}_c{c:ddd}.tif'
+        
+        files = fp.FilePattern(path, filepattern, recursive=True)
+
+        result = []
+
+        for file in files():
+            result.append(file)
+
+        # test that same number of files are returned
+        assert len(result) == len(fp_data.test_recursive_directory_fp) 
+
+        # test that each variable and path are equal for each file in list
+        for i in range(len(fp_data.test_recursive_directory_fp)): 
+            print(result[i])
+            assert fp_data.test_recursive_directory_fp[i][0]["r"] == result[i][0]["r"]
+            assert fp_data.test_recursive_directory_fp[i][0]["c"] == result[i][0]["c"]
+            assert fp_data.test_recursive_directory_fp[i][0]["directory"] == result[i][0]["directory"]
+            assert str(os.path.basename(fp_data.test_recursive_directory_fp[i][1][0])) == os.path.basename(result[i][1][0])
+            
+    def test_recursive_directory_regex_fp(self):
+        
+        path = self.root_directory + '/test_data/recursive_data'
+        
+        filepattern = '/(?P<directory>[a-zA-Z]+)/img_r{r:ddd}_c{c:ddd}.tif'
+        
+        files = fp.FilePattern(path, filepattern, recursive=True)
 
+        result = []
+
+        for file in files():
+            result.append(file)
+
+        # test that same number of files are returned
+        assert len(result) == len(fp_data.test_recursive_directory_fp) 
+
+        # test that each variable and path are equal for each file in list
+        for i in range(len(fp_data.test_recursive_directory_fp)): 
+            print(result[i])
+            assert fp_data.test_recursive_directory_fp[i][0]["r"] == result[i][0]["r"]
+            assert fp_data.test_recursive_directory_fp[i][0]["c"] == result[i][0]["c"]
+            assert fp_data.test_recursive_directory_fp[i][0]["directory"] == result[i][0]["directory"]
+            assert str(os.path.basename(fp_data.test_recursive_directory_fp[i][1][0])) == os.path.basename(result[i][1][0])
 
 
 
+    def test_recursive_multi_directory_regex_fp(self):
+        
+        path = self.root_directory + '/test_data'
+        
+        filepattern = '/.*/{directory:c+}/img_r{r:ddd}_c{c:ddd}.tif'
+        
+        files = fp.FilePattern(path, filepattern, recursive=True)
+
+        result = []
+
+        for file in files():
+            result.append(file)
+
+        # test that same number of files are returned
+        assert len(result) == len(fp_data.test_recursive_directory_fp) 
+
+        # test that each variable and path are equal for each file in list
+        for i in range(len(fp_data.test_recursive_directory_fp)): 
+            print(result[i])
+            assert fp_data.test_recursive_directory_fp[i][0]["r"] == result[i][0]["r"]
+            assert fp_data.test_recursive_directory_fp[i][0]["c"] == result[i][0]["c"]
+            assert fp_data.test_recursive_directory_fp[i][0]["directory"] == result[i][0]["directory"]
+            assert str(os.path.basename(fp_data.test_recursive_directory_fp[i][1][0])) == os.path.basename(result[i][1][0])
+
 # Todo: These tests need new data to be added after replacing the old version of filepattern.
 """
     def test_group_by_multi(self):

tests/test_filepattern_data.py

@@ -616,3 +616,34 @@
 ({'c': 2, 'dir': 'TXREAD', 'r': 2},
  ['test_data/recursive_data/TXREAD/img_r002_c002.tif'])
 ]
+
+
+test_recursive_directory_fp = [
+({'c': 0, 'directory': 'DAPI', 'r': 0}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/DAPI/img_r000_c000.tif']),
+({'c': 1, 'directory': 'DAPI', 'r': 0}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/DAPI/img_r000_c001.tif']),
+({'c': 2, 'directory': 'DAPI', 'r': 0}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/DAPI/img_r000_c002.tif']),
+({'c': 0, 'directory': 'DAPI', 'r': 1}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/DAPI/img_r001_c000.tif']),
+({'c': 1, 'directory': 'DAPI', 'r': 1}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/DAPI/img_r001_c001.tif']),
+({'c': 2, 'directory': 'DAPI', 'r': 1}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/DAPI/img_r001_c002.tif']),
+({'c': 0, 'directory': 'DAPI', 'r': 2}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/DAPI/img_r002_c000.tif']),
+({'c': 1, 'directory': 'DAPI', 'r': 2}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/DAPI/img_r002_c001.tif']),
+({'c': 2, 'directory': 'DAPI', 'r': 2}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/DAPI/img_r002_c002.tif']),
+({'c': 0, 'directory': 'GFP', 'r': 0}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/GFP/img_r000_c000.tif']),
+({'c': 1, 'directory': 'GFP', 'r': 0}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/GFP/img_r000_c001.tif']),
+({'c': 2, 'directory': 'GFP', 'r': 0}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/GFP/img_r000_c002.tif']),
+({'c': 0, 'directory': 'GFP', 'r': 1}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/GFP/img_r001_c000.tif']),
+({'c': 1, 'directory': 'GFP', 'r': 1}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/GFP/img_r001_c001.tif']),
+({'c': 2, 'directory': 'GFP', 'r': 1}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/GFP/img_r001_c002.tif']),
+({'c': 0, 'directory': 'GFP', 'r': 2}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/GFP/img_r002_c000.tif']),
+({'c': 1, 'directory': 'GFP', 'r': 2}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/GFP/img_r002_c001.tif']),
+({'c': 2, 'directory': 'GFP', 'r': 2}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/GFP/img_r002_c002.tif']),
+({'c': 0, 'directory': 'TXREAD', 'r': 0}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/TXREAD/img_r000_c000.tif']),
+({'c': 1, 'directory': 'TXREAD', 'r': 0}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/TXREAD/img_r000_c001.tif']),
+({'c': 2, 'directory': 'TXREAD', 'r': 0}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/TXREAD/img_r000_c002.tif']),
+({'c': 0, 'directory': 'TXREAD', 'r': 1}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/TXREAD/img_r001_c000.tif']),
+({'c': 1, 'directory': 'TXREAD', 'r': 1}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/TXREAD/img_r001_c001.tif']),
+({'c': 2, 'directory': 'TXREAD', 'r': 1}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/TXREAD/img_r001_c002.tif']),
+({'c': 0, 'directory': 'TXREAD', 'r': 2}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/TXREAD/img_r002_c000.tif']),
+({'c': 1, 'directory': 'TXREAD', 'r': 2}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/TXREAD/img_r002_c001.tif']),
+({'c': 2, 'directory': 'TXREAD', 'r': 2}, ['/Users/jmckinzie/Documents/GitHub/filepattern-1/tests/test_data/recursive_data/TXREAD/img_r002_c002.tif']),
+]