Merge pull request #85 from JesseMckinzie/sort_flag

Add flag to avoid sorting captured files

Author: sameeul

.github/workflows/build_and_test_mac.yml

@@ -12,17 +12,18 @@ env:
 jobs:
   Build_and_Run_GTest:
     name: Build and Run GoogleTest
-    runs-on: macos-12
+    runs-on: macos-13
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
     - uses: actions/checkout@v3
 
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: anaconda-client-env
+        miniconda-version: "latest"
         python-version: 3.8
         channels: conda-forge
         auto-activate-base: false
@@ -45,18 +46,19 @@ jobs:
 
   Build_and_Run_PyTest:
     name: Build and Run PyTest
-    runs-on: macos-12
+    runs-on: macos-13
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
     - uses: actions/checkout@v3
 
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: anaconda-client-env
         python-version: 3.8
+        miniconda-version: "latest"
         channels: conda-forge
         auto-activate-base: false
 

.github/workflows/build_wheels.yml

@@ -12,7 +12,7 @@ jobs:
       MACOSX_DEPLOYMENT_TARGET: "10.15"
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-12, windows-latest]
+        os: [ubuntu-20.04, macos-13, windows-latest]
         cibw_archs: ["auto64"]
         cibw_build: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"]
 

.github/workflows/publish_pypi.yml

@@ -14,7 +14,7 @@ jobs:
       MACOSX_DEPLOYMENT_TARGET: "10.15"
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-12, windows-latest]
+        os: [ubuntu-20.04, macos-13, windows-latest]
         cibw_archs: ["auto64"]
         cibw_build: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"]
 

src/filepattern/cpp/bindings.cpp

@@ -22,10 +22,12 @@ PYBIND11_MODULE(backend, m){
                       const std::string&, 
                       const std::string&,  
                       bool, 
+                      bool,
                       bool>())
         .def(py::init<const std::vector<std::string>&,
                       const std::string&, 
                       bool, 
+                      bool,
                       bool>())
         .def("getMatching", &FilePattern::getMatching)
         .def("getOccurrences", &FilePattern::getOccurrences)

src/filepattern/cpp/external/external_filepattern.cpp

@@ -2,7 +2,7 @@
 
 using namespace std;
 
-ExternalFilePattern::ExternalFilePattern(const string& path, const string& filePattern, const string& block_size, bool recursive, bool suppressWarnings):
+ExternalFilePattern::ExternalFilePattern(const string& path, const string& filePattern, const string& block_size, bool recursive, bool suppressWarnings, bool sorted):
 ExternalPattern(path, block_size, recursive) {
 
     this->setSuppressWarnings(suppressWarnings);
@@ -23,13 +23,17 @@ ExternalPattern(path, block_size, recursive) {
     this->setFirstCall(true); // first call to next() has not occurred
 
     this->matchFiles(); // match files to pattern
-
-    ExternalMergeSort sort = ExternalMergeSort(std_map,
-                                               this->getValidFilesPath(),
-                                               this->getValidFilesPath(),
-                                               this->stream_.getBlockSizeStr(),
-                                               "",
-                                               this->stream_.map_size_);
+    
+    this->setIsSorted(sorted);
+
+    if (isSorted()) {
+        ExternalMergeSort sort = ExternalMergeSort(std_map,
+                                                this->getValidFilesPath(),
+                                                this->getValidFilesPath(),
+                                                this->stream_.getBlockSizeStr(),
+                                                "",
+                                                this->stream_.map_size_);
+    }
 
     this->group_stream_.open(this->stream_.getValidFilesPath());
     this->infile_.open(this->getValidFilesPath()); // open temp file for the valid files

src/filepattern/cpp/external/external_filepattern.hpp

@@ -33,7 +33,7 @@ class ExternalFilePattern : public ExternalPattern {
          * @param recursive Iterate over all subdirectories if true
          */
 
-        ExternalFilePattern(const std::string& path, const std::string& file_pattern, const std::string& block_size="50 MB", bool recursive=false, bool suppress_warnings=false);
+        ExternalFilePattern(const std::string& path, const std::string& file_pattern, const std::string& block_size="50 MB", bool recursive=false, bool suppress_warnings=false, bool sorted=true);
 
         ExternalFilePattern(){}
 

src/filepattern/cpp/external/external_pattern.cpp

@@ -185,9 +185,12 @@ void ExternalPattern::groupByHelper(){
             grouped_variables.clear();
             for(auto& g: vec.first) grouped_variables.push_back(g);
             // Sort the matched files by the group_by parameter
-            sort(vec.second.begin(), vec.second.end(), [&group_by = as_const(group_by)](Tuple& p1, Tuple& p2){
-                return get<0>(p1)[group_by] < get<0>(p2)[group_by];
-            });
+
+            if (isSorted()) {
+                sort(vec.second.begin(), vec.second.end(), [&group_by = as_const(group_by)](Tuple& p1, Tuple& p2){
+                    return get<0>(p1)[group_by] < get<0>(p2)[group_by];
+                });
+            }
 
             Types current_value = get<0>(vec.second[0])[group_by]; // get the value of variable
             vector<Tuple> empty_vec;
@@ -207,9 +210,12 @@ void ExternalPattern::groupByHelper(){
 
                 grouped_variables.push_back(make_pair(group_by, current_value));
                 temp_group.push_back(make_pair(grouped_variables, temp_vec));
-                sort(temp_group[group_ptr].second.begin(), temp_group[group_ptr].second.end(), [](Tuple& m1, Tuple& m2){
-                    return get<1>(m1)[0] < get<1>(m2)[0];
-                });
+
+                if (isSorted()) {
+                    sort(temp_group[group_ptr].second.begin(), temp_group[group_ptr].second.end(), [](Tuple& m1, Tuple& m2){
+                        return get<1>(m1)[0] < get<1>(m2)[0];
+                    });
+                }
                 temp_vec.clear();
 
                 if (i < vec.second.size()){
@@ -290,10 +296,12 @@ void ExternalPattern::nextGroup(){
             } else {
 
                 // update variable value and end loop on variable value change
-                        // sort block by basename
-                sort(this->current_group_[0].second.begin(), this->current_group_[0].second.end(), [](Tuple& m1, Tuple& m2){
-                    return get<1>(m1)[0] < get<1>(m2)[0];
-                });
+                // sort block by basename
+                if (isSorted()) {
+                    sort(this->current_group_[0].second.begin(), this->current_group_[0].second.end(), [](Tuple& m1, Tuple& m2){
+                        return get<1>(m1)[0] < get<1>(m2)[0];
+                    });
+                }
                 this->current_value_ = get<0>(this->temp_)[this->group_[0]];
                 value_added = false;
 
@@ -302,9 +310,12 @@ void ExternalPattern::nextGroup(){
             };
         }
     }
-    sort(this->current_group_[0].second.begin(), this->current_group_[0].second.end(), [](Tuple& m1, Tuple& m2){
-        return get<1>(m1)[0] < get<1>(m2)[0];
-    });
+
+    if (isSorted()) {
+        sort(this->current_group_[0].second.begin(), this->current_group_[0].second.end(), [](Tuple& m1, Tuple& m2){
+            return get<1>(m1)[0] < get<1>(m2)[0];
+        });
+    }
     this->groupByHelper();
 }
 
@@ -334,12 +345,15 @@ void ExternalPattern::groupBy(vector<string>& group_by) {
     // sort valid files externally
     string path = this->stream_.getValidFilesPath();
     this->tmp_directories_.push_back(path);
-    ExternalMergeSort sort = ExternalMergeSort(std_map,
-                                               path,
-                                               path,
-                                               this->stream_.getBlockSizeStr(),
-                                               group_by[0],
-                                               this->stream_.map_size_);
+
+    if (isSorted()) {
+        ExternalMergeSort sort = ExternalMergeSort(std_map,
+                                                path,
+                                                path,
+                                                this->stream_.getBlockSizeStr(),
+                                                group_by[0],
+                                                this->stream_.map_size_);
+    }
 }
 
 string ExternalPattern::externalOutPutName(){

src/filepattern/cpp/external/external_stringpattern.cpp

@@ -2,7 +2,7 @@
 
 using namespace std;
 
-ExternalStringPattern::ExternalStringPattern(const string& path, const string& file_pattern, const string& block_size, bool suppress_warnings):
+ExternalStringPattern::ExternalStringPattern(const string& path, const string& file_pattern, const string& block_size, bool suppress_warnings, bool sorted):
 ExternalPattern(path, block_size, false) {
     this->setSuppressWarnings(suppress_warnings);
     this->setPath(path); // store path to target directory
@@ -23,6 +23,8 @@ ExternalPattern(path, block_size, false) {
 
     this->setFirstCall(true); // first call to next() has not occurred
 
+    this->setIsSorted(sorted);
+
     this->matchFiles(); // match files to pattern
     this->group_stream_.open(this->stream_.getValidFilesPath());
     this->infile_.open(this->getValidFilesPath()); // open temp file for the valid files

src/filepattern/cpp/external/external_stringpattern.hpp

@@ -34,7 +34,7 @@ class ExternalStringPattern : public ExternalPattern {
          * @param recursive Iterate over all subdirectories if true
          */
 
-        ExternalStringPattern(const std::string& path, const std::string& file_pattern, const std::string& block_size="50 MB", bool suppress_warnings=false);
+        ExternalStringPattern(const std::string& path, const std::string& file_pattern, const std::string& block_size="50 MB", bool suppress_warnings=false, bool sorted=true);
 
         ~ExternalStringPattern();
 

src/filepattern/cpp/external/external_vectorpattern.cpp

@@ -6,7 +6,7 @@ const std::regex ExternalVectorPattern::STITCH_REGEX_ = std::regex("(corr): (.*)
 const std::vector<std::regex> ExternalVectorPattern::STITCH_REGEX_VECTOR_ = {std::regex("(corr):\\s*(.*?);"), std::regex("(position):\\s*\\((.*?),\\s*(.*?)\\);"), std::regex("(grid):\\s*\\((.*),\\s*(.*)\\);")};
 const std::vector<std::string> ExternalVectorPattern::STITCH_VARIABLES_ = {"correlation","posX","posY","gridX","gridY"}; // stitching vector variables
 
-ExternalVectorPattern::ExternalVectorPattern(const string& path, const string& file_pattern, const string& block_size, bool suppress_warnings):
+ExternalVectorPattern::ExternalVectorPattern(const string& path, const string& file_pattern, const string& block_size, bool suppress_warnings, bool sorted):
 ExternalPattern(path, block_size, false){
     this->setSuppressWarnings(suppress_warnings);
     this->path_ = path; // store path to target directory
@@ -26,6 +26,7 @@ ExternalPattern(path, block_size, false){
     this->setFirstCall(true); // first call to next() has not occurred
 
     this->matchFiles();
+    this->setIsSorted(sorted);
 
     this->group_stream_.open(this->stream_.getValidFilesPath());
     this->infile_.open(this->getValidFilesPath()); // open temp file for the valid files