Merge pull request #65 from JesseMckinzie/fix_pybind_iterator_v2

Fix pybind11 iterator

Author: sameeul

ci-utils/envs/conda_cpp.txt

@@ -1 +1 @@
-pybind11 >= 2.10.0, <=2.11.1
+pybind11

environment.yml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.7
   - compilers
   - cmake
-  - pybind11[version='>=2.10.0']
+  - pybind11
   - pydantic
   - pip:
     - sphinx

src/filepattern/cpp/bindings.cpp

@@ -36,36 +36,54 @@ PYBIND11_MODULE(backend, m){
         .def_static("getRegex", &FilePattern::getRegex)
         .def_static("inferPattern", py::overload_cast<const std::string&, std::string&, const std::string&>(&FilePattern::inferPattern))
         .def_static("inferPattern", py::overload_cast<std::vector<std::string>&, std::string&>(&FilePattern::inferPattern))
-        .def("__iter__", [](FilePattern &v){
+        .def("isGrouped", [](FilePattern &v){
             auto& pattern_obj = v.getPatternObject();
+            return !(pattern_obj->group_.size() == 0 || (pattern_obj->group_.size() != 0 && pattern_obj->group_[0] == ""));
+        })
+        .def("iterator", [](FilePattern &v){
+
+            auto& pattern_obj = v.getPatternObject();
+
             if(pattern_obj != nullptr) {
-                if(pattern_obj->external) {
 
-                    if(pattern_obj->group_.size() == 0 || (pattern_obj->group_.size() != 0 && pattern_obj->group_[0] == "")) {
+                return py::make_iterator(pattern_obj->valid_files_.begin(), pattern_obj->valid_files_.end());
+
+            }
+
+        }, py::keep_alive<0, 1>())
+        .def("iteratorExternal", [](FilePattern &v){
 
-                        v.next();
-                        return py::make_iterator(pattern_obj->current_block_.begin(), pattern_obj->current_block_.end());
+            auto& pattern_obj = v.getPatternObject();
 
-                    } else {
+            if(pattern_obj != nullptr) {
+                
+                v.next();
+                return py::make_iterator(pattern_obj->current_block_.begin(), pattern_obj->current_block_.end());
 
-                        v.nextGroup();
-                        return py::make_iterator(pattern_obj->current_group_.begin(), pattern_obj->current_group_.end());
-                    }
+            }
 
-                } else {
+        }, py::keep_alive<0, 1>())
+        .def("iteratorGrouped", [](FilePattern &v){
 
-                    if(pattern_obj->group_.size() == 0 || (pattern_obj->group_.size() != 0 && pattern_obj->group_[0] == "")){
+            auto& pattern_obj = v.getPatternObject();
+
+            if(pattern_obj != nullptr) {
+
+                return py::make_iterator(pattern_obj->valid_grouped_files_.begin(), pattern_obj->valid_grouped_files_.end());
+                
+            }
+
+        }, py::keep_alive<0, 1>())
+        .def("iteratorGroupedExternal", [](FilePattern &v){
+
+            auto& pattern_obj = v.getPatternObject();
+
+            if(pattern_obj != nullptr) {
 
-                        return py::make_iterator(pattern_obj->valid_files_.begin(), pattern_obj->valid_files_.end());
-                        //return py::make_iterator(pattern_obj->valid_grouped_files_.begin(), pattern_obj->valid_grouped_files_.end());
-                    }
-                    else{
+                    v.nextGroup();
+                    return py::make_iterator(pattern_obj->current_group_.begin(), pattern_obj->current_group_.end());
 
-                        return py::make_iterator(pattern_obj->valid_grouped_files_.begin(), pattern_obj->valid_grouped_files_.end());
-                    }
-                }
             }
-            },
-            py::keep_alive<0, 1>()); // Keep vector alive while iterator is used
 
+        }, py::keep_alive<0, 1>());
 }

src/filepattern/filepattern.py

@@ -259,7 +259,13 @@ def __iter__(self) -> Union[List[Tuple[List[Tuple[str, Union[str, int, float]]],
         """
 
         if self._block_size == "":
-            for file in self._file_pattern.__iter__():
+            
+            if (self._file_pattern.isGrouped()):
+                iterator = self._file_pattern.iteratorGrouped()
+            else:
+                iterator = self._file_pattern.iterator()
+            
+            for file in iterator:
                 if (self.pydantic_iterator):
 
                     if (isinstance(file[0], dict)):
@@ -280,8 +286,15 @@ def __iter__(self) -> Union[List[Tuple[List[Tuple[str, Union[str, int, float]]],
                 else:
                     yield file
         else:
+                
             while True:
-                for block in self._file_pattern.__iter__():
+                
+                if (self._file_pattern.isGrouped()):
+                    iterator = self._file_pattern.iteratorGroupedExternal()
+                else:
+                    iterator = self._file_pattern.iteratorExternal()
+                    
+                for block in iterator:
 
                     if self._length() == 0:
                         break