Skip to content

Commit f7d188f

Browse files
rogerdahlrogerdahl
committed
Add tests for file iterator
1 parent 20611fb commit f7d188f

File tree

1 file changed

+180
-18
lines changed

1 file changed

+180
-18
lines changed

‎lib_common/src/d1_common/tests/test_file_iterator.py‎

Lines changed: 180 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,29 +17,191 @@
1717
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1818
# See the License for the specific language governing permissions and
1919
# limitations under the License.
20-
20+
import os
2121
import tempfile
2222

2323
import d1_common.iter.dir
24+
import pytest
2425

2526
import d1_test.d1_test_case
2627

2728

29+
TEST_TREE_PATH = {
30+
"dir11": {
31+
"file11": None,
32+
"file12": None,
33+
"dir111": {
34+
"file1111": None,
35+
"file1112": None,
36+
"file1113": None,
37+
"file1114": None,
38+
},
39+
"dir112": {"dir1121": None},
40+
"dir113": {
41+
"file1131": None,
42+
"dir1131": {
43+
"file11311": None,
44+
"file11312": None,
45+
"dir11311": {},
46+
"dir11312": {},
47+
},
48+
"dir1132": {
49+
"file11321": None,
50+
"file11322": None,
51+
"dir11321": {
52+
"dir113211": {
53+
"dir1132111": {
54+
"file11321111": None,
55+
"file11321112": None,
56+
"dir11321111": {"dir113211111": None},
57+
}
58+
}
59+
},
60+
},
61+
},
62+
},
63+
"dir12": {
64+
"dir121": {
65+
"file1211": None,
66+
"file1212": None,
67+
"file1213": None,
68+
"file1214": None,
69+
},
70+
"dir122": {"file1221": None, "file1222": None},
71+
},
72+
"dir13": {},
73+
"dir15": {"file151": None, "dir151": {}},
74+
}
75+
76+
77+
@pytest.fixture(scope="function")
78+
def tree_path(request):
79+
def r(dir_path, branch_dict):
80+
for k, v in branch_dict.items():
81+
p = os.path.join(dir_path, k)
82+
if v is None:
83+
with open(p, "w") as f:
84+
f.write(k)
85+
else:
86+
os.mkdir(p)
87+
r(p, v)
88+
89+
with tempfile.TemporaryDirectory() as tmp_dir_path:
90+
r(tmp_dir_path, TEST_TREE_PATH)
91+
92+
yield tmp_dir_path
93+
94+
95+
@pytest.mark.parametrize("return_dir_paths", [True, False])
96+
@pytest.mark.parametrize("recursive", [True, False])
2897
class TestFileIterator(d1_test.d1_test_case.D1TestCase):
29-
def _create_test_dir(self):
30-
tmp_dir_path = tempfile.mkdtemp()
31-
tmp_file_list = []
32-
for i in range(10):
33-
with tempfile.NamedTemporaryFile(
34-
dir=tmp_dir_path, suffix='.test', delete=False
35-
) as tmp_file:
36-
tmp_file.write('test_file_{}\n'.format(i).encode('utf-8'))
37-
tmp_file_list.append(tmp_file.name)
38-
return tmp_dir_path, tmp_file_list
39-
40-
def test_1000(self):
41-
"""file_iter(): Returns all files in dir"""
42-
tmp_dir_path, tmp_file_list = self._create_test_dir()
43-
itr = d1_common.iter.dir.dir_iter([tmp_dir_path])
44-
iter_file_list = list(itr)
45-
assert sorted(iter_file_list) == sorted(tmp_file_list)
98+
# /dir11
99+
# /dir11/dir111
100+
# /dir11/dir111/file1111
101+
# /dir11/dir111/file1112
102+
# /dir11/dir111/file1113
103+
# /dir11/dir111/file1114
104+
# /dir11/dir112
105+
# /dir11/dir112/dir1121
106+
# /dir11/dir113
107+
# /dir11/dir113/dir1131
108+
# /dir11/dir113/dir1131/dir11311
109+
# /dir11/dir113/dir1131/dir11312
110+
# /dir11/dir113/dir1131/file11311
111+
# /dir11/dir113/dir1131/file11312
112+
# /dir11/dir113/dir1132
113+
# /dir11/dir113/dir1132/dir11321
114+
# /dir11/dir113/dir1132/dir11321/dir113211
115+
# /dir11/dir113/dir1132/dir11321/dir113211/dir1132111
116+
# /dir11/dir113/dir1132/dir11321/dir113211/dir1132111/dir11321111
117+
# /dir11/dir113/dir1132/dir11321/dir113211/dir1132111/dir11321111/dir113211111
118+
# /dir11/dir113/dir1132/dir11321/dir113211/dir1132111/file11321111
119+
# /dir11/dir113/dir1132/dir11321/dir113211/dir1132111/file11321112
120+
# /dir11/dir113/dir1132/file11321
121+
# /dir11/dir113/dir1132/file11322
122+
# /dir11/dir113/file1131
123+
# /dir11/file11
124+
# /dir11/file12
125+
# /dir12
126+
# /dir12/dir121
127+
# /dir12/dir121/file1211
128+
# /dir12/dir121/file1212
129+
# /dir12/dir121/file1213
130+
# /dir12/dir121/file1214
131+
# /dir12/dir122
132+
# /dir12/dir122/file1221
133+
# /dir12/dir122/file1222
134+
# /dir13
135+
# /dir15
136+
# /dir15/dir151
137+
# /dir15/file151
138+
139+
def _check(self, test_path, postfix_str, *path_list, **param_dict):
140+
tmp_path_list = self._add_tmp(path_list, test_path)
141+
found_path_list = self._normalize(
142+
d1_common.iter.dir.dir_iter(tmp_path_list, **param_dict), test_path
143+
)
144+
postfix_str += "_incdirs" if param_dict["return_dir_paths"] else ""
145+
postfix_str += "_recursive" if param_dict["recursive"] else ""
146+
self.sample.assert_equals(found_path_list, postfix_str)
147+
148+
def _add_tmp(self, path_list, test_path):
149+
"""Add tmp root to paths"""
150+
return [os.path.join(test_path, p) for p in path_list]
151+
152+
def _normalize(self, itr, test_path):
153+
"""Trim the /tmp/*/ section and sort for reproducibility
154+
"""
155+
return sorted([v[len(test_path) :] for v in itr])
156+
157+
def test_1000(self, tree_path, return_dir_paths, recursive):
158+
"""file_iter(): Empty dir"""
159+
self._check(
160+
tree_path,
161+
"empty",
162+
"dir13",
163+
return_dir_paths=return_dir_paths,
164+
recursive=recursive,
165+
)
166+
167+
def test_1010(self, tree_path, return_dir_paths, recursive):
168+
"""file_iter(): Dir with dir and file"""
169+
self._check(
170+
tree_path,
171+
"dir_file",
172+
"dir15",
173+
return_dir_paths=return_dir_paths,
174+
recursive=recursive,
175+
)
176+
177+
def test_1020(self, tree_path, return_dir_paths, recursive):
178+
"""file_iter(): Root of deeply nested"""
179+
self._check(
180+
tree_path,
181+
"dir_root_nested",
182+
"dir11",
183+
return_dir_paths=return_dir_paths,
184+
recursive=recursive,
185+
)
186+
187+
@pytest.mark.parametrize(
188+
"sample_postfix,exclude_glob_list",
189+
[
190+
("exclude_files", ["*"]),
191+
("exclude_dirs", ["*/"]),
192+
("exclude_named_dir", ["dir1*/"]),
193+
("exclude_named_file", ["*11*"]),
194+
],
195+
)
196+
def test_1030(
197+
self, tree_path, return_dir_paths, recursive, sample_postfix, exclude_glob_list
198+
):
199+
"""file_iter(): Root of deeply nested"""
200+
self._check(
201+
tree_path,
202+
"dir_root_exclude_{}".format(sample_postfix),
203+
"dir11",
204+
return_dir_paths=return_dir_paths,
205+
recursive=recursive,
206+
exclude_glob_list=exclude_glob_list,
207+
)

0 commit comments

Comments
 (0)