|
1 | 1 | import functools
|
2 | 2 | import pickle
|
3 |
| -import time |
4 |
| -from multiprocessing import Manager, Pool |
| 3 | +from multiprocessing import Manager |
5 | 4 |
|
6 | 5 |
|
7 | 6 | class SharedLRUCache:
|
@@ -57,253 +56,3 @@ def wrapper(*args, **kwargs):
|
57 | 56 |
|
58 | 57 | def shared_lru_cache(maxsize=128):
|
59 | 58 | return SharedLRUCache(maxsize)
|
60 |
| - |
61 |
| - |
62 |
| -def test_shared_lru_cache_basic(): |
63 |
| - @shared_lru_cache(maxsize=2) |
64 |
| - def func(x): |
65 |
| - return x * 2 |
66 |
| - |
67 |
| - assert func(2) == 4 |
68 |
| - assert func(3) == 6 |
69 |
| - assert func(2) == 4 # Should be cached |
70 |
| - |
71 |
| - |
72 |
| -def test_shared_lru_cache_maxsize(): |
73 |
| - @shared_lru_cache(maxsize=2) |
74 |
| - def func(x): |
75 |
| - return x * 2 |
76 |
| - |
77 |
| - func(1) |
78 |
| - func(2) |
79 |
| - func(3) |
80 |
| - |
81 |
| - # 1 should have been evicted |
82 |
| - assert len(func.cache) == 2 |
83 |
| - assert "((1,), frozenset())" not in func.cache |
84 |
| - |
85 |
| - |
86 |
| -def worker(x): |
87 |
| - @shared_lru_cache(maxsize=10) |
88 |
| - def func(x): |
89 |
| - return x * 2 |
90 |
| - |
91 |
| - return func(x) |
92 |
| - |
93 |
| - |
94 |
| -def test_shared_lru_cache_large_data(): |
95 |
| - @shared_lru_cache(maxsize=2) |
96 |
| - def func(x): |
97 |
| - return b"0" * (10**6) # 1MB of data |
98 |
| - |
99 |
| - data1 = func(1) |
100 |
| - data2 = func(2) |
101 |
| - |
102 |
| - assert len(data1) == 10**6 |
103 |
| - assert len(data2) == 10**6 |
104 |
| - assert len(func.cache) == 2 |
105 |
| - |
106 |
| - |
107 |
| -shared_cache = shared_lru_cache(maxsize=10) |
108 |
| - |
109 |
| - |
110 |
| -def worker(x): |
111 |
| - @shared_cache |
112 |
| - def func(x): |
113 |
| - return x * 2 |
114 |
| - |
115 |
| - return func(x) |
116 |
| - |
117 |
| - |
118 |
| -def test_shared_lru_cache_multiprocessing(): |
119 |
| - with Pool(processes=4) as pool: |
120 |
| - results = pool.map(worker, range(20)) |
121 |
| - assert results == [x * 2 for x in range(20)] |
122 |
| - |
123 |
| - |
124 |
| -def test_shared_lru_cache_different_arg_types(): |
125 |
| - @shared_lru_cache(maxsize=5) |
126 |
| - def func(x): |
127 |
| - return str(x) |
128 |
| - |
129 |
| - assert func(1) == "1" |
130 |
| - assert func("a") == "a" |
131 |
| - assert func(2.5) == "2.5" |
132 |
| - assert func((1, 2)) == "(1, 2)" |
133 |
| - assert func({"a": 1}) == "{'a': 1}" |
134 |
| - |
135 |
| - |
136 |
| -def test_shared_lru_cache_with_kwargs(): |
137 |
| - @shared_lru_cache(maxsize=3) |
138 |
| - def func(x, y=10): |
139 |
| - return x + y |
140 |
| - |
141 |
| - assert func(1) == 11 |
142 |
| - assert func(2, y=20) == 22 |
143 |
| - assert func(3, y=30) == 33 |
144 |
| - assert func(1) == 11 # Should be cached |
145 |
| - assert len(func.cache) == 3 |
146 |
| - |
147 |
| - |
148 |
| -def test_shared_lru_cache_eviction_order(): |
149 |
| - @shared_lru_cache(maxsize=3) |
150 |
| - def func(x): |
151 |
| - return x * 2 |
152 |
| - |
153 |
| - func(1) |
154 |
| - func(2) |
155 |
| - func(3) |
156 |
| - func(4) # This should evict 1 |
157 |
| - |
158 |
| - assert "((1,), frozenset())" not in func.cache |
159 |
| - assert "((2,), frozenset())" in func.cache |
160 |
| - assert "((3,), frozenset())" in func.cache |
161 |
| - assert "((4,), frozenset())" in func.cache |
162 |
| - |
163 |
| - |
164 |
| -def fibonacci(n): |
165 |
| - if n < 2: |
166 |
| - return n |
167 |
| - return fibonacci(n - 1) + fibonacci(n - 2) |
168 |
| - |
169 |
| - |
170 |
| -cached_fibonacci = shared_lru_cache(maxsize=100)(fibonacci) |
171 |
| - |
172 |
| - |
173 |
| -def test_shared_lru_cache_performance(): |
174 |
| - start_time = time.time() |
175 |
| - result = cached_fibonacci(30) # Increased from 4 to 30 for a more meaningful test |
176 |
| - end_time = time.time() |
177 |
| - |
178 |
| - assert result == 832040 # The 30th Fibonacci number |
179 |
| - assert end_time - start_time < 1 # Should be very fast due to caching |
180 |
| - |
181 |
| - |
182 |
| -def test_shared_lru_cache_vs_functools_lru_cache(): |
183 |
| - import functools |
184 |
| - import time |
185 |
| - |
186 |
| - # Define functions with shared_lru_cache and functools.lru_cache |
187 |
| - shared_fib = shared_lru_cache(maxsize=100)(fibonacci) |
188 |
| - |
189 |
| - functools_fib = functools.lru_cache(maxsize=100)(fibonacci) |
190 |
| - |
191 |
| - # Test performance for a smaller Fibonacci number |
192 |
| - n = 32 |
193 |
| - |
194 |
| - # Measure time for shared_lru_cache |
195 |
| - start_time = time.time() |
196 |
| - shared_result = shared_fib(n) |
197 |
| - shared_time = time.time() - start_time |
198 |
| - |
199 |
| - # Measure time for functools.lru_cache |
200 |
| - start_time = time.time() |
201 |
| - functools_result = functools_fib(n) |
202 |
| - functools_time = time.time() - start_time |
203 |
| - |
204 |
| - # Assert that both functions return the same result |
205 |
| - assert shared_result == functools_result |
206 |
| - |
207 |
| - # Print performance comparison |
208 |
| - print(f"shared_lru_cache time: {shared_time:.6f} seconds") |
209 |
| - print(f"functools.lru_cache time: {functools_time:.6f} seconds") |
210 |
| - |
211 |
| - # Assert that the performance difference is within an acceptable range |
212 |
| - # This is a flexible assertion as performance can vary between runs |
213 |
| - assert ( |
214 |
| - abs(shared_time - functools_time) < 0.1 |
215 |
| - ), "Performance difference is too large" |
216 |
| - |
217 |
| - |
218 |
| -# Simulate loading an image from disk |
219 |
| -def load_image(filename): |
220 |
| - import numpy as np |
221 |
| - |
222 |
| - time.sleep(0.2) # Simulate I/O delay |
223 |
| - return np.random.rand(2, 2) # Return a random 1000x1000 array |
224 |
| - |
225 |
| - |
226 |
| -shared_load = shared_lru_cache(maxsize=100)(load_image) |
227 |
| -functools_load = functools.lru_cache(maxsize=100)(load_image) |
228 |
| - |
229 |
| - |
230 |
| -# Helper function for multiprocessing |
231 |
| -def worker_shared(filename): |
232 |
| - return shared_load(filename) |
233 |
| - |
234 |
| - |
235 |
| -def worker_functools(filename): |
236 |
| - return functools_load(filename) |
237 |
| - |
238 |
| - |
239 |
| -def test_shared_lru_cache_vs_lru_cache_multiprocessing_all_miss(): |
240 |
| - import time |
241 |
| - from multiprocessing import Pool |
242 |
| - |
243 |
| - # Define functions with shared_lru_cache and functools.lru_cache |
244 |
| - # Test parameters |
245 |
| - n_workers = 4 |
246 |
| - n_images = 20 |
247 |
| - filenames = [f"image_{i}.jpg" for i in range(n_images)] |
248 |
| - |
249 |
| - # Test shared_lru_cache |
250 |
| - start_time = time.time() |
251 |
| - with Pool(n_workers) as pool: |
252 |
| - shared_results = pool.map(worker_shared, filenames) |
253 |
| - shared_time = time.time() - start_time |
254 |
| - |
255 |
| - # Test functools.lru_cache |
256 |
| - start_time = time.time() |
257 |
| - with Pool(n_workers) as pool: |
258 |
| - functools_results = pool.map(worker_functools, filenames) |
259 |
| - functools_time = time.time() - start_time |
260 |
| - |
261 |
| - # Assert that both methods return the same number of results |
262 |
| - assert len(shared_results) == len(functools_results) == n_images |
263 |
| - |
264 |
| - # Print performance comparison |
265 |
| - print(f"shared_lru_cache time: {shared_time:.6f} seconds") |
266 |
| - print(f"functools.lru_cache time: {functools_time:.6f} seconds") |
267 |
| - |
268 |
| - # Assert that shared_lru_cache is faster |
269 |
| - assert ( |
270 |
| - shared_time < functools_time * 1.5 |
271 |
| - ), "shared_lru_cache should be faster in multiprocessing scenario" |
272 |
| - |
273 |
| - |
274 |
| -def test_shared_lru_cache_vs_lru_cache_multiprocessing_with_hits(): |
275 |
| - import time |
276 |
| - from multiprocessing import Pool |
277 |
| - |
278 |
| - # Test parameters |
279 |
| - n_workers = 4 |
280 |
| - n_images = 20 |
281 |
| - n_repeats = 3 # Number of times to repeat the process to ensure cache hits |
282 |
| - filenames = [f"image_{i}.jpg" for i in range(n_images)] |
283 |
| - |
284 |
| - def run_test(worker_func): |
285 |
| - total_time = 0 |
286 |
| - for _ in range(n_repeats): |
287 |
| - start_time = time.time() |
288 |
| - with Pool(n_workers) as pool: |
289 |
| - results = pool.map(worker_func, filenames) |
290 |
| - total_time += time.time() - start_time |
291 |
| - return total_time / n_repeats, results |
292 |
| - |
293 |
| - # Test shared_lru_cache |
294 |
| - shared_time, shared_results = run_test(worker_shared) |
295 |
| - |
296 |
| - # Test functools.lru_cache |
297 |
| - functools_time, functools_results = run_test(worker_functools) |
298 |
| - |
299 |
| - # Assert that both methods return the same number of results |
300 |
| - assert len(shared_results) == len(functools_results) == n_images |
301 |
| - |
302 |
| - # Print performance comparison |
303 |
| - print(f"shared_lru_cache average time: {shared_time:.6f} seconds") |
304 |
| - print(f"functools.lru_cache average time: {functools_time:.6f} seconds") |
305 |
| - |
306 |
| - # Assert that shared_lru_cache is faster |
307 |
| - assert ( |
308 |
| - shared_time < functools_time |
309 |
| - ), "shared_lru_cache should be faster in multiprocessing scenario with cache hits" |
0 commit comments