Skip to content

Commit dfcc848

Browse files
fselmofselmo
committed
Validate certain cached requests against a validation threshold:
- Validate requests deemed unsafe to cache when they are not yet within a validation threshold of the chain. These requests rely on data from blocks and are only cached if the blocks they rely on are within this configured threshold. - Allow threshold to be configurable to `finalized`, `safe`, or turned off by setting to `None`. Default this value to finalized (safest option). Configuration is done via an enum. - Never cache requests that make use of block identifiers: `finalized`, `safe`, `latest`, and `pending`. These are moving targets and will never be static, thus should never be cached. `earliest` should not be an issue. - Add tests around the validation threshold boundary to make sure we cache only when appropriate, based on the configured options.
1 parent b7730ac commit dfcc848

File tree

8 files changed

+619
-36
lines changed

8 files changed

+619
-36
lines changed

‎tests/core/caching-utils/test_request_caching.py

Lines changed: 339 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@
1919
CACHEABLE_REQUESTS,
2020
generate_cache_key,
2121
)
22+
from web3._utils.caching.caching_utils import (
23+
ASYNC_INTERNAL_VALIDATION_MAP,
24+
BLOCKHASH_IN_PARAMS,
25+
BLOCKNUM_IN_PARAMS,
26+
BLOCKNUM_IN_RESULT,
27+
INTERNAL_VALIDATION_MAP,
28+
)
2229
from web3.exceptions import (
2330
Web3RPCError,
2431
)
@@ -34,6 +41,7 @@
3441
RPCEndpoint,
3542
)
3643
from web3.utils import (
44+
RequestCacheValidationThreshold,
3745
SimpleCache,
3846
)
3947

@@ -187,9 +195,176 @@ def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(pr
187195
assert _provider_set_on_init.cacheable_requests == {RPCEndpoint("fake_endpoint")}
188196

189197

198+
@pytest.mark.parametrize(
199+
"threshold",
200+
(RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE),
201+
)
202+
@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT)
203+
@pytest.mark.parametrize(
204+
"blocknum,should_cache",
205+
(
206+
("0x0", True),
207+
("0x1", True),
208+
("0x2", True),
209+
("0x3", False),
210+
("0x4", False),
211+
("0x5", False),
212+
),
213+
)
214+
def test_blocknum_validation_against_validation_threshold_when_caching(
215+
threshold, endpoint, blocknum, should_cache, request_mocker
216+
):
217+
w3 = Web3(
218+
HTTPProvider(
219+
cache_allowed_requests=True, request_cache_validation_threshold=threshold
220+
)
221+
)
222+
with request_mocker(
223+
w3,
224+
mock_results={
225+
endpoint: (
226+
# mock the result to requests that return blocks
227+
{"number": blocknum}
228+
if "getBlock" in endpoint
229+
# mock the result to requests that return transactions
230+
else {"blockNumber": blocknum}
231+
),
232+
"eth_getBlockByNumber": lambda _method, params: (
233+
# mock the threshold block to be blocknum "0x2", return
234+
# blocknum otherwise
235+
{"number": "0x2"}
236+
if params[0] == threshold.value
237+
else {"number": params[0]}
238+
),
239+
},
240+
):
241+
assert len(w3.provider._request_cache.items()) == 0
242+
w3.manager.request_blocking(endpoint, [blocknum, False])
243+
assert len(w3.provider._request_cache.items()) == int(should_cache)
244+
245+
246+
@pytest.mark.parametrize(
247+
"threshold",
248+
(RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE),
249+
)
250+
@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS)
251+
@pytest.mark.parametrize(
252+
"block_id,blocknum,should_cache",
253+
(
254+
("earliest", "0x0", True),
255+
("earliest", "0x2", True),
256+
("finalized", "0x2", False),
257+
("safe", "0x2", False),
258+
("latest", "0x2", False),
259+
("pending", None, False),
260+
),
261+
)
262+
def test_block_id_param_caching(
263+
threshold, endpoint, block_id, blocknum, should_cache, request_mocker
264+
):
265+
w3 = Web3(
266+
HTTPProvider(
267+
cache_allowed_requests=True, request_cache_validation_threshold=threshold
268+
)
269+
)
270+
with request_mocker(
271+
w3,
272+
mock_results={
273+
endpoint: "0x0",
274+
"eth_getBlockByNumber": lambda _method, params: (
275+
# mock the threshold block to be blocknum "0x2" for all test cases
276+
{"number": "0x2"}
277+
if params[0] == threshold.value
278+
else {"number": blocknum}
279+
),
280+
},
281+
):
282+
assert len(w3.provider._request_cache.items()) == 0
283+
w3.manager.request_blocking(RPCEndpoint(endpoint), [block_id, False])
284+
assert len(w3.provider._request_cache.items()) == int(should_cache)
285+
286+
287+
@pytest.mark.parametrize(
288+
"threshold",
289+
(RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE),
290+
)
291+
@pytest.mark.parametrize("endpoint", BLOCKHASH_IN_PARAMS)
292+
@pytest.mark.parametrize(
293+
"blocknum,should_cache",
294+
(
295+
("0x0", True),
296+
("0x1", True),
297+
("0x2", True),
298+
("0x3", False),
299+
("0x4", False),
300+
("0x5", False),
301+
),
302+
)
303+
def test_blockhash_validation_against_validation_threshold_when_caching(
304+
threshold, endpoint, blocknum, should_cache, request_mocker
305+
):
306+
w3 = Web3(
307+
HTTPProvider(
308+
cache_allowed_requests=True, request_cache_validation_threshold=threshold
309+
)
310+
)
311+
with request_mocker(
312+
w3,
313+
mock_results={
314+
"eth_getBlockByNumber": lambda _method, params: (
315+
# mock the threshold block to be blocknum "0x2"
316+
{"number": "0x2"}
317+
if params[0] == threshold.value
318+
else {"number": params[0]}
319+
),
320+
"eth_getBlockByHash": {"number": blocknum},
321+
endpoint: "0x0",
322+
},
323+
):
324+
assert len(w3.provider._request_cache.items()) == 0
325+
w3.manager.request_blocking(endpoint, [b"\x00" * 32, False])
326+
cached_items = len(w3.provider._request_cache.items())
327+
assert cached_items == 2 if should_cache else cached_items == 0
328+
329+
330+
def test_request_caching_validation_threshold_is_finalized_by_default():
331+
w3 = Web3(HTTPProvider(cache_allowed_requests=True))
332+
assert (
333+
w3.provider.request_cache_validation_threshold
334+
== RequestCacheValidationThreshold.FINALIZED
335+
)
336+
337+
338+
@pytest.mark.parametrize(
339+
"endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT | BLOCKHASH_IN_PARAMS
340+
)
341+
@pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5"))
342+
def test_request_caching_with_validation_threshold_set_to_none(
343+
endpoint, blocknum, request_mocker
344+
):
345+
w3 = Web3(
346+
HTTPProvider(
347+
cache_allowed_requests=True,
348+
request_cache_validation_threshold=None,
349+
)
350+
)
351+
with request_mocker(w3, mock_results={endpoint: {"number": blocknum}}):
352+
assert len(w3.provider._request_cache.items()) == 0
353+
w3.manager.request_blocking(endpoint, [blocknum, False])
354+
assert len(w3.provider._request_cache.items()) == 1
355+
356+
190357
# -- async -- #
191358

192359

360+
def test_async_cacheable_requests_are_the_same_as_sync():
361+
assert (
362+
set(CACHEABLE_REQUESTS)
363+
== set(INTERNAL_VALIDATION_MAP.keys())
364+
== set(ASYNC_INTERNAL_VALIDATION_MAP.keys())
365+
), "make sure the async and sync cacheable requests are the same"
366+
367+
193368
@pytest_asyncio.fixture
194369
async def async_w3(request_mocker):
195370
_async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True))
@@ -307,3 +482,167 @@ async def test_async_request_caching_does_not_share_state_between_providers(
307482
assert result_b == 22222
308483
assert result_c == 33333
309484
assert result_a_shared_cache == 11111
485+
486+
487+
@pytest.mark.asyncio
488+
@pytest.mark.parametrize(
489+
"threshold",
490+
(RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE),
491+
)
492+
@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT)
493+
@pytest.mark.parametrize(
494+
"blocknum,should_cache",
495+
(
496+
("0x0", True),
497+
("0x1", True),
498+
("0x2", True),
499+
("0x3", False),
500+
("0x4", False),
501+
("0x5", False),
502+
),
503+
)
504+
async def test_async_blocknum_validation_against_validation_threshold(
505+
threshold, endpoint, blocknum, should_cache, request_mocker
506+
):
507+
async_w3 = AsyncWeb3(
508+
AsyncHTTPProvider(
509+
cache_allowed_requests=True, request_cache_validation_threshold=threshold
510+
)
511+
)
512+
async with request_mocker(
513+
async_w3,
514+
mock_results={
515+
endpoint: (
516+
# mock the result to requests that return blocks
517+
{"number": blocknum}
518+
if "getBlock" in endpoint
519+
# mock the result to requests that return transactions
520+
else {"blockNumber": blocknum}
521+
),
522+
"eth_getBlockByNumber": lambda _method, params: (
523+
# mock the threshold block to be blocknum "0x2", return
524+
# blocknum otherwise
525+
{"number": "0x2"}
526+
if params[0] == threshold.value
527+
else {"number": params[0]}
528+
),
529+
},
530+
):
531+
assert len(async_w3.provider._request_cache.items()) == 0
532+
await async_w3.manager.coro_request(endpoint, [blocknum, False])
533+
assert len(async_w3.provider._request_cache.items()) == int(should_cache)
534+
535+
536+
@pytest.mark.asyncio
537+
@pytest.mark.parametrize(
538+
"threshold",
539+
(RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE),
540+
)
541+
@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS)
542+
@pytest.mark.parametrize(
543+
"block_id,blocknum,should_cache",
544+
(
545+
("earliest", "0x0", True),
546+
("earliest", "0x2", True),
547+
("finalized", "0x2", False),
548+
("safe", "0x2", False),
549+
("latest", "0x2", False),
550+
("pending", None, False),
551+
),
552+
)
553+
async def test_async_block_id_param_caching(
554+
threshold, endpoint, block_id, blocknum, should_cache, request_mocker
555+
):
556+
async_w3 = AsyncWeb3(
557+
AsyncHTTPProvider(
558+
cache_allowed_requests=True, request_cache_validation_threshold=threshold
559+
)
560+
)
561+
async with request_mocker(
562+
async_w3,
563+
mock_results={
564+
endpoint: "0x0",
565+
"eth_getBlockByNumber": lambda _method, params: (
566+
# mock the threshold block to be blocknum "0x2" for all test cases
567+
{"number": "0x2"}
568+
if params[0] == threshold.value
569+
else {"number": blocknum}
570+
),
571+
},
572+
):
573+
assert len(async_w3.provider._request_cache.items()) == 0
574+
await async_w3.manager.coro_request(RPCEndpoint(endpoint), [block_id, False])
575+
assert len(async_w3.provider._request_cache.items()) == int(should_cache)
576+
577+
578+
@pytest.mark.asyncio
579+
@pytest.mark.parametrize(
580+
"threshold",
581+
(RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE),
582+
)
583+
@pytest.mark.parametrize("endpoint", BLOCKHASH_IN_PARAMS)
584+
@pytest.mark.parametrize(
585+
"blocknum,should_cache",
586+
(
587+
("0x0", True),
588+
("0x1", True),
589+
("0x2", True),
590+
("0x3", False),
591+
("0x4", False),
592+
("0x5", False),
593+
),
594+
)
595+
async def test_async_blockhash_validation_against_validation_threshold(
596+
threshold, endpoint, blocknum, should_cache, request_mocker
597+
):
598+
async_w3 = AsyncWeb3(
599+
AsyncHTTPProvider(
600+
cache_allowed_requests=True, request_cache_validation_threshold=threshold
601+
)
602+
)
603+
async with request_mocker(
604+
async_w3,
605+
mock_results={
606+
"eth_getBlockByNumber": lambda _method, params: (
607+
# mock the threshold block to be blocknum "0x2"
608+
{"number": "0x2"}
609+
if params[0] == threshold.value
610+
else {"number": params[0]}
611+
),
612+
"eth_getBlockByHash": {"number": blocknum},
613+
endpoint: "0x0",
614+
},
615+
):
616+
assert len(async_w3.provider._request_cache.items()) == 0
617+
await async_w3.manager.coro_request(endpoint, [b"\x00" * 32, False])
618+
cached_items = len(async_w3.provider._request_cache.items())
619+
assert cached_items == 2 if should_cache else cached_items == 0
620+
621+
622+
@pytest.mark.asyncio
623+
async def test_async_request_caching_validation_threshold_is_finalized_by_default():
624+
async_w3 = AsyncWeb3(AsyncHTTPProvider(cache_allowed_requests=True))
625+
assert (
626+
async_w3.provider.request_cache_validation_threshold
627+
== RequestCacheValidationThreshold.FINALIZED
628+
)
629+
630+
631+
@pytest.mark.asyncio
632+
@pytest.mark.parametrize(
633+
"endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT | BLOCKHASH_IN_PARAMS
634+
)
635+
@pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5"))
636+
async def test_async_request_caching_with_validation_threshold_set_to_none(
637+
endpoint, blocknum, request_mocker
638+
):
639+
async_w3 = AsyncWeb3(
640+
AsyncHTTPProvider(
641+
cache_allowed_requests=True,
642+
request_cache_validation_threshold=None,
643+
)
644+
)
645+
async with request_mocker(async_w3, mock_results={endpoint: {"number": blocknum}}):
646+
assert len(async_w3.provider._request_cache.items()) == 0
647+
await async_w3.manager.coro_request(endpoint, [blocknum, False])
648+
assert len(async_w3.provider._request_cache.items()) == 1

‎web3/_utils/caching/__init__.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from .request_caching_validation import (
2+
ASYNC_PROVIDER_TYPE,
3+
SYNC_PROVIDER_TYPE,
4+
)
5+
from .caching_utils import (
6+
CACHEABLE_REQUESTS,
7+
async_handle_request_caching,
8+
generate_cache_key,
9+
handle_request_caching,
10+
is_cacheable_request,
11+
RequestInformation,
12+
)

0 commit comments

Comments
 (0)