diff --git a/chia/_tests/core/data_layer/conftest.py b/chia/_tests/core/data_layer/conftest.py index cf7f1ea18bfc..8bfe569c6eaf 100644 --- a/chia/_tests/core/data_layer/conftest.py +++ b/chia/_tests/core/data_layer/conftest.py @@ -61,8 +61,13 @@ def store_id_fixture() -> bytes32: @pytest.fixture(name="raw_data_store", scope="function") -async def raw_data_store_fixture(database_uri: str) -> AsyncIterable[DataStore]: - async with DataStore.managed(database=database_uri, uri=True) as store: +async def raw_data_store_fixture(database_uri: str, tmp_path: pathlib.Path) -> AsyncIterable[DataStore]: + async with DataStore.managed( + database=database_uri, + uri=True, + merkle_blobs_path=tmp_path.joinpath("merkle-blobs"), + key_value_blobs_path=tmp_path.joinpath("key-value-blobs"), + ) as store: yield store diff --git a/chia/_tests/core/data_layer/old_format/__init__.py b/chia/_tests/core/data_layer/old_format/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-005876c1cdc4d5f1726551b207b9f63efc9cd2f72df80a3a26a1ba73d40d6745-delta-23-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-005876c1cdc4d5f1726551b207b9f63efc9cd2f72df80a3a26a1ba73d40d6745-delta-23-v1.0.dat new file mode 100644 index 000000000000..5fbf94fb25f6 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-005876c1cdc4d5f1726551b207b9f63efc9cd2f72df80a3a26a1ba73d40d6745-delta-23-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-005876c1cdc4d5f1726551b207b9f63efc9cd2f72df80a3a26a1ba73d40d6745-full-23-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-005876c1cdc4d5f1726551b207b9f63efc9cd2f72df80a3a26a1ba73d40d6745-full-23-v1.0.dat new file mode 100644 index 000000000000..1acdd9b448c1 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-005876c1cdc4d5f1726551b207b9f63efc9cd2f72df80a3a26a1ba73d40d6745-full-23-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-01b36e72a975cdc00d6514eea81668d19e8ea3150217ae98cb3361688a016fab-delta-9-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-01b36e72a975cdc00d6514eea81668d19e8ea3150217ae98cb3361688a016fab-delta-9-v1.0.dat new file mode 100644 index 000000000000..fb7b0f2cc2f3 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-01b36e72a975cdc00d6514eea81668d19e8ea3150217ae98cb3361688a016fab-delta-9-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-01b36e72a975cdc00d6514eea81668d19e8ea3150217ae98cb3361688a016fab-full-9-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-01b36e72a975cdc00d6514eea81668d19e8ea3150217ae98cb3361688a016fab-full-9-v1.0.dat new file mode 100644 index 000000000000..8b51f87bd145 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-01b36e72a975cdc00d6514eea81668d19e8ea3150217ae98cb3361688a016fab-full-9-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-06147c3b12d73e9b83b686a8c10b4a36a513c8a93c0ff99ae197f06326278be9-delta-5-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-06147c3b12d73e9b83b686a8c10b4a36a513c8a93c0ff99ae197f06326278be9-delta-5-v1.0.dat new file mode 100644 index 000000000000..9f5307adc257 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-06147c3b12d73e9b83b686a8c10b4a36a513c8a93c0ff99ae197f06326278be9-delta-5-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-06147c3b12d73e9b83b686a8c10b4a36a513c8a93c0ff99ae197f06326278be9-full-5-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-06147c3b12d73e9b83b686a8c10b4a36a513c8a93c0ff99ae197f06326278be9-full-5-v1.0.dat new file mode 100644 index 000000000000..0d415adfe9ed Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-06147c3b12d73e9b83b686a8c10b4a36a513c8a93c0ff99ae197f06326278be9-full-5-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-073c051a5934ad3b8db39eee2189e4300e55f48aaa17ff4ae30eeae088ff544a-delta-22-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-073c051a5934ad3b8db39eee2189e4300e55f48aaa17ff4ae30eeae088ff544a-delta-22-v1.0.dat new file mode 100644 index 000000000000..24f8b153643b Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-073c051a5934ad3b8db39eee2189e4300e55f48aaa17ff4ae30eeae088ff544a-delta-22-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-073c051a5934ad3b8db39eee2189e4300e55f48aaa17ff4ae30eeae088ff544a-full-22-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-073c051a5934ad3b8db39eee2189e4300e55f48aaa17ff4ae30eeae088ff544a-full-22-v1.0.dat new file mode 100644 index 000000000000..216b027e2f23 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-073c051a5934ad3b8db39eee2189e4300e55f48aaa17ff4ae30eeae088ff544a-full-22-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-0cc077559b9c7b4aefe8f8f591c195e0779bebdf89f2ad8285a00ea5f859d965-delta-1-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-0cc077559b9c7b4aefe8f8f591c195e0779bebdf89f2ad8285a00ea5f859d965-delta-1-v1.0.dat new file mode 100644 index 000000000000..24af67833845 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-0cc077559b9c7b4aefe8f8f591c195e0779bebdf89f2ad8285a00ea5f859d965-delta-1-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-0cc077559b9c7b4aefe8f8f591c195e0779bebdf89f2ad8285a00ea5f859d965-full-1-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-0cc077559b9c7b4aefe8f8f591c195e0779bebdf89f2ad8285a00ea5f859d965-full-1-v1.0.dat new file mode 100644 index 000000000000..24af67833845 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-0cc077559b9c7b4aefe8f8f591c195e0779bebdf89f2ad8285a00ea5f859d965-full-1-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-16377275567b723b20936d3f1ec0a2fd83f6ac379b922351a5e4c54949069f3b-delta-2-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-16377275567b723b20936d3f1ec0a2fd83f6ac379b922351a5e4c54949069f3b-delta-2-v1.0.dat new file mode 100644 index 000000000000..3150750103bd Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-16377275567b723b20936d3f1ec0a2fd83f6ac379b922351a5e4c54949069f3b-delta-2-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-16377275567b723b20936d3f1ec0a2fd83f6ac379b922351a5e4c54949069f3b-full-2-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-16377275567b723b20936d3f1ec0a2fd83f6ac379b922351a5e4c54949069f3b-full-2-v1.0.dat new file mode 100644 index 000000000000..1a53726373dc Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-16377275567b723b20936d3f1ec0a2fd83f6ac379b922351a5e4c54949069f3b-full-2-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-1cb824a7a5f02cd30ac6c38e8f6216780d9bfa2d24811d282a368dcd541438a7-delta-29-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-1cb824a7a5f02cd30ac6c38e8f6216780d9bfa2d24811d282a368dcd541438a7-delta-29-v1.0.dat new file mode 100644 index 000000000000..adec0e07927c Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-1cb824a7a5f02cd30ac6c38e8f6216780d9bfa2d24811d282a368dcd541438a7-delta-29-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-1cb824a7a5f02cd30ac6c38e8f6216780d9bfa2d24811d282a368dcd541438a7-full-29-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-1cb824a7a5f02cd30ac6c38e8f6216780d9bfa2d24811d282a368dcd541438a7-full-29-v1.0.dat new file mode 100644 index 000000000000..8d21f3a04f91 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-1cb824a7a5f02cd30ac6c38e8f6216780d9bfa2d24811d282a368dcd541438a7-full-29-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-27b89dc4809ebc5a3b87757d35e95e2761d978cf121e44fa2773a5c06e4cc7b5-delta-28-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-27b89dc4809ebc5a3b87757d35e95e2761d978cf121e44fa2773a5c06e4cc7b5-delta-28-v1.0.dat new file mode 100644 index 000000000000..abba94518e8a Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-27b89dc4809ebc5a3b87757d35e95e2761d978cf121e44fa2773a5c06e4cc7b5-delta-28-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-27b89dc4809ebc5a3b87757d35e95e2761d978cf121e44fa2773a5c06e4cc7b5-full-28-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-27b89dc4809ebc5a3b87757d35e95e2761d978cf121e44fa2773a5c06e4cc7b5-full-28-v1.0.dat new file mode 100644 index 000000000000..1e4044c97305 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-27b89dc4809ebc5a3b87757d35e95e2761d978cf121e44fa2773a5c06e4cc7b5-full-28-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-28a6b7c134abfaeb0ab58a018313f6c87a61a40a4d9ec9bedf53aa1d12f3ee37-delta-7-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-28a6b7c134abfaeb0ab58a018313f6c87a61a40a4d9ec9bedf53aa1d12f3ee37-delta-7-v1.0.dat new file mode 100644 index 000000000000..9fe9f5386560 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-28a6b7c134abfaeb0ab58a018313f6c87a61a40a4d9ec9bedf53aa1d12f3ee37-delta-7-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-28a6b7c134abfaeb0ab58a018313f6c87a61a40a4d9ec9bedf53aa1d12f3ee37-full-7-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-28a6b7c134abfaeb0ab58a018313f6c87a61a40a4d9ec9bedf53aa1d12f3ee37-full-7-v1.0.dat new file mode 100644 index 000000000000..15275a609767 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-28a6b7c134abfaeb0ab58a018313f6c87a61a40a4d9ec9bedf53aa1d12f3ee37-full-7-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-30a6bfe7cecbeda259a295dc6de3a436357f52388c3b03d86901e7da68565aeb-delta-19-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-30a6bfe7cecbeda259a295dc6de3a436357f52388c3b03d86901e7da68565aeb-delta-19-v1.0.dat new file mode 100644 index 000000000000..413e71349de3 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-30a6bfe7cecbeda259a295dc6de3a436357f52388c3b03d86901e7da68565aeb-delta-19-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-30a6bfe7cecbeda259a295dc6de3a436357f52388c3b03d86901e7da68565aeb-full-19-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-30a6bfe7cecbeda259a295dc6de3a436357f52388c3b03d86901e7da68565aeb-full-19-v1.0.dat new file mode 100644 index 000000000000..c5ff94f5b3c4 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-30a6bfe7cecbeda259a295dc6de3a436357f52388c3b03d86901e7da68565aeb-full-19-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-343a2bf9add798e3ac2e6a571823cf9fa7e8a1bed532143354ead2648bd036ef-delta-10-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-343a2bf9add798e3ac2e6a571823cf9fa7e8a1bed532143354ead2648bd036ef-delta-10-v1.0.dat new file mode 100644 index 000000000000..b078e315534e Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-343a2bf9add798e3ac2e6a571823cf9fa7e8a1bed532143354ead2648bd036ef-delta-10-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-343a2bf9add798e3ac2e6a571823cf9fa7e8a1bed532143354ead2648bd036ef-full-10-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-343a2bf9add798e3ac2e6a571823cf9fa7e8a1bed532143354ead2648bd036ef-full-10-v1.0.dat new file mode 100644 index 000000000000..bb804bd56e63 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-343a2bf9add798e3ac2e6a571823cf9fa7e8a1bed532143354ead2648bd036ef-full-10-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4d90efbc1fb3df324193831ea4a57dd5e10e67d9653343eb18d178272adb0447-delta-17-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4d90efbc1fb3df324193831ea4a57dd5e10e67d9653343eb18d178272adb0447-delta-17-v1.0.dat new file mode 100644 index 000000000000..4d58ce1f961b Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4d90efbc1fb3df324193831ea4a57dd5e10e67d9653343eb18d178272adb0447-delta-17-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4d90efbc1fb3df324193831ea4a57dd5e10e67d9653343eb18d178272adb0447-full-17-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4d90efbc1fb3df324193831ea4a57dd5e10e67d9653343eb18d178272adb0447-full-17-v1.0.dat new file mode 100644 index 000000000000..f55a269095f9 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4d90efbc1fb3df324193831ea4a57dd5e10e67d9653343eb18d178272adb0447-full-17-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4dd2ea099e91635c441f40b36d3f84078a2d818d2dc601c7278e72cbdfe3eca8-delta-20-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4dd2ea099e91635c441f40b36d3f84078a2d818d2dc601c7278e72cbdfe3eca8-delta-20-v1.0.dat new file mode 100644 index 000000000000..bc15c9db353b Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4dd2ea099e91635c441f40b36d3f84078a2d818d2dc601c7278e72cbdfe3eca8-delta-20-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4dd2ea099e91635c441f40b36d3f84078a2d818d2dc601c7278e72cbdfe3eca8-full-20-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4dd2ea099e91635c441f40b36d3f84078a2d818d2dc601c7278e72cbdfe3eca8-full-20-v1.0.dat new file mode 100644 index 000000000000..8830dab7112d Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-4dd2ea099e91635c441f40b36d3f84078a2d818d2dc601c7278e72cbdfe3eca8-full-20-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-509effbdca78639023b933ce6c08a0465fb247e1cd5329e9e9c553940e4b6e46-delta-31-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-509effbdca78639023b933ce6c08a0465fb247e1cd5329e9e9c553940e4b6e46-delta-31-v1.0.dat new file mode 100644 index 000000000000..b441889524d8 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-509effbdca78639023b933ce6c08a0465fb247e1cd5329e9e9c553940e4b6e46-delta-31-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-509effbdca78639023b933ce6c08a0465fb247e1cd5329e9e9c553940e4b6e46-full-31-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-509effbdca78639023b933ce6c08a0465fb247e1cd5329e9e9c553940e4b6e46-full-31-v1.0.dat new file mode 100644 index 000000000000..a7e94e400bb5 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-509effbdca78639023b933ce6c08a0465fb247e1cd5329e9e9c553940e4b6e46-full-31-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5379a4d9ff29c29d1ef0906d22e82c52472753d31806189ab813c43365341b78-delta-40-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5379a4d9ff29c29d1ef0906d22e82c52472753d31806189ab813c43365341b78-delta-40-v1.0.dat new file mode 100644 index 000000000000..ee239d032440 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5379a4d9ff29c29d1ef0906d22e82c52472753d31806189ab813c43365341b78-delta-40-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5379a4d9ff29c29d1ef0906d22e82c52472753d31806189ab813c43365341b78-full-40-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5379a4d9ff29c29d1ef0906d22e82c52472753d31806189ab813c43365341b78-full-40-v1.0.dat new file mode 100644 index 000000000000..b56a69a27015 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5379a4d9ff29c29d1ef0906d22e82c52472753d31806189ab813c43365341b78-full-40-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-55908eda5686a8f89e4c50672cbe893ec1734fb23449dc03325efe7c414f9aa4-delta-49-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-55908eda5686a8f89e4c50672cbe893ec1734fb23449dc03325efe7c414f9aa4-delta-49-v1.0.dat new file mode 100644 index 000000000000..4d989aabec0c Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-55908eda5686a8f89e4c50672cbe893ec1734fb23449dc03325efe7c414f9aa4-delta-49-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-55908eda5686a8f89e4c50672cbe893ec1734fb23449dc03325efe7c414f9aa4-full-49-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-55908eda5686a8f89e4c50672cbe893ec1734fb23449dc03325efe7c414f9aa4-full-49-v1.0.dat new file mode 100644 index 000000000000..2ac50696c2e6 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-55908eda5686a8f89e4c50672cbe893ec1734fb23449dc03325efe7c414f9aa4-full-49-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-57cc2691fb1fb986c99a58bcb0e029d0cd0cff41553d703147c54196d7d9ca63-delta-14-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-57cc2691fb1fb986c99a58bcb0e029d0cd0cff41553d703147c54196d7d9ca63-delta-14-v1.0.dat new file mode 100644 index 000000000000..532608dc28c9 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-57cc2691fb1fb986c99a58bcb0e029d0cd0cff41553d703147c54196d7d9ca63-delta-14-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-57cc2691fb1fb986c99a58bcb0e029d0cd0cff41553d703147c54196d7d9ca63-full-14-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-57cc2691fb1fb986c99a58bcb0e029d0cd0cff41553d703147c54196d7d9ca63-full-14-v1.0.dat new file mode 100644 index 000000000000..71ab54a84bee Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-57cc2691fb1fb986c99a58bcb0e029d0cd0cff41553d703147c54196d7d9ca63-full-14-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5943bf8ae4f5e59969d8570e4f40a8223299febdcfbcf188b3b3e2ab11044e18-delta-34-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5943bf8ae4f5e59969d8570e4f40a8223299febdcfbcf188b3b3e2ab11044e18-delta-34-v1.0.dat new file mode 100644 index 000000000000..fc10a09a6870 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5943bf8ae4f5e59969d8570e4f40a8223299febdcfbcf188b3b3e2ab11044e18-delta-34-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5943bf8ae4f5e59969d8570e4f40a8223299febdcfbcf188b3b3e2ab11044e18-full-34-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5943bf8ae4f5e59969d8570e4f40a8223299febdcfbcf188b3b3e2ab11044e18-full-34-v1.0.dat new file mode 100644 index 000000000000..6d676572e1bf Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-5943bf8ae4f5e59969d8570e4f40a8223299febdcfbcf188b3b3e2ab11044e18-full-34-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6518527b7c939bee60ce6b024cbe90d3b9d8913c56b8ce11a4df5da7ff7db1c8-delta-8-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6518527b7c939bee60ce6b024cbe90d3b9d8913c56b8ce11a4df5da7ff7db1c8-delta-8-v1.0.dat new file mode 100644 index 000000000000..62788e819736 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6518527b7c939bee60ce6b024cbe90d3b9d8913c56b8ce11a4df5da7ff7db1c8-delta-8-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6518527b7c939bee60ce6b024cbe90d3b9d8913c56b8ce11a4df5da7ff7db1c8-full-8-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6518527b7c939bee60ce6b024cbe90d3b9d8913c56b8ce11a4df5da7ff7db1c8-full-8-v1.0.dat new file mode 100644 index 000000000000..c43281fb991a Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6518527b7c939bee60ce6b024cbe90d3b9d8913c56b8ce11a4df5da7ff7db1c8-full-8-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-66ff26a26620379e14a7c91252d27ee4dbe06ad69a3a390a88642fe757f2b288-delta-45-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-66ff26a26620379e14a7c91252d27ee4dbe06ad69a3a390a88642fe757f2b288-delta-45-v1.0.dat new file mode 100644 index 000000000000..f659eea2e9b2 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-66ff26a26620379e14a7c91252d27ee4dbe06ad69a3a390a88642fe757f2b288-delta-45-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-66ff26a26620379e14a7c91252d27ee4dbe06ad69a3a390a88642fe757f2b288-full-45-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-66ff26a26620379e14a7c91252d27ee4dbe06ad69a3a390a88642fe757f2b288-full-45-v1.0.dat new file mode 100644 index 000000000000..40bd84f9eac5 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-66ff26a26620379e14a7c91252d27ee4dbe06ad69a3a390a88642fe757f2b288-full-45-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6bd0a508ee2c4afbe9d4daa811139fd6e54e7f4e16850cbce999fa30f8bdccd2-delta-6-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6bd0a508ee2c4afbe9d4daa811139fd6e54e7f4e16850cbce999fa30f8bdccd2-delta-6-v1.0.dat new file mode 100644 index 000000000000..501b44cac1ba Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6bd0a508ee2c4afbe9d4daa811139fd6e54e7f4e16850cbce999fa30f8bdccd2-delta-6-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6bd0a508ee2c4afbe9d4daa811139fd6e54e7f4e16850cbce999fa30f8bdccd2-full-6-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6bd0a508ee2c4afbe9d4daa811139fd6e54e7f4e16850cbce999fa30f8bdccd2-full-6-v1.0.dat new file mode 100644 index 000000000000..0dccb52867c3 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6bd0a508ee2c4afbe9d4daa811139fd6e54e7f4e16850cbce999fa30f8bdccd2-full-6-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6ce850d0d77ca743fcc2fc792747472e5d2c1c0813aa43abbb370554428fc897-delta-48-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6ce850d0d77ca743fcc2fc792747472e5d2c1c0813aa43abbb370554428fc897-delta-48-v1.0.dat new file mode 100644 index 000000000000..f0c2cd0e9974 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6ce850d0d77ca743fcc2fc792747472e5d2c1c0813aa43abbb370554428fc897-delta-48-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6ce850d0d77ca743fcc2fc792747472e5d2c1c0813aa43abbb370554428fc897-full-48-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6ce850d0d77ca743fcc2fc792747472e5d2c1c0813aa43abbb370554428fc897-full-48-v1.0.dat new file mode 100644 index 000000000000..021b52339cc0 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6ce850d0d77ca743fcc2fc792747472e5d2c1c0813aa43abbb370554428fc897-full-48-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6eb4ca2e1552b156c5969396b49070eb08ad6c96b347359387519be59f7ccaed-delta-26-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6eb4ca2e1552b156c5969396b49070eb08ad6c96b347359387519be59f7ccaed-delta-26-v1.0.dat new file mode 100644 index 000000000000..b3585f712f34 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6eb4ca2e1552b156c5969396b49070eb08ad6c96b347359387519be59f7ccaed-delta-26-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6eb4ca2e1552b156c5969396b49070eb08ad6c96b347359387519be59f7ccaed-full-26-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6eb4ca2e1552b156c5969396b49070eb08ad6c96b347359387519be59f7ccaed-full-26-v1.0.dat new file mode 100644 index 000000000000..f394ced0f224 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-6eb4ca2e1552b156c5969396b49070eb08ad6c96b347359387519be59f7ccaed-full-26-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-71c797fb7592d3f0a5a20c79ab8497ddaa0fd9ec17712e109d25c91b3f3c76e5-delta-3-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-71c797fb7592d3f0a5a20c79ab8497ddaa0fd9ec17712e109d25c91b3f3c76e5-delta-3-v1.0.dat new file mode 100644 index 000000000000..4c7d38f54bf6 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-71c797fb7592d3f0a5a20c79ab8497ddaa0fd9ec17712e109d25c91b3f3c76e5-delta-3-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-71c797fb7592d3f0a5a20c79ab8497ddaa0fd9ec17712e109d25c91b3f3c76e5-full-3-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-71c797fb7592d3f0a5a20c79ab8497ddaa0fd9ec17712e109d25c91b3f3c76e5-full-3-v1.0.dat new file mode 100644 index 000000000000..efe4fad98654 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-71c797fb7592d3f0a5a20c79ab8497ddaa0fd9ec17712e109d25c91b3f3c76e5-full-3-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-73357026053d5a4969e7a6b9aeeef91c14cc6d5f32fc700fe6d21d2a1b22496c-delta-25-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-73357026053d5a4969e7a6b9aeeef91c14cc6d5f32fc700fe6d21d2a1b22496c-delta-25-v1.0.dat new file mode 100644 index 000000000000..7d8fae210bc4 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-73357026053d5a4969e7a6b9aeeef91c14cc6d5f32fc700fe6d21d2a1b22496c-delta-25-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-73357026053d5a4969e7a6b9aeeef91c14cc6d5f32fc700fe6d21d2a1b22496c-full-25-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-73357026053d5a4969e7a6b9aeeef91c14cc6d5f32fc700fe6d21d2a1b22496c-full-25-v1.0.dat new file mode 100644 index 000000000000..778fed297742 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-73357026053d5a4969e7a6b9aeeef91c14cc6d5f32fc700fe6d21d2a1b22496c-full-25-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-7c897e5c46e834ced65bde7de87716acfaa5dffbdb30b5cd9377d8c319df2034-delta-35-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-7c897e5c46e834ced65bde7de87716acfaa5dffbdb30b5cd9377d8c319df2034-delta-35-v1.0.dat new file mode 100644 index 000000000000..9c35a014ee09 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-7c897e5c46e834ced65bde7de87716acfaa5dffbdb30b5cd9377d8c319df2034-delta-35-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-7c897e5c46e834ced65bde7de87716acfaa5dffbdb30b5cd9377d8c319df2034-full-35-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-7c897e5c46e834ced65bde7de87716acfaa5dffbdb30b5cd9377d8c319df2034-full-35-v1.0.dat new file mode 100644 index 000000000000..58987cf93c96 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-7c897e5c46e834ced65bde7de87716acfaa5dffbdb30b5cd9377d8c319df2034-full-35-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-87b8394d80d08117a5a1cd04ed8a682564eab7197a2c090159863591b5108874-delta-4-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-87b8394d80d08117a5a1cd04ed8a682564eab7197a2c090159863591b5108874-delta-4-v1.0.dat new file mode 100644 index 000000000000..b3e9c2a07eae Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-87b8394d80d08117a5a1cd04ed8a682564eab7197a2c090159863591b5108874-delta-4-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-87b8394d80d08117a5a1cd04ed8a682564eab7197a2c090159863591b5108874-full-4-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-87b8394d80d08117a5a1cd04ed8a682564eab7197a2c090159863591b5108874-full-4-v1.0.dat new file mode 100644 index 000000000000..cd8335a032b3 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-87b8394d80d08117a5a1cd04ed8a682564eab7197a2c090159863591b5108874-full-4-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-89eb40b9cc0921c5f5c3feb20927c13a9ada5760f82d219dcee153b7d400165c-delta-41-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-89eb40b9cc0921c5f5c3feb20927c13a9ada5760f82d219dcee153b7d400165c-delta-41-v1.0.dat new file mode 100644 index 000000000000..9ed8b4f295a7 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-89eb40b9cc0921c5f5c3feb20927c13a9ada5760f82d219dcee153b7d400165c-delta-41-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-89eb40b9cc0921c5f5c3feb20927c13a9ada5760f82d219dcee153b7d400165c-full-41-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-89eb40b9cc0921c5f5c3feb20927c13a9ada5760f82d219dcee153b7d400165c-full-41-v1.0.dat new file mode 100644 index 000000000000..37131894dff7 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-89eb40b9cc0921c5f5c3feb20927c13a9ada5760f82d219dcee153b7d400165c-full-41-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8b649433156b8c924436cdec9c6de26106fd6f73a0528570f48748f7b40d7f8a-delta-21-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8b649433156b8c924436cdec9c6de26106fd6f73a0528570f48748f7b40d7f8a-delta-21-v1.0.dat new file mode 100644 index 000000000000..0879ef4771ec Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8b649433156b8c924436cdec9c6de26106fd6f73a0528570f48748f7b40d7f8a-delta-21-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8b649433156b8c924436cdec9c6de26106fd6f73a0528570f48748f7b40d7f8a-full-21-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8b649433156b8c924436cdec9c6de26106fd6f73a0528570f48748f7b40d7f8a-full-21-v1.0.dat new file mode 100644 index 000000000000..6e539f404ed6 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8b649433156b8c924436cdec9c6de26106fd6f73a0528570f48748f7b40d7f8a-full-21-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8d364023a0834c8c3077e236a465493acbf488e4f9d1f4c6cc230343c10a8f7d-delta-42-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8d364023a0834c8c3077e236a465493acbf488e4f9d1f4c6cc230343c10a8f7d-delta-42-v1.0.dat new file mode 100644 index 000000000000..ff5a231bb49f Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8d364023a0834c8c3077e236a465493acbf488e4f9d1f4c6cc230343c10a8f7d-delta-42-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8d364023a0834c8c3077e236a465493acbf488e4f9d1f4c6cc230343c10a8f7d-full-42-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8d364023a0834c8c3077e236a465493acbf488e4f9d1f4c6cc230343c10a8f7d-full-42-v1.0.dat new file mode 100644 index 000000000000..c0e7c22204a5 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-8d364023a0834c8c3077e236a465493acbf488e4f9d1f4c6cc230343c10a8f7d-full-42-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-925689e24a3d98d98676d816cdd8b73e7b2df057d9d4503da9b27bf91d79666c-delta-38-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-925689e24a3d98d98676d816cdd8b73e7b2df057d9d4503da9b27bf91d79666c-delta-38-v1.0.dat new file mode 100644 index 000000000000..aa1fd09c672c Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-925689e24a3d98d98676d816cdd8b73e7b2df057d9d4503da9b27bf91d79666c-delta-38-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-925689e24a3d98d98676d816cdd8b73e7b2df057d9d4503da9b27bf91d79666c-full-38-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-925689e24a3d98d98676d816cdd8b73e7b2df057d9d4503da9b27bf91d79666c-full-38-v1.0.dat new file mode 100644 index 000000000000..25a3ed3eb974 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-925689e24a3d98d98676d816cdd8b73e7b2df057d9d4503da9b27bf91d79666c-full-38-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-937be3d428b19f521be4f98faecc3307ae11ee731c76992f417fa4268d13859e-delta-11-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-937be3d428b19f521be4f98faecc3307ae11ee731c76992f417fa4268d13859e-delta-11-v1.0.dat new file mode 100644 index 000000000000..9b8e7eb7b474 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-937be3d428b19f521be4f98faecc3307ae11ee731c76992f417fa4268d13859e-delta-11-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-937be3d428b19f521be4f98faecc3307ae11ee731c76992f417fa4268d13859e-full-11-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-937be3d428b19f521be4f98faecc3307ae11ee731c76992f417fa4268d13859e-full-11-v1.0.dat new file mode 100644 index 000000000000..c48709891413 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-937be3d428b19f521be4f98faecc3307ae11ee731c76992f417fa4268d13859e-full-11-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-97f34af499b79e2111fc296a598fc9654c2467ea038dfea41fd58241fb3642de-delta-32-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-97f34af499b79e2111fc296a598fc9654c2467ea038dfea41fd58241fb3642de-delta-32-v1.0.dat new file mode 100644 index 000000000000..d2e4ec16ac29 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-97f34af499b79e2111fc296a598fc9654c2467ea038dfea41fd58241fb3642de-delta-32-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-97f34af499b79e2111fc296a598fc9654c2467ea038dfea41fd58241fb3642de-full-32-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-97f34af499b79e2111fc296a598fc9654c2467ea038dfea41fd58241fb3642de-full-32-v1.0.dat new file mode 100644 index 000000000000..d226e15d693e Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-97f34af499b79e2111fc296a598fc9654c2467ea038dfea41fd58241fb3642de-full-32-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-9d1b737243b8a1d0022f2b36ac53333c6280354a74d77f2a3642dcab35204e59-delta-33-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-9d1b737243b8a1d0022f2b36ac53333c6280354a74d77f2a3642dcab35204e59-delta-33-v1.0.dat new file mode 100644 index 000000000000..fdb00bcc863f Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-9d1b737243b8a1d0022f2b36ac53333c6280354a74d77f2a3642dcab35204e59-delta-33-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-9d1b737243b8a1d0022f2b36ac53333c6280354a74d77f2a3642dcab35204e59-full-33-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-9d1b737243b8a1d0022f2b36ac53333c6280354a74d77f2a3642dcab35204e59-full-33-v1.0.dat new file mode 100644 index 000000000000..5edbc5104f9f Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-9d1b737243b8a1d0022f2b36ac53333c6280354a74d77f2a3642dcab35204e59-full-33-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-a6663f98ef6ddf6db55f01163e34bb2e87aa82f0347e79ce31e8dbfa390c480c-delta-47-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-a6663f98ef6ddf6db55f01163e34bb2e87aa82f0347e79ce31e8dbfa390c480c-delta-47-v1.0.dat new file mode 100644 index 000000000000..f234e57238b9 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-a6663f98ef6ddf6db55f01163e34bb2e87aa82f0347e79ce31e8dbfa390c480c-delta-47-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-a6663f98ef6ddf6db55f01163e34bb2e87aa82f0347e79ce31e8dbfa390c480c-full-47-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-a6663f98ef6ddf6db55f01163e34bb2e87aa82f0347e79ce31e8dbfa390c480c-full-47-v1.0.dat new file mode 100644 index 000000000000..067474b23eaf Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-a6663f98ef6ddf6db55f01163e34bb2e87aa82f0347e79ce31e8dbfa390c480c-full-47-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-aa77376d1ccd3664e5c6366e010c52a978fedbf40f5ce262fee71b2e7fe0c6a9-delta-50-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-aa77376d1ccd3664e5c6366e010c52a978fedbf40f5ce262fee71b2e7fe0c6a9-delta-50-v1.0.dat new file mode 100644 index 000000000000..5d22567acaf1 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-aa77376d1ccd3664e5c6366e010c52a978fedbf40f5ce262fee71b2e7fe0c6a9-delta-50-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-aa77376d1ccd3664e5c6366e010c52a978fedbf40f5ce262fee71b2e7fe0c6a9-full-50-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-aa77376d1ccd3664e5c6366e010c52a978fedbf40f5ce262fee71b2e7fe0c6a9-full-50-v1.0.dat new file mode 100644 index 000000000000..81be8e48bf08 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-aa77376d1ccd3664e5c6366e010c52a978fedbf40f5ce262fee71b2e7fe0c6a9-full-50-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b0f28514741ed1a71f5c6544bf92f9e0e493c5f3cf28328909771d8404eff626-delta-24-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b0f28514741ed1a71f5c6544bf92f9e0e493c5f3cf28328909771d8404eff626-delta-24-v1.0.dat new file mode 100644 index 000000000000..7254580846e9 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b0f28514741ed1a71f5c6544bf92f9e0e493c5f3cf28328909771d8404eff626-delta-24-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b0f28514741ed1a71f5c6544bf92f9e0e493c5f3cf28328909771d8404eff626-full-24-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b0f28514741ed1a71f5c6544bf92f9e0e493c5f3cf28328909771d8404eff626-full-24-v1.0.dat new file mode 100644 index 000000000000..70590b56fbb5 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b0f28514741ed1a71f5c6544bf92f9e0e493c5f3cf28328909771d8404eff626-full-24-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b3efee5358e6eb89ab3b60db2d128d57eef39e8538fb63c5632412d4f8e7d09e-delta-44-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b3efee5358e6eb89ab3b60db2d128d57eef39e8538fb63c5632412d4f8e7d09e-delta-44-v1.0.dat new file mode 100644 index 000000000000..1f25d685eaa6 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b3efee5358e6eb89ab3b60db2d128d57eef39e8538fb63c5632412d4f8e7d09e-delta-44-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b3efee5358e6eb89ab3b60db2d128d57eef39e8538fb63c5632412d4f8e7d09e-full-44-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b3efee5358e6eb89ab3b60db2d128d57eef39e8538fb63c5632412d4f8e7d09e-full-44-v1.0.dat new file mode 100644 index 000000000000..5b3be76c3bcd Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-b3efee5358e6eb89ab3b60db2d128d57eef39e8538fb63c5632412d4f8e7d09e-full-44-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bb0b56b6eb7acbb4e80893b04c72412fe833418232e1ed7b06d97d7a7f08b4e1-delta-16-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bb0b56b6eb7acbb4e80893b04c72412fe833418232e1ed7b06d97d7a7f08b4e1-delta-16-v1.0.dat new file mode 100644 index 000000000000..ddf393650846 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bb0b56b6eb7acbb4e80893b04c72412fe833418232e1ed7b06d97d7a7f08b4e1-delta-16-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bb0b56b6eb7acbb4e80893b04c72412fe833418232e1ed7b06d97d7a7f08b4e1-full-16-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bb0b56b6eb7acbb4e80893b04c72412fe833418232e1ed7b06d97d7a7f08b4e1-full-16-v1.0.dat new file mode 100644 index 000000000000..6f7a4d94a9fe Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bb0b56b6eb7acbb4e80893b04c72412fe833418232e1ed7b06d97d7a7f08b4e1-full-16-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bc45262b757ff494b53bd2a8fba0f5511cc1f9c2a2c5360e04ea8cebbf6409df-delta-13-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bc45262b757ff494b53bd2a8fba0f5511cc1f9c2a2c5360e04ea8cebbf6409df-delta-13-v1.0.dat new file mode 100644 index 000000000000..1cb6245829ac Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bc45262b757ff494b53bd2a8fba0f5511cc1f9c2a2c5360e04ea8cebbf6409df-delta-13-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bc45262b757ff494b53bd2a8fba0f5511cc1f9c2a2c5360e04ea8cebbf6409df-full-13-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bc45262b757ff494b53bd2a8fba0f5511cc1f9c2a2c5360e04ea8cebbf6409df-full-13-v1.0.dat new file mode 100644 index 000000000000..88602da4fc6c Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bc45262b757ff494b53bd2a8fba0f5511cc1f9c2a2c5360e04ea8cebbf6409df-full-13-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bd0494ba430aff13458b557113b073d226eaf11257dfe26ff3323fa1cfe1335b-delta-39-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bd0494ba430aff13458b557113b073d226eaf11257dfe26ff3323fa1cfe1335b-delta-39-v1.0.dat new file mode 100644 index 000000000000..5b395176dda1 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bd0494ba430aff13458b557113b073d226eaf11257dfe26ff3323fa1cfe1335b-delta-39-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bd0494ba430aff13458b557113b073d226eaf11257dfe26ff3323fa1cfe1335b-full-39-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bd0494ba430aff13458b557113b073d226eaf11257dfe26ff3323fa1cfe1335b-full-39-v1.0.dat new file mode 100644 index 000000000000..b127cba0e8fb Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-bd0494ba430aff13458b557113b073d226eaf11257dfe26ff3323fa1cfe1335b-full-39-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cd04f5fbba1553fa728b4dd8131d4723aaac288e0c7dc080447fbf0872c0a6eb-delta-36-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cd04f5fbba1553fa728b4dd8131d4723aaac288e0c7dc080447fbf0872c0a6eb-delta-36-v1.0.dat new file mode 100644 index 000000000000..a60e870db953 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cd04f5fbba1553fa728b4dd8131d4723aaac288e0c7dc080447fbf0872c0a6eb-delta-36-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cd04f5fbba1553fa728b4dd8131d4723aaac288e0c7dc080447fbf0872c0a6eb-full-36-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cd04f5fbba1553fa728b4dd8131d4723aaac288e0c7dc080447fbf0872c0a6eb-full-36-v1.0.dat new file mode 100644 index 000000000000..be684480740a Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cd04f5fbba1553fa728b4dd8131d4723aaac288e0c7dc080447fbf0872c0a6eb-full-36-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cdd2399557fb3163a848f08831fdc833703354edb19a0d32a965fdb140f160c2-delta-18-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cdd2399557fb3163a848f08831fdc833703354edb19a0d32a965fdb140f160c2-delta-18-v1.0.dat new file mode 100644 index 000000000000..7272e143b336 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cdd2399557fb3163a848f08831fdc833703354edb19a0d32a965fdb140f160c2-delta-18-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cdd2399557fb3163a848f08831fdc833703354edb19a0d32a965fdb140f160c2-full-18-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cdd2399557fb3163a848f08831fdc833703354edb19a0d32a965fdb140f160c2-full-18-v1.0.dat new file mode 100644 index 000000000000..3e7a99dbf8b0 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cdd2399557fb3163a848f08831fdc833703354edb19a0d32a965fdb140f160c2-full-18-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cf7a08fca7b1332095242e4d9800f4b94a3f4eaae88fe8407da42736d54b9e18-delta-37-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cf7a08fca7b1332095242e4d9800f4b94a3f4eaae88fe8407da42736d54b9e18-delta-37-v1.0.dat new file mode 100644 index 000000000000..537947903540 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cf7a08fca7b1332095242e4d9800f4b94a3f4eaae88fe8407da42736d54b9e18-delta-37-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cf7a08fca7b1332095242e4d9800f4b94a3f4eaae88fe8407da42736d54b9e18-full-37-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cf7a08fca7b1332095242e4d9800f4b94a3f4eaae88fe8407da42736d54b9e18-full-37-v1.0.dat new file mode 100644 index 000000000000..f08a438dd83b Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-cf7a08fca7b1332095242e4d9800f4b94a3f4eaae88fe8407da42736d54b9e18-full-37-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-d1f97465a9f52187e2ef3a0d811a1258f52380a65340c55f3e8e65b92753bc13-delta-15-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-d1f97465a9f52187e2ef3a0d811a1258f52380a65340c55f3e8e65b92753bc13-delta-15-v1.0.dat new file mode 100644 index 000000000000..9787996669ba Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-d1f97465a9f52187e2ef3a0d811a1258f52380a65340c55f3e8e65b92753bc13-delta-15-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-d1f97465a9f52187e2ef3a0d811a1258f52380a65340c55f3e8e65b92753bc13-full-15-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-d1f97465a9f52187e2ef3a0d811a1258f52380a65340c55f3e8e65b92753bc13-full-15-v1.0.dat new file mode 100644 index 000000000000..da1bed276d80 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-d1f97465a9f52187e2ef3a0d811a1258f52380a65340c55f3e8e65b92753bc13-full-15-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e475eccd4ee597e5ff67b1a249e37d65d6e3f754c3f0379fdb43692513588fef-delta-46-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e475eccd4ee597e5ff67b1a249e37d65d6e3f754c3f0379fdb43692513588fef-delta-46-v1.0.dat new file mode 100644 index 000000000000..943925f33329 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e475eccd4ee597e5ff67b1a249e37d65d6e3f754c3f0379fdb43692513588fef-delta-46-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e475eccd4ee597e5ff67b1a249e37d65d6e3f754c3f0379fdb43692513588fef-full-46-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e475eccd4ee597e5ff67b1a249e37d65d6e3f754c3f0379fdb43692513588fef-full-46-v1.0.dat new file mode 100644 index 000000000000..d27d130e98f0 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e475eccd4ee597e5ff67b1a249e37d65d6e3f754c3f0379fdb43692513588fef-full-46-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e82e63517d78fd65b23a05c3b9a98cf905ddad7026995a238bfe634006b84cd0-delta-27-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e82e63517d78fd65b23a05c3b9a98cf905ddad7026995a238bfe634006b84cd0-delta-27-v1.0.dat new file mode 100644 index 000000000000..1a8f75dbfbe1 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e82e63517d78fd65b23a05c3b9a98cf905ddad7026995a238bfe634006b84cd0-delta-27-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e82e63517d78fd65b23a05c3b9a98cf905ddad7026995a238bfe634006b84cd0-full-27-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e82e63517d78fd65b23a05c3b9a98cf905ddad7026995a238bfe634006b84cd0-full-27-v1.0.dat new file mode 100644 index 000000000000..4cb7e9768c52 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-e82e63517d78fd65b23a05c3b9a98cf905ddad7026995a238bfe634006b84cd0-full-27-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-ed2cf0fd6c0f6237c87c161e1fca303b3fbe6c04e01f652b88720b4572143349-delta-12-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-ed2cf0fd6c0f6237c87c161e1fca303b3fbe6c04e01f652b88720b4572143349-delta-12-v1.0.dat new file mode 100644 index 000000000000..fc5ba60691c3 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-ed2cf0fd6c0f6237c87c161e1fca303b3fbe6c04e01f652b88720b4572143349-delta-12-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-ed2cf0fd6c0f6237c87c161e1fca303b3fbe6c04e01f652b88720b4572143349-full-12-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-ed2cf0fd6c0f6237c87c161e1fca303b3fbe6c04e01f652b88720b4572143349-full-12-v1.0.dat new file mode 100644 index 000000000000..f0ae2f3a5c39 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-ed2cf0fd6c0f6237c87c161e1fca303b3fbe6c04e01f652b88720b4572143349-full-12-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f6e454eaf24a83c46a7bed4c19260a0a3ce0ed5c51739cb6d748d4913dc2ef58-delta-30-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f6e454eaf24a83c46a7bed4c19260a0a3ce0ed5c51739cb6d748d4913dc2ef58-delta-30-v1.0.dat new file mode 100644 index 000000000000..de04419f1302 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f6e454eaf24a83c46a7bed4c19260a0a3ce0ed5c51739cb6d748d4913dc2ef58-delta-30-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f6e454eaf24a83c46a7bed4c19260a0a3ce0ed5c51739cb6d748d4913dc2ef58-full-30-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f6e454eaf24a83c46a7bed4c19260a0a3ce0ed5c51739cb6d748d4913dc2ef58-full-30-v1.0.dat new file mode 100644 index 000000000000..273f4e1a97aa Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f6e454eaf24a83c46a7bed4c19260a0a3ce0ed5c51739cb6d748d4913dc2ef58-full-30-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f7ad2bdf86d9609b4d6381086ec1e296bf558e2ff467ead29dd7fa6e31bacc56-delta-43-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f7ad2bdf86d9609b4d6381086ec1e296bf558e2ff467ead29dd7fa6e31bacc56-delta-43-v1.0.dat new file mode 100644 index 000000000000..6813376f2ee5 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f7ad2bdf86d9609b4d6381086ec1e296bf558e2ff467ead29dd7fa6e31bacc56-delta-43-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f7ad2bdf86d9609b4d6381086ec1e296bf558e2ff467ead29dd7fa6e31bacc56-full-43-v1.0.dat b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f7ad2bdf86d9609b4d6381086ec1e296bf558e2ff467ead29dd7fa6e31bacc56-full-43-v1.0.dat new file mode 100644 index 000000000000..1f1a7ef1d223 Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/files/2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964-f7ad2bdf86d9609b4d6381086ec1e296bf558e2ff467ead29dd7fa6e31bacc56-full-43-v1.0.dat differ diff --git a/chia/_tests/core/data_layer/old_format/files/__init__.py b/chia/_tests/core/data_layer/old_format/files/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/chia/_tests/core/data_layer/old_format/old_db.sqlite b/chia/_tests/core/data_layer/old_format/old_db.sqlite new file mode 100644 index 000000000000..8a6ce930fb1b Binary files /dev/null and b/chia/_tests/core/data_layer/old_format/old_db.sqlite differ diff --git a/chia/_tests/core/data_layer/test_data_layer_util.py b/chia/_tests/core/data_layer/test_data_layer_util.py index dd40ab33ccfb..cb351d699ad9 100644 --- a/chia/_tests/core/data_layer/test_data_layer_util.py +++ b/chia/_tests/core/data_layer/test_data_layer_util.py @@ -7,6 +7,7 @@ # TODO: update after resolution in https://github.com/pytest-dev/pytest/issues/7469 from _pytest.fixtures import SubRequest +from chia_rs.datalayer import ProofOfInclusion, ProofOfInclusionLayer from chia_rs.sized_bytes import bytes32 from chia._tests.util.misc import Marks, datacases, measure_runtime @@ -14,8 +15,6 @@ from chia.data_layer.data_layer_util import ( ClearPendingRootsRequest, ClearPendingRootsResponse, - ProofOfInclusion, - ProofOfInclusionLayer, Root, Side, Status, @@ -38,10 +37,15 @@ def create_valid_proof_of_inclusion(layer_count: int, other_hash_side: Side) -> other_hashes = [bytes32([i] * 32) for i in range(layer_count)] for other_hash in other_hashes: - new_layer = ProofOfInclusionLayer.from_hashes( - primary_hash=existing_hash, + if other_hash_side == Side.LEFT: + combined_hash = internal_hash(other_hash, existing_hash) + else: + combined_hash = internal_hash(existing_hash, other_hash) + + new_layer = ProofOfInclusionLayer( other_hash_side=other_hash_side, other_hash=other_hash, + combined_hash=combined_hash, ) layers.append(new_layer) @@ -72,16 +76,16 @@ def invalid_proof_of_inclusion_fixture(request: SubRequest, side: Side) -> Proof a_hash = bytes32(b"f" * 32) if request.param == "bad root hash": - layers[-1] = dataclasses.replace(layers[-1], combined_hash=a_hash) - return dataclasses.replace(valid_proof_of_inclusion, layers=layers) + layers[-1] = layers[-1].replace(combined_hash=a_hash) + return valid_proof_of_inclusion.replace(layers=layers) elif request.param == "bad other hash": - layers[1] = dataclasses.replace(layers[1], other_hash=a_hash) - return dataclasses.replace(valid_proof_of_inclusion, layers=layers) + layers[1] = layers[1].replace(other_hash=a_hash) + return valid_proof_of_inclusion.replace(layers=layers) elif request.param == "bad other side": - layers[1] = dataclasses.replace(layers[1], other_hash_side=layers[1].other_hash_side.other()) - return dataclasses.replace(valid_proof_of_inclusion, layers=layers) + layers[1] = layers[1].replace(other_hash_side=Side(layers[1].other_hash_side).other()) + return valid_proof_of_inclusion.replace(layers=layers) elif request.param == "bad node hash": - return dataclasses.replace(valid_proof_of_inclusion, node_hash=a_hash) + return valid_proof_of_inclusion.replace(node_hash=a_hash) raise Exception(f"Unhandled parametrization: {request.param!r}") # pragma: no cover diff --git a/chia/_tests/core/data_layer/test_data_rpc.py b/chia/_tests/core/data_layer/test_data_rpc.py index 1f62a63878ec..bae6c8cf2219 100644 --- a/chia/_tests/core/data_layer/test_data_rpc.py +++ b/chia/_tests/core/data_layer/test_data_rpc.py @@ -19,6 +19,7 @@ from typing import Any, Optional, cast import anyio +import chia_rs.datalayer import pytest from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint8, uint16, uint32, uint64 @@ -51,12 +52,13 @@ ProofLayer, Status, StoreProofs, + get_delta_filename_path, + get_full_tree_filename_path, key_hash, leaf_hash, ) from chia.data_layer.data_layer_wallet import DataLayerWallet, verify_offer from chia.data_layer.data_store import DataStore -from chia.data_layer.download_data import get_delta_filename_path, get_full_tree_filename_path from chia.data_layer.start_data_layer import create_data_layer_service from chia.simulator.block_tools import BlockTools from chia.simulator.full_node_simulator import FullNodeSimulator @@ -133,6 +135,7 @@ async def init_data_layer( manage_data_interval: int = 5, maximum_full_file_count: Optional[int] = None, group_files_by_store: bool = False, + enable_batch_autoinsert: bool = True, ) -> AsyncIterator[DataLayer]: async with init_data_layer_service( wallet_rpc_port, @@ -141,7 +144,7 @@ async def init_data_layer( wallet_service, manage_data_interval, maximum_full_file_count, - True, + enable_batch_autoinsert, group_files_by_store, ) as data_layer_service: yield data_layer_service._api.data_layer @@ -256,6 +259,7 @@ def create_mnemonic(seed: bytes = b"ab") -> str: @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_create_insert_get( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -335,6 +339,7 @@ async def test_create_insert_get( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_upsert( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -365,6 +370,7 @@ async def test_upsert( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_create_double_insert( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -402,6 +408,7 @@ async def test_create_double_insert( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_keys_values_ancestors( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -415,20 +422,26 @@ async def test_keys_values_ancestors( assert res is not None store_id = bytes32.from_hexstr(res["id"]) await farm_block_check_singleton(data_layer, full_node_api, ph, store_id, wallet=wallet_rpc_api.service) + reference_hashes = [] key1 = b"a" value1 = b"\x01\x02" + reference_hashes.append(leaf_hash(key=key1, value=value1)) changelist: list[dict[str, str]] = [{"action": "insert", "key": key1.hex(), "value": value1.hex()}] key2 = b"b" value2 = b"\x03\x02" + reference_hashes.append(leaf_hash(key=key2, value=value2)) changelist.append({"action": "insert", "key": key2.hex(), "value": value2.hex()}) key3 = b"c" value3 = b"\x04\x05" + reference_hashes.append(leaf_hash(key=key3, value=value3)) changelist.append({"action": "insert", "key": key3.hex(), "value": value3.hex()}) key4 = b"d" value4 = b"\x06\x03" + reference_hashes.append(leaf_hash(key=key4, value=value4)) changelist.append({"action": "insert", "key": key4.hex(), "value": value4.hex()}) key5 = b"e" value5 = b"\x07\x01" + reference_hashes.append(leaf_hash(key=key5, value=value5)) changelist.append({"action": "insert", "key": key5.hex(), "value": value5.hex()}) res = await data_rpc_api.batch_update({"id": store_id.hex(), "changelist": changelist}) update_tx_rec0 = res["tx_id"] @@ -446,9 +459,9 @@ async def test_keys_values_ancestors( assert len(keys["keys"]) == len(dic) for key in keys["keys"]: assert key in dic - val = await data_rpc_api.get_ancestors({"id": store_id.hex(), "hash": val["keys_values"][4]["hash"]}) + val = await data_rpc_api.get_ancestors({"id": store_id.hex(), "hash": reference_hashes[4].hex()}) # todo better assertions for get_ancestors result - assert len(val["ancestors"]) == 3 + assert len(val["ancestors"]) == 2 res_before = await data_rpc_api.get_root({"id": store_id.hex()}) assert res_before["confirmed"] is True assert res_before["timestamp"] > 0 @@ -478,6 +491,7 @@ async def test_keys_values_ancestors( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_get_roots( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -531,6 +545,7 @@ async def test_get_roots( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_get_root_history( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -585,6 +600,7 @@ async def test_get_root_history( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_get_kv_diff( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -652,13 +668,19 @@ async def test_get_kv_diff( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_batch_update_matches_single_operations( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: wallet_rpc_api, full_node_api, wallet_rpc_port, ph, bt = await init_wallet_and_node( self_hostname, one_wallet_and_one_simulator_services ) - async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: + async with init_data_layer( + wallet_rpc_port=wallet_rpc_port, + bt=bt, + db_path=tmp_path, + enable_batch_autoinsert=False, + ) as data_layer: data_rpc_api = DataLayerRpcApi(data_layer) res = await data_rpc_api.create_data_store({}) assert res is not None @@ -724,6 +746,7 @@ async def test_batch_update_matches_single_operations( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_get_owned_stores( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -765,6 +788,7 @@ async def test_get_owned_stores( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_subscriptions( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -1005,10 +1029,8 @@ async def process_for_data_layer_keys( for sleep_time in backoff_times(): try: value = await data_layer.get_value(store_id=store_id, key=expected_key) - except Exception as e: - # TODO: more specific exceptions... - if "Key not found" not in str(e): - raise # pragma: no cover + except chia_rs.datalayer.UnknownKeyError: + pass else: if expected_value is None or value == expected_value: break @@ -1597,6 +1619,7 @@ class MakeAndTakeReference: indirect=["offer_setup"], ) @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_make_and_take_offer(offer_setup: OfferSetup, reference: MakeAndTakeReference) -> None: offer_setup = await populate_offer_setup(offer_setup=offer_setup, count=reference.entries_to_insert) @@ -1709,6 +1732,7 @@ async def test_make_and_then_take_offer_invalid_inclusion_key( @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_verify_offer_rpc_valid(bare_data_layer_api: DataLayerRpcApi) -> None: reference = make_one_take_one_reference @@ -1727,6 +1751,7 @@ async def test_verify_offer_rpc_valid(bare_data_layer_api: DataLayerRpcApi) -> N @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_verify_offer_rpc_invalid(bare_data_layer_api: DataLayerRpcApi) -> None: reference = make_one_take_one_reference broken_taker_offer = copy.deepcopy(reference.make_offer_response) @@ -1747,6 +1772,7 @@ async def test_verify_offer_rpc_invalid(bare_data_layer_api: DataLayerRpcApi) -> @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_make_offer_failure_rolls_back_db(offer_setup: OfferSetup) -> None: # TODO: only needs the maker and db? wallet? reference = make_one_take_one_reference @@ -1789,6 +1815,7 @@ async def test_make_offer_failure_rolls_back_db(offer_setup: OfferSetup) -> None ], ) @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_make_and_cancel_offer(offer_setup: OfferSetup, reference: MakeAndTakeReference) -> None: offer_setup = await populate_offer_setup(offer_setup=offer_setup, count=reference.entries_to_insert) @@ -1865,6 +1892,7 @@ async def test_make_and_cancel_offer(offer_setup: OfferSetup, reference: MakeAnd ], ) @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_make_and_cancel_offer_then_update( offer_setup: OfferSetup, reference: MakeAndTakeReference, secure: bool ) -> None: @@ -1954,6 +1982,7 @@ async def test_make_and_cancel_offer_then_update( ], ) @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_make_and_cancel_offer_not_secure_clears_pending_roots( offer_setup: OfferSetup, reference: MakeAndTakeReference, @@ -1996,6 +2025,7 @@ async def test_make_and_cancel_offer_not_secure_clears_pending_roots( @pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") @pytest.mark.anyio +@pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") async def test_get_sync_status( self_hostname: str, one_wallet_and_one_simulator_services: SimulatorsAndWalletsServices, tmp_path: Path ) -> None: @@ -2975,18 +3005,18 @@ async def test_pagination_rpcs( "total_pages": 1, "total_bytes": 8, "diff": [ - {"type": "DELETE", "key": key6.hex(), "value": value6.hex()}, {"type": "INSERT", "key": key6.hex(), "value": new_value.hex()}, + {"type": "DELETE", "key": key6.hex(), "value": value6.hex()}, ], } assert diff_res == diff_reference - with pytest.raises(Exception, match="Can't find keys"): + with pytest.raises(Exception, match="Cannot find merkle blob"): await data_rpc_api.get_keys( {"id": store_id.hex(), "page": 0, "max_page_size": 100, "root_hash": bytes32([0] * 31 + [1]).hex()} ) - with pytest.raises(Exception, match="Can't find keys and values"): + with pytest.raises(Exception, match="Cannot find merkle blob"): await data_rpc_api.get_keys_values( {"id": store_id.hex(), "page": 0, "max_page_size": 100, "root_hash": bytes32([0] * 31 + [1]).hex()} ) @@ -3026,7 +3056,12 @@ async def test_pagination_cmds( wallet_rpc_api, full_node_api, wallet_rpc_port, ph, bt = await init_wallet_and_node( self_hostname, one_wallet_and_one_simulator_services ) - async with init_data_layer_service(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer_service: + async with init_data_layer_service( + wallet_rpc_port=wallet_rpc_port, + bt=bt, + db_path=tmp_path, + enable_batch_autoinsert=False, + ) as data_layer_service: assert data_layer_service.rpc_server is not None rpc_port = data_layer_service.rpc_server.listen_port data_layer = data_layer_service._api.data_layer @@ -3174,7 +3209,7 @@ async def test_pagination_cmds( if max_page_size is None or max_page_size == 100: assert keys == { "keys": ["0x61616161", "0x6161"], - "root_hash": "0x889a4a61b17be799ae9d36831246672ef857a24091f54481431a83309d4e890e", + "root_hash": "0x3f4ae7b8e10ef48b3114843537d5def989ee0a3b6568af7e720a71730f260fa1", "success": True, "total_bytes": 6, "total_pages": 1, @@ -3194,7 +3229,7 @@ async def test_pagination_cmds( "value": "0x6161", }, ], - "root_hash": "0x889a4a61b17be799ae9d36831246672ef857a24091f54481431a83309d4e890e", + "root_hash": "0x3f4ae7b8e10ef48b3114843537d5def989ee0a3b6568af7e720a71730f260fa1", "success": True, "total_bytes": 9, "total_pages": 1, @@ -3211,7 +3246,7 @@ async def test_pagination_cmds( elif max_page_size == 5: assert keys == { "keys": ["0x61616161"], - "root_hash": "0x889a4a61b17be799ae9d36831246672ef857a24091f54481431a83309d4e890e", + "root_hash": "0x3f4ae7b8e10ef48b3114843537d5def989ee0a3b6568af7e720a71730f260fa1", "success": True, "total_bytes": 6, "total_pages": 2, @@ -3225,7 +3260,7 @@ async def test_pagination_cmds( "value": "0x61", } ], - "root_hash": "0x889a4a61b17be799ae9d36831246672ef857a24091f54481431a83309d4e890e", + "root_hash": "0x3f4ae7b8e10ef48b3114843537d5def989ee0a3b6568af7e720a71730f260fa1", "success": True, "total_bytes": 9, "total_pages": 2, @@ -3412,7 +3447,10 @@ async def test_unsubmitted_batch_update( count=NUM_BLOCKS_WITHOUT_SUBMIT, guarantee_transaction_blocks=True ) keys_values = await data_rpc_api.get_keys_values({"id": store_id.hex()}) - assert keys_values == prev_keys_values + # order agnostic comparison of the list of dicts + assert {item["key"]: item for item in keys_values["keys_values"]} == { + item["key"]: item for item in prev_keys_values["keys_values"] + } pending_root = await data_layer.data_store.get_pending_root(store_id=store_id) assert pending_root is not None @@ -3745,7 +3783,11 @@ class ModifiedStatus(IntEnum): await data_rpc_api.batch_update({"id": store_id.hex(), "changelist": changelist, "submit_on_chain": False}) # Artificially remove the first migration. - async with DataStore.managed(database=tmp_path.joinpath("db.sqlite")) as data_store: + async with DataStore.managed( + database=tmp_path.joinpath("db.sqlite"), + merkle_blobs_path=tmp_path.joinpath("merkle-blobs"), + key_value_blobs_path=tmp_path.joinpath("key-value-blobs"), + ) as data_store: async with data_store.db_wrapper.writer() as writer: await writer.execute("DELETE FROM schema") @@ -3762,7 +3804,9 @@ class ModifiedStatus(IntEnum): update_tx_rec1 = res["tx_id"] await farm_block_with_spend(full_node_api, ph, update_tx_rec1, wallet_rpc_api) keys = await data_rpc_api.get_keys({"id": store_id.hex()}) - assert keys == {"keys": ["0x30303031", "0x30303030"]} + # order agnostic comparison of the list + keys["keys"] = set(keys["keys"]) + assert keys == {"keys": {"0x30303031", "0x30303030"}} @pytest.mark.limit_consensus_modes(reason="does not depend on consensus rules") diff --git a/chia/_tests/core/data_layer/test_data_store.py b/chia/_tests/core/data_layer/test_data_store.py index a61862ee576e..226b1a0d348b 100644 --- a/chia/_tests/core/data_layer/test_data_store.py +++ b/chia/_tests/core/data_layer/test_data_store.py @@ -1,54 +1,54 @@ from __future__ import annotations +import contextlib +import importlib.resources as importlib_resources import itertools import logging import os import random import re +import shutil import statistics import time from collections.abc import Awaitable from dataclasses import dataclass +from hashlib import sha256 from pathlib import Path from random import Random -from typing import Any, Callable, Optional, cast +from typing import Any, BinaryIO, Callable, Optional import aiohttp -import aiosqlite +import chia_rs.datalayer import pytest +from chia_rs.datalayer import KeyAlreadyPresentError, MerkleBlob, TreeIndex from chia_rs.sized_bytes import bytes32 from chia._tests.core.data_layer.util import Example, add_0123_example, add_01234567_example from chia._tests.util.misc import BenchmarkRunner, Marks, boolean_datacases, datacases -from chia.data_layer.data_layer_errors import KeyNotFoundError, NodeHashError, TreeGenerationIncrementingError +from chia.data_layer.data_layer_errors import KeyNotFoundError, TreeGenerationIncrementingError from chia.data_layer.data_layer_util import ( DiffData, InternalNode, - Node, - NodeType, OperationType, - ProofOfInclusion, - ProofOfInclusionLayer, Root, + SerializedNode, ServerInfo, Side, Status, Subscription, TerminalNode, _debug_dump, - leaf_hash, -) -from chia.data_layer.data_store import DataStore -from chia.data_layer.download_data import ( get_delta_filename_path, get_full_tree_filename_path, - insert_from_delta_file, - insert_into_data_store_from_file, - write_files_for_root, + leaf_hash, ) +from chia.data_layer.data_store import DataStore +from chia.data_layer.download_data import insert_from_delta_file, write_files_for_root +from chia.data_layer.util.benchmark import generate_datastore from chia.types.blockchain_format.program import Program from chia.util.byte_types import hexstr_to_bytes from chia.util.db_wrapper import DBWrapper2, generate_in_memory_db_uri +from chia.util.lru_cache import LRUCache log = logging.getLogger(__name__) @@ -57,31 +57,48 @@ table_columns: dict[str, list[str]] = { - "node": ["hash", "node_type", "left", "right", "key", "value"], "root": ["tree_id", "generation", "node_hash", "status"], + "subscriptions": ["tree_id", "url", "ignore_till", "num_consecutive_failures", "from_wallet"], + "schema": ["version_id", "applied_at"], + "ids": ["kv_id", "hash", "blob", "store_id"], + "nodes": ["store_id", "hash", "root_hash", "generation", "idx"], } -# TODO: Someday add tests for malformed DB data to make sure we handle it gracefully -# and with good error messages. - - @pytest.mark.anyio -async def test_valid_node_values_fixture_are_valid(data_store: DataStore, valid_node_values: dict[str, Any]) -> None: - async with data_store.db_wrapper.writer() as writer: - await writer.execute( - """ - INSERT INTO node(hash, node_type, left, right, key, value) - VALUES(:hash, :node_type, :left, :right, :key, :value) - """, - valid_node_values, - ) +async def test_migrate_from_old_format(store_id: bytes32, tmp_path: Path) -> None: + old_format_resources = importlib_resources.files(__name__.rpartition(".")[0]).joinpath("old_format") + db_uri = tmp_path / "old_db.sqlite" + db_uri.write_bytes(old_format_resources.joinpath("old_db.sqlite").read_bytes()) + files_resources = old_format_resources.joinpath("files") + + with importlib_resources.as_file(files_resources) as files_path: + async with DataStore.managed( + database=db_uri, + uri=True, + merkle_blobs_path=tmp_path.joinpath("merkle-blobs"), + key_value_blobs_path=tmp_path.joinpath("key-value-blobs"), + ) as data_store: + await data_store.migrate_db(files_path) + root = await data_store.get_tree_root(store_id=store_id) + expected = Root( + store_id=bytes32.fromhex("2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e612073746f7265206964"), + node_hash=bytes32.fromhex("aa77376d1ccd3664e5c6366e010c52a978fedbf40f5ce262fee71b2e7fe0c6a9"), + generation=50, + status=Status.COMMITTED, + ) + assert root == expected +# TODO: Someday add tests for malformed DB data to make sure we handle it gracefully +# and with good error messages. @pytest.mark.parametrize(argnames=["table_name", "expected_columns"], argvalues=table_columns.items()) @pytest.mark.anyio async def test_create_creates_tables_and_columns( - database_uri: str, table_name: str, expected_columns: list[str] + database_uri: str, + table_name: str, + expected_columns: list[str], + tmp_path: Path, ) -> None: # Never string-interpolate sql queries... Except maybe in tests when it does not # allow you to parametrize the query. @@ -93,7 +110,12 @@ async def test_create_creates_tables_and_columns( columns = await cursor.fetchall() assert columns == [] - async with DataStore.managed(database=database_uri, uri=True): + async with DataStore.managed( + database=database_uri, + uri=True, + merkle_blobs_path=tmp_path.joinpath("merkle-blobs"), + key_value_blobs_path=tmp_path.joinpath("key-value-blobs"), + ): async with db_wrapper.reader() as reader: cursor = await reader.execute(query) columns = await cursor.fetchall() @@ -211,46 +233,6 @@ async def test_get_tree_generation_returns_none_when_none_available( await raw_data_store.get_tree_generation(store_id=store_id) -@pytest.mark.anyio -async def test_insert_internal_node_does_nothing_if_matching(data_store: DataStore, store_id: bytes32) -> None: - await add_01234567_example(data_store=data_store, store_id=store_id) - - kv_node = await data_store.get_node_by_key(key=b"\x04", store_id=store_id) - ancestors = await data_store.get_ancestors(node_hash=kv_node.hash, store_id=store_id) - parent = ancestors[0] - - async with data_store.db_wrapper.reader() as reader: - cursor = await reader.execute("SELECT * FROM node") - before = await cursor.fetchall() - - await data_store._insert_internal_node(left_hash=parent.left_hash, right_hash=parent.right_hash) - - async with data_store.db_wrapper.reader() as reader: - cursor = await reader.execute("SELECT * FROM node") - after = await cursor.fetchall() - - assert after == before - - -@pytest.mark.anyio -async def test_insert_terminal_node_does_nothing_if_matching(data_store: DataStore, store_id: bytes32) -> None: - await add_01234567_example(data_store=data_store, store_id=store_id) - - kv_node = await data_store.get_node_by_key(key=b"\x04", store_id=store_id) - - async with data_store.db_wrapper.reader() as reader: - cursor = await reader.execute("SELECT * FROM node") - before = await cursor.fetchall() - - await data_store._insert_terminal_node(key=kv_node.key, value=kv_node.value) - - async with data_store.db_wrapper.reader() as reader: - cursor = await reader.execute("SELECT * FROM node") - after = await cursor.fetchall() - - assert after == before - - @pytest.mark.anyio async def test_build_a_tree( data_store: DataStore, @@ -293,7 +275,7 @@ async def test_get_ancestors(data_store: DataStore, store_id: bytes32) -> None: "c852ecd8fb61549a0a42f9eb9dde65e6c94a01934dbd9c1d35ab94e2a0ae58e2", ] - ancestors_2 = await data_store.get_ancestors_optimized(node_hash=reference_node_hash, store_id=store_id) + ancestors_2 = await data_store.get_ancestors(node_hash=reference_node_hash, store_id=store_id) assert ancestors == ancestors_2 @@ -306,6 +288,10 @@ async def test_get_ancestors_optimized(data_store: DataStore, store_id: bytes32) first_insertions = [True, False, True, False, True, True, False, True, False, True, True, False, False, True, False] deleted_all = False node_count = 0 + node_hashes: list[bytes32] = [] + hash_to_key: dict[bytes32, bytes] = {} + node_hash: Optional[bytes32] + for i in range(1000): is_insert = False if i <= 14: @@ -318,12 +304,10 @@ async def test_get_ancestors_optimized(data_store: DataStore, store_id: bytes32) if not deleted_all: while node_count > 0: node_count -= 1 - seed = bytes32(b"0" * 32) - node_hash = await data_store.get_terminal_node_for_seed(store_id, seed) + node_hash = random.choice(node_hashes) assert node_hash is not None - node = await data_store.get_node(node_hash) - assert isinstance(node, TerminalNode) - await data_store.delete(key=node.key, store_id=store_id, status=Status.COMMITTED) + await data_store.delete(key=hash_to_key[node_hash], store_id=store_id, status=Status.COMMITTED) + node_hashes.remove(node_hash) deleted_all = True is_insert = True else: @@ -335,10 +319,10 @@ async def test_get_ancestors_optimized(data_store: DataStore, store_id: bytes32) key = (i % 200).to_bytes(4, byteorder="big") value = (i % 200).to_bytes(4, byteorder="big") seed = Program.to((key, value)).get_tree_hash() - node_hash = await data_store.get_terminal_node_for_seed(store_id, seed) + node_hash = None if len(node_hashes) == 0 else random.choice(node_hashes) if is_insert: node_count += 1 - side = None if node_hash is None else data_store.get_side_for_seed(seed) + side = None if node_hash is None else (Side.LEFT if seed[0] < 128 else Side.RIGHT) insert_result = await data_store.insert( key=key, @@ -346,10 +330,11 @@ async def test_get_ancestors_optimized(data_store: DataStore, store_id: bytes32) store_id=store_id, reference_node_hash=node_hash, side=side, - use_optimized=False, status=Status.COMMITTED, ) node_hash = insert_result.node_hash + hash_to_key[node_hash] = key + node_hashes.append(node_hash) if node_hash is not None: generation = await data_store.get_tree_generation(store_id=store_id) current_ancestors = await data_store.get_ancestors(node_hash=node_hash, store_id=store_id) @@ -357,39 +342,38 @@ async def test_get_ancestors_optimized(data_store: DataStore, store_id: bytes32) else: node_count -= 1 assert node_hash is not None - node = await data_store.get_node(node_hash) - assert isinstance(node, TerminalNode) - await data_store.delete(key=node.key, store_id=store_id, use_optimized=False, status=Status.COMMITTED) + node_hashes.remove(node_hash) + await data_store.delete(key=hash_to_key[node_hash], store_id=store_id, status=Status.COMMITTED) for generation, node_hash, expected_ancestors in ancestors: - current_ancestors = await data_store.get_ancestors_optimized( + current_ancestors = await data_store.get_ancestors( node_hash=node_hash, store_id=store_id, generation=generation ) assert current_ancestors == expected_ancestors @pytest.mark.anyio -@pytest.mark.parametrize( - "use_optimized", - [True, False], -) @pytest.mark.parametrize( "num_batches", [1, 5, 10, 25], ) -async def test_batch_update( +async def test_batch_update_against_single_operations( data_store: DataStore, store_id: bytes32, - use_optimized: bool, tmp_path: Path, num_batches: int, ) -> None: - total_operations = 1000 if use_optimized else 100 + total_operations = 1000 num_ops_per_batch = total_operations // num_batches saved_batches: list[list[dict[str, Any]]] = [] saved_kv: list[list[TerminalNode]] = [] db_uri = generate_in_memory_db_uri() - async with DataStore.managed(database=db_uri, uri=True) as single_op_data_store: + async with DataStore.managed( + database=db_uri, + uri=True, + merkle_blobs_path=tmp_path.joinpath("merkle-blobs"), + key_value_blobs_path=tmp_path.joinpath("key-value-blobs"), + ) as single_op_data_store: await single_op_data_store.create_tree(store_id, status=Status.COMMITTED) random = Random() random.seed(100, version=2) @@ -412,7 +396,6 @@ async def test_batch_update( key=key, value=value, store_id=store_id, - use_optimized=use_optimized, status=Status.COMMITTED, ) else: @@ -420,7 +403,6 @@ async def test_batch_update( key=key, new_value=value, store_id=store_id, - use_optimized=use_optimized, status=Status.COMMITTED, ) action = "insert" if op_type == "insert" else "upsert" @@ -432,7 +414,6 @@ async def test_batch_update( await single_op_data_store.delete( key=key, store_id=store_id, - use_optimized=use_optimized, status=Status.COMMITTED, ) batch.append({"action": "delete", "key": key}) @@ -446,7 +427,6 @@ async def test_batch_update( key=key, new_value=new_value, store_id=store_id, - use_optimized=use_optimized, status=Status.COMMITTED, ) keys_values[key] = new_value @@ -469,38 +449,13 @@ async def test_batch_update( assert {node.key: node.value for node in current_kv} == { node.key: node.value for node in saved_kv[batch_number] } - queue: list[bytes32] = [root.node_hash] - ancestors: dict[bytes32, bytes32] = {} - while len(queue) > 0: - node_hash = queue.pop(0) - expected_ancestors = [] - ancestor = node_hash - while ancestor in ancestors: - ancestor = ancestors[ancestor] - expected_ancestors.append(ancestor) - result_ancestors = await data_store.get_ancestors_optimized(node_hash, store_id) - assert [node.hash for node in result_ancestors] == expected_ancestors - node = await data_store.get_node(node_hash) - if isinstance(node, InternalNode): - queue.append(node.left_hash) - queue.append(node.right_hash) - ancestors[node.left_hash] = node_hash - ancestors[node.right_hash] = node_hash all_kv = await data_store.get_keys_values(store_id) assert {node.key: node.value for node in all_kv} == keys_values @pytest.mark.anyio -@pytest.mark.parametrize( - "use_optimized", - [True, False], -) -async def test_upsert_ignores_existing_arguments( - data_store: DataStore, - store_id: bytes32, - use_optimized: bool, -) -> None: +async def test_upsert_ignores_existing_arguments(data_store: DataStore, store_id: bytes32) -> None: key = b"key" value = b"value1" @@ -508,7 +463,6 @@ async def test_upsert_ignores_existing_arguments( key=key, value=value, store_id=store_id, - use_optimized=use_optimized, status=Status.COMMITTED, ) node = await data_store.get_node_by_key(key, store_id) @@ -519,7 +473,6 @@ async def test_upsert_ignores_existing_arguments( key=key, new_value=new_value, store_id=store_id, - use_optimized=use_optimized, status=Status.COMMITTED, ) node = await data_store.get_node_by_key(key, store_id) @@ -529,7 +482,6 @@ async def test_upsert_ignores_existing_arguments( key=key, new_value=new_value, store_id=store_id, - use_optimized=use_optimized, status=Status.COMMITTED, ) node = await data_store.get_node_by_key(key, store_id) @@ -540,7 +492,6 @@ async def test_upsert_ignores_existing_arguments( key=key2, new_value=value, store_id=store_id, - use_optimized=use_optimized, status=Status.COMMITTED, ) node = await data_store.get_node_by_key(key2, store_id) @@ -575,30 +526,24 @@ async def test_insert_batch_reference_and_side( ) assert new_root_hash is not None, "batch insert failed or failed to update root" - parent = await data_store.get_node(new_root_hash) - assert isinstance(parent, InternalNode) + merkle_blob = await data_store.get_merkle_blob(store_id=store_id, root_hash=new_root_hash) + nodes_with_indexes = merkle_blob.get_nodes_with_indexes() + nodes = [pair[1] for pair in nodes_with_indexes] + assert len(nodes) == 3 + assert isinstance(nodes[1], chia_rs.datalayer.LeafNode) + assert isinstance(nodes[2], chia_rs.datalayer.LeafNode) + left_terminal_node = await data_store.get_terminal_node(nodes[1].key, nodes[1].value, store_id) + right_terminal_node = await data_store.get_terminal_node(nodes[2].key, nodes[2].value, store_id) if side == Side.LEFT: - child = await data_store.get_node(parent.left_hash) - assert parent.left_hash == child.hash + assert left_terminal_node.key == b"key2" + assert right_terminal_node.key == b"key1" elif side == Side.RIGHT: - child = await data_store.get_node(parent.right_hash) - assert parent.right_hash == child.hash + assert left_terminal_node.key == b"key1" + assert right_terminal_node.key == b"key2" else: # pragma: no cover raise Exception("invalid side for test") -@pytest.mark.anyio -async def test_ancestor_table_unique_inserts(data_store: DataStore, store_id: bytes32) -> None: - await add_0123_example(data_store=data_store, store_id=store_id) - hash_1 = bytes32.from_hexstr("0763561814685fbf92f6ca71fbb1cb11821951450d996375c239979bd63e9535") - hash_2 = bytes32.from_hexstr("924be8ff27e84cba17f5bc918097f8410fab9824713a4668a21c8e060a8cab40") - await data_store._insert_ancestor_table(hash_1, hash_2, store_id, 2) - await data_store._insert_ancestor_table(hash_1, hash_2, store_id, 2) - with pytest.raises(Exception, match=r"^Requested insertion of ancestor"): - await data_store._insert_ancestor_table(hash_1, hash_1, store_id, 2) - await data_store._insert_ancestor_table(hash_1, hash_2, store_id, 2) - - @pytest.mark.anyio async def test_get_pairs( data_store: DataStore, @@ -609,7 +554,7 @@ async def test_get_pairs( pairs = await data_store.get_keys_values(store_id=store_id) - assert [node.hash for node in pairs] == example.terminal_nodes + assert {node.hash for node in pairs} == set(example.terminal_nodes) @pytest.mark.anyio @@ -662,37 +607,6 @@ async def test_inserting_duplicate_key_fails( ) -@pytest.mark.anyio() -async def test_inserting_invalid_length_hash_raises_original_exception( - data_store: DataStore, -) -> None: - with pytest.raises(aiosqlite.IntegrityError): - # casting since we are testing an invalid case - await data_store._insert_node( - node_hash=cast(bytes32, b"\x05"), - node_type=NodeType.TERMINAL, - left_hash=None, - right_hash=None, - key=b"\x06", - value=b"\x07", - ) - - -@pytest.mark.anyio() -async def test_inserting_invalid_length_ancestor_hash_raises_original_exception( - data_store: DataStore, - store_id: bytes32, -) -> None: - with pytest.raises(aiosqlite.IntegrityError): - # casting since we are testing an invalid case - await data_store._insert_ancestor_table( - left_hash=bytes32(b"\x01" * 32), - right_hash=bytes32(b"\x02" * 32), - store_id=store_id, - generation=0, - ) - - @pytest.mark.anyio() async def test_autoinsert_balances_from_scratch(data_store: DataStore, store_id: bytes32) -> None: random = Random() @@ -705,7 +619,7 @@ async def test_autoinsert_balances_from_scratch(data_store: DataStore, store_id: insert_result = await data_store.autoinsert(key, value, store_id, status=Status.COMMITTED) hashes.append(insert_result.node_hash) - heights = {node_hash: len(await data_store.get_ancestors_optimized(node_hash, store_id)) for node_hash in hashes} + heights = {node_hash: len(await data_store.get_ancestors(node_hash, store_id)) for node_hash in hashes} too_tall = {hash: height for hash, height in heights.items() if height > 14} assert too_tall == {} assert 11 <= statistics.mean(heights.values()) <= 12 @@ -715,7 +629,7 @@ async def test_autoinsert_balances_from_scratch(data_store: DataStore, store_id: async def test_autoinsert_balances_gaps(data_store: DataStore, store_id: bytes32) -> None: random = Random() random.seed(101, version=2) - hashes = [] + hashes: list[bytes32] = [] for i in range(2000): key = (i + 100).to_bytes(4, byteorder="big") @@ -723,7 +637,7 @@ async def test_autoinsert_balances_gaps(data_store: DataStore, store_id: bytes32 if i == 0 or i > 10: insert_result = await data_store.autoinsert(key, value, store_id, status=Status.COMMITTED) else: - reference_node_hash = await data_store.get_terminal_node_for_seed(store_id, bytes32.zeros) + reference_node_hash = hashes[-1] insert_result = await data_store.insert( key=key, value=value, @@ -732,11 +646,11 @@ async def test_autoinsert_balances_gaps(data_store: DataStore, store_id: bytes32 side=Side.LEFT, status=Status.COMMITTED, ) - ancestors = await data_store.get_ancestors_optimized(insert_result.node_hash, store_id) + ancestors = await data_store.get_ancestors(insert_result.node_hash, store_id) assert len(ancestors) == i hashes.append(insert_result.node_hash) - heights = {node_hash: len(await data_store.get_ancestors_optimized(node_hash, store_id)) for node_hash in hashes} + heights = {node_hash: len(await data_store.get_ancestors(node_hash, store_id)) for node_hash in hashes} too_tall = {hash: height for hash, height in heights.items() if height > 14} assert too_tall == {} assert 11 <= statistics.mean(heights.values()) <= 12 @@ -874,24 +788,24 @@ async def test_proof_of_inclusion_by_hash(data_store: DataStore, store_id: bytes await _debug_dump(db=data_store.db_wrapper) expected_layers = [ - ProofOfInclusionLayer( + chia_rs.datalayer.ProofOfInclusionLayer( other_hash_side=Side.RIGHT, other_hash=bytes32.fromhex("fb66fe539b3eb2020dfbfadfd601fa318521292b41f04c2057c16fca6b947ca1"), combined_hash=bytes32.fromhex("36cb1fc56017944213055da8cb0178fb0938c32df3ec4472f5edf0dff85ba4a3"), ), - ProofOfInclusionLayer( + chia_rs.datalayer.ProofOfInclusionLayer( other_hash_side=Side.RIGHT, other_hash=bytes32.fromhex("6d3af8d93db948e8b6aa4386958e137c6be8bab726db86789594b3588b35adcd"), combined_hash=bytes32.fromhex("5f67a0ab1976e090b834bf70e5ce2a0f0a9cd474e19a905348c44ae12274d30b"), ), - ProofOfInclusionLayer( + chia_rs.datalayer.ProofOfInclusionLayer( other_hash_side=Side.LEFT, other_hash=bytes32.fromhex("c852ecd8fb61549a0a42f9eb9dde65e6c94a01934dbd9c1d35ab94e2a0ae58e2"), combined_hash=bytes32.fromhex("7a5193a4e31a0a72f6623dfeb2876022ab74a48abb5966088a1c6f5451cc5d81"), ), ] - assert proof == ProofOfInclusion(node_hash=node.hash, layers=expected_layers) + assert proof == chia_rs.datalayer.ProofOfInclusion(node_hash=node.hash, layers=expected_layers) @pytest.mark.anyio @@ -904,26 +818,7 @@ async def test_proof_of_inclusion_by_hash_no_ancestors(data_store: DataStore, st proof = await data_store.get_proof_of_inclusion_by_hash(node_hash=node.hash, store_id=store_id) - assert proof == ProofOfInclusion(node_hash=node.hash, layers=[]) - - -@pytest.mark.anyio -async def test_proof_of_inclusion_by_hash_program(data_store: DataStore, store_id: bytes32) -> None: - """The proof of inclusion program has the expected Python equivalence.""" - - await add_01234567_example(data_store=data_store, store_id=store_id) - node = await data_store.get_node_by_key(key=b"\x04", store_id=store_id) - - proof = await data_store.get_proof_of_inclusion_by_hash(node_hash=node.hash, store_id=store_id) - - assert proof.as_program() == [ - b"\x04", - [ - bytes32.fromhex("fb66fe539b3eb2020dfbfadfd601fa318521292b41f04c2057c16fca6b947ca1"), - bytes32.fromhex("6d3af8d93db948e8b6aa4386958e137c6be8bab726db86789594b3588b35adcd"), - bytes32.fromhex("c852ecd8fb61549a0a42f9eb9dde65e6c94a01934dbd9c1d35ab94e2a0ae58e2"), - ], - ] + assert proof == chia_rs.datalayer.ProofOfInclusion(node_hash=node.hash, layers=[]) @pytest.mark.anyio @@ -939,27 +834,6 @@ async def test_proof_of_inclusion_by_hash_equals_by_key(data_store: DataStore, s assert proof_by_hash == proof_by_key -@pytest.mark.anyio -async def test_proof_of_inclusion_by_hash_bytes(data_store: DataStore, store_id: bytes32) -> None: - """The proof of inclusion provided by the data store is able to be converted to a - program and subsequently to bytes. - """ - await add_01234567_example(data_store=data_store, store_id=store_id) - node = await data_store.get_node_by_key(key=b"\x04", store_id=store_id) - - proof = await data_store.get_proof_of_inclusion_by_hash(node_hash=node.hash, store_id=store_id) - - expected = ( - b"\xff\x04\xff\xff\xa0\xfbf\xfeS\x9b>\xb2\x02\r\xfb\xfa\xdf\xd6\x01\xfa1\x85!)" - b"+A\xf0L W\xc1o\xcak\x94|\xa1\xff\xa0m:\xf8\xd9=\xb9H\xe8\xb6\xaaC\x86\x95" - b"\x8e\x13|k\xe8\xba\xb7&\xdb\x86x\x95\x94\xb3X\x8b5\xad\xcd\xff\xa0\xc8R\xec" - b"\xd8\xfbaT\x9a\nB\xf9\xeb\x9d\xdee\xe6\xc9J\x01\x93M\xbd\x9c\x1d5\xab\x94" - b"\xe2\xa0\xaeX\xe2\x80\x80" - ) - - assert bytes(proof.as_program()) == expected - - # @pytest.mark.anyio # async def test_create_first_pair(data_store: DataStore, store_id: bytes) -> None: # key = SExp.to([1, 2]) @@ -1036,46 +910,6 @@ async def test_check_roots_are_incrementing_gap(raw_data_store: DataStore) -> No await raw_data_store._check_roots_are_incrementing() -@pytest.mark.anyio -async def test_check_hashes_internal(raw_data_store: DataStore) -> None: - async with raw_data_store.db_wrapper.writer() as writer: - await writer.execute( - "INSERT INTO node(hash, node_type, left, right) VALUES(:hash, :node_type, :left, :right)", - { - "hash": a_bytes_32, - "node_type": NodeType.INTERNAL, - "left": a_bytes_32, - "right": a_bytes_32, - }, - ) - - with pytest.raises( - NodeHashError, - match=r"\n +000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f$", - ): - await raw_data_store._check_hashes() - - -@pytest.mark.anyio -async def test_check_hashes_terminal(raw_data_store: DataStore) -> None: - async with raw_data_store.db_wrapper.writer() as writer: - await writer.execute( - "INSERT INTO node(hash, node_type, key, value) VALUES(:hash, :node_type, :key, :value)", - { - "hash": a_bytes_32, - "node_type": NodeType.TERMINAL, - "key": Program.to((1, 2)).as_bin(), - "value": Program.to((1, 2)).as_bin(), - }, - ) - - with pytest.raises( - NodeHashError, - match=r"\n +000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f$", - ): - await raw_data_store._check_hashes() - - @pytest.mark.anyio async def test_root_state(data_store: DataStore, store_id: bytes32) -> None: key = b"\x01\x02" @@ -1127,28 +961,29 @@ async def test_kv_diff(data_store: DataStore, store_id: bytes32) -> None: insertions = 0 expected_diff: set[DiffData] = set() root_start = None + for i in range(500): key = (i + 100).to_bytes(4, byteorder="big") value = (i + 200).to_bytes(4, byteorder="big") seed = leaf_hash(key=key, value=value) - node_hash = await data_store.get_terminal_node_for_seed(store_id, seed) + node = await data_store.get_terminal_node_for_seed(seed, store_id) + side_seed = bytes(seed)[0] + side = None if node is None else (Side.LEFT if side_seed < 128 else Side.RIGHT) + if random.randint(0, 4) > 0 or insertions < 10: insertions += 1 - side = None if node_hash is None else data_store.get_side_for_seed(seed) - + reference_node_hash = node.hash if node is not None else None await data_store.insert( key=key, value=value, store_id=store_id, - reference_node_hash=node_hash, - side=side, status=Status.COMMITTED, + reference_node_hash=reference_node_hash, + side=side, ) if i > 200: expected_diff.add(DiffData(OperationType.INSERT, key, value)) else: - assert node_hash is not None - node = await data_store.get_node(node_hash) assert isinstance(node, TerminalNode) await data_store.delete(key=node.key, store_id=store_id, status=Status.COMMITTED) if i > 200: @@ -1275,6 +1110,39 @@ async def test_subscribe_unsubscribe(data_store: DataStore, store_id: bytes32) - ] +@pytest.mark.anyio +async def test_unsubscribe_clears_databases(data_store: DataStore, store_id: bytes32) -> None: + num_inserts = 100 + await data_store.subscribe(Subscription(store_id, [])) + for value in range(num_inserts): + await data_store.insert( + key=value.to_bytes(4, byteorder="big"), + value=value.to_bytes(4, byteorder="big"), + store_id=store_id, + reference_node_hash=None, + side=None, + status=Status.COMMITTED, + ) + await data_store.add_node_hashes(store_id) + + tables = ["ids", "nodes"] + for table in tables: + async with data_store.db_wrapper.reader() as reader: + async with reader.execute(f"SELECT COUNT(*) FROM {table}") as cursor: + row_count = await cursor.fetchone() + assert row_count is not None + assert row_count[0] > 0 + + await data_store.unsubscribe(store_id) + + for table in tables: + async with data_store.db_wrapper.reader() as reader: + async with reader.execute(f"SELECT COUNT(*) FROM {table}") as cursor: + row_count = await cursor.fetchone() + assert row_count is not None + assert row_count[0] == 0 + + @pytest.mark.anyio async def test_server_selection(data_store: DataStore, store_id: bytes32) -> None: start_timestamp = 1000 @@ -1410,16 +1278,71 @@ async def mock_http_download( assert sinfo.ignore_till == start_timestamp # we don't increase on second failure -@pytest.mark.parametrize( - "test_delta", - [True, False], -) +async def get_first_generation(data_store: DataStore, node_hash: bytes32, store_id: bytes32) -> Optional[int]: + async with data_store.db_wrapper.reader() as reader: + cursor = await reader.execute( + "SELECT generation FROM nodes WHERE hash = ? AND store_id = ?", + ( + node_hash, + store_id, + ), + ) + + row = await cursor.fetchone() + if row is None: + return None + + return int(row[0]) + + +async def write_tree_to_file_old_format( + data_store: DataStore, + root: Root, + node_hash: bytes32, + store_id: bytes32, + writer: BinaryIO, + merkle_blob: Optional[MerkleBlob] = None, + hash_to_index: Optional[dict[bytes32, TreeIndex]] = None, +) -> None: + if node_hash == bytes32.zeros: + return + + if merkle_blob is None: + merkle_blob = await data_store.get_merkle_blob(store_id=store_id, root_hash=root.node_hash) + if hash_to_index is None: + hash_to_index = merkle_blob.get_hashes_indexes() + + generation = await get_first_generation(data_store, node_hash, store_id) + # Root's generation is not the first time we see this hash, so it's not a new delta. + if root.generation != generation: + return + + raw_index = hash_to_index[node_hash] + raw_node = merkle_blob.get_raw_node(raw_index) + + if isinstance(raw_node, chia_rs.datalayer.InternalNode): + left_hash = merkle_blob.get_hash_at_index(raw_node.left) + right_hash = merkle_blob.get_hash_at_index(raw_node.right) + await write_tree_to_file_old_format(data_store, root, left_hash, store_id, writer, merkle_blob, hash_to_index) + await write_tree_to_file_old_format(data_store, root, right_hash, store_id, writer, merkle_blob, hash_to_index) + to_write = bytes(SerializedNode(False, bytes(left_hash), bytes(right_hash))) + elif isinstance(raw_node, chia_rs.datalayer.LeafNode): + node = await data_store.get_terminal_node(raw_node.key, raw_node.value, store_id) + to_write = bytes(SerializedNode(True, node.key, node.value)) + else: + raise Exception(f"Node is neither InternalNode nor TerminalNode: {raw_node}") + + writer.write(len(to_write).to_bytes(4, byteorder="big")) + writer.write(to_write) + + +@pytest.mark.parametrize(argnames="test_delta", argvalues=["full", "delta", "old"]) @boolean_datacases(name="group_files_by_store", false="group by singleton", true="don't group by singleton") @pytest.mark.anyio async def test_data_server_files( data_store: DataStore, store_id: bytes32, - test_delta: bool, + test_delta: str, group_files_by_store: bool, tmp_path: Path, ) -> None: @@ -1428,45 +1351,70 @@ async def test_data_server_files( num_ops_per_batch = 100 db_uri = generate_in_memory_db_uri() - async with DataStore.managed(database=db_uri, uri=True) as data_store_server: + async with DataStore.managed( + database=db_uri, + uri=True, + merkle_blobs_path=tmp_path.joinpath("merkle-blobs"), + key_value_blobs_path=tmp_path.joinpath("key-value-blobs"), + ) as data_store_server: await data_store_server.create_tree(store_id, status=Status.COMMITTED) random = Random() random.seed(100, version=2) keys: list[bytes] = [] counter = 0 - - for batch in range(num_batches): - changelist: list[dict[str, Any]] = [] - for operation in range(num_ops_per_batch): - if random.randint(0, 4) > 0 or len(keys) == 0: - key = counter.to_bytes(4, byteorder="big") - value = (2 * counter).to_bytes(4, byteorder="big") - keys.append(key) - changelist.append({"action": "insert", "key": key, "value": value}) + num_repeats = 2 + + # Repeat twice to guarantee there will be hashes from the old file format + for _ in range(num_repeats): + for batch in range(num_batches): + changelist: list[dict[str, Any]] = [] + if batch == num_batches - 1: + for key in keys: + changelist.append({"action": "delete", "key": key}) + keys = [] + counter = 0 else: - key = random.choice(keys) - keys.remove(key) - changelist.append({"action": "delete", "key": key}) - counter += 1 - await data_store_server.insert_batch(store_id, changelist, status=Status.COMMITTED) - root = await data_store_server.get_tree_root(store_id) - await write_files_for_root( - data_store_server, store_id, root, tmp_path, 0, group_by_store=group_files_by_store - ) - roots.append(root) + for operation in range(num_ops_per_batch): + if random.randint(0, 4) > 0 or len(keys) == 0: + key = counter.to_bytes(4, byteorder="big") + value = (2 * counter).to_bytes(4, byteorder="big") + keys.append(key) + changelist.append({"action": "insert", "key": key, "value": value}) + else: + key = random.choice(keys) + keys.remove(key) + changelist.append({"action": "delete", "key": key}) + counter += 1 + + await data_store_server.insert_batch(store_id, changelist, status=Status.COMMITTED) + root = await data_store_server.get_tree_root(store_id) + await data_store_server.add_node_hashes(store_id) + if test_delta == "old": + node_hash = root.node_hash if root.node_hash is not None else bytes32.zeros + filename = get_delta_filename_path( + tmp_path, store_id, node_hash, root.generation, group_files_by_store + ) + filename.parent.mkdir(parents=True, exist_ok=True) + with open(filename, "xb") as writer: + await write_tree_to_file_old_format(data_store_server, root, node_hash, store_id, writer) + else: + await write_files_for_root( + data_store_server, store_id, root, tmp_path, 0, group_by_store=group_files_by_store + ) + roots.append(root) generation = 1 - assert len(roots) == num_batches + assert len(roots) == num_batches * num_repeats for root in roots: - assert root.node_hash is not None - if not test_delta: - filename = get_full_tree_filename_path(tmp_path, store_id, root.node_hash, generation, group_files_by_store) + node_hash = root.node_hash if root.node_hash is not None else bytes32.zeros + if test_delta == "full": + filename = get_full_tree_filename_path(tmp_path, store_id, node_hash, generation, group_files_by_store) assert filename.exists() else: - filename = get_delta_filename_path(tmp_path, store_id, root.node_hash, generation, group_files_by_store) + filename = get_delta_filename_path(tmp_path, store_id, node_hash, generation, group_files_by_store) assert filename.exists() - await insert_into_data_store_from_file(data_store, store_id, root.node_hash, tmp_path.joinpath(filename)) + await data_store.insert_into_data_store_from_file(store_id, root.node_hash, tmp_path.joinpath(filename)) current_root = await data_store.get_tree_root(store_id=store_id) assert current_root.node_hash == root.node_hash generation += 1 @@ -1551,6 +1499,20 @@ def id(self) -> str: return f"count={self.count},batch_count={self.batch_count}" +@dataclass +class BatchUpdateBenchmarkCase: + pre: int + num_inserts: int + num_deletes: int + num_upserts: int + limit: float + marks: Marks = () + + @property + def id(self) -> str: + return f"pre={self.pre},inserts={self.num_inserts},deletes={self.num_deletes},upserts={self.num_upserts}" + + @datacases( BatchInsertBenchmarkCase( pre=0, @@ -1610,6 +1572,103 @@ async def test_benchmark_batch_insert_speed( ) +@datacases( + BatchUpdateBenchmarkCase( + pre=1_000, + num_inserts=1_000, + num_deletes=500, + num_upserts=500, + limit=36, + ), +) +@pytest.mark.anyio +async def test_benchmark_batch_update_speed( + data_store: DataStore, + store_id: bytes32, + benchmark_runner: BenchmarkRunner, + case: BatchUpdateBenchmarkCase, +) -> None: + r = random.Random() + r.seed("shadowlands", version=2) + + pre = [ + { + "action": "insert", + "key": x.to_bytes(32, byteorder="big", signed=False), + "value": bytes(r.getrandbits(8) for _ in range(1200)), + } + for x in range(case.pre) + ] + batch = [] + + if case.pre > 0: + await data_store.insert_batch( + store_id=store_id, + changelist=pre, + status=Status.COMMITTED, + ) + + keys = [x.to_bytes(32, byteorder="big", signed=False) for x in range(case.pre)] + for operation in range(case.num_inserts): + key = (operation + case.pre).to_bytes(32, byteorder="big", signed=False) + batch.append( + { + "action": "insert", + "key": key, + "value": bytes(r.getrandbits(8) for _ in range(1200)), + } + ) + keys.append(key) + + if case.num_deletes > 0: + r.shuffle(keys) + assert len(keys) >= case.num_deletes + batch.extend( + { + "action": "delete", + "key": key, + } + for key in keys[: case.num_deletes] + ) + keys = keys[case.num_deletes :] + + if case.num_upserts > 0: + assert len(keys) > 0 + r.shuffle(keys) + batch.extend( + [ + { + "action": "upsert", + "key": keys[operation % len(keys)], + "value": bytes(r.getrandbits(8) for _ in range(1200)), + } + for operation in range(case.num_upserts) + ] + ) + + with benchmark_runner.assert_runtime(seconds=case.limit): + await data_store.insert_batch( + store_id=store_id, + changelist=batch, + ) + + +@datacases( + BatchInsertBenchmarkCase( + pre=0, + count=50, + limit=2, + ), +) +@pytest.mark.anyio +async def test_benchmark_tool( + benchmark_runner: BenchmarkRunner, + case: BatchInsertBenchmarkCase, +) -> None: + with benchmark_runner.assert_runtime(seconds=case.limit): + await generate_datastore(case.count) + + @datacases( BatchesInsertBenchmarkCase( count=50, @@ -1644,189 +1703,6 @@ async def test_benchmark_batch_insert_speed_multiple_batches( ) -@pytest.mark.anyio -async def test_delete_store_data(raw_data_store: DataStore) -> None: - store_id = bytes32.zeros - store_id_2 = bytes32(b"\0" * 31 + b"\1") - await raw_data_store.create_tree(store_id=store_id, status=Status.COMMITTED) - await raw_data_store.create_tree(store_id=store_id_2, status=Status.COMMITTED) - total_keys = 4 - keys = [key.to_bytes(4, byteorder="big") for key in range(total_keys)] - batch1 = [ - {"action": "insert", "key": keys[0], "value": keys[0]}, - {"action": "insert", "key": keys[1], "value": keys[1]}, - ] - batch2 = batch1.copy() - batch1.append({"action": "insert", "key": keys[2], "value": keys[2]}) - batch2.append({"action": "insert", "key": keys[3], "value": keys[3]}) - assert batch1 != batch2 - await raw_data_store.insert_batch(store_id, batch1, status=Status.COMMITTED) - await raw_data_store.insert_batch(store_id_2, batch2, status=Status.COMMITTED) - keys_values_before = await raw_data_store.get_keys_values(store_id_2) - async with raw_data_store.db_wrapper.reader() as reader: - result = await reader.execute("SELECT * FROM node") - nodes = await result.fetchall() - kv_nodes_before = {} - for node in nodes: - if node["key"] is not None: - kv_nodes_before[node["key"]] = node["value"] - assert [kv_nodes_before[key] for key in keys] == keys - await raw_data_store.delete_store_data(store_id) - # Deleting from `node` table doesn't alter other stores. - keys_values_after = await raw_data_store.get_keys_values(store_id_2) - assert keys_values_before == keys_values_after - async with raw_data_store.db_wrapper.reader() as reader: - result = await reader.execute("SELECT * FROM node") - nodes = await result.fetchall() - kv_nodes_after = {} - for node in nodes: - if node["key"] is not None: - kv_nodes_after[node["key"]] = node["value"] - for i in range(total_keys): - if i != 2: - assert kv_nodes_after[keys[i]] == keys[i] - else: - # `keys[2]` was only present in the first store. - assert keys[i] not in kv_nodes_after - assert not await raw_data_store.store_id_exists(store_id) - await raw_data_store.delete_store_data(store_id_2) - async with raw_data_store.db_wrapper.reader() as reader: - async with reader.execute("SELECT COUNT(*) FROM node") as cursor: - row_count = await cursor.fetchone() - assert row_count is not None - assert row_count[0] == 0 - assert not await raw_data_store.store_id_exists(store_id_2) - - -@pytest.mark.anyio -async def test_delete_store_data_multiple_stores(raw_data_store: DataStore) -> None: - # Make sure inserting and deleting the same data works - for repetition in range(2): - num_stores = 50 - total_keys = 150 - keys_deleted_per_store = 3 - store_ids = [bytes32(i.to_bytes(32, byteorder="big")) for i in range(num_stores)] - for store_id in store_ids: - await raw_data_store.create_tree(store_id=store_id, status=Status.COMMITTED) - original_keys = [key.to_bytes(4, byteorder="big") for key in range(total_keys)] - batches = [] - for i in range(num_stores): - batch = [ - {"action": "insert", "key": key, "value": key} for key in original_keys[i * keys_deleted_per_store :] - ] - batches.append(batch) - - for store_id, batch in zip(store_ids, batches): - await raw_data_store.insert_batch(store_id, batch, status=Status.COMMITTED) - - for tree_index in range(num_stores): - async with raw_data_store.db_wrapper.reader() as reader: - result = await reader.execute("SELECT * FROM node") - nodes = await result.fetchall() - - keys = {node["key"] for node in nodes if node["key"] is not None} - assert len(keys) == total_keys - tree_index * keys_deleted_per_store - keys_after_index = set(original_keys[tree_index * keys_deleted_per_store :]) - keys_before_index = set(original_keys[: tree_index * keys_deleted_per_store]) - assert keys_after_index.issubset(keys) - assert keys.isdisjoint(keys_before_index) - await raw_data_store.delete_store_data(store_ids[tree_index]) - - async with raw_data_store.db_wrapper.reader() as reader: - async with reader.execute("SELECT COUNT(*) FROM node") as cursor: - row_count = await cursor.fetchone() - assert row_count is not None - assert row_count[0] == 0 - - -@pytest.mark.parametrize("common_keys_count", [1, 250, 499]) -@pytest.mark.anyio -async def test_delete_store_data_with_common_values(raw_data_store: DataStore, common_keys_count: int) -> None: - store_id_1 = bytes32(b"\x00" * 31 + b"\x01") - store_id_2 = bytes32(b"\x00" * 31 + b"\x02") - - await raw_data_store.create_tree(store_id=store_id_1, status=Status.COMMITTED) - await raw_data_store.create_tree(store_id=store_id_2, status=Status.COMMITTED) - - key_offset = 1000 - total_keys_per_store = 500 - assert common_keys_count < key_offset - common_keys = {key.to_bytes(4, byteorder="big") for key in range(common_keys_count)} - unique_keys_1 = { - (key + key_offset).to_bytes(4, byteorder="big") for key in range(total_keys_per_store - common_keys_count) - } - unique_keys_2 = { - (key + (2 * key_offset)).to_bytes(4, byteorder="big") for key in range(total_keys_per_store - common_keys_count) - } - - batch1 = [{"action": "insert", "key": key, "value": key} for key in common_keys.union(unique_keys_1)] - batch2 = [{"action": "insert", "key": key, "value": key} for key in common_keys.union(unique_keys_2)] - - await raw_data_store.insert_batch(store_id_1, batch1, status=Status.COMMITTED) - await raw_data_store.insert_batch(store_id_2, batch2, status=Status.COMMITTED) - - await raw_data_store.delete_store_data(store_id_1) - async with raw_data_store.db_wrapper.reader() as reader: - result = await reader.execute("SELECT * FROM node") - nodes = await result.fetchall() - - keys = {node["key"] for node in nodes if node["key"] is not None} - # Since one store got all its keys deleted, we're left only with the keys of the other store. - assert len(keys) == total_keys_per_store - assert keys.intersection(unique_keys_1) == set() - assert keys.symmetric_difference(common_keys.union(unique_keys_2)) == set() - - -@pytest.mark.anyio -@pytest.mark.parametrize("pending_status", [Status.PENDING, Status.PENDING_BATCH]) -async def test_delete_store_data_protects_pending_roots(raw_data_store: DataStore, pending_status: Status) -> None: - num_stores = 5 - total_keys = 15 - store_ids = [bytes32(i.to_bytes(32, byteorder="big")) for i in range(num_stores)] - for store_id in store_ids: - await raw_data_store.create_tree(store_id=store_id, status=Status.COMMITTED) - original_keys = [key.to_bytes(4, byteorder="big") for key in range(total_keys)] - batches = [] - keys_per_pending_root = 2 - - for i in range(num_stores - 1): - start_index = i * keys_per_pending_root - end_index = (i + 1) * keys_per_pending_root - batch = [{"action": "insert", "key": key, "value": key} for key in original_keys[start_index:end_index]] - batches.append(batch) - for store_id, batch in zip(store_ids, batches): - await raw_data_store.insert_batch(store_id, batch, status=pending_status) - - store_id = store_ids[-1] - batch = [{"action": "insert", "key": key, "value": key} for key in original_keys] - await raw_data_store.insert_batch(store_id, batch, status=Status.COMMITTED) - - async with raw_data_store.db_wrapper.reader() as reader: - result = await reader.execute("SELECT * FROM node") - nodes = await result.fetchall() - - keys = {node["key"] for node in nodes if node["key"] is not None} - assert keys == set(original_keys) - - await raw_data_store.delete_store_data(store_id) - async with raw_data_store.db_wrapper.reader() as reader: - result = await reader.execute("SELECT * FROM node") - nodes = await result.fetchall() - - keys = {node["key"] for node in nodes if node["key"] is not None} - assert keys == set(original_keys[: (num_stores - 1) * keys_per_pending_root]) - - for index in range(num_stores - 1): - store_id = store_ids[index] - root = await raw_data_store.get_pending_root(store_id) - assert root is not None - await raw_data_store.change_root_status(root, Status.COMMITTED) - kv = await raw_data_store.get_keys_values(store_id=store_id) - start_index = index * keys_per_pending_root - end_index = (index + 1) * keys_per_pending_root - assert {pair.key for pair in kv} == set(original_keys[start_index:end_index]) - - @pytest.mark.anyio @boolean_datacases(name="group_files_by_store", true="group by singleton", false="don't group by singleton") @pytest.mark.parametrize("max_full_files", [1, 2, 5]) @@ -1839,7 +1715,6 @@ async def test_insert_from_delta_file( group_files_by_store: bool, max_full_files: int, ) -> None: - await data_store.create_tree(store_id=store_id, status=Status.COMMITTED) num_files = 5 for generation in range(num_files): key = generation.to_bytes(4, byteorder="big") @@ -1850,30 +1725,34 @@ async def test_insert_from_delta_file( store_id=store_id, status=Status.COMMITTED, ) + await data_store.add_node_hashes(store_id) root = await data_store.get_tree_root(store_id=store_id) - assert root.generation == num_files + 1 + assert root.generation == num_files root_hashes = [] tmp_path_1 = tmp_path.joinpath("1") tmp_path_2 = tmp_path.joinpath("2") - for generation in range(1, num_files + 2): + for generation in range(1, num_files + 1): root = await data_store.get_tree_root(store_id=store_id, generation=generation) await write_files_for_root(data_store, store_id, root, tmp_path_1, 0, False, group_files_by_store) root_hashes.append(bytes32.zeros if root.node_hash is None else root.node_hash) store_path = tmp_path_1.joinpath(f"{store_id}") if group_files_by_store else tmp_path_1 with os.scandir(store_path) as entries: filenames = {entry.name for entry in entries} - assert len(filenames) == 2 * (num_files + 1) + assert len(filenames) == 2 * num_files for filename in filenames: if "full" in filename: store_path.joinpath(filename).unlink() with os.scandir(store_path) as entries: filenames = {entry.name for entry in entries} - assert len(filenames) == num_files + 1 + assert len(filenames) == num_files kv_before = await data_store.get_keys_values(store_id=store_id) await data_store.rollback_to_generation(store_id, 0) + with contextlib.suppress(FileNotFoundError): + shutil.rmtree(data_store.merkle_blobs_path) + root = await data_store.get_tree_root(store_id=store_id) assert root.generation == 0 os.rename(store_path, tmp_path_2) @@ -1946,12 +1825,13 @@ async def mock_http_download_2( assert success root = await data_store.get_tree_root(store_id=store_id) - assert root.generation == num_files + 1 + assert root.generation == num_files with os.scandir(store_path) as entries: filenames = {entry.name for entry in entries} - assert len(filenames) == num_files + 1 + max_full_files # 6 deltas and max_full_files full files + assert len(filenames) == num_files + max_full_files - 1 kv = await data_store.get_keys_values(store_id=store_id) - assert kv == kv_before + # order agnostic comparison of the list + assert set(kv) == set(kv_before) @pytest.mark.anyio @@ -1989,7 +1869,7 @@ async def test_get_node_by_key_with_overlapping_keys(raw_data_store: DataStore) if random.randint(0, 4) == 0: batch = [{"action": "delete", "key": key}] await raw_data_store.insert_batch(store_id, batch, status=Status.COMMITTED) - with pytest.raises(KeyNotFoundError, match=f"Key not found: {key.hex()}"): + with pytest.raises(chia_rs.datalayer.UnknownKeyError): await raw_data_store.get_node_by_key(store_id=store_id, key=key) @@ -2009,6 +1889,7 @@ async def test_insert_from_delta_file_correct_file_exists( store_id=store_id, status=Status.COMMITTED, ) + await data_store.add_node_hashes(store_id) root = await data_store.get_tree_root(store_id=store_id) assert root.generation == num_files + 1 @@ -2019,18 +1900,20 @@ async def test_insert_from_delta_file_correct_file_exists( root_hashes.append(bytes32.zeros if root.node_hash is None else root.node_hash) store_path = tmp_path.joinpath(f"{store_id}") if group_files_by_store else tmp_path with os.scandir(store_path) as entries: - filenames = {entry.name for entry in entries} + filenames = {entry.name for entry in entries if entry.name.endswith(".dat")} assert len(filenames) == 2 * (num_files + 1) for filename in filenames: if "full" in filename: store_path.joinpath(filename).unlink() with os.scandir(store_path) as entries: - filenames = {entry.name for entry in entries} + filenames = {entry.name for entry in entries if entry.name.endswith(".dat")} assert len(filenames) == num_files + 1 kv_before = await data_store.get_keys_values(store_id=store_id) await data_store.rollback_to_generation(store_id, 0) root = await data_store.get_tree_root(store_id=store_id) assert root.generation == 0 + with contextlib.suppress(FileNotFoundError): + shutil.rmtree(data_store.merkle_blobs_path) sinfo = ServerInfo("http://127.0.0.1/8003", 0, 0) success = await insert_from_delta_file( @@ -2052,10 +1935,11 @@ async def test_insert_from_delta_file_correct_file_exists( root = await data_store.get_tree_root(store_id=store_id) assert root.generation == num_files + 1 with os.scandir(store_path) as entries: - filenames = {entry.name for entry in entries} + filenames = {entry.name for entry in entries if entry.name.endswith(".dat")} assert len(filenames) == num_files + 2 # 1 full and 6 deltas kv = await data_store.get_keys_values(store_id=store_id) - assert kv == kv_before + # order agnostic comparison of the list + assert set(kv) == set(kv_before) @pytest.mark.anyio @@ -2075,6 +1959,7 @@ async def test_insert_from_delta_file_incorrect_file_exists( store_id=store_id, status=Status.COMMITTED, ) + await data_store.add_node_hashes(store_id) root = await data_store.get_tree_root(store_id=store_id) assert root.generation == 2 @@ -2083,7 +1968,7 @@ async def test_insert_from_delta_file_incorrect_file_exists( incorrect_root_hash = bytes32([0] * 31 + [1]) store_path = tmp_path.joinpath(f"{store_id}") if group_files_by_store else tmp_path with os.scandir(store_path) as entries: - filenames = [entry.name for entry in entries] + filenames = [entry.name for entry in entries if entry.name.endswith(".dat")] assert len(filenames) == 2 os.rename( store_path.joinpath(filenames[0]), @@ -2115,7 +2000,7 @@ async def test_insert_from_delta_file_incorrect_file_exists( root = await data_store.get_tree_root(store_id=store_id) assert root.generation == 1 with os.scandir(store_path) as entries: - filenames = [entry.name for entry in entries] + filenames = [entry.name for entry in entries if entry.name.endswith(".dat")] assert len(filenames) == 0 @@ -2126,7 +2011,7 @@ async def test_insert_key_already_present(data_store: DataStore, store_id: bytes await data_store.insert( key=key, value=value, store_id=store_id, reference_node_hash=None, side=None, status=Status.COMMITTED ) - with pytest.raises(Exception, match=f"Key already present: {key.hex()}"): + with pytest.raises(KeyAlreadyPresentError): await data_store.insert(key=key, value=value, store_id=store_id, reference_node_hash=None, side=None) @@ -2141,7 +2026,7 @@ async def test_batch_insert_key_already_present( value = b"bar" changelist = [{"action": "insert", "key": key, "value": value}] await data_store.insert_batch(store_id, changelist, Status.COMMITTED, use_batch_autoinsert) - with pytest.raises(Exception, match=f"Key already present: {key.hex()}"): + with pytest.raises(KeyAlreadyPresentError): await data_store.insert_batch(store_id, changelist, Status.COMMITTED, use_batch_autoinsert) @@ -2184,7 +2069,7 @@ async def test_update_keys(data_store: DataStore, store_id: bytes32, use_upsert: @pytest.mark.anyio -async def test_migration_unknown_version(data_store: DataStore) -> None: +async def test_migration_unknown_version(data_store: DataStore, tmp_path: Path) -> None: async with data_store.db_wrapper.writer() as writer: await writer.execute( "INSERT INTO schema(version_id) VALUES(:version_id)", @@ -2193,227 +2078,284 @@ async def test_migration_unknown_version(data_store: DataStore) -> None: }, ) with pytest.raises(Exception, match="Unknown version"): - await data_store.migrate_db() - - -async def _check_ancestors( - data_store: DataStore, store_id: bytes32, root_hash: bytes32 -) -> dict[bytes32, Optional[bytes32]]: - ancestors: dict[bytes32, Optional[bytes32]] = {} - root_node: Node = await data_store.get_node(root_hash) - queue: list[Node] = [root_node] - - while queue: - node = queue.pop(0) - if isinstance(node, InternalNode): - left_node = await data_store.get_node(node.left_hash) - right_node = await data_store.get_node(node.right_hash) - ancestors[left_node.hash] = node.hash - ancestors[right_node.hash] = node.hash - queue.append(left_node) - queue.append(right_node) - - ancestors[root_hash] = None - for node_hash, ancestor_hash in ancestors.items(): - ancestor_node = await data_store._get_one_ancestor(node_hash, store_id) - if ancestor_hash is None: - assert ancestor_node is None - else: - assert ancestor_node is not None - assert ancestor_node.hash == ancestor_hash + await data_store.migrate_db(tmp_path) + + +@boolean_datacases(name="group_files_by_store", false="group by singleton", true="don't group by singleton") +@pytest.mark.anyio +async def test_migration( + data_store: DataStore, + store_id: bytes32, + group_files_by_store: bool, + tmp_path: Path, +) -> None: + num_batches = 10 + num_ops_per_batch = 100 + keys: list[bytes] = [] + counter = 0 + random = Random() + random.seed(100, version=2) + + for batch in range(num_batches): + changelist: list[dict[str, Any]] = [] + for operation in range(num_ops_per_batch): + if random.randint(0, 4) > 0 or len(keys) == 0: + key = counter.to_bytes(4, byteorder="big") + value = (2 * counter).to_bytes(4, byteorder="big") + keys.append(key) + changelist.append({"action": "insert", "key": key, "value": value}) + else: + key = random.choice(keys) + keys.remove(key) + changelist.append({"action": "delete", "key": key}) + counter += 1 + await data_store.insert_batch(store_id, changelist, status=Status.COMMITTED) + root = await data_store.get_tree_root(store_id) + await data_store.add_node_hashes(store_id) + await write_files_for_root(data_store, store_id, root, tmp_path, 0, group_by_store=group_files_by_store) + + kv_before = await data_store.get_keys_values(store_id=store_id) + async with data_store.db_wrapper.writer(foreign_key_enforcement_enabled=False) as writer: + tables = [table for table in table_columns.keys() if table != "root"] + for table in tables: + await writer.execute(f"DELETE FROM {table}") + + with contextlib.suppress(FileNotFoundError): + shutil.rmtree(data_store.merkle_blobs_path) + with contextlib.suppress(FileNotFoundError): + shutil.rmtree(data_store.key_value_blobs_path) - return ancestors + data_store.recent_merkle_blobs = LRUCache(capacity=128) + assert await data_store.get_keys_values(store_id=store_id) == [] + await data_store.migrate_db(tmp_path) + # order agnostic comparison of the list + assert set(await data_store.get_keys_values(store_id=store_id)) == set(kv_before) @pytest.mark.anyio -async def test_build_ancestor_table(data_store: DataStore, store_id: bytes32) -> None: - num_values = 1000 +@pytest.mark.parametrize("num_keys", [10, 1000]) +async def test_get_existing_hashes( + data_store: DataStore, + store_id: bytes32, + num_keys: int, +) -> None: changelist: list[dict[str, Any]] = [] - for value in range(num_values): - value_bytes = value.to_bytes(4, byteorder="big") - changelist.append({"action": "upsert", "key": value_bytes, "value": value_bytes}) - await data_store.insert_batch( - store_id=store_id, - changelist=changelist, - status=Status.PENDING, - ) + for i in range(num_keys): + key = i.to_bytes(4, byteorder="big") + value = (2 * i).to_bytes(4, byteorder="big") + changelist.append({"action": "insert", "key": key, "value": value}) + await data_store.insert_batch(store_id, changelist, status=Status.COMMITTED) + await data_store.add_node_hashes(store_id) - pending_root = await data_store.get_pending_root(store_id=store_id) - assert pending_root is not None - assert pending_root.node_hash is not None - await data_store.change_root_status(pending_root, Status.COMMITTED) - await data_store.build_ancestor_table_for_latest_root(store_id=store_id) - - assert pending_root.node_hash is not None - await _check_ancestors(data_store, store_id, pending_root.node_hash) + root = await data_store.get_tree_root(store_id=store_id) + merkle_blob = await data_store.get_merkle_blob(store_id=store_id, root_hash=root.node_hash) + hash_to_index = merkle_blob.get_hashes_indexes() + existing_hashes = list(hash_to_index.keys()) + not_existing_hashes = [bytes32(i.to_bytes(32, byteorder="big")) for i in range(num_keys)] + result = await data_store.get_existing_hashes(existing_hashes + not_existing_hashes, store_id) + assert result == set(existing_hashes) @pytest.mark.anyio -async def test_sparse_ancestor_table(data_store: DataStore, store_id: bytes32) -> None: - num_values = 100 - for value in range(num_values): - value_bytes = value.to_bytes(4, byteorder="big") - await data_store.autoinsert( - key=value_bytes, - value=value_bytes, - store_id=store_id, - status=Status.COMMITTED, - ) - root = await data_store.get_tree_root(store_id=store_id) - assert root.node_hash is not None - ancestors = await _check_ancestors(data_store, store_id, root.node_hash) +@pytest.mark.parametrize(argnames="size_offset", argvalues=[-1, 0, 1]) +async def test_basic_key_value_db_vs_disk_cutoff( + data_store: DataStore, + store_id: bytes32, + seeded_random: random.Random, + size_offset: int, +) -> None: + size = data_store.prefer_db_kv_blob_length + size_offset - # Check the ancestor table is sparse - root_generation = root.generation - current_generation_count = 0 - previous_generation_count = 0 - for node_hash, ancestor_hash in ancestors.items(): - async with data_store.db_wrapper.reader() as reader: - if ancestor_hash is not None: - cursor = await reader.execute( - "SELECT MAX(generation) AS generation FROM ancestors WHERE hash == :hash AND ancestor == :ancestor", - {"hash": node_hash, "ancestor": ancestor_hash}, - ) - else: - cursor = await reader.execute( - "SELECT MAX(generation) AS generation FROM ancestors WHERE hash == :hash AND ancestor IS NULL", - {"hash": node_hash}, - ) + blob = bytes(seeded_random.getrandbits(8) for _ in range(size)) + blob_hash = bytes32(sha256(blob).digest()) + async with data_store.db_wrapper.writer() as writer: + with data_store.manage_kv_files(store_id): + await data_store.add_kvid(blob=blob, store_id=store_id, writer=writer) + + file_exists = data_store.get_key_value_path(store_id=store_id, blob_hash=blob_hash).exists() + async with data_store.db_wrapper.writer() as writer: + async with writer.execute( + "SELECT blob FROM ids WHERE hash = :blob_hash", + {"blob_hash": blob_hash}, + ) as cursor: row = await cursor.fetchone() assert row is not None - generation = row["generation"] - assert generation <= root_generation - if generation == root_generation: - current_generation_count += 1 - else: - previous_generation_count += 1 + db_blob: Optional[bytes] = row["blob"] - assert current_generation_count == 15 - assert previous_generation_count == 184 + if size_offset <= 0: + assert not file_exists + assert db_blob == blob + else: + assert file_exists + assert db_blob is None -async def get_all_nodes(data_store: DataStore, store_id: bytes32) -> list[Node]: - root = await data_store.get_tree_root(store_id) - assert root.node_hash is not None - root_node = await data_store.get_node(root.node_hash) - nodes: list[Node] = [] - queue: list[Node] = [root_node] +@pytest.mark.anyio +@pytest.mark.parametrize(argnames="size_offset", argvalues=[-1, 0, 1]) +@pytest.mark.parametrize(argnames="limit_change", argvalues=[-2, -1, 1, 2]) +async def test_changing_key_value_db_vs_disk_cutoff( + data_store: DataStore, + store_id: bytes32, + seeded_random: random.Random, + size_offset: int, + limit_change: int, +) -> None: + size = data_store.prefer_db_kv_blob_length + size_offset + + blob = bytes(seeded_random.getrandbits(8) for _ in range(size)) + async with data_store.db_wrapper.writer() as writer: + with data_store.manage_kv_files(store_id): + kv_id = await data_store.add_kvid(blob=blob, store_id=store_id, writer=writer) - while len(queue) > 0: - node = queue.pop(0) - nodes.append(node) - if isinstance(node, InternalNode): - left_node = await data_store.get_node(node.left_hash) - right_node = await data_store.get_node(node.right_hash) - queue.append(left_node) - queue.append(right_node) + data_store.prefer_db_kv_blob_length += limit_change + retrieved_blob = await data_store.get_blob_from_kvid(kv_id=kv_id, store_id=store_id) - return nodes + assert blob == retrieved_blob @pytest.mark.anyio -async def test_get_nodes(data_store: DataStore, store_id: bytes32) -> None: - num_values = 50 - changelist: list[dict[str, Any]] = [] +async def test_get_keys_both_disk_and_db( + data_store: DataStore, + store_id: bytes32, + seeded_random: random.Random, +) -> None: + inserted_keys: set[bytes] = set() - for value in range(num_values): - value_bytes = value.to_bytes(4, byteorder="big") - changelist.append({"action": "upsert", "key": value_bytes, "value": value_bytes}) - await data_store.insert_batch( - store_id=store_id, - changelist=changelist, - status=Status.COMMITTED, - ) + for size_offset in [-1, 0, 1]: + size = data_store.prefer_db_kv_blob_length + size_offset + + blob = bytes(seeded_random.getrandbits(8) for _ in range(size)) + await data_store.insert(key=blob, value=b"", store_id=store_id, status=Status.COMMITTED) + inserted_keys.add(blob) - expected_nodes = await get_all_nodes(data_store, store_id) - nodes = await data_store.get_nodes([node.hash for node in expected_nodes]) - assert nodes == expected_nodes + retrieved_keys = set(await data_store.get_keys(store_id=store_id)) - node_hash = bytes32.zeros - node_hash_2 = bytes32([0] * 31 + [1]) - with pytest.raises(Exception, match=f"^Nodes not found for hashes: {node_hash.hex()}, {node_hash_2.hex()}"): - await data_store.get_nodes([node_hash, node_hash_2] + [node.hash for node in expected_nodes]) + assert retrieved_keys == inserted_keys @pytest.mark.anyio -@pytest.mark.parametrize("pre", [0, 2048]) -@pytest.mark.parametrize("batch_size", [25, 100, 500]) -async def test_get_leaf_at_minimum_height( +async def test_get_keys_values_both_disk_and_db( data_store: DataStore, store_id: bytes32, - pre: int, - batch_size: int, + seeded_random: random.Random, ) -> None: - num_values = 1000 - value_offset = 1000000 - all_min_leafs: set[TerminalNode] = set() + inserted_keys_values: dict[bytes, bytes] = {} - if pre > 0: - # This builds a complete binary tree, in order to test more than one batch in the queue before finding the leaf + for size_offset in [-1, 0, 1]: + size = data_store.prefer_db_kv_blob_length + size_offset + + key = bytes(seeded_random.getrandbits(8) for _ in range(size)) + value = bytes(seeded_random.getrandbits(8) for _ in range(size)) + await data_store.insert(key=key, value=value, store_id=store_id, status=Status.COMMITTED) + inserted_keys_values[key] = value + + terminal_nodes = await data_store.get_keys_values(store_id=store_id) + retrieved_keys_values = {node.key: node.value for node in terminal_nodes} + + assert retrieved_keys_values == inserted_keys_values + + +@pytest.mark.anyio +@boolean_datacases(name="success", false="invalid file", true="valid file") +async def test_db_data_insert_from_file( + data_store: DataStore, + store_id: bytes32, + tmp_path: Path, + seeded_random: random.Random, + success: bool, +) -> None: + num_keys = 1000 + db_uri = generate_in_memory_db_uri() + + async with DataStore.managed( + database=db_uri, + uri=True, + merkle_blobs_path=tmp_path.joinpath("merkle-blobs-tmp"), + key_value_blobs_path=tmp_path.joinpath("key-value-blobs-tmp"), + ) as tmp_data_store: + await tmp_data_store.create_tree(store_id, status=Status.COMMITTED) changelist: list[dict[str, Any]] = [] + for _ in range(num_keys): + use_file = seeded_random.choice([True, False]) + assert tmp_data_store.prefer_db_kv_blob_length > 7 + size = tmp_data_store.prefer_db_kv_blob_length + 1 if use_file else 8 + key = seeded_random.randbytes(size) + value = seeded_random.randbytes(size) + changelist.append({"action": "insert", "key": key, "value": value}) + + await tmp_data_store.insert_batch(store_id, changelist, status=Status.COMMITTED) + root = await tmp_data_store.get_tree_root(store_id) + files_path = tmp_path.joinpath("files") + await write_files_for_root(tmp_data_store, store_id, root, files_path, 1000) + assert root.node_hash is not None + filename = get_delta_filename_path(files_path, store_id, root.node_hash, 1) + assert filename.exists() - for value in range(pre): - value_bytes = (value * value).to_bytes(8, byteorder="big") - changelist.append({"action": "upsert", "key": value_bytes, "value": value_bytes}) - await data_store.insert_batch( - store_id=store_id, - changelist=changelist, - status=Status.COMMITTED, - ) + root_hash = bytes32([0] * 31 + [1]) if not success else root.node_hash + sinfo = ServerInfo("http://127.0.0.1/8003", 0, 0) - for value in range(num_values): - value_bytes = value.to_bytes(4, byteorder="big") - # Use autoinsert instead of `insert_batch` to get a more randomly shaped tree - await data_store.autoinsert( - key=value_bytes, - value=value_bytes, - store_id=store_id, - status=Status.COMMITTED, - ) + if not success: + target_filename_path = get_delta_filename_path(files_path, store_id, root_hash, 1) + shutil.copyfile(filename, target_filename_path) + assert target_filename_path.exists() - if (value + 1) % batch_size == 0: - hash_to_parent: dict[bytes32, InternalNode] = {} - root = await data_store.get_tree_root(store_id) - assert root.node_hash is not None - min_leaf = await data_store.get_leaf_at_minimum_height(root.node_hash, hash_to_parent) - all_nodes = await get_all_nodes(data_store, store_id) - heights: dict[bytes32, int] = {} - heights[root.node_hash] = 0 - min_leaf_height = None - - for node in all_nodes: - if isinstance(node, InternalNode): - heights[node.left_hash] = heights[node.hash] + 1 - heights[node.right_hash] = heights[node.hash] + 1 - elif min_leaf_height is not None: - min_leaf_height = min(min_leaf_height, heights[node.hash]) - else: - min_leaf_height = heights[node.hash] - - assert min_leaf_height is not None - if pre > 0: - assert min_leaf_height >= 11 - for node in all_nodes: - if isinstance(node, TerminalNode): - assert node == min_leaf - assert heights[min_leaf.hash] == min_leaf_height - break - if node.left_hash in hash_to_parent: - assert hash_to_parent[node.left_hash] == node - if node.right_hash in hash_to_parent: - assert hash_to_parent[node.right_hash] == node - - # Push down the min height leaf, so on the next iteration we get a different leaf - pushdown_height = 20 - for repeat in range(pushdown_height): - value_bytes = (value + (repeat + 1) * value_offset).to_bytes(4, byteorder="big") - await data_store.insert( - key=value_bytes, - value=value_bytes, - store_id=store_id, - reference_node_hash=min_leaf.hash, - side=Side.RIGHT, - status=Status.COMMITTED, - ) - assert min_leaf not in all_min_leafs - all_min_leafs.add(min_leaf) + keys_value_path = data_store.key_value_blobs_path.joinpath(store_id.hex()) + assert sum(1 for path in keys_value_path.rglob("*") if path.is_file()) == 0 + + is_success = await insert_from_delta_file( + data_store=data_store, + store_id=store_id, + existing_generation=0, + target_generation=1, + root_hashes=[root_hash], + server_info=sinfo, + client_foldername=files_path, + timeout=aiohttp.ClientTimeout(total=15, sock_connect=5), + log=log, + proxy_url="", + downloader=None, + ) + assert is_success == success + + async with data_store.db_wrapper.reader() as reader: + async with reader.execute("SELECT COUNT(*) FROM ids") as cursor: + row_count = await cursor.fetchone() + assert row_count is not None + if success: + assert row_count[0] > 0 + else: + assert row_count[0] == 0 + + if success: + assert sum(1 for path in keys_value_path.rglob("*") if path.is_file()) > 0 + else: + assert sum(1 for path in keys_value_path.rglob("*") if path.is_file()) == 0 + + +@pytest.mark.anyio +async def test_manage_kv_files( + data_store: DataStore, + store_id: bytes32, + seeded_random: random.Random, +) -> None: + num_keys = 1000 + num_files = 0 + keys_value_path = data_store.key_value_blobs_path.joinpath(store_id.hex()) + + with pytest.raises(Exception, match="Test exception"): + async with data_store.db_wrapper.writer() as writer: + with data_store.manage_kv_files(store_id): + for _ in range(num_keys): + use_file = seeded_random.choice([True, False]) + assert data_store.prefer_db_kv_blob_length > 7 + size = data_store.prefer_db_kv_blob_length + 1 if use_file else 8 + key = seeded_random.randbytes(size) + value = seeded_random.randbytes(size) + await data_store.add_key_value(key, value, store_id, writer) + num_files += 2 * use_file + + assert num_files > 0 + assert sum(1 for path in keys_value_path.rglob("*") if path.is_file()) == num_files + raise Exception("Test exception") + + assert sum(1 for path in keys_value_path.rglob("*") if path.is_file()) == 0 diff --git a/chia/_tests/core/data_layer/test_data_store_schema.py b/chia/_tests/core/data_layer/test_data_store_schema.py index 5f39426a8ed6..1d7d021571ec 100644 --- a/chia/_tests/core/data_layer/test_data_store_schema.py +++ b/chia/_tests/core/data_layer/test_data_store_schema.py @@ -1,171 +1,17 @@ from __future__ import annotations import sqlite3 -from typing import Any import pytest from chia_rs.sized_bytes import bytes32 -from chia._tests.core.data_layer.util import add_01234567_example, create_valid_node_values -from chia.data_layer.data_layer_util import NodeType, Side, Status +from chia._tests.core.data_layer.util import add_01234567_example +from chia.data_layer.data_layer_util import Status from chia.data_layer.data_store import DataStore pytestmark = pytest.mark.data_layer -@pytest.mark.anyio -async def test_node_update_fails(data_store: DataStore, store_id: bytes32) -> None: - await add_01234567_example(data_store=data_store, store_id=store_id) - node = await data_store.get_node_by_key(key=b"\x04", store_id=store_id) - - async with data_store.db_wrapper.writer() as writer: - with pytest.raises(sqlite3.IntegrityError, match=r"^updates not allowed to the node table$"): - await writer.execute( - "UPDATE node SET value = :value WHERE hash == :hash", - { - "hash": node.hash, - "value": node.value, - }, - ) - - -@pytest.mark.parametrize(argnames="length", argvalues=sorted(set(range(50)) - {32})) -@pytest.mark.anyio -async def test_node_hash_must_be_32( - data_store: DataStore, - store_id: bytes32, - length: int, - valid_node_values: dict[str, Any], -) -> None: - valid_node_values["hash"] = bytes([0] * length) - - async with data_store.db_wrapper.writer() as writer: - with pytest.raises(sqlite3.IntegrityError, match=r"^CHECK constraint failed:"): - await writer.execute( - """ - INSERT INTO node(hash, node_type, left, right, key, value) - VALUES(:hash, :node_type, :left, :right, :key, :value) - """, - valid_node_values, - ) - - -@pytest.mark.anyio -async def test_node_hash_must_not_be_null( - data_store: DataStore, - store_id: bytes32, - valid_node_values: dict[str, Any], -) -> None: - valid_node_values["hash"] = None - - async with data_store.db_wrapper.writer() as writer: - with pytest.raises(sqlite3.IntegrityError, match=r"^NOT NULL constraint failed: node.hash$"): - await writer.execute( - """ - INSERT INTO node(hash, node_type, left, right, key, value) - VALUES(:hash, :node_type, :left, :right, :key, :value) - """, - valid_node_values, - ) - - -@pytest.mark.anyio -async def test_node_type_must_be_valid( - data_store: DataStore, - node_type: NodeType, - bad_node_type: int, - valid_node_values: dict[str, Any], -) -> None: - valid_node_values["node_type"] = bad_node_type - - async with data_store.db_wrapper.writer() as writer: - with pytest.raises(sqlite3.IntegrityError, match=r"^CHECK constraint failed:"): - await writer.execute( - """ - INSERT INTO node(hash, node_type, left, right, key, value) - VALUES(:hash, :node_type, :left, :right, :key, :value) - """, - valid_node_values, - ) - - -@pytest.mark.parametrize(argnames="side", argvalues=Side) -@pytest.mark.anyio -async def test_node_internal_child_not_null(data_store: DataStore, store_id: bytes32, side: Side) -> None: - await add_01234567_example(data_store=data_store, store_id=store_id) - node_a = await data_store.get_node_by_key(key=b"\x02", store_id=store_id) - node_b = await data_store.get_node_by_key(key=b"\x04", store_id=store_id) - - values = create_valid_node_values(node_type=NodeType.INTERNAL, left_hash=node_a.hash, right_hash=node_b.hash) - - if side == Side.LEFT: - values["left"] = None - elif side == Side.RIGHT: - values["right"] = None - - async with data_store.db_wrapper.writer() as writer: - with pytest.raises(sqlite3.IntegrityError, match=r"^CHECK constraint failed:"): - await writer.execute( - """ - INSERT INTO node(hash, node_type, left, right, key, value) - VALUES(:hash, :node_type, :left, :right, :key, :value) - """, - values, - ) - - -@pytest.mark.parametrize(argnames="bad_child_hash", argvalues=[b"\x01" * 32, b"\0" * 31, b""]) -@pytest.mark.parametrize(argnames="side", argvalues=Side) -@pytest.mark.anyio -async def test_node_internal_must_be_valid_reference( - data_store: DataStore, - store_id: bytes32, - bad_child_hash: bytes, - side: Side, -) -> None: - await add_01234567_example(data_store=data_store, store_id=store_id) - node_a = await data_store.get_node_by_key(key=b"\x02", store_id=store_id) - node_b = await data_store.get_node_by_key(key=b"\x04", store_id=store_id) - - values = create_valid_node_values(node_type=NodeType.INTERNAL, left_hash=node_a.hash, right_hash=node_b.hash) - - if side == Side.LEFT: - values["left"] = bad_child_hash - elif side == Side.RIGHT: - values["right"] = bad_child_hash - else: # pragma: no cover - assert False - - async with data_store.db_wrapper.writer() as writer: - with pytest.raises(sqlite3.IntegrityError, match=r"^FOREIGN KEY constraint failed$"): - await writer.execute( - """ - INSERT INTO node(hash, node_type, left, right, key, value) - VALUES(:hash, :node_type, :left, :right, :key, :value) - """, - values, - ) - - -@pytest.mark.parametrize(argnames="key_or_value", argvalues=["key", "value"]) -@pytest.mark.anyio -async def test_node_terminal_key_value_not_null(data_store: DataStore, store_id: bytes32, key_or_value: str) -> None: - await add_01234567_example(data_store=data_store, store_id=store_id) - - values = create_valid_node_values(node_type=NodeType.TERMINAL) - values[key_or_value] = None - - async with data_store.db_wrapper.writer() as writer: - with pytest.raises(sqlite3.IntegrityError, match=r"^CHECK constraint failed:"): - await writer.execute( - """ - INSERT INTO node(hash, node_type, left, right, key, value) - VALUES(:hash, :node_type, :left, :right, :key, :value) - """, - values, - ) - - @pytest.mark.parametrize(argnames="length", argvalues=sorted(set(range(50)) - {32})) @pytest.mark.anyio async def test_root_store_id_must_be_32(data_store: DataStore, store_id: bytes32, length: int) -> None: @@ -250,21 +96,6 @@ async def test_root_generation_must_not_be_null(data_store: DataStore, store_id: ) -@pytest.mark.anyio -async def test_root_node_hash_must_reference(data_store: DataStore) -> None: - values = {"tree_id": bytes32.zeros, "generation": 0, "node_hash": bytes32.zeros, "status": Status.PENDING} - - async with data_store.db_wrapper.writer() as writer: - with pytest.raises(sqlite3.IntegrityError, match=r"^FOREIGN KEY constraint failed$"): - await writer.execute( - """ - INSERT INTO root(tree_id, generation, node_hash, status) - VALUES(:tree_id, :generation, :node_hash, :status) - """, - values, - ) - - @pytest.mark.parametrize(argnames="bad_status", argvalues=sorted(set(range(-20, 20)) - {*Status})) @pytest.mark.anyio async def test_root_status_must_be_valid(data_store: DataStore, store_id: bytes32, bad_status: int) -> None: @@ -319,44 +150,6 @@ async def test_root_store_id_generation_must_be_unique(data_store: DataStore, st ) -@pytest.mark.parametrize(argnames="length", argvalues=sorted(set(range(50)) - {32})) -@pytest.mark.anyio -async def test_ancestors_ancestor_must_be_32( - data_store: DataStore, - store_id: bytes32, - length: int, -) -> None: - async with data_store.db_wrapper.writer() as writer: - node_hash = await data_store._insert_terminal_node(key=b"\x00", value=b"\x01") - with pytest.raises(sqlite3.IntegrityError, match=r"^CHECK constraint failed:"): - await writer.execute( - """ - INSERT INTO ancestors(hash, ancestor, tree_id, generation) - VALUES(:hash, :ancestor, :tree_id, :generation) - """, - {"hash": node_hash, "ancestor": bytes([0] * length), "tree_id": bytes32.zeros, "generation": 0}, - ) - - -@pytest.mark.parametrize(argnames="length", argvalues=sorted(set(range(50)) - {32})) -@pytest.mark.anyio -async def test_ancestors_store_id_must_be_32( - data_store: DataStore, - store_id: bytes32, - length: int, -) -> None: - async with data_store.db_wrapper.writer() as writer: - node_hash = await data_store._insert_terminal_node(key=b"\x00", value=b"\x01") - with pytest.raises(sqlite3.IntegrityError, match=r"^CHECK constraint failed:"): - await writer.execute( - """ - INSERT INTO ancestors(hash, ancestor, tree_id, generation) - VALUES(:hash, :ancestor, :tree_id, :generation) - """, - {"hash": node_hash, "ancestor": bytes32.zeros, "tree_id": bytes([0] * length), "generation": 0}, - ) - - @pytest.mark.parametrize(argnames="length", argvalues=sorted(set(range(50)) - {32})) @pytest.mark.anyio async def test_subscriptions_store_id_must_be_32( diff --git a/chia/cmds/dev/data.py b/chia/cmds/dev/data.py index 15a71a7c183c..7fec13372c8c 100644 --- a/chia/cmds/dev/data.py +++ b/chia/cmds/dev/data.py @@ -23,7 +23,7 @@ from chia.cmds.cmd_helpers import NeedsWalletRPC from chia.data_layer.data_layer import server_files_path_from_config from chia.data_layer.data_layer_util import ServerInfo, Status, Subscription -from chia.data_layer.data_store import DataStore +from chia.data_layer.data_store import DataStore, default_prefer_file_kv_blob_length from chia.data_layer.download_data import insert_from_delta_file from chia.util.chia_logging import initialize_logging from chia.util.config import load_config @@ -78,6 +78,11 @@ class SyncTimeCommand: profile_tasks: bool = option("--profile-tasks/--no-profile-tasks") restart_all: bool = option("--restart-all/--no-restart-all") working_path: Optional[Path] = option("--working-path", default=None) + prefer_db_kv_blob_length: int = option( + "--prefer-db-kv-blob-length", + default=default_prefer_file_kv_blob_length, + type=int, + ) async def run(self) -> None: config = load_config(self.context.root_path, "config.yaml", "data_layer", fill_missing_services=True) @@ -104,14 +109,28 @@ async def run(self) -> None: working_path = self.working_path working_path.mkdir(parents=True, exist_ok=True) + print_date(f"working in: {working_path}") + database_path = working_path.joinpath("datalayer.sqlite") - print_date(f"working with database at: {database_path}") + + merkle_blob_path = working_path.joinpath("merkle-blobs") + merkle_blob_path.mkdir(parents=True, exist_ok=True) + + key_value_blob_path = working_path.joinpath("key-value-blobs") + key_value_blob_path.mkdir(parents=True, exist_ok=True) wallet_client_info = await exit_stack.enter_async_context(self.wallet_rpc_info.wallet_rpc()) wallet_rpc = wallet_client_info.client await wallet_rpc.dl_track_new(DLTrackNew(launcher_id=self.store_id)) - data_store = await exit_stack.enter_async_context(DataStore.managed(database=database_path)) + data_store = await exit_stack.enter_async_context( + DataStore.managed( + database=database_path, + merkle_blobs_path=merkle_blob_path, + key_value_blobs_path=key_value_blob_path, + prefer_db_kv_blob_length=self.prefer_db_kv_blob_length, + ) + ) await data_store.subscribe(subscription=Subscription(store_id=self.store_id, servers_info=[])) diff --git a/chia/data_layer/data_layer.py b/chia/data_layer/data_layer.py index f5b74c3fe494..f0bfedf5c52d 100644 --- a/chia/data_layer/data_layer.py +++ b/chia/data_layer/data_layer.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union, cast, final import aiohttp +from chia_rs.datalayer import ProofOfInclusion, ProofOfInclusionLayer from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint32, uint64 @@ -32,10 +33,9 @@ PluginRemote, PluginStatus, Proof, - ProofOfInclusion, - ProofOfInclusionLayer, Root, ServerInfo, + Side, Status, StoreProofs, Subscription, @@ -43,18 +43,16 @@ TerminalNode, Unspecified, UnsubscribeData, + calculate_sibling_sides_integer, + collect_sibling_hashes, + get_delta_filename_path, + get_full_tree_filename_path, leaf_hash, unspecified, ) from chia.data_layer.data_layer_wallet import DataLayerWallet, Mirror, verify_offer from chia.data_layer.data_store import DataStore -from chia.data_layer.download_data import ( - delete_full_file_if_exists, - get_delta_filename_path, - get_full_tree_filename_path, - insert_from_delta_file, - write_files_for_root, -) +from chia.data_layer.download_data import delete_full_file_if_exists, insert_from_delta_file, write_files_for_root from chia.data_layer.singleton_record import SingletonRecord from chia.protocols.outbound_message import NodeType from chia.rpc.rpc_server import StateChangedProtocol, default_get_connections @@ -120,6 +118,8 @@ class DataLayer: _protocol_check: ClassVar[RpcServiceProtocol] = cast("DataLayer", None) db_path: Path + merkle_blobs_path: Path + key_value_blobs_path: Path config: dict[str, Any] root_path: Path log: logging.Logger @@ -189,6 +189,12 @@ def create( server_files_replaced = server_files_path_from_config(config, root_path) db_path_replaced: str = config["database_path"].replace("CHALLENGE", config["selected_network"]) + merkle_blobs_path_replaced: str = config.get( + "merkle_blobs_path", "data_layer/db/merkle_blobs_CHALLENGE" + ).replace("CHALLENGE", config["selected_network"]) + key_value_blobs_path_replaced: str = config.get( + "key_value_blobs_path", "data_layer/db/key_value_blobs_CHALLENGE" + ).replace("CHALLENGE", config["selected_network"]) self = cls( config=config, @@ -196,6 +202,8 @@ def create( wallet_rpc_init=wallet_rpc_init, log=logging.getLogger(name if name is None else __name__), db_path=path_from_root(root_path, db_path_replaced), + merkle_blobs_path=path_from_root(root_path, merkle_blobs_path_replaced), + key_value_blobs_path=path_from_root(root_path, key_value_blobs_path_replaced), server_files_location=server_files_replaced, downloaders=downloaders, uploaders=uploaders, @@ -209,6 +217,8 @@ def create( ) self.db_path.parent.mkdir(parents=True, exist_ok=True) + self.merkle_blobs_path.mkdir(parents=True, exist_ok=True) + self.key_value_blobs_path.mkdir(parents=True, exist_ok=True) self.server_files_location.mkdir(parents=True, exist_ok=True) return self @@ -219,11 +229,18 @@ async def manage(self) -> AsyncIterator[None]: if self.config.get("log_sqlite_cmds", False): sql_log_path = path_from_root(self.root_path, "log/data_sql.log") self.log.info(f"logging SQL commands to {sql_log_path}") - - async with DataStore.managed(database=self.db_path, sql_log_path=sql_log_path) as self._data_store: + cache_capacity = self.config.get("merkle_blobs_cache_size", 1) + + async with DataStore.managed( + database=self.db_path, + merkle_blobs_path=self.merkle_blobs_path, + key_value_blobs_path=self.key_value_blobs_path, + sql_log_path=sql_log_path, + cache_capacity=cache_capacity, + ) as self._data_store: self._wallet_rpc = await self.wallet_rpc_init - await self._data_store.migrate_db() + await self._data_store.migrate_db(self.server_files_location) self.periodically_manage_data_task = create_referenced_task(self.periodically_manage_data()) try: yield @@ -278,7 +295,6 @@ async def batch_update( ) -> Optional[TransactionRecord]: status = Status.PENDING if submit_on_chain else Status.PENDING_BATCH await self.batch_insert(store_id=store_id, changelist=changelist, status=status) - await self.data_store.clean_node_table() if submit_on_chain: return await self.publish_update(store_id=store_id, fee=fee) @@ -312,8 +328,6 @@ async def multistore_batch_update( status = Status.PENDING if submit_on_chain else Status.PENDING_BATCH await self.batch_insert(store_id=store_id, changelist=changelist, status=status) - await self.data_store.clean_node_table() - if submit_on_chain: updates: list[LauncherRootPair] = [] for store_id in store_ids: @@ -570,7 +584,6 @@ async def _update_confirmation_status(self, store_id: bytes32) -> None: and pending_root.status == Status.PENDING ): await self.data_store.change_root_status(pending_root, Status.COMMITTED) - await self.data_store.build_ancestor_table_for_latest_root(store_id=store_id) await self.data_store.clear_pending_roots(store_id=store_id) async def fetch_and_validate(self, store_id: bytes32) -> None: @@ -877,8 +890,6 @@ async def process_unsubscribe(self, store_id: bytes32, retain_data: bool) -> Non # stop tracking first, then unsubscribe from the data store await self.wallet_rpc.dl_stop_tracking(DLStopTracking(store_id)) await self.data_store.unsubscribe(store_id) - if not retain_data: - await self.data_store.delete_store_data(store_id) self.log.info(f"Unsubscribed to {store_id}") for file_path in paths: @@ -1116,7 +1127,7 @@ async def process_offered_stores(self, offer_stores: tuple[OfferStore, ...]) -> node_hash=proof_of_inclusion.node_hash, layers=tuple( Layer( - other_hash_side=layer.other_hash_side, + other_hash_side=Side(layer.other_hash_side), other_hash=layer.other_hash, combined_hash=layer.combined_hash, ) @@ -1183,7 +1194,6 @@ async def make_offer( verify_offer(maker=offer.maker, taker=offer.taker, summary=summary) - await self.data_store.clean_node_table() return offer async def take_offer( @@ -1216,12 +1226,12 @@ async def take_offer( for layer in proof.layers ] proof_of_inclusion = ProofOfInclusion(node_hash=proof.node_hash, layers=layers) - sibling_sides_integer = proof_of_inclusion.sibling_sides_integer() + sibling_sides_integer = calculate_sibling_sides_integer(proof_of_inclusion) proofs_of_inclusion.append( ( root.hex(), str(sibling_sides_integer), - ["0x" + sibling_hash.hex() for sibling_hash in proof_of_inclusion.sibling_hashes()], + ["0x" + sibling_hash.hex() for sibling_hash in collect_sibling_hashes(proof_of_inclusion)], ) ) @@ -1242,8 +1252,6 @@ async def take_offer( }, } - await self.data_store.clean_node_table() - # Excluding wallet from transaction since failures in the wallet may occur # after the transaction is submitted to the chain. If we roll back data we # may lose published data. diff --git a/chia/data_layer/data_layer_errors.py b/chia/data_layer/data_layer_errors.py index 76416e948393..d40301e8a180 100644 --- a/chia/data_layer/data_layer_errors.py +++ b/chia/data_layer/data_layer_errors.py @@ -38,6 +38,11 @@ def __init__(self, key: bytes) -> None: super().__init__(f"Key not found: {key.hex()}") +class MerkleBlobNotFoundError(Exception): + def __init__(self, root_hash: bytes32) -> None: + super().__init__(f"Cannot find merkle blob for root hash {root_hash.hex()}") + + class OfferIntegrityError(Exception): pass diff --git a/chia/data_layer/data_layer_rpc_api.py b/chia/data_layer/data_layer_rpc_api.py index 92f4a4423b1e..ef4212e2261c 100644 --- a/chia/data_layer/data_layer_rpc_api.py +++ b/chia/data_layer/data_layer_rpc_api.py @@ -609,7 +609,7 @@ async def get_proof(self, request: GetProofRequest) -> GetProofResponse: for key in request.keys: node = await self.service.data_store.get_node_by_key(store_id=request.store_id, key=key) pi = await self.service.data_store.get_proof_of_inclusion_by_hash( - store_id=request.store_id, node_hash=node.hash, use_optimized=True + store_id=request.store_id, node_hash=node.hash ) proof = HashOnlyProof.from_key_value( diff --git a/chia/data_layer/data_layer_util.py b/chia/data_layer/data_layer_util.py index 517b763517a0..264fd10c769c 100644 --- a/chia/data_layer/data_layer_util.py +++ b/chia/data_layer/data_layer_util.py @@ -4,9 +4,11 @@ from dataclasses import dataclass, field from enum import Enum, IntEnum from hashlib import sha256 +from pathlib import Path from typing import TYPE_CHECKING, Any, Optional, Union import aiosqlite +from chia_rs.datalayer import ProofOfInclusion, ProofOfInclusionLayer from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint8, uint64 from typing_extensions import final @@ -48,6 +50,45 @@ def key_hash(key: bytes) -> bytes32: return bytes32(sha256(b"\1" + key).digest()) +# TODO: allow Optional[bytes32] for `node_hash` and resolve the filenames here +def get_full_tree_filename(store_id: bytes32, node_hash: bytes32, generation: int, group_by_store: bool = False) -> str: + if group_by_store: + return f"{store_id}/{node_hash}-full-{generation}-v1.0.dat" + return f"{store_id}-{node_hash}-full-{generation}-v1.0.dat" + + +def get_delta_filename(store_id: bytes32, node_hash: bytes32, generation: int, group_by_store: bool = False) -> str: + if group_by_store: + return f"{store_id}/{node_hash}-delta-{generation}-v1.0.dat" + return f"{store_id}-{node_hash}-delta-{generation}-v1.0.dat" + + +def get_full_tree_filename_path( + foldername: Path, + store_id: bytes32, + node_hash: bytes32, + generation: int, + group_by_store: bool = False, +) -> Path: + if group_by_store: + path = foldername.joinpath(f"{store_id}") + return path.joinpath(f"{node_hash}-full-{generation}-v1.0.dat") + return foldername.joinpath(f"{store_id}-{node_hash}-full-{generation}-v1.0.dat") + + +def get_delta_filename_path( + foldername: Path, + store_id: bytes32, + node_hash: bytes32, + generation: int, + group_by_store: bool = False, +) -> Path: + if group_by_store: + path = foldername.joinpath(f"{store_id}") + return path.joinpath(f"{node_hash}-delta-{generation}-v1.0.dat") + return foldername.joinpath(f"{store_id}-{node_hash}-delta-{generation}-v1.0.dat") + + @dataclasses.dataclass(frozen=True) class PaginationData: total_pages: int @@ -94,7 +135,6 @@ async def _dot_dump( root_hash: bytes32, ) -> str: terminal_nodes = await data_store.get_keys_values(store_id=store_id, root_hash=root_hash) - internal_nodes = await data_store.get_internal_nodes(store_id=store_id, root_hash=root_hash) n = 8 @@ -108,16 +148,7 @@ async def _dot_dump( value = terminal_node.value.hex() dot_nodes.append(f"""node_{hash} [shape=box, label="{hash[:n]}\\nkey: {key}\\nvalue: {value}"];""") - for internal_node in internal_nodes: - hash = internal_node.hash.hex() - left = internal_node.left_hash.hex() - right = internal_node.right_hash.hex() - dot_nodes.append(f"""node_{hash} [label="{hash[:n]}"]""") - dot_connections.append(f"""node_{hash} -> node_{left} [label="L"];""") - dot_connections.append(f"""node_{hash} -> node_{right} [label="R"];""") - dot_pair_boxes.append( - f"node [shape = box]; {{rank = same; node_{left}->node_{right}[style=invis]; rankdir = LR}}" - ) + # TODO: implement for internal nodes. currently this prints only terminal nodes lines = [ "digraph {", @@ -130,11 +161,6 @@ async def _dot_dump( return "\n".join(lines) -def row_to_node(row: aiosqlite.Row) -> Node: - cls = node_type_to_class[row["node_type"]] - return cls.from_row(row=row) - - class Status(IntEnum): PENDING = 1 COMMITTED = 2 @@ -147,9 +173,9 @@ class NodeType(IntEnum): @final -class Side(IntEnum): - LEFT = 0 - RIGHT = 1 +class Side(uint8, Enum): + LEFT = uint8(0) + RIGHT = uint8(1) def other(self) -> Side: if self == Side.LEFT: @@ -208,78 +234,12 @@ def from_row(cls, row: aiosqlite.Row) -> TerminalNode: ) -@final -@dataclass(frozen=True) -class ProofOfInclusionLayer: - other_hash_side: Side - other_hash: bytes32 - combined_hash: bytes32 - - @classmethod - def from_internal_node( - cls, - internal_node: InternalNode, - traversal_child_hash: bytes32, - ) -> ProofOfInclusionLayer: - return ProofOfInclusionLayer( - other_hash_side=internal_node.other_child_side(hash=traversal_child_hash), - other_hash=internal_node.other_child_hash(hash=traversal_child_hash), - combined_hash=internal_node.hash, - ) - - @classmethod - def from_hashes(cls, primary_hash: bytes32, other_hash_side: Side, other_hash: bytes32) -> ProofOfInclusionLayer: - combined_hash = calculate_internal_hash( - hash=primary_hash, - other_hash_side=other_hash_side, - other_hash=other_hash, - ) - - return cls(other_hash_side=other_hash_side, other_hash=other_hash, combined_hash=combined_hash) - - -other_side_to_bit = {Side.LEFT: 1, Side.RIGHT: 0} - - -@dataclass(frozen=True) -class ProofOfInclusion: - node_hash: bytes32 - # children before parents - layers: list[ProofOfInclusionLayer] - - @property - def root_hash(self) -> bytes32: - if len(self.layers) == 0: - return self.node_hash - - return self.layers[-1].combined_hash - - def sibling_sides_integer(self) -> int: - return sum(other_side_to_bit[layer.other_hash_side] << index for index, layer in enumerate(self.layers)) - - def sibling_hashes(self) -> list[bytes32]: - return [layer.other_hash for layer in self.layers] - - def as_program(self) -> Program: - return Program.to([self.sibling_sides_integer(), self.sibling_hashes()]) - - def valid(self) -> bool: - existing_hash = self.node_hash - - for layer in self.layers: - calculated_hash = calculate_internal_hash( - hash=existing_hash, other_hash_side=layer.other_hash_side, other_hash=layer.other_hash - ) - - if calculated_hash != layer.combined_hash: - return False - - existing_hash = calculated_hash +def calculate_sibling_sides_integer(proof: ProofOfInclusion) -> int: + return sum((1 << index if layer.other_hash_side == Side.LEFT else 0) for index, layer in enumerate(proof.layers)) - if existing_hash != self.root_hash: - return False - return True +def collect_sibling_hashes(proof: ProofOfInclusion) -> list[bytes32]: + return [layer.other_hash for layer in proof.layers] @final diff --git a/chia/data_layer/data_layer_wallet.py b/chia/data_layer/data_layer_wallet.py index 847b6e8c0116..6a6abb9a931e 100644 --- a/chia/data_layer/data_layer_wallet.py +++ b/chia/data_layer/data_layer_wallet.py @@ -6,13 +6,14 @@ from typing import TYPE_CHECKING, Any, ClassVar, Optional, cast from chia_rs import BlockRecord, CoinSpend, CoinState, G1Element, G2Element +from chia_rs.datalayer import ProofOfInclusion, ProofOfInclusionLayer from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint8, uint32, uint64, uint128 from clvm.EvalError import EvalError from typing_extensions import Unpack, final from chia.data_layer.data_layer_errors import LauncherCoinNotFoundError, OfferIntegrityError -from chia.data_layer.data_layer_util import OfferStore, ProofOfInclusion, ProofOfInclusionLayer, StoreProofs, leaf_hash +from chia.data_layer.data_layer_util import OfferStore, StoreProofs, leaf_hash from chia.data_layer.singleton_record import SingletonRecord from chia.server.ws_connection import WSChiaConnection from chia.types.blockchain_format.coin import Coin @@ -1226,7 +1227,7 @@ def verify_offer( raise OfferIntegrityError("maker: invalid proof of inclusion found") # TODO: verify each kv hash to the proof's node hash - roots = {proof.root_hash for proof in proofs} + roots = {proof.root_hash() for proof in proofs} if len(roots) > 1: raise OfferIntegrityError("maker: multiple roots referenced for a single store id") if len(roots) < 1: diff --git a/chia/data_layer/data_store.py b/chia/data_layer/data_store.py index c2357aa52e5e..882579f69a5e 100644 --- a/chia/data_layer/data_store.py +++ b/chia/data_layer/data_store.py @@ -1,18 +1,37 @@ from __future__ import annotations import contextlib +import copy +import itertools import logging +import shutil +import sqlite3 from collections import defaultdict -from collections.abc import AsyncIterator, Awaitable -from contextlib import asynccontextmanager -from dataclasses import dataclass, replace +from collections.abc import AsyncIterator, Awaitable, Iterable, Iterator, Sequence +from contextlib import asynccontextmanager, contextmanager +from dataclasses import dataclass, field, replace +from hashlib import sha256 from pathlib import Path from typing import Any, BinaryIO, Callable, Optional, Union import aiosqlite +import anyio.to_thread +import chia_rs.datalayer +import zstd +from chia_rs.datalayer import ( + DeltaFileCache, + DeltaReader, + KeyAlreadyPresentError, + KeyId, + MerkleBlob, + ProofOfInclusion, + TreeIndex, + ValueId, +) from chia_rs.sized_bytes import bytes32 +from chia_rs.sized_ints import int64 -from chia.data_layer.data_layer_errors import KeyNotFoundError, NodeHashError, TreeGenerationIncrementingError +from chia.data_layer.data_layer_errors import KeyNotFoundError, MerkleBlobNotFoundError, TreeGenerationIncrementingError from chia.data_layer.data_layer_util import ( DiffData, InsertResult, @@ -24,8 +43,6 @@ Node, NodeType, OperationType, - ProofOfInclusion, - ProofOfInclusionLayer, Root, SerializedNode, ServerInfo, @@ -34,15 +51,18 @@ Subscription, TerminalNode, Unspecified, + get_delta_filename_path, get_hashes_for_page, internal_hash, key_hash, leaf_hash, - row_to_node, unspecified, ) -from chia.types.blockchain_format.program import Program +from chia.util.batches import to_batches +from chia.util.cpu import available_logical_cores from chia.util.db_wrapper import SQLITE_MAX_VARIABLE_NUMBER, DBWrapper2 +from chia.util.log_exceptions import log_exceptions +from chia.util.lru_cache import LRUCache log = logging.getLogger(__name__) @@ -50,17 +70,33 @@ # TODO: review exceptions for values that shouldn't be displayed # TODO: pick exception types other than Exception +KeyOrValueId = int64 + +default_prefer_file_kv_blob_length: int = 4096 + @dataclass class DataStore: """A key/value store with the pairs being terminal nodes in a CLVM object tree.""" db_wrapper: DBWrapper2 + recent_merkle_blobs: LRUCache[bytes32, MerkleBlob] + merkle_blobs_path: Path + key_value_blobs_path: Path + unconfirmed_keys_values: dict[bytes32, list[bytes32]] = field(default_factory=dict) + prefer_db_kv_blob_length: int = default_prefer_file_kv_blob_length @classmethod @contextlib.asynccontextmanager async def managed( - cls, database: Union[str, Path], uri: bool = False, sql_log_path: Optional[Path] = None + cls, + database: Union[str, Path], + merkle_blobs_path: Path, + key_value_blobs_path: Path, + uri: bool = False, + sql_log_path: Optional[Path] = None, + cache_capacity: int = 1, + prefer_db_kv_blob_length: int = default_prefer_file_kv_blob_length, ) -> AsyncIterator[DataStore]: async with DBWrapper2.managed( database=database, @@ -75,46 +111,16 @@ async def managed( row_factory=aiosqlite.Row, log_path=sql_log_path, ) as db_wrapper: - self = cls(db_wrapper=db_wrapper) + recent_merkle_blobs: LRUCache[bytes32, MerkleBlob] = LRUCache(capacity=cache_capacity) + self = cls( + db_wrapper=db_wrapper, + recent_merkle_blobs=recent_merkle_blobs, + merkle_blobs_path=merkle_blobs_path, + key_value_blobs_path=key_value_blobs_path, + prefer_db_kv_blob_length=prefer_db_kv_blob_length, + ) async with db_wrapper.writer() as writer: - await writer.execute( - f""" - CREATE TABLE IF NOT EXISTS node( - hash BLOB PRIMARY KEY NOT NULL CHECK(length(hash) == 32), - node_type INTEGER NOT NULL CHECK( - ( - node_type == {int(NodeType.INTERNAL)} - AND left IS NOT NULL - AND right IS NOT NULL - AND key IS NULL - AND value IS NULL - ) - OR - ( - node_type == {int(NodeType.TERMINAL)} - AND left IS NULL - AND right IS NULL - AND key IS NOT NULL - AND value IS NOT NULL - ) - ), - left BLOB REFERENCES node, - right BLOB REFERENCES node, - key BLOB, - value BLOB - ) - """ - ) - await writer.execute( - """ - CREATE TRIGGER IF NOT EXISTS no_node_updates - BEFORE UPDATE ON node - BEGIN - SELECT RAISE(FAIL, 'updates not allowed to the node table'); - END - """ - ) await writer.execute( f""" CREATE TABLE IF NOT EXISTS root( @@ -124,25 +130,7 @@ async def managed( status INTEGER NOT NULL CHECK( {" OR ".join(f"status == {status}" for status in Status)} ), - PRIMARY KEY(tree_id, generation), - FOREIGN KEY(node_hash) REFERENCES node(hash) - ) - """ - ) - # TODO: Add ancestor -> hash relationship, this might involve temporarily - # deferring the foreign key enforcement due to the insertion order - # and the node table also enforcing a similar relationship in the - # other direction. - # FOREIGN KEY(ancestor) REFERENCES ancestors(ancestor) - await writer.execute( - """ - CREATE TABLE IF NOT EXISTS ancestors( - hash BLOB NOT NULL REFERENCES node, - ancestor BLOB CHECK(length(ancestor) == 32), - tree_id BLOB NOT NULL CHECK(length(tree_id) == 32), - generation INTEGER NOT NULL, - PRIMARY KEY(hash, tree_id, generation), - FOREIGN KEY(ancestor) REFERENCES node(hash) + PRIMARY KEY(tree_id, generation) ) """ ) @@ -173,7 +161,34 @@ async def managed( ) await writer.execute( """ - CREATE INDEX IF NOT EXISTS node_key_index ON node(key) + CREATE TABLE IF NOT EXISTS ids( + kv_id INTEGER PRIMARY KEY, + hash BLOB NOT NULL CHECK(length(store_id) == 32), + blob BLOB, + store_id BLOB NOT NULL CHECK(length(store_id) == 32) + ) + """ + ) + await writer.execute( + """ + CREATE TABLE IF NOT EXISTS nodes( + store_id BLOB NOT NULL CHECK(length(store_id) == 32), + hash BLOB NOT NULL, + root_hash BLOB NOT NULL, + generation INTEGER NOT NULL CHECK(generation >= 0), + idx INTEGER NOT NULL, + PRIMARY KEY(store_id, hash) + ) + """ + ) + await writer.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS ids_hash_index ON ids(hash, store_id) + """ + ) + await writer.execute( + """ + CREATE INDEX IF NOT EXISTS nodes_generation_index ON nodes(generation) """ ) @@ -184,19 +199,228 @@ async def transaction(self) -> AsyncIterator[None]: async with self.db_wrapper.writer(): yield - async def migrate_db(self) -> None: + async def insert_into_data_store_from_file( + self, + store_id: bytes32, + root_hash: Optional[bytes32], + filename: Path, + delta_reader: Optional[DeltaReader] = None, + ) -> Optional[DeltaReader]: + async with self.db_wrapper.writer(): + with self.manage_kv_files(store_id): + if root_hash is None: + merkle_blob = MerkleBlob(b"") + else: + root = await self.get_tree_root(store_id=store_id) + if delta_reader is None: + delta_reader = DeltaReader(internal_nodes={}, leaf_nodes={}) + if root.node_hash is not None: + delta_reader.collect_from_merkle_blob( + self.get_merkle_path(store_id=store_id, root_hash=root.node_hash), + indexes=[TreeIndex(0)], + ) + + internal_nodes, terminal_nodes = await self.read_from_file(filename, store_id) + delta_reader.add_internal_nodes(internal_nodes) + delta_reader.add_leaf_nodes(terminal_nodes) + + missing_hashes = await anyio.to_thread.run_sync(delta_reader.get_missing_hashes, root_hash) + + if len(missing_hashes) > 0: + # TODO: consider adding transactions around this code + merkle_blob_queries = await self.build_merkle_blob_queries_for_missing_hashes( + missing_hashes, store_id + ) + if len(merkle_blob_queries) > 0: + jobs = [ + (self.get_merkle_path(store_id=store_id, root_hash=old_root_hash), indexes) + for old_root_hash, indexes in merkle_blob_queries.items() + ] + await anyio.to_thread.run_sync(delta_reader.collect_from_merkle_blobs, jobs) + await self.build_cache_and_collect_missing_hashes(root, root_hash, store_id, delta_reader) + + merkle_blob = delta_reader.create_merkle_blob_and_filter_unused_nodes(root_hash, set()) + + # Don't store these blob objects into cache, since their data structures are not calculated yet. + await self.insert_root_from_merkle_blob(merkle_blob, store_id, Status.COMMITTED, update_cache=False) + return delta_reader + + async def build_merkle_blob_queries_for_missing_hashes( + self, + missing_hashes: set[bytes32], + store_id: bytes32, + ) -> defaultdict[bytes32, list[TreeIndex]]: + queries = defaultdict[bytes32, list[TreeIndex]](list) + + batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10) + + async with self.db_wrapper.reader() as reader: + for batch in to_batches(missing_hashes, batch_size): + placeholders = ",".join(["?"] * len(batch.entries)) + query = f""" + SELECT hash, root_hash, idx + FROM nodes + WHERE store_id = ? AND hash IN ({placeholders}) + LIMIT {len(batch.entries)} + """ + + async with reader.execute(query, (store_id, *batch.entries)) as cursor: + rows = await cursor.fetchall() + for row in rows: + root_hash_blob = bytes32(row["root_hash"]) + index = TreeIndex(row["idx"]) + queries[root_hash_blob].append(index) + + return queries + + async def build_cache_and_collect_missing_hashes( + self, + root: Root, + root_hash: bytes32, + store_id: bytes32, + delta_reader: DeltaReader, + ) -> None: + missing_hashes = delta_reader.get_missing_hashes(root_hash) + + if len(missing_hashes) == 0: + return + + async with self.db_wrapper.reader() as reader: + cursor = await reader.execute( + # TODO: the INDEXED BY seems like it shouldn't be needed, figure out why it is + # https://sqlite.org/lang_indexedby.html: admonished to omit all use of INDEXED BY + # https://sqlite.org/queryplanner-ng.html#howtofix + "SELECT MAX(generation) FROM nodes INDEXED BY nodes_generation_index WHERE store_id = ?", + (store_id,), + ) + generation_row = await cursor.fetchone() + if generation_row is None or generation_row[0] is None: + current_generation = 0 + else: + current_generation = generation_row[0] + generations: Sequence[int] = [current_generation] + + while missing_hashes: + if current_generation >= root.generation: + raise Exception("Invalid delta file, cannot find all the required hashes") + + current_generation = generations[-1] + 1 + + batch_size = available_logical_cores() + generations = range( + current_generation, + min(current_generation + batch_size, root.generation), + ) + jobs: list[tuple[bytes32, Path]] = [] + generations_by_root_hash: dict[bytes32, int] = {} + for generation in generations: + generation_root = await self.get_tree_root(store_id=store_id, generation=generation) + if generation_root.node_hash is None: + # no need to process an empty generation + continue + path = self.get_merkle_path(store_id=store_id, root_hash=generation_root.node_hash) + jobs.append((generation_root.node_hash, path)) + generations_by_root_hash[generation_root.node_hash] = generation + + found = await anyio.to_thread.run_sync( + delta_reader.collect_and_return_from_merkle_blobs, + jobs, + missing_hashes, + ) + async with self.db_wrapper.writer() as writer: + await writer.executemany( + """ + INSERT + OR IGNORE INTO nodes(store_id, hash, root_hash, generation, idx) + VALUES (?, ?, ?, ?, ?) + """, + ( + (store_id, hash, root_hash, generations_by_root_hash[root_hash], index.raw) + for root_hash, map in found + for hash, index in map.items() + ), + ) + + missing_hashes = delta_reader.get_missing_hashes(root_hash) + + log.info(f"Missing hashes: added old hashes from generation {current_generation}") + + async def read_from_file( + self, filename: Path, store_id: bytes32 + ) -> tuple[dict[bytes32, tuple[bytes32, bytes32]], dict[bytes32, tuple[KeyId, ValueId]]]: + internal_nodes: dict[bytes32, tuple[bytes32, bytes32]] = {} + terminal_nodes: dict[bytes32, tuple[KeyId, ValueId]] = {} + + with open(filename, "rb") as reader: + async with self.db_wrapper.writer() as writer: + while True: + chunk = b"" + while len(chunk) < 4: + size_to_read = 4 - len(chunk) + cur_chunk = reader.read(size_to_read) + if cur_chunk is None or cur_chunk == b"": + if size_to_read < 4: + raise Exception("Incomplete read of length.") + break + chunk += cur_chunk + if chunk == b"": + break + + size = int.from_bytes(chunk, byteorder="big") + serialize_nodes_bytes = b"" + while len(serialize_nodes_bytes) < size: + size_to_read = size - len(serialize_nodes_bytes) + cur_chunk = reader.read(size_to_read) + if cur_chunk is None or cur_chunk == b"": + raise Exception("Incomplete read of blob.") + serialize_nodes_bytes += cur_chunk + serialized_node = SerializedNode.from_bytes(serialize_nodes_bytes) + + node_type = NodeType.TERMINAL if serialized_node.is_terminal else NodeType.INTERNAL + if node_type == NodeType.INTERNAL: + node_hash = internal_hash(bytes32(serialized_node.value1), bytes32(serialized_node.value2)) + internal_nodes[node_hash] = (bytes32(serialized_node.value1), bytes32(serialized_node.value2)) + else: + kid, vid = await self.add_key_value( + serialized_node.value1, + serialized_node.value2, + store_id, + writer=writer, + ) + node_hash = leaf_hash(serialized_node.value1, serialized_node.value2) + terminal_nodes[node_hash] = (kid, vid) + + return internal_nodes, terminal_nodes + + async def migrate_db(self, server_files_location: Path) -> None: async with self.db_wrapper.reader() as reader: cursor = await reader.execute("SELECT * FROM schema") - row = await cursor.fetchone() - if row is not None: + rows = await cursor.fetchall() + all_versions = {"v1.0", "v2.0"} + + for row in rows: version = row["version_id"] - if version != "v1.0": + if version not in all_versions: raise Exception("Unknown version") - log.info(f"Found DB schema version {version}. No migration needed.") - return + if version == "v2.0": + log.info(f"Found DB schema version {version}. No migration needed.") + return + + version = "v2.0" + old_tables = ["node", "root", "ancestors"] + all_stores = await self.get_store_ids() + all_roots: list[list[Root]] = [] + for store_id in all_stores: + try: + root = await self.get_tree_root(store_id=store_id) + roots = await self.get_roots_between(store_id, 1, root.generation) + all_roots.append([*roots, root]) + except Exception as e: + if "unable to find root for id, generation" in str(e): + log.error(f"Cannot find roots for {store_id}. Skipping it.") + + log.info(f"Initiating migration to version {version}. Found {len(all_roots)} stores to migrate") - version = "v1.0" - log.info(f"Initiating migration to version {version}") async with self.db_wrapper.writer(foreign_key_enforcement_enabled=False) as writer: await writer.execute( f""" @@ -207,16 +431,360 @@ async def migrate_db(self) -> None: status INTEGER NOT NULL CHECK( {" OR ".join(f"status == {status}" for status in Status)} ), - PRIMARY KEY(tree_id, generation), - FOREIGN KEY(node_hash) REFERENCES node(hash) + PRIMARY KEY(tree_id, generation) ) """ ) - await writer.execute("INSERT INTO new_root SELECT * FROM root") - await writer.execute("DROP TABLE root") + for old_table in old_tables: + await writer.execute(f"DROP TABLE IF EXISTS {old_table}") await writer.execute("ALTER TABLE new_root RENAME TO root") await writer.execute("INSERT INTO schema (version_id) VALUES (?)", (version,)) - log.info(f"Finished migrating DB to version {version}") + log.info(f"Initialized new DB schema {version}.") + + total_generations = 0 + synced_generations = 0 + for roots in all_roots: + assert len(roots) > 0 + total_generations += len(roots) + + for roots in all_roots: + store_id = roots[0].store_id + await self.create_tree(store_id=store_id, status=Status.COMMITTED) + + expected_synced_generations = synced_generations + len(roots) + for root in roots: + recovery_filename: Optional[Path] = None + + for group_by_store in (True, False): + filename = get_delta_filename_path( + server_files_location, + store_id, + bytes32.zeros if root.node_hash is None else root.node_hash, + root.generation, + group_by_store, + ) + + if filename.exists(): + log.info(f"Found filename {filename}. Recovering data from it") + recovery_filename = filename + break + + if recovery_filename is None: + log.error(f"Cannot find any recovery file for root {root}") + break + + try: + await self.insert_into_data_store_from_file(store_id, root.node_hash, recovery_filename) + synced_generations += 1 + log.info( + f"Successfully recovered root from {filename}. " + f"Total roots processed: {(synced_generations / total_generations * 100):.2f}%" + ) + except Exception as e: + log.error(f"Cannot recover data from {filename}: {e}") + break + + if synced_generations < expected_synced_generations: + log.error( + f"Could not recover {expected_synced_generations - synced_generations} generations. " + f"Consider resyncing the store {store_id} once the migration is complete." + ) + # Reset the counter as if we synced correctly, so the percentages add to 100% at the end. + synced_generations = expected_synced_generations + + async def get_merkle_blob( + self, + store_id: bytes32, + root_hash: Optional[bytes32], + read_only: bool = False, + update_cache: bool = True, + ) -> MerkleBlob: + if root_hash is None: + return MerkleBlob(blob=b"") + if self.recent_merkle_blobs.get_capacity() == 0: + update_cache = False + + existing_blob = self.recent_merkle_blobs.get(root_hash) + if existing_blob is not None: + return existing_blob if read_only else copy.deepcopy(existing_blob) + + try: + with log_exceptions(log=log, message="Error while getting merkle blob"): + path = self.get_merkle_path(store_id=store_id, root_hash=root_hash) + # TODO: consider file-system based locking of either the file or the store directory + merkle_blob = MerkleBlob.from_path(path) + except Exception as e: + raise MerkleBlobNotFoundError(root_hash=root_hash) from e + + if update_cache: + self.recent_merkle_blobs.put(root_hash, copy.deepcopy(merkle_blob)) + + return merkle_blob + + def get_bytes_path(self, bytes_: bytes) -> Path: + raw = bytes_.hex() + segment_sizes = [2, 2, 2] + start = 0 + segments = [] + for size in segment_sizes: + segments.append(raw[start : start + size]) + start += size + + return Path(*segments, raw) + + def get_merkle_path(self, store_id: bytes32, root_hash: Optional[bytes32]) -> Path: + store_root = self.merkle_blobs_path.joinpath(store_id.hex()) + if root_hash is None: + return store_root + + return store_root.joinpath(self.get_bytes_path(bytes_=root_hash)) + + def get_key_value_path(self, store_id: bytes32, blob_hash: Optional[bytes32]) -> Path: + store_root = self.key_value_blobs_path.joinpath(store_id.hex()) + if blob_hash is None: + return store_root + + return store_root.joinpath(self.get_bytes_path(bytes_=blob_hash)) + + async def insert_root_from_merkle_blob( + self, + merkle_blob: MerkleBlob, + store_id: bytes32, + status: Status, + old_root: Optional[Root] = None, + update_cache: bool = True, + ) -> Root: + if not merkle_blob.empty(): + merkle_blob.calculate_lazy_hashes() + if self.recent_merkle_blobs.get_capacity() == 0: + update_cache = False + + root_hash = merkle_blob.get_root_hash() + if old_root is not None and old_root.node_hash == root_hash: + raise ValueError("Changelist resulted in no change to tree data") + + if root_hash is not None: + log.info(f"inserting merkle blob: {len(merkle_blob)} bytes {root_hash.hex()}") + blob_path = self.get_merkle_path(store_id=store_id, root_hash=merkle_blob.get_root_hash()) + if not blob_path.exists(): + blob_path.parent.mkdir(parents=True, exist_ok=True) + # TODO: consider file-system based locking of either the file or the store directory + merkle_blob.to_path(blob_path) + + if update_cache: + self.recent_merkle_blobs.put(root_hash, copy.deepcopy(merkle_blob)) + + return await self._insert_root(store_id, root_hash, status) + + def _use_file_for_new_kv_blob(self, blob: bytes) -> bool: + return len(blob) > self.prefer_db_kv_blob_length + + async def get_kvid(self, blob: bytes, store_id: bytes32) -> Optional[KeyOrValueId]: + blob_hash = bytes32(sha256(blob).digest()) + + async with self.db_wrapper.reader() as reader: + cursor = await reader.execute( + "SELECT kv_id FROM ids WHERE hash = ? AND store_id = ?", + ( + blob_hash, + store_id, + ), + ) + row = await cursor.fetchone() + + if row is None: + return None + + return KeyOrValueId(row[0]) + + def get_blob_from_file(self, blob_hash: bytes32, store_id: bytes32) -> bytes: + # TODO: seems that zstd needs hinting + # TODO: consider file-system based locking of either the file or the store directory + return zstd.decompress(self.get_key_value_path(store_id=store_id, blob_hash=blob_hash).read_bytes()) # type: ignore[no-any-return] + + async def get_blob_from_kvid(self, kv_id: KeyOrValueId, store_id: bytes32) -> Optional[bytes]: + async with self.db_wrapper.reader() as reader: + cursor = await reader.execute( + "SELECT hash, blob FROM ids WHERE kv_id = ? AND store_id = ?", + ( + kv_id, + store_id, + ), + ) + row = await cursor.fetchone() + + if row is None: + return None + + blob: bytes = row["blob"] + if blob is not None: + return blob + + blob_hash = bytes32(row["hash"]) + return self.get_blob_from_file(blob_hash, store_id) + + async def get_terminal_node(self, kid: KeyId, vid: ValueId, store_id: bytes32) -> TerminalNode: + key = await self.get_blob_from_kvid(kid.raw, store_id) + value = await self.get_blob_from_kvid(vid.raw, store_id) + if key is None or value is None: + raise Exception("Cannot find the key/value pair") + + return TerminalNode(hash=leaf_hash(key, value), key=key, value=value) + + async def add_kvid(self, blob: bytes, store_id: bytes32, writer: aiosqlite.Connection) -> KeyOrValueId: + use_file = self._use_file_for_new_kv_blob(blob) + blob_hash = bytes32(sha256(blob).digest()) + if use_file: + table_blob = None + else: + table_blob = blob + try: + row = await writer.execute_insert( + "INSERT INTO ids (hash, blob, store_id) VALUES (?, ?, ?)", + ( + blob_hash, + table_blob, + store_id, + ), + ) + except sqlite3.IntegrityError as e: + if "UNIQUE constraint failed" in str(e): + kv_id = await self.get_kvid(blob, store_id) + if kv_id is None: + raise Exception("Internal error") from e + return kv_id + + raise + + if row is None: + raise Exception("Internal error") + if use_file: + path = self.get_key_value_path(store_id=store_id, blob_hash=blob_hash) + path.parent.mkdir(parents=True, exist_ok=True) + self.unconfirmed_keys_values[store_id].append(blob_hash) + # TODO: consider file-system based locking of either the file or the store directory + path.write_bytes(zstd.compress(blob)) + return KeyOrValueId(row[0]) + + def delete_unconfirmed_kvids(self, store_id: bytes32) -> None: + for blob_hash in self.unconfirmed_keys_values[store_id]: + with log_exceptions(log=log, consume=True): + path = self.get_key_value_path(store_id=store_id, blob_hash=blob_hash) + try: + path.unlink() + except FileNotFoundError: + log.error(f"Cannot find key/value path {path} for hash {blob_hash}") + del self.unconfirmed_keys_values[store_id] + + def confirm_all_kvids(self, store_id: bytes32) -> None: + del self.unconfirmed_keys_values[store_id] + + @contextmanager + def manage_kv_files(self, store_id: bytes32) -> Iterator[None]: + if store_id not in self.unconfirmed_keys_values: + self.unconfirmed_keys_values[store_id] = [] + else: + raise Exception("Internal error: unconfirmed keys values cache not cleaned") + + try: + yield + except: + self.delete_unconfirmed_kvids(store_id) + raise + else: + self.confirm_all_kvids(store_id) + + async def add_key_value( + self, key: bytes, value: bytes, store_id: bytes32, writer: aiosqlite.Connection + ) -> tuple[KeyId, ValueId]: + kid = KeyId(await self.add_kvid(key, store_id, writer=writer)) + vid = ValueId(await self.add_kvid(value, store_id, writer=writer)) + + return (kid, vid) + + async def get_terminal_node_by_hash( + self, + node_hash: bytes32, + store_id: bytes32, + root_hash: Union[bytes32, Unspecified] = unspecified, + ) -> TerminalNode: + resolved_root_hash: Optional[bytes32] + if root_hash is unspecified: + root = await self.get_tree_root(store_id=store_id) + resolved_root_hash = root.node_hash + else: + resolved_root_hash = root_hash + + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash) + kid, vid = merkle_blob.get_node_by_hash(node_hash) + return await self.get_terminal_node(kid, vid, store_id) + + async def get_terminal_nodes_by_hashes( + self, + node_hashes: list[bytes32], + store_id: bytes32, + root_hash: Union[bytes32, Unspecified] = unspecified, + ) -> list[TerminalNode]: + resolved_root_hash: Optional[bytes32] + if root_hash is unspecified: + root = await self.get_tree_root(store_id=store_id) + resolved_root_hash = root.node_hash + else: + resolved_root_hash = root_hash + + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash) + kv_ids: list[tuple[KeyId, ValueId]] = [] + for node_hash in node_hashes: + kid, vid = merkle_blob.get_node_by_hash(node_hash) + kv_ids.append((kid, vid)) + kv_ids_unpacked = (KeyOrValueId(id.raw) for kv_id in kv_ids for id in kv_id) + table_blobs = await self.get_table_blobs(kv_ids_unpacked, store_id) + + terminal_nodes: list[TerminalNode] = [] + for kid, vid in kv_ids: + terminal_nodes.append(self.get_terminal_node_from_table_blobs(kid, vid, table_blobs, store_id)) + + return terminal_nodes + + async def get_existing_hashes(self, node_hashes: list[bytes32], store_id: bytes32) -> set[bytes32]: + result: set[bytes32] = set() + batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10) + + async with self.db_wrapper.reader() as reader: + for i in range(0, len(node_hashes), batch_size): + chunk = node_hashes[i : i + batch_size] + placeholders = ",".join(["?"] * len(chunk)) + query = f"SELECT hash FROM nodes WHERE store_id = ? AND hash IN ({placeholders}) LIMIT {len(chunk)}" + + async with reader.execute(query, (store_id, *chunk)) as cursor: + rows = await cursor.fetchall() + result.update(row["hash"] for row in rows) + + return result + + async def add_node_hashes(self, store_id: bytes32, generation: Optional[int] = None) -> None: + root = await self.get_tree_root(store_id=store_id, generation=generation) + if root.node_hash is None: + return + + merkle_blob = await self.get_merkle_blob( + store_id=store_id, root_hash=root.node_hash, read_only=True, update_cache=False + ) + hash_to_index = merkle_blob.get_hashes_indexes() + + existing_hashes = await self.get_existing_hashes(list(hash_to_index.keys()), store_id) + async with self.db_wrapper.writer() as writer: + await writer.executemany( + """ + INSERT INTO nodes(store_id, hash, root_hash, generation, idx) + VALUES (?, ?, ?, ?, ?) + """, + ( + (store_id, hash, root.node_hash, root.generation, index.raw) + for hash, index in hash_to_index.items() + if hash not in existing_hashes + ), + ) async def _insert_root( self, @@ -225,11 +793,7 @@ async def _insert_root( status: Status, generation: Optional[int] = None, ) -> Root: - # This should be replaced by an SQLite schema level check. - # https://github.com/Chia-Network/chia-blockchain/pull/9284 - store_id = bytes32(store_id) - - async with self.db_wrapper.writer() as writer: + async with self.db_wrapper.writer_maybe_transaction() as writer: if generation is None: try: existing_generation = await self.get_tree_generation(store_id=store_id) @@ -254,173 +818,8 @@ async def _insert_root( """, new_root.to_row(), ) - - # `node_hash` is now a root, so it has no ancestor. - # Don't change the ancestor table unless the root is committed. - if node_hash is not None and status == Status.COMMITTED: - values = { - "hash": node_hash, - "tree_id": store_id, - "generation": generation, - } - await writer.execute( - """ - INSERT INTO ancestors(hash, ancestor, tree_id, generation) - VALUES (:hash, NULL, :tree_id, :generation) - """, - values, - ) - return new_root - async def _insert_node( - self, - node_hash: bytes32, - node_type: NodeType, - left_hash: Optional[bytes32], - right_hash: Optional[bytes32], - key: Optional[bytes], - value: Optional[bytes], - ) -> None: - # TODO: can we get sqlite to do this check? - values = { - "hash": node_hash, - "node_type": node_type, - "left": left_hash, - "right": right_hash, - "key": key, - "value": value, - } - - async with self.db_wrapper.writer() as writer: - try: - await writer.execute( - """ - INSERT INTO node(hash, node_type, left, right, key, value) - VALUES(:hash, :node_type, :left, :right, :key, :value) - """, - values, - ) - except aiosqlite.IntegrityError as e: - if not e.args[0].startswith("UNIQUE constraint"): - # UNIQUE constraint failed: node.hash - raise - - async with writer.execute( - "SELECT * FROM node WHERE hash == :hash LIMIT 1", - {"hash": node_hash}, - ) as cursor: - result = await cursor.fetchone() - - if result is None: - # some ideas for causes: - # an sqlite bug - # bad queries in this function - # unexpected db constraints - raise Exception("Unable to find conflicting row") from e # pragma: no cover - - result_dict = dict(result) - if result_dict != values: - raise Exception( - f"Requested insertion of node with matching hash but other values differ: {node_hash}" - ) from None - - async def insert_node(self, node_type: NodeType, value1: bytes, value2: bytes) -> None: - if node_type == NodeType.INTERNAL: - left_hash = bytes32(value1) - right_hash = bytes32(value2) - node_hash = internal_hash(left_hash, right_hash) - await self._insert_node(node_hash, node_type, bytes32(value1), bytes32(value2), None, None) - else: - node_hash = leaf_hash(key=value1, value=value2) - await self._insert_node(node_hash, node_type, None, None, value1, value2) - - async def _insert_internal_node(self, left_hash: bytes32, right_hash: bytes32) -> bytes32: - node_hash: bytes32 = internal_hash(left_hash=left_hash, right_hash=right_hash) - - await self._insert_node( - node_hash=node_hash, - node_type=NodeType.INTERNAL, - left_hash=left_hash, - right_hash=right_hash, - key=None, - value=None, - ) - - return node_hash - - async def _insert_ancestor_table( - self, - left_hash: bytes32, - right_hash: bytes32, - store_id: bytes32, - generation: int, - ) -> None: - node_hash = internal_hash(left_hash=left_hash, right_hash=right_hash) - - async with self.db_wrapper.writer() as writer: - for hash in (left_hash, right_hash): - values = { - "hash": hash, - "ancestor": node_hash, - "tree_id": store_id, - "generation": generation, - } - try: - await writer.execute( - """ - INSERT INTO ancestors(hash, ancestor, tree_id, generation) - VALUES (:hash, :ancestor, :tree_id, :generation) - """, - values, - ) - except aiosqlite.IntegrityError as e: - if not e.args[0].startswith("UNIQUE constraint"): - # UNIQUE constraint failed: ancestors.hash, ancestors.tree_id, ancestors.generation - raise - - async with writer.execute( - """ - SELECT * - FROM ancestors - WHERE hash == :hash AND generation == :generation AND tree_id == :tree_id - LIMIT 1 - """, - {"hash": hash, "generation": generation, "tree_id": store_id}, - ) as cursor: - result = await cursor.fetchone() - - if result is None: - # some ideas for causes: - # an sqlite bug - # bad queries in this function - # unexpected db constraints - raise Exception("Unable to find conflicting row") from e # pragma: no cover - - result_dict = dict(result) - if result_dict != values: - raise Exception( - "Requested insertion of ancestor, where ancestor differ, but other values are identical: " - f"{hash} {generation} {store_id}" - ) from None - - async def _insert_terminal_node(self, key: bytes, value: bytes) -> bytes32: - # forcing type hint here for: - # https://github.com/Chia-Network/clvm/pull/102 - # https://github.com/Chia-Network/clvm/pull/106 - node_hash: bytes32 = Program.to((key, value)).get_tree_hash() - - await self._insert_node( - node_hash=node_hash, - node_type=NodeType.TERMINAL, - left_hash=None, - right_hash=None, - key=key, - value=value, - ) - - return node_hash - async def get_pending_root(self, store_id: bytes32) -> Optional[Root]: async with self.db_wrapper.reader() as reader: cursor = await reader.execute( @@ -478,21 +877,6 @@ async def change_root_status(self, root: Root, status: Status = Status.PENDING) root.generation, ), ) - # `node_hash` is now a root, so it has no ancestor. - # Don't change the ancestor table unless the root is committed. - if root.node_hash is not None and status == Status.COMMITTED: - values = { - "hash": root.node_hash, - "tree_id": root.store_id, - "generation": root.generation, - } - await writer.execute( - """ - INSERT INTO ancestors(hash, ancestor, tree_id, generation) - VALUES (:hash, NULL, :tree_id, :generation) - """, - values, - ) async def check(self) -> None: for check in self._checks: @@ -518,30 +902,7 @@ async def _check_roots_are_incrementing(self) -> None: if len(bad_trees) > 0: raise TreeGenerationIncrementingError(store_ids=bad_trees) - async def _check_hashes(self) -> None: - async with self.db_wrapper.reader() as reader: - cursor = await reader.execute("SELECT * FROM node") - - bad_node_hashes: list[bytes32] = [] - async for row in cursor: - node = row_to_node(row=row) - if isinstance(node, InternalNode): - expected_hash = internal_hash(left_hash=node.left_hash, right_hash=node.right_hash) - elif isinstance(node, TerminalNode): - expected_hash = Program.to((node.key, node.value)).get_tree_hash() - else: - raise Exception(f"Internal error, unknown node type: {node!r}") - - if node.hash != expected_hash: - bad_node_hashes.append(node.hash) - - if len(bad_node_hashes) > 0: - raise NodeHashError(node_hashes=bad_node_hashes) - - _checks: tuple[Callable[[DataStore], Awaitable[None]], ...] = ( - _check_roots_are_incrementing, - _check_hashes, - ) + _checks: tuple[Callable[[DataStore], Awaitable[None]], ...] = (_check_roots_are_incrementing,) async def create_tree(self, store_id: bytes32, status: Status = Status.PENDING) -> bool: await self._insert_root(store_id=store_id, node_hash=None, status=status) @@ -634,165 +995,62 @@ async def get_roots_between(self, store_id: bytes32, generation_begin: int, gene return roots - async def get_last_tree_root_by_hash( - self, store_id: bytes32, hash: Optional[bytes32], max_generation: Optional[int] = None - ) -> Optional[Root]: - async with self.db_wrapper.reader() as reader: - max_generation_str = "AND generation < :max_generation " if max_generation is not None else "" - node_hash_str = "AND node_hash == :node_hash " if hash is not None else "AND node_hash is NULL " - cursor = await reader.execute( - "SELECT * FROM root WHERE tree_id == :tree_id " - f"{max_generation_str}" - f"{node_hash_str}" - "ORDER BY generation DESC LIMIT 1", - {"tree_id": store_id, "node_hash": hash, "max_generation": max_generation}, - ) - row = await cursor.fetchone() - - if row is None: - return None - return Root.from_row(row=row) - async def get_ancestors( self, node_hash: bytes32, store_id: bytes32, root_hash: Optional[bytes32] = None, - ) -> list[InternalNode]: - async with self.db_wrapper.reader() as reader: - if root_hash is None: - root = await self.get_tree_root(store_id=store_id) - root_hash = root.node_hash - if root_hash is None: - raise Exception(f"Root hash is unspecified for store ID: {store_id.hex()}") - cursor = await reader.execute( - """ - WITH RECURSIVE - tree_from_root_hash(hash, node_type, left, right, key, value, depth) AS ( - SELECT node.*, 0 AS depth FROM node WHERE node.hash == :root_hash - UNION ALL - SELECT node.*, tree_from_root_hash.depth + 1 AS depth FROM node, tree_from_root_hash - WHERE node.hash == tree_from_root_hash.left OR node.hash == tree_from_root_hash.right - ), - ancestors(hash, node_type, left, right, key, value, depth) AS ( - SELECT node.*, NULL AS depth FROM node - WHERE node.left == :reference_hash OR node.right == :reference_hash - UNION ALL - SELECT node.*, NULL AS depth FROM node, ancestors - WHERE node.left == ancestors.hash OR node.right == ancestors.hash - ) - SELECT * FROM tree_from_root_hash INNER JOIN ancestors - WHERE tree_from_root_hash.hash == ancestors.hash - ORDER BY tree_from_root_hash.depth DESC - """, - {"reference_hash": node_hash, "root_hash": root_hash}, - ) - - # The resulting rows must represent internal nodes. InternalNode.from_row() - # does some amount of validation in the sense that it will fail if left - # or right can't turn into a bytes32 as expected. There is room for more - # validation here if desired. - ancestors = [InternalNode.from_row(row=row) async for row in cursor] - - return ancestors - - async def get_ancestors_optimized( - self, - node_hash: bytes32, - store_id: bytes32, generation: Optional[int] = None, - root_hash: Optional[bytes32] = None, ) -> list[InternalNode]: async with self.db_wrapper.reader(): - nodes = [] if root_hash is None: root = await self.get_tree_root(store_id=store_id, generation=generation) root_hash = root.node_hash - if root_hash is None: - return [] - - while True: - internal_node = await self._get_one_ancestor(node_hash, store_id, generation) - if internal_node is None: - break - nodes.append(internal_node) - node_hash = internal_node.hash - - if len(nodes) > 0: - if root_hash != nodes[-1].hash: - raise RuntimeError("Ancestors list didn't produce the root as top result.") - - return nodes + raise Exception(f"Root hash is unspecified for store ID: {store_id.hex()}") - async def get_internal_nodes(self, store_id: bytes32, root_hash: Optional[bytes32] = None) -> list[InternalNode]: - async with self.db_wrapper.reader() as reader: - if root_hash is None: - root = await self.get_tree_root(store_id=store_id) - root_hash = root.node_hash - cursor = await reader.execute( - """ - WITH RECURSIVE - tree_from_root_hash(hash, node_type, left, right, key, value) AS ( - SELECT node.* FROM node WHERE node.hash == :root_hash - UNION ALL - SELECT node.* FROM node, tree_from_root_hash WHERE node.hash == tree_from_root_hash.left - OR node.hash == tree_from_root_hash.right - ) - SELECT * FROM tree_from_root_hash - WHERE node_type == :node_type - """, - {"root_hash": root_hash, "node_type": NodeType.INTERNAL}, + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root_hash) + reference_kid, _ = merkle_blob.get_node_by_hash(node_hash) + + reference_index = merkle_blob.get_key_index(reference_kid) + lineage = merkle_blob.get_lineage_with_indexes(reference_index) + result: list[InternalNode] = [] + for index, node in itertools.islice(lineage, 1, None): + assert isinstance(node, chia_rs.datalayer.InternalNode) + result.append( + InternalNode( + hash=node.hash, + left_hash=merkle_blob.get_hash_at_index(node.left), + right_hash=merkle_blob.get_hash_at_index(node.right), + ) ) + return result - internal_nodes: list[InternalNode] = [] - async for row in cursor: - node = row_to_node(row=row) - if not isinstance(node, InternalNode): - raise Exception(f"Unexpected internal node found: {node.hash.hex()}") - internal_nodes.append(node) + def get_terminal_node_from_table_blobs( + self, + kid: KeyId, + vid: ValueId, + table_blobs: dict[KeyOrValueId, tuple[bytes32, Optional[bytes]]], + store_id: bytes32, + ) -> TerminalNode: + key = table_blobs[KeyOrValueId(kid.raw)][1] + if key is None: + key_hash = table_blobs[KeyOrValueId(kid.raw)][0] + key = self.get_blob_from_file(key_hash, store_id) - return internal_nodes + value = table_blobs[KeyOrValueId(vid.raw)][1] + if value is None: + value_hash = table_blobs[KeyOrValueId(vid.raw)][0] + value = self.get_blob_from_file(value_hash, store_id) - async def get_keys_values_cursor( - self, - reader: aiosqlite.Connection, - root_hash: Optional[bytes32], - only_keys: bool = False, - ) -> aiosqlite.Cursor: - select_clause = "SELECT hash, key" if only_keys else "SELECT *" - maybe_value = "" if only_keys else "value, " - select_node_clause = "node.hash, node.node_type, node.left, node.right, node.key" if only_keys else "node.*" - return await reader.execute( - f""" - WITH RECURSIVE - tree_from_root_hash(hash, node_type, left, right, key, {maybe_value}depth, rights) AS ( - SELECT {select_node_clause}, 0 AS depth, 0 AS rights FROM node WHERE node.hash == :root_hash - UNION ALL - SELECT - {select_node_clause}, - tree_from_root_hash.depth + 1 AS depth, - CASE - WHEN node.hash == tree_from_root_hash.right - THEN tree_from_root_hash.rights + (1 << (62 - tree_from_root_hash.depth)) - ELSE tree_from_root_hash.rights - END AS rights - FROM node, tree_from_root_hash - WHERE node.hash == tree_from_root_hash.left OR node.hash == tree_from_root_hash.right - ) - {select_clause} FROM tree_from_root_hash - WHERE node_type == :node_type - ORDER BY depth ASC, rights ASC - """, - {"root_hash": root_hash, "node_type": NodeType.TERMINAL}, - ) + return TerminalNode(hash=leaf_hash(key, value), key=key, value=value) async def get_keys_values( self, store_id: bytes32, root_hash: Union[bytes32, Unspecified] = unspecified, ) -> list[TerminalNode]: - async with self.db_wrapper.reader() as reader: + async with self.db_wrapper.reader(): resolved_root_hash: Optional[bytes32] if root_hash is unspecified: root = await self.get_tree_root(store_id=store_id) @@ -800,25 +1058,18 @@ async def get_keys_values( else: resolved_root_hash = root_hash - cursor = await self.get_keys_values_cursor(reader, resolved_root_hash) + try: + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash) + except MerkleBlobNotFoundError: + return [] + + kv_ids = merkle_blob.get_keys_values() + kv_ids_unpacked = (KeyOrValueId(id.raw) for pair in kv_ids.items() for id in pair) + table_blobs = await self.get_table_blobs(kv_ids_unpacked, store_id) + terminal_nodes: list[TerminalNode] = [] - async for row in cursor: - if row["depth"] > 62: - # TODO: Review the value and implementation of left-to-right order - # reporting. Initial use is for balanced insertion with the - # work done in the query. - - # This is limited based on the choice of 63 for the maximum left - # shift in the query. This is in turn based on the SQLite integers - # ranging in size up to signed 8 bytes, 64 bits. If we exceed this then - # we no longer guarantee the left-to-right ordering of the node - # list. While 63 allows for a lot of nodes in a balanced tree, in - # the worst case it allows only 62 terminal nodes. - raise Exception("Tree depth exceeded 62, unable to guarantee left-to-right node order.") - node = row_to_node(row=row) - if not isinstance(node, TerminalNode): - raise Exception(f"Unexpected internal node found: {node.hash.hex()}") - terminal_nodes.append(node) + for kid, vid in kv_ids.items(): + terminal_nodes.append(self.get_terminal_node_from_table_blobs(kid, vid, table_blobs, store_id)) return terminal_nodes @@ -827,7 +1078,7 @@ async def get_keys_values_compressed( store_id: bytes32, root_hash: Union[bytes32, Unspecified] = unspecified, ) -> KeysValuesCompressed: - async with self.db_wrapper.reader() as reader: + async with self.db_wrapper.reader(): resolved_root_hash: Optional[bytes32] if root_hash is unspecified: root = await self.get_tree_root(store_id=store_id) @@ -835,36 +1086,26 @@ async def get_keys_values_compressed( else: resolved_root_hash = root_hash - cursor = await self.get_keys_values_cursor(reader, resolved_root_hash) keys_values_hashed: dict[bytes32, bytes32] = {} key_hash_to_length: dict[bytes32, int] = {} leaf_hash_to_length: dict[bytes32, int] = {} - async for row in cursor: - if row["depth"] > 62: - raise Exception("Tree depth exceeded 62, unable to guarantee left-to-right node order.") - node = row_to_node(row=row) - if not isinstance(node, TerminalNode): - raise Exception(f"Unexpected internal node found: {node.hash.hex()}") - keys_values_hashed[key_hash(node.key)] = leaf_hash(node.key, node.value) - key_hash_to_length[key_hash(node.key)] = len(node.key) - leaf_hash_to_length[leaf_hash(node.key, node.value)] = len(node.key) + len(node.value) + if resolved_root_hash is not None: + try: + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash) + except MerkleBlobNotFoundError: + return KeysValuesCompressed({}, {}, {}, resolved_root_hash) - return KeysValuesCompressed(keys_values_hashed, key_hash_to_length, leaf_hash_to_length, resolved_root_hash) + kv_ids = merkle_blob.get_keys_values() + kv_ids_unpacked = (KeyOrValueId(id.raw) for pair in kv_ids.items() for id in pair) + table_blobs = await self.get_table_blobs(kv_ids_unpacked, store_id) - async def get_leaf_hashes_by_hashed_key( - self, store_id: bytes32, root_hash: Optional[bytes32] = None - ) -> dict[bytes32, bytes32]: - result: dict[bytes32, bytes32] = {} - async with self.db_wrapper.reader() as reader: - if root_hash is None: - root = await self.get_tree_root(store_id=store_id) - root_hash = root.node_hash - - cursor = await self.get_keys_values_cursor(reader, root_hash, True) - async for row in cursor: - result[key_hash(row["key"])] = bytes32(row["hash"]) + for kid, vid in kv_ids.items(): + node = self.get_terminal_node_from_table_blobs(kid, vid, table_blobs, store_id) + keys_values_hashed[key_hash(node.key)] = leaf_hash(node.key, node.value) + key_hash_to_length[key_hash(node.key)] = len(node.key) + leaf_hash_to_length[leaf_hash(node.key, node.value)] = len(node.key) + len(node.value) - return result + return KeysValuesCompressed(keys_values_hashed, key_hash_to_length, leaf_hash_to_length, resolved_root_hash) async def get_keys_paginated( self, @@ -877,11 +1118,12 @@ async def get_keys_paginated( pagination_data = get_hashes_for_page(page, keys_values_compressed.key_hash_to_length, max_page_size) keys: list[bytes] = [] + leaf_hashes: list[bytes32] = [] for hash in pagination_data.hashes: leaf_hash = keys_values_compressed.keys_values_hashed[hash] - node = await self.get_node(leaf_hash) - assert isinstance(node, TerminalNode) - keys.append(node.key) + leaf_hashes.append(leaf_hash) + nodes = await self.get_terminal_nodes_by_hashes(leaf_hashes, store_id, root_hash) + keys = [node.key for node in nodes] return KeysPaginationData( pagination_data.total_pages, @@ -900,12 +1142,7 @@ async def get_keys_values_paginated( keys_values_compressed = await self.get_keys_values_compressed(store_id, root_hash) pagination_data = get_hashes_for_page(page, keys_values_compressed.leaf_hash_to_length, max_page_size) - keys_values: list[TerminalNode] = [] - for hash in pagination_data.hashes: - node = await self.get_node(hash) - assert isinstance(node, TerminalNode) - keys_values.append(node) - + keys_values = await self.get_terminal_nodes_by_hashes(pagination_data.hashes, store_id, root_hash) return KeysValuesPaginationData( pagination_data.total_pages, pagination_data.total_bytes, @@ -942,14 +1179,25 @@ async def get_kv_diff_paginated( pagination_data = get_hashes_for_page(page, lengths, max_page_size) kv_diff: list[DiffData] = [] - + insertion_hashes: list[bytes32] = [] + deletion_hashes: list[bytes32] = [] for hash in pagination_data.hashes: - node = await self.get_node(hash) - assert isinstance(node, TerminalNode) if hash in insertions: - kv_diff.append(DiffData(OperationType.INSERT, node.key, node.value)) + insertion_hashes.append(hash) else: - kv_diff.append(DiffData(OperationType.DELETE, node.key, node.value)) + deletion_hashes.append(hash) + if hash2 != bytes32.zeros: + insertion_nodes = await self.get_terminal_nodes_by_hashes(insertion_hashes, store_id, hash2) + else: + insertion_nodes = [] + if hash1 != bytes32.zeros: + deletion_nodes = await self.get_terminal_nodes_by_hashes(deletion_hashes, store_id, hash1) + else: + deletion_nodes = [] + for node in insertion_nodes: + kv_diff.append(DiffData(OperationType.INSERT, node.key, node.value)) + for node in deletion_nodes: + kv_diff.append(DiffData(OperationType.DELETE, node.key, node.value)) return KVDiffPaginationData( pagination_data.total_pages, @@ -957,380 +1205,128 @@ async def get_kv_diff_paginated( kv_diff, ) - async def get_node_type(self, node_hash: bytes32) -> NodeType: - async with self.db_wrapper.reader() as reader: - cursor = await reader.execute( - "SELECT node_type FROM node WHERE hash == :hash LIMIT 1", - {"hash": node_hash}, - ) - raw_node_type = await cursor.fetchone() - - if raw_node_type is None: - raise Exception(f"No node found for specified hash: {node_hash.hex()}") - - return NodeType(raw_node_type["node_type"]) - - async def get_terminal_node_for_seed( - self, store_id: bytes32, seed: bytes32, root_hash: Optional[bytes32] = None - ) -> Optional[bytes32]: - path = "".join(reversed("".join(f"{b:08b}" for b in seed))) - async with self.db_wrapper.reader() as reader: - if root_hash is None: - root = await self.get_tree_root(store_id) - root_hash = root.node_hash - if root_hash is None: - return None - - async with reader.execute( - """ - WITH RECURSIVE - random_leaf(hash, node_type, left, right, depth, side) AS ( - SELECT - node.hash AS hash, - node.node_type AS node_type, - node.left AS left, - node.right AS right, - 1 AS depth, - SUBSTR(:path, 1, 1) as side - FROM node - WHERE node.hash == :root_hash - UNION ALL - SELECT - node.hash AS hash, - node.node_type AS node_type, - node.left AS left, - node.right AS right, - random_leaf.depth + 1 AS depth, - SUBSTR(:path, random_leaf.depth + 1, 1) as side - FROM node, random_leaf - WHERE ( - (random_leaf.side == "0" AND node.hash == random_leaf.left) - OR (random_leaf.side != "0" AND node.hash == random_leaf.right) - ) - ) - SELECT hash AS hash FROM random_leaf - WHERE node_type == :node_type - LIMIT 1 - """, - {"root_hash": root_hash, "node_type": NodeType.TERMINAL, "path": path}, - ) as cursor: - row = await cursor.fetchone() - if row is None: - # No cover since this is an error state that should be unreachable given the code - # above has already verified that there is a non-empty tree. - raise Exception("No terminal node found for seed") # pragma: no cover - return bytes32(row["hash"]) - - def get_side_for_seed(self, seed: bytes32) -> Side: - side_seed = bytes(seed)[0] - return Side.LEFT if side_seed < 128 else Side.RIGHT - async def autoinsert( self, key: bytes, value: bytes, store_id: bytes32, - use_optimized: bool = True, status: Status = Status.PENDING, root: Optional[Root] = None, ) -> InsertResult: - async with self.db_wrapper.writer(): - if root is None: - root = await self.get_tree_root(store_id=store_id) - - was_empty = root.node_hash is None - - if was_empty: - reference_node_hash = None - side = None - else: - seed = leaf_hash(key=key, value=value) - reference_node_hash = await self.get_terminal_node_for_seed(store_id, seed, root_hash=root.node_hash) - side = self.get_side_for_seed(seed) - - return await self.insert( - key=key, - value=value, - store_id=store_id, - reference_node_hash=reference_node_hash, - side=side, - use_optimized=use_optimized, - status=status, - root=root, - ) - - async def get_keys_values_dict( - self, - store_id: bytes32, - root_hash: Union[bytes32, Unspecified] = unspecified, - ) -> dict[bytes, bytes]: - pairs = await self.get_keys_values(store_id=store_id, root_hash=root_hash) - return {node.key: node.value for node in pairs} + return await self.insert( + key=key, + value=value, + store_id=store_id, + reference_node_hash=None, + side=None, + status=status, + root=root, + ) async def get_keys( self, store_id: bytes32, root_hash: Union[bytes32, Unspecified] = unspecified, ) -> list[bytes]: - async with self.db_wrapper.reader() as reader: + async with self.db_wrapper.reader(): if root_hash is unspecified: root = await self.get_tree_root(store_id=store_id) resolved_root_hash = root.node_hash else: resolved_root_hash = root_hash - cursor = await reader.execute( - """ - WITH RECURSIVE - tree_from_root_hash(hash, node_type, left, right, key) AS ( - SELECT node.hash, node.node_type, node.left, node.right, node.key - FROM node WHERE node.hash == :root_hash - UNION ALL - SELECT - node.hash, node.node_type, node.left, node.right, node.key FROM node, tree_from_root_hash - WHERE node.hash == tree_from_root_hash.left OR node.hash == tree_from_root_hash.right - ) - SELECT key FROM tree_from_root_hash WHERE node_type == :node_type - """, - {"root_hash": resolved_root_hash, "node_type": NodeType.TERMINAL}, - ) - - keys: list[bytes] = [row["key"] async for row in cursor] - return keys - - async def get_ancestors_common( - self, - node_hash: bytes32, - store_id: bytes32, - root_hash: Optional[bytes32], - generation: Optional[int] = None, - use_optimized: bool = True, - ) -> list[InternalNode]: - if use_optimized: - ancestors: list[InternalNode] = await self.get_ancestors_optimized( - node_hash=node_hash, - store_id=store_id, - generation=generation, - root_hash=root_hash, - ) - else: - ancestors = await self.get_ancestors_optimized( - node_hash=node_hash, - store_id=store_id, - generation=generation, - root_hash=root_hash, - ) - ancestors_2: list[InternalNode] = await self.get_ancestors( - node_hash=node_hash, store_id=store_id, root_hash=root_hash - ) - if ancestors != ancestors_2: - raise RuntimeError("Ancestors optimized didn't produce the expected result.") + try: + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash) + except MerkleBlobNotFoundError: + return [] - if len(ancestors) >= 62: - raise RuntimeError("Tree exceeds max height of 62.") - return ancestors + kv_ids = merkle_blob.get_keys_values() + raw_key_ids = (KeyOrValueId(id.raw) for id in kv_ids.keys()) + table_blobs = await self.get_table_blobs(raw_key_ids, store_id) + keys: list[bytes] = [] + for kid in kv_ids.keys(): + blob_hash, blob = table_blobs[KeyOrValueId(kid.raw)] + if blob is None: + blob = self.get_blob_from_file(blob_hash, store_id) + keys.append(blob) - async def update_ancestor_hashes_on_insert( - self, - store_id: bytes32, - left: bytes32, - right: bytes32, - traversal_node_hash: bytes32, - ancestors: list[InternalNode], - status: Status, - root: Root, - ) -> Root: - # update ancestors after inserting root, to keep table constraints. - insert_ancestors_cache: list[tuple[bytes32, bytes32, bytes32]] = [] - new_generation = root.generation + 1 - # create first new internal node - new_hash = await self._insert_internal_node(left_hash=left, right_hash=right) - insert_ancestors_cache.append((left, right, store_id)) - - # create updated replacements for the rest of the internal nodes - for ancestor in ancestors: - if not isinstance(ancestor, InternalNode): - raise Exception(f"Expected an internal node but got: {type(ancestor).__name__}") - - if ancestor.left_hash == traversal_node_hash: - left = new_hash - right = ancestor.right_hash - elif ancestor.right_hash == traversal_node_hash: - left = ancestor.left_hash - right = new_hash - - traversal_node_hash = ancestor.hash - - new_hash = await self._insert_internal_node(left_hash=left, right_hash=right) - insert_ancestors_cache.append((left, right, store_id)) - - new_root = await self._insert_root( - store_id=store_id, - node_hash=new_hash, - status=status, - generation=new_generation, - ) + return keys - if status == Status.COMMITTED: - for left_hash, right_hash, cache_store_id in insert_ancestors_cache: - await self._insert_ancestor_table(left_hash, right_hash, cache_store_id, new_generation) + def get_reference_kid_side(self, merkle_blob: MerkleBlob, seed: bytes32) -> tuple[KeyId, Side]: + side_seed = bytes(seed)[0] + side = Side.LEFT if side_seed < 128 else Side.RIGHT + reference_node = merkle_blob.get_random_leaf_node(seed) + kid = reference_node.key + return (kid, side) + + async def get_terminal_node_from_kid(self, merkle_blob: MerkleBlob, kid: KeyId, store_id: bytes32) -> TerminalNode: + index = merkle_blob.get_key_index(kid) + raw_node = merkle_blob.get_raw_node(index) + assert isinstance(raw_node, chia_rs.datalayer.LeafNode) + return await self.get_terminal_node(raw_node.key, raw_node.value, store_id) + + async def get_terminal_node_for_seed(self, seed: bytes32, store_id: bytes32) -> Optional[TerminalNode]: + root = await self.get_tree_root(store_id=store_id) + if root is None or root.node_hash is None: + return None - return new_root + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash) + assert not merkle_blob.empty() + kid, _ = self.get_reference_kid_side(merkle_blob, seed) + return await self.get_terminal_node_from_kid(merkle_blob, kid, store_id) async def insert( self, key: bytes, value: bytes, store_id: bytes32, - reference_node_hash: Optional[bytes32], - side: Optional[Side], - use_optimized: bool = True, + reference_node_hash: Optional[bytes32] = None, + side: Optional[Side] = None, status: Status = Status.PENDING, root: Optional[Root] = None, ) -> InsertResult: - async with self.db_wrapper.writer(): - if root is None: - root = await self.get_tree_root(store_id=store_id) + async with self.db_wrapper.writer() as writer: + with self.manage_kv_files(store_id): + if root is None: + root = await self.get_tree_root(store_id=store_id) + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash) - try: - await self.get_node_by_key(key=key, store_id=store_id) - raise Exception(f"Key already present: {key.hex()}") - except KeyNotFoundError: - pass - - was_empty = root.node_hash is None - if reference_node_hash is None: - if not was_empty: - raise Exception(f"Reference node hash must be specified for non-empty tree: {store_id.hex()}") - else: - reference_node_type = await self.get_node_type(node_hash=reference_node_hash) - if reference_node_type == NodeType.INTERNAL: - raise Exception("can not insert a new key/value on an internal node") + kid, vid = await self.add_key_value(key, value, store_id, writer=writer) + hash = leaf_hash(key, value) + reference_kid = None + if reference_node_hash is not None: + reference_kid, _ = merkle_blob.get_node_by_hash(reference_node_hash) - # create new terminal node - new_terminal_node_hash = await self._insert_terminal_node(key=key, value=value) + was_empty = root.node_hash is None + if not was_empty and reference_kid is None: + if side is not None: + raise Exception("Side specified without reference node hash") - if was_empty: - if side is not None: - raise Exception(f"Tree was empty so side must be unspecified, got: {side!r}") + seed = leaf_hash(key=key, value=value) + reference_kid, side = self.get_reference_kid_side(merkle_blob, seed) - new_root = await self._insert_root( - store_id=store_id, - node_hash=new_terminal_node_hash, - status=status, - ) - else: - if side is None: - raise Exception("Tree was not empty, side must be specified.") - if reference_node_hash is None: - raise Exception("Tree was not empty, reference node hash must be specified.") - if root.node_hash is None: - raise Exception("Internal error.") - - if side == Side.LEFT: - left = new_terminal_node_hash - right = reference_node_hash - elif side == Side.RIGHT: - left = reference_node_hash - right = new_terminal_node_hash - else: - raise Exception(f"Internal error, unknown side: {side!r}") - - ancestors = await self.get_ancestors_common( - node_hash=reference_node_hash, - store_id=store_id, - root_hash=root.node_hash, - generation=root.generation, - use_optimized=use_optimized, - ) - new_root = await self.update_ancestor_hashes_on_insert( - store_id=store_id, - left=left, - right=right, - traversal_node_hash=reference_node_hash, - ancestors=ancestors, - status=status, - root=root, - ) + merkle_blob.insert(kid, vid, hash, reference_kid, side) - return InsertResult(node_hash=new_terminal_node_hash, root=new_root) + new_root = await self.insert_root_from_merkle_blob(merkle_blob, store_id, status) + return InsertResult(node_hash=hash, root=new_root) async def delete( self, key: bytes, store_id: bytes32, - use_optimized: bool = True, status: Status = Status.PENDING, root: Optional[Root] = None, ) -> Optional[Root]: - root_hash = None if root is None else root.node_hash async with self.db_wrapper.writer(): - try: - node = await self.get_node_by_key(key=key, store_id=store_id) - node_hash = node.hash - assert isinstance(node, TerminalNode) - except KeyNotFoundError: - log.debug(f"Request to delete an unknown key ignored: {key.hex()}") - return root - - ancestors: list[InternalNode] = await self.get_ancestors_common( - node_hash=node_hash, - store_id=store_id, - root_hash=root_hash, - use_optimized=use_optimized, - ) - - if len(ancestors) == 0: - # the only node is being deleted - return await self._insert_root( - store_id=store_id, - node_hash=None, - status=status, - ) - - parent = ancestors[0] - other_hash = parent.other_child_hash(hash=node_hash) - - if len(ancestors) == 1: - # the parent is the root so the other side will become the new root - return await self._insert_root( - store_id=store_id, - node_hash=other_hash, - status=status, - ) - - old_child_hash = parent.hash - new_child_hash = other_hash if root is None: - new_generation = await self.get_tree_generation(store_id) + 1 - else: - new_generation = root.generation + 1 - # update ancestors after inserting root, to keep table constraints. - insert_ancestors_cache: list[tuple[bytes32, bytes32, bytes32]] = [] - # more parents to handle so let's traverse them - for ancestor in ancestors[1:]: - if ancestor.left_hash == old_child_hash: - left_hash = new_child_hash - right_hash = ancestor.right_hash - elif ancestor.right_hash == old_child_hash: - left_hash = ancestor.left_hash - right_hash = new_child_hash - else: - raise Exception("Internal error.") + root = await self.get_tree_root(store_id=store_id) + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash) - new_child_hash = await self._insert_internal_node(left_hash=left_hash, right_hash=right_hash) - insert_ancestors_cache.append((left_hash, right_hash, store_id)) - old_child_hash = ancestor.hash + kid = await self.get_kvid(key, store_id) + if kid is not None: + merkle_blob.delete(KeyId(kid)) - new_root = await self._insert_root( - store_id=store_id, - node_hash=new_child_hash, - status=status, - generation=new_generation, - ) - if status == Status.COMMITTED: - for left_hash, right_hash, cache_store_id in insert_ancestors_cache: - await self._insert_ancestor_table(left_hash, right_hash, cache_store_id, new_generation) + new_root = await self.insert_root_from_merkle_blob(merkle_blob, store_id, status) return new_root @@ -1339,151 +1335,21 @@ async def upsert( key: bytes, new_value: bytes, store_id: bytes32, - use_optimized: bool = True, status: Status = Status.PENDING, root: Optional[Root] = None, ) -> InsertResult: - async with self.db_wrapper.writer(): - if root is None: - root = await self.get_tree_root(store_id=store_id) - - try: - old_node = await self.get_node_by_key(key=key, store_id=store_id) - except KeyNotFoundError: - log.debug(f"Key not found: {key.hex()}. Doing an autoinsert instead") - return await self.autoinsert( - key=key, - value=new_value, - store_id=store_id, - use_optimized=use_optimized, - status=status, - root=root, - ) - if old_node.value == new_value: - log.debug(f"New value matches old value in upsert operation: {key.hex()}. Ignoring upsert") - return InsertResult(leaf_hash(key, new_value), root) - - # create new terminal node - new_terminal_node_hash = await self._insert_terminal_node(key=key, value=new_value) - - ancestors = await self.get_ancestors_common( - node_hash=old_node.hash, - store_id=store_id, - root_hash=root.node_hash, - generation=root.generation, - use_optimized=use_optimized, - ) - - # Store contains only the old root, replace it with a new root having the terminal node. - if len(ancestors) == 0: - new_root = await self._insert_root( - store_id=store_id, - node_hash=new_terminal_node_hash, - status=status, - ) - else: - parent = ancestors[0] - if parent.left_hash == old_node.hash: - left = new_terminal_node_hash - right = parent.right_hash - elif parent.right_hash == old_node.hash: - left = parent.left_hash - right = new_terminal_node_hash - else: - raise Exception("Internal error.") - - new_root = await self.update_ancestor_hashes_on_insert( - store_id=store_id, - left=left, - right=right, - traversal_node_hash=parent.hash, - ancestors=ancestors[1:], - status=status, - root=root, - ) - - return InsertResult(node_hash=new_terminal_node_hash, root=new_root) - - async def clean_node_table(self, writer: Optional[aiosqlite.Connection] = None) -> None: - query = """ - WITH RECURSIVE pending_nodes AS ( - SELECT node_hash AS hash FROM root - WHERE status IN (:pending_status, :pending_batch_status) - UNION ALL - SELECT n.left FROM node n - INNER JOIN pending_nodes pn ON n.hash = pn.hash - WHERE n.left IS NOT NULL - UNION ALL - SELECT n.right FROM node n - INNER JOIN pending_nodes pn ON n.hash = pn.hash - WHERE n.right IS NOT NULL - ) - DELETE FROM node - WHERE hash IN ( - SELECT n.hash FROM node n - LEFT JOIN ancestors a ON n.hash = a.hash - LEFT JOIN pending_nodes pn ON n.hash = pn.hash - WHERE a.hash IS NULL AND pn.hash IS NULL - ) - """ - params = {"pending_status": Status.PENDING.value, "pending_batch_status": Status.PENDING_BATCH.value} - if writer is None: - async with self.db_wrapper.writer(foreign_key_enforcement_enabled=False) as sub_writer: - await sub_writer.execute(query, params) - else: - await writer.execute(query, params) - - async def get_nodes(self, node_hashes: list[bytes32]) -> list[Node]: - query_parameter_place_holders = ",".join("?" for _ in node_hashes) - async with self.db_wrapper.reader() as reader: - # TODO: handle SQLITE_MAX_VARIABLE_NUMBER - cursor = await reader.execute( - f"SELECT * FROM node WHERE hash IN ({query_parameter_place_holders})", - [*node_hashes], - ) - rows = await cursor.fetchall() - - hash_to_node = {row["hash"]: row_to_node(row=row) for row in rows} - - missing_hashes = [node_hash.hex() for node_hash in node_hashes if node_hash not in hash_to_node] - if missing_hashes: - raise Exception(f"Nodes not found for hashes: {', '.join(missing_hashes)}") - - return [hash_to_node[node_hash] for node_hash in node_hashes] - - async def get_leaf_at_minimum_height( - self, root_hash: bytes32, hash_to_parent: dict[bytes32, InternalNode] - ) -> TerminalNode: - queue: list[bytes32] = [root_hash] - batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10) - - while True: - assert len(queue) > 0 - nodes = await self.get_nodes(queue[:batch_size]) - queue = queue[batch_size:] + async with self.db_wrapper.writer() as writer: + with self.manage_kv_files(store_id): + if root is None: + root = await self.get_tree_root(store_id=store_id) + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash) - for node in nodes: - if isinstance(node, TerminalNode): - return node - hash_to_parent[node.left_hash] = node - hash_to_parent[node.right_hash] = node - queue.append(node.left_hash) - queue.append(node.right_hash) + kid, vid = await self.add_key_value(key, new_value, store_id, writer=writer) + hash = leaf_hash(key, new_value) + merkle_blob.upsert(kid, vid, hash) - async def batch_upsert( - self, - hash: bytes32, - to_update_hashes: set[bytes32], - pending_upsert_new_hashes: dict[bytes32, bytes32], - ) -> bytes32: - if hash not in to_update_hashes: - return hash - node = await self.get_node(hash) - if isinstance(node, TerminalNode): - return pending_upsert_new_hashes[hash] - new_left_hash = await self.batch_upsert(node.left_hash, to_update_hashes, pending_upsert_new_hashes) - new_right_hash = await self.batch_upsert(node.right_hash, to_update_hashes, pending_upsert_new_hashes) - return await self._insert_internal_node(new_left_hash, new_right_hash) + new_root = await self.insert_root_from_merkle_blob(merkle_blob, store_id, status) + return InsertResult(node_hash=hash, root=new_root) async def insert_batch( self, @@ -1492,337 +1358,90 @@ async def insert_batch( status: Status = Status.PENDING, enable_batch_autoinsert: bool = True, ) -> Optional[bytes32]: - async with self.transaction(): - old_root = await self.get_tree_root(store_id) - pending_root = await self.get_pending_root(store_id=store_id) - if pending_root is None: - latest_local_root: Optional[Root] = old_root - elif pending_root.status == Status.PENDING_BATCH: - # We have an unfinished batch, continue the current batch on top of it. - if pending_root.generation != old_root.generation + 1: - raise Exception("Internal error") - await self.change_root_status(pending_root, Status.COMMITTED) - await self.build_ancestor_table_for_latest_root(store_id=store_id) - latest_local_root = pending_root - else: - raise Exception("Internal error") - - assert latest_local_root is not None - - key_hash_frequency: dict[bytes32, int] = {} - first_action: dict[bytes32, str] = {} - last_action: dict[bytes32, str] = {} + async with self.db_wrapper.writer() as writer: + with self.manage_kv_files(store_id): + old_root = await self.get_tree_root(store_id=store_id) + pending_root = await self.get_pending_root(store_id=store_id) + if pending_root is not None: + if pending_root.status == Status.PENDING_BATCH: + # We have an unfinished batch, continue the current batch on top of it. + if pending_root.generation != old_root.generation + 1: + raise Exception("Internal error") + old_root = pending_root + await self.clear_pending_roots(store_id) + else: + raise Exception("Internal error") - for change in changelist: - key = change["key"] - hash = key_hash(key) - key_hash_frequency[hash] = key_hash_frequency.get(hash, 0) + 1 - if hash not in first_action: - first_action[hash] = change["action"] - last_action[hash] = change["action"] + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=old_root.node_hash) - pending_autoinsert_hashes: list[bytes32] = [] - pending_upsert_new_hashes: dict[bytes32, bytes32] = {} - leaf_hashes = await self.get_leaf_hashes_by_hashed_key(store_id) + key_hash_frequency: dict[bytes32, int] = {} + first_action: dict[bytes32, str] = {} + last_action: dict[bytes32, str] = {} - for change in changelist: - if change["action"] == "insert": - key = change["key"] - value = change["value"] - reference_node_hash = change.get("reference_node_hash", None) - side = change.get("side", None) - if reference_node_hash is None and side is None: - hash = key_hash(key) - # The key is not referenced in any other operation but this autoinsert, hence the order - # of performing these should not matter. We perform all these autoinserts as a batch - # at the end, to speed up the tree processing operations. - # Additionally, if the first action is a delete, we can still perform the autoinsert at the - # end, since the order will be preserved. - if enable_batch_autoinsert: - if key_hash_frequency[hash] == 1 or ( - key_hash_frequency[hash] == 2 and first_action[hash] == "delete" - ): - old_node = await self.maybe_get_node_from_key_hash(leaf_hashes, hash) - terminal_node_hash = await self._insert_terminal_node(key, value) - - if old_node is None: - pending_autoinsert_hashes.append(terminal_node_hash) - elif key_hash_frequency[hash] == 1: - raise Exception(f"Key already present: {key.hex()}") - else: - pending_upsert_new_hashes[old_node.hash] = terminal_node_hash - continue - insert_result = await self.autoinsert( - key, value, store_id, True, Status.COMMITTED, root=latest_local_root - ) - latest_local_root = insert_result.root - else: - if reference_node_hash is None or side is None: - raise Exception("Provide both reference_node_hash and side or neither.") - insert_result = await self.insert( - key, - value, - store_id, - reference_node_hash, - side, - True, - Status.COMMITTED, - root=latest_local_root, - ) - latest_local_root = insert_result.root - elif change["action"] == "delete": + for change in changelist: key = change["key"] hash = key_hash(key) - if key_hash_frequency[hash] == 2 and last_action[hash] == "insert" and enable_batch_autoinsert: - continue - latest_local_root = await self.delete(key, store_id, True, Status.COMMITTED, root=latest_local_root) - elif change["action"] == "upsert": - key = change["key"] - new_value = change["value"] - hash = key_hash(key) - if key_hash_frequency[hash] == 1 and enable_batch_autoinsert: - terminal_node_hash = await self._insert_terminal_node(key, new_value) - old_node = await self.maybe_get_node_from_key_hash(leaf_hashes, hash) - if old_node is not None: - pending_upsert_new_hashes[old_node.hash] = terminal_node_hash + key_hash_frequency[hash] = key_hash_frequency.get(hash, 0) + 1 + if hash not in first_action: + first_action[hash] = change["action"] + last_action[hash] = change["action"] + + batch_keys_values: list[tuple[KeyId, ValueId]] = [] + batch_hashes: list[bytes32] = [] + + for change in changelist: + if change["action"] == "insert": + key = change["key"] + value = change["value"] + + reference_node_hash = change.get("reference_node_hash", None) + side = change.get("side", None) + reference_kid: Optional[KeyId] = None + if reference_node_hash is not None: + reference_kid, _ = merkle_blob.get_node_by_hash(reference_node_hash) + + key_hashed = key_hash(key) + kid, vid = await self.add_key_value(key, value, store_id, writer=writer) + try: + merkle_blob.get_key_index(kid) + except chia_rs.datalayer.UnknownKeyError: + pass else: - pending_autoinsert_hashes.append(terminal_node_hash) - continue - insert_result = await self.upsert( - key, new_value, store_id, True, Status.COMMITTED, root=latest_local_root - ) - latest_local_root = insert_result.root - else: - raise Exception(f"Operation in batch is not insert or delete: {change}") - - if len(pending_upsert_new_hashes) > 0: - to_update_hashes: set[bytes32] = set(pending_upsert_new_hashes.keys()) - to_update_queue: list[bytes32] = list(pending_upsert_new_hashes.keys()) - batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10) - - while len(to_update_queue) > 0: - nodes = await self._get_one_ancestor_multiple_hashes(to_update_queue[:batch_size], store_id) - to_update_queue = to_update_queue[batch_size:] - for node in nodes: - if node.hash not in to_update_hashes: - to_update_hashes.add(node.hash) - to_update_queue.append(node.hash) - - assert latest_local_root is not None - assert latest_local_root.node_hash is not None - new_root_hash = await self.batch_upsert( - latest_local_root.node_hash, - to_update_hashes, - pending_upsert_new_hashes, - ) - latest_local_root = await self._insert_root(store_id, new_root_hash, Status.COMMITTED) - - # Start with the leaf nodes and pair them to form new nodes at the next level up, repeating this process - # in a bottom-up fashion until a single root node remains. This constructs a balanced tree from the leaves. - while len(pending_autoinsert_hashes) > 1: - new_hashes: list[bytes32] = [] - for i in range(0, len(pending_autoinsert_hashes) - 1, 2): - internal_node_hash = await self._insert_internal_node( - pending_autoinsert_hashes[i], pending_autoinsert_hashes[i + 1] - ) - new_hashes.append(internal_node_hash) - if len(pending_autoinsert_hashes) % 2 != 0: - new_hashes.append(pending_autoinsert_hashes[-1]) - - pending_autoinsert_hashes = new_hashes - - if len(pending_autoinsert_hashes): - subtree_hash = pending_autoinsert_hashes[0] - if latest_local_root is None or latest_local_root.node_hash is None: - await self._insert_root(store_id=store_id, node_hash=subtree_hash, status=Status.COMMITTED) - else: - hash_to_parent: dict[bytes32, InternalNode] = {} - min_height_leaf = await self.get_leaf_at_minimum_height(latest_local_root.node_hash, hash_to_parent) - ancestors: list[InternalNode] = [] - hash = min_height_leaf.hash - while hash in hash_to_parent: - node = hash_to_parent[hash] - ancestors.append(node) - hash = node.hash - - await self.update_ancestor_hashes_on_insert( - store_id=store_id, - left=min_height_leaf.hash, - right=subtree_hash, - traversal_node_hash=min_height_leaf.hash, - ancestors=ancestors, - status=Status.COMMITTED, - root=latest_local_root, - ) - - root = await self.get_tree_root(store_id=store_id) - if root.node_hash == old_root.node_hash: - if len(changelist) != 0: - await self.rollback_to_generation(store_id, old_root.generation) - raise ValueError("Changelist resulted in no change to tree data") - # We delete all "temporary" records stored in root and ancestor tables and store only the final result. - await self.rollback_to_generation(store_id, old_root.generation) - await self.insert_root_with_ancestor_table(store_id=store_id, node_hash=root.node_hash, status=status) - if status in {Status.PENDING, Status.PENDING_BATCH}: - new_root = await self.get_pending_root(store_id=store_id) - assert new_root is not None - elif status == Status.COMMITTED: - new_root = await self.get_tree_root(store_id=store_id) - else: - raise Exception(f"No known status: {status}") - if new_root.node_hash != root.node_hash: - raise RuntimeError( - f"Tree root mismatches after batch update: Expected: {root.node_hash}. Got: {new_root.node_hash}" - ) - if new_root.generation != old_root.generation + 1: - raise RuntimeError( - "Didn't get the expected generation after batch update: " - f"Expected: {old_root.generation + 1}. Got: {new_root.generation}" - ) - return root.node_hash - - async def _get_one_ancestor( - self, - node_hash: bytes32, - store_id: bytes32, - generation: Optional[int] = None, - ) -> Optional[InternalNode]: - async with self.db_wrapper.reader() as reader: - if generation is None: - generation = await self.get_tree_generation(store_id=store_id) - cursor = await reader.execute( - """ - SELECT * from node INNER JOIN ( - SELECT ancestors.ancestor AS hash, MAX(ancestors.generation) AS generation - FROM ancestors - WHERE ancestors.hash == :hash - AND ancestors.tree_id == :tree_id - AND ancestors.generation <= :generation - GROUP BY hash - ) asc on asc.hash == node.hash - """, - {"hash": node_hash, "tree_id": store_id, "generation": generation}, - ) - row = await cursor.fetchone() - if row is None: - return None - return InternalNode.from_row(row=row) - - async def _get_one_ancestor_multiple_hashes( - self, - node_hashes: list[bytes32], - store_id: bytes32, - generation: Optional[int] = None, - ) -> list[InternalNode]: - async with self.db_wrapper.reader() as reader: - node_hashes_place_holders = ",".join("?" for _ in node_hashes) - if generation is None: - generation = await self.get_tree_generation(store_id=store_id) - cursor = await reader.execute( - f""" - SELECT * from node INNER JOIN ( - SELECT ancestors.ancestor AS hash, MAX(ancestors.generation) AS generation - FROM ancestors - WHERE ancestors.hash IN ({node_hashes_place_holders}) - AND ancestors.tree_id == ? - AND ancestors.generation <= ? - GROUP BY hash - ) asc on asc.hash == node.hash - """, - [*node_hashes, store_id, generation], - ) - rows = await cursor.fetchall() - return [InternalNode.from_row(row=row) for row in rows] - - async def build_ancestor_table_for_latest_root(self, store_id: bytes32) -> None: - async with self.db_wrapper.writer(): - root = await self.get_tree_root(store_id=store_id) - if root.node_hash is None: - return - previous_root = await self.get_tree_root( - store_id=store_id, - generation=max(root.generation - 1, 0), - ) - - if previous_root.node_hash is not None: - previous_internal_nodes: list[InternalNode] = await self.get_internal_nodes( - store_id=store_id, - root_hash=previous_root.node_hash, - ) - known_hashes: set[bytes32] = {node.hash for node in previous_internal_nodes} - else: - known_hashes = set() - internal_nodes: list[InternalNode] = await self.get_internal_nodes( - store_id=store_id, - root_hash=root.node_hash, - ) - for node in internal_nodes: - # We already have the same values in ancestor tables, if we have the same internal node. - # Don't reinsert it so we can save DB space. - if node.hash not in known_hashes: - await self._insert_ancestor_table(node.left_hash, node.right_hash, store_id, root.generation) - - async def insert_root_with_ancestor_table( - self, store_id: bytes32, node_hash: Optional[bytes32], status: Status = Status.PENDING - ) -> None: - async with self.db_wrapper.writer(): - await self._insert_root(store_id=store_id, node_hash=node_hash, status=status) - # Don't update the ancestor table for non-committed status. - if status == Status.COMMITTED: - await self.build_ancestor_table_for_latest_root(store_id=store_id) - - async def get_node_by_key_latest_generation(self, key: bytes, store_id: bytes32) -> TerminalNode: - async with self.db_wrapper.reader() as reader: - root = await self.get_tree_root(store_id=store_id) - if root.node_hash is None: - raise KeyNotFoundError(key=key) - - cursor = await reader.execute( - """ - SELECT a.hash FROM ancestors a - JOIN node n ON a.hash = n.hash - WHERE n.key = :key - AND a.tree_id = :tree_id - ORDER BY a.generation DESC - LIMIT 1 - """, - {"key": key, "tree_id": store_id}, - ) - - row = await cursor.fetchone() - if row is None: - raise KeyNotFoundError(key=key) - - node = await self.get_node(row["hash"]) - node_hash = node.hash - while True: - internal_node = await self._get_one_ancestor(node_hash, store_id) - if internal_node is None: - break - node_hash = internal_node.hash - - if node_hash != root.node_hash: - raise KeyNotFoundError(key=key) - assert isinstance(node, TerminalNode) - return node - - async def maybe_get_node_from_key_hash( - self, leaf_hashes: dict[bytes32, bytes32], hash: bytes32 - ) -> Optional[TerminalNode]: - if hash in leaf_hashes: - leaf_hash = leaf_hashes[hash] - node = await self.get_node(leaf_hash) - assert isinstance(node, TerminalNode) - return node + raise KeyAlreadyPresentError(kid) + hash = leaf_hash(key, value) + + if reference_node_hash is None and side is None: + if enable_batch_autoinsert and reference_kid is None: + if key_hash_frequency[key_hashed] == 1 or ( + key_hash_frequency[key_hashed] == 2 and first_action[key_hashed] == "delete" + ): + batch_keys_values.append((kid, vid)) + batch_hashes.append(hash) + continue + if not merkle_blob.empty(): + seed = leaf_hash(key=key, value=value) + reference_kid, side = self.get_reference_kid_side(merkle_blob, seed) + + merkle_blob.insert(kid, vid, hash, reference_kid, side) + elif change["action"] == "delete": + key = change["key"] + deletion_kid = await self.get_kvid(key, store_id) + if deletion_kid is not None: + merkle_blob.delete(KeyId(deletion_kid)) + elif change["action"] == "upsert": + key = change["key"] + new_value = change["value"] + kid, vid = await self.add_key_value(key, new_value, store_id, writer=writer) + hash = leaf_hash(key, new_value) + merkle_blob.upsert(kid, vid, hash) + else: + raise Exception(f"Operation in batch is not insert or delete: {change}") - return None + if len(batch_keys_values) > 0: + merkle_blob.batch_insert(batch_keys_values, batch_hashes) - async def maybe_get_node_by_key(self, key: bytes, store_id: bytes32) -> Optional[TerminalNode]: - try: - node = await self.get_node_by_key_latest_generation(key, store_id) - return node - except KeyNotFoundError: - return None + new_root = await self.insert_root_from_merkle_blob(merkle_blob, store_id, status, old_root) + return new_root.node_hash async def get_node_by_key( self, @@ -1830,60 +1449,49 @@ async def get_node_by_key( store_id: bytes32, root_hash: Union[bytes32, Unspecified] = unspecified, ) -> TerminalNode: - if root_hash is unspecified: - return await self.get_node_by_key_latest_generation(key, store_id) - - nodes = await self.get_keys_values(store_id=store_id, root_hash=root_hash) - - for node in nodes: - if node.key == key: - return node - - raise KeyNotFoundError(key=key) - - async def get_node(self, node_hash: bytes32) -> Node: - async with self.db_wrapper.reader() as reader: - cursor = await reader.execute("SELECT * FROM node WHERE hash == :hash LIMIT 1", {"hash": node_hash}) - row = await cursor.fetchone() + async with self.db_wrapper.reader(): + resolved_root_hash: Optional[bytes32] + if root_hash is unspecified: + root = await self.get_tree_root(store_id=store_id) + resolved_root_hash = root.node_hash + else: + resolved_root_hash = root_hash - if row is None: - raise Exception(f"Node not found for requested hash: {node_hash.hex()}") + try: + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash) + except MerkleBlobNotFoundError: + raise KeyNotFoundError(key=key) - node = row_to_node(row=row) - return node + kvid = await self.get_kvid(key, store_id) + if kvid is None: + raise KeyNotFoundError(key=key) + kid = KeyId(kvid) + return await self.get_terminal_node_from_kid(merkle_blob, kid, store_id) async def get_tree_as_nodes(self, store_id: bytes32) -> Node: - async with self.db_wrapper.reader() as reader: + async with self.db_wrapper.reader(): root = await self.get_tree_root(store_id=store_id) # TODO: consider actual proper behavior assert root.node_hash is not None - root_node = await self.get_node(node_hash=root.node_hash) - cursor = await reader.execute( - """ - WITH RECURSIVE - tree_from_root_hash(hash, node_type, left, right, key, value) AS ( - SELECT node.* FROM node WHERE node.hash == :root_hash - UNION ALL - SELECT node.* FROM node, tree_from_root_hash - WHERE node.hash == tree_from_root_hash.left OR node.hash == tree_from_root_hash.right - ) - SELECT * FROM tree_from_root_hash - """, - {"root_hash": root_node.hash}, - ) - nodes = [row_to_node(row=row) async for row in cursor] + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash) + + nodes = merkle_blob.get_nodes_with_indexes() hash_to_node: dict[bytes32, Node] = {} - for node in reversed(nodes): - if isinstance(node, InternalNode): - node_entry: Node = replace( - node, left=hash_to_node[node.left_hash], right=hash_to_node[node.right_hash] + tree_node: Node + for _, node in reversed(nodes): + if isinstance(node, chia_rs.datalayer.InternalNode): + left_hash = merkle_blob.get_hash_at_index(node.left) + right_hash = merkle_blob.get_hash_at_index(node.right) + tree_node = InternalNode.from_child_nodes( + left=hash_to_node[left_hash], right=hash_to_node[right_hash] ) else: - node_entry = node - hash_to_node[node_entry.hash] = node_entry + assert isinstance(node, chia_rs.datalayer.LeafNode) + tree_node = await self.get_terminal_node(node.key, node.value, store_id) + hash_to_node[node.hash] = tree_node - root_node = hash_to_node[root_node.hash] + root_node = hash_to_node[root.node_hash] return root_node @@ -1892,66 +1500,86 @@ async def get_proof_of_inclusion_by_hash( node_hash: bytes32, store_id: bytes32, root_hash: Optional[bytes32] = None, - use_optimized: bool = False, ) -> ProofOfInclusion: - """Collect the information for a proof of inclusion of a hash in the Merkle - tree. - """ - - # Ideally this would use get_ancestors_common, but this _common function has this interesting property - # when used with use_optimized=False - it will compare both methods in this case and raise an exception. - # this is undesirable in the DL Offers flow where PENDING roots can cause the optimized code to fail. - if use_optimized: - ancestors = await self.get_ancestors_optimized(node_hash=node_hash, store_id=store_id, root_hash=root_hash) - else: - ancestors = await self.get_ancestors(node_hash=node_hash, store_id=store_id, root_hash=root_hash) - - layers: list[ProofOfInclusionLayer] = [] - child_hash = node_hash - for parent in ancestors: - layer = ProofOfInclusionLayer.from_internal_node(internal_node=parent, traversal_child_hash=child_hash) - layers.append(layer) - child_hash = parent.hash - - proof_of_inclusion = ProofOfInclusion(node_hash=node_hash, layers=layers) - - if len(ancestors) > 0: - expected_root = ancestors[-1].hash - else: - expected_root = node_hash - - if expected_root != proof_of_inclusion.root_hash: - raise Exception( - f"Incorrect root, expected: {expected_root.hex()}" - f"\n has: {proof_of_inclusion.root_hash.hex()}" - ) - - return proof_of_inclusion + if root_hash is None: + root = await self.get_tree_root(store_id=store_id) + root_hash = root.node_hash + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root_hash) + kid, _ = merkle_blob.get_node_by_hash(node_hash) + return merkle_blob.get_proof_of_inclusion(kid) async def get_proof_of_inclusion_by_key( self, key: bytes, store_id: bytes32, ) -> ProofOfInclusion: - """Collect the information for a proof of inclusion of a key and its value in - the Merkle tree. - """ - async with self.db_wrapper.reader(): - node = await self.get_node_by_key(key=key, store_id=store_id) - return await self.get_proof_of_inclusion_by_hash(node_hash=node.hash, store_id=store_id) + root = await self.get_tree_root(store_id=store_id) + merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash) + kvid = await self.get_kvid(key, store_id) + if kvid is None: + raise Exception(f"Cannot find key: {key.hex()}") + kid = KeyId(kvid) + return merkle_blob.get_proof_of_inclusion(kid) + + async def get_nodes_for_file( + self, + root: Root, + node_hash: bytes32, + store_id: bytes32, + deltas_only: bool, + delta_file_cache: DeltaFileCache, + tree_nodes: list[SerializedNode], + ) -> None: + if deltas_only: + if delta_file_cache.seen_previous_hash(node_hash): + return - async def get_first_generation(self, node_hash: bytes32, store_id: bytes32) -> int: - async with self.db_wrapper.reader() as reader: - cursor = await reader.execute( - "SELECT MIN(generation) AS generation FROM ancestors WHERE hash == :hash AND tree_id == :tree_id", - {"hash": node_hash, "tree_id": store_id}, + raw_index = delta_file_cache.get_index(node_hash) + raw_node = delta_file_cache.get_raw_node(raw_index) + + if isinstance(raw_node, chia_rs.datalayer.InternalNode): + left_hash = delta_file_cache.get_hash_at_index(raw_node.left) + right_hash = delta_file_cache.get_hash_at_index(raw_node.right) + await self.get_nodes_for_file(root, left_hash, store_id, deltas_only, delta_file_cache, tree_nodes) + await self.get_nodes_for_file(root, right_hash, store_id, deltas_only, delta_file_cache, tree_nodes) + tree_nodes.append(SerializedNode(False, bytes(left_hash), bytes(right_hash))) + elif isinstance(raw_node, chia_rs.datalayer.LeafNode): + tree_nodes.append( + SerializedNode( + True, + raw_node.key.to_bytes(), + raw_node.value.to_bytes(), + ) ) - row = await cursor.fetchone() - if row is None: - raise RuntimeError("Hash not found in ancestor table.") + else: + raise Exception(f"Node is neither InternalNode nor TerminalNode: {raw_node}") + + async def get_table_blobs( + self, kv_ids_iter: Iterable[KeyOrValueId], store_id: bytes32 + ) -> dict[KeyOrValueId, tuple[bytes32, Optional[bytes]]]: + result: dict[KeyOrValueId, tuple[bytes32, Optional[bytes]]] = {} + batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10) + kv_ids = list(dict.fromkeys(kv_ids_iter)) + + async with self.db_wrapper.reader() as reader: + for i in range(0, len(kv_ids), batch_size): + chunk = kv_ids[i : i + batch_size] + placeholders = ",".join(["?"] * len(chunk)) + query = f""" + SELECT hash, blob, kv_id + FROM ids + WHERE store_id = ? AND kv_id IN ({placeholders}) + LIMIT {len(chunk)} + """ + + async with reader.execute(query, (store_id, *chunk)) as cursor: + rows = await cursor.fetchall() + result.update({row["kv_id"]: (row["hash"], row["blob"]) for row in rows}) - generation = row["generation"] - return int(generation) + if len(result) != len(kv_ids): + raise Exception("Cannot retrieve all the requested kv_ids") + + return result async def write_tree_to_file( self, @@ -1964,24 +1592,42 @@ async def write_tree_to_file( if node_hash == bytes32.zeros: return - if deltas_only: - generation = await self.get_first_generation(node_hash, store_id) - # Root's generation is not the first time we see this hash, so it's not a new delta. - if root.generation != generation: - return - node = await self.get_node(node_hash) - to_write = b"" - if isinstance(node, InternalNode): - await self.write_tree_to_file(root, node.left_hash, store_id, deltas_only, writer) - await self.write_tree_to_file(root, node.right_hash, store_id, deltas_only, writer) - to_write = bytes(SerializedNode(False, bytes(node.left_hash), bytes(node.right_hash))) - elif isinstance(node, TerminalNode): - to_write = bytes(SerializedNode(True, node.key, node.value)) - else: - raise Exception(f"Node is neither InternalNode nor TerminalNode: {node}") + with log_exceptions(log=log, message="Error while getting merkle blob"): + root_path = self.get_merkle_path(store_id=store_id, root_hash=root.node_hash) + delta_file_cache = DeltaFileCache(root_path) - writer.write(len(to_write).to_bytes(4, byteorder="big")) - writer.write(to_write) + if root.generation > 0: + previous_root = await self.get_tree_root(store_id=store_id, generation=root.generation - 1) + if previous_root.node_hash is not None: + with log_exceptions(log=log, message="Error while getting previous merkle blob"): + previous_root_path = self.get_merkle_path(store_id=store_id, root_hash=previous_root.node_hash) + delta_file_cache.load_previous_hashes(previous_root_path) + + tree_nodes: list[SerializedNode] = [] + + await self.get_nodes_for_file(root, node_hash, store_id, deltas_only, delta_file_cache, tree_nodes) + kv_ids = ( + KeyOrValueId.from_bytes(raw_id) + for node in tree_nodes + if node.is_terminal + for raw_id in (node.value1, node.value2) + ) + table_blobs = await self.get_table_blobs(kv_ids, store_id) + + for node in tree_nodes: + if node.is_terminal: + blobs = [] + for raw_id in (node.value1, node.value2): + id = KeyOrValueId.from_bytes(raw_id) + blob_hash, blob = table_blobs[id] + if blob is None: + blob = self.get_blob_from_file(blob_hash, store_id) + blobs.append(blob) + to_write = bytes(SerializedNode(True, blobs[0], blobs[1])) + else: + to_write = bytes(node) + writer.write(len(to_write).to_bytes(4, byteorder="big")) + writer.write(to_write) async def update_subscriptions_from_wallet(self, store_id: bytes32, new_urls: list[str]) -> None: async with self.db_wrapper.writer() as writer: @@ -2066,94 +1712,36 @@ async def remove_subscriptions(self, store_id: bytes32, urls: list[str]) -> None }, ) - async def delete_store_data(self, store_id: bytes32) -> None: - async with self.db_wrapper.writer(foreign_key_enforcement_enabled=False) as writer: - await self.clean_node_table(writer) - cursor = await writer.execute( - """ - WITH RECURSIVE all_nodes AS ( - SELECT a.hash, n.left, n.right - FROM ancestors AS a - JOIN node AS n ON a.hash = n.hash - WHERE a.tree_id = :tree_id - ), - pending_nodes AS ( - SELECT node_hash AS hash FROM root - WHERE status IN (:pending_status, :pending_batch_status) - UNION ALL - SELECT n.left FROM node n - INNER JOIN pending_nodes pn ON n.hash = pn.hash - WHERE n.left IS NOT NULL - UNION ALL - SELECT n.right FROM node n - INNER JOIN pending_nodes pn ON n.hash = pn.hash - WHERE n.right IS NOT NULL - ) - - SELECT hash, left, right - FROM all_nodes - WHERE hash NOT IN (SELECT hash FROM ancestors WHERE tree_id != :tree_id) - AND hash NOT IN (SELECT hash from pending_nodes) - """, - { - "tree_id": store_id, - "pending_status": Status.PENDING.value, - "pending_batch_status": Status.PENDING_BATCH.value, - }, - ) - to_delete: dict[bytes, tuple[bytes, bytes]] = {} - ref_counts: dict[bytes, int] = {} - async for row in cursor: - hash = row["hash"] - left = row["left"] - right = row["right"] - if hash in to_delete: - prev_left, prev_right = to_delete[hash] - assert prev_left == left - assert prev_right == right - continue - to_delete[hash] = (left, right) - if left is not None: - ref_counts[left] = ref_counts.get(left, 0) + 1 - if right is not None: - ref_counts[right] = ref_counts.get(right, 0) + 1 - - await writer.execute("DELETE FROM ancestors WHERE tree_id == ?", (store_id,)) - await writer.execute("DELETE FROM root WHERE tree_id == ?", (store_id,)) - queue = [hash for hash in to_delete if ref_counts.get(hash, 0) == 0] - while queue: - hash = queue.pop(0) - if hash not in to_delete: - continue - await writer.execute("DELETE FROM node WHERE hash == ?", (hash,)) - - left, right = to_delete[hash] - if left is not None: - ref_counts[left] -= 1 - if ref_counts[left] == 0: - queue.append(left) - - if right is not None: - ref_counts[right] -= 1 - if ref_counts[right] == 0: - queue.append(right) - async def unsubscribe(self, store_id: bytes32) -> None: async with self.db_wrapper.writer() as writer: await writer.execute( "DELETE FROM subscriptions WHERE tree_id == :tree_id", {"tree_id": store_id}, ) + await writer.execute( + "DELETE FROM ids WHERE store_id == :store_id", + {"store_id": store_id}, + ) + await writer.execute( + "DELETE FROM nodes WHERE store_id == :store_id", + {"store_id": store_id}, + ) + + with contextlib.suppress(FileNotFoundError): + shutil.rmtree(self.get_merkle_path(store_id=store_id, root_hash=None)) + + with contextlib.suppress(FileNotFoundError): + shutil.rmtree(self.get_key_value_path(store_id=store_id, blob_hash=None)) async def rollback_to_generation(self, store_id: bytes32, target_generation: int) -> None: async with self.db_wrapper.writer() as writer: await writer.execute( - "DELETE FROM ancestors WHERE tree_id == :tree_id AND generation > :target_generation", + "DELETE FROM root WHERE tree_id == :tree_id AND generation > :target_generation", {"tree_id": store_id, "target_generation": target_generation}, ) await writer.execute( - "DELETE FROM root WHERE tree_id == :tree_id AND generation > :target_generation", - {"tree_id": store_id, "target_generation": target_generation}, + "DELETE FROM nodes WHERE store_id == :store_id AND generation > :target_generation", + {"store_id": store_id, "target_generation": target_generation}, ) async def update_server_info(self, store_id: bytes32, server_info: ServerInfo) -> None: diff --git a/chia/data_layer/download_data.py b/chia/data_layer/download_data.py index fcd78f14998a..3e89684eb4fb 100644 --- a/chia/data_layer/download_data.py +++ b/chia/data_layer/download_data.py @@ -8,49 +8,21 @@ from typing import Optional import aiohttp +from chia_rs.datalayer import DeltaReader from chia_rs.sized_bytes import bytes32 from typing_extensions import Literal -from chia.data_layer.data_layer_util import NodeType, PluginRemote, Root, SerializedNode, ServerInfo, Status +from chia.data_layer.data_layer_util import ( + PluginRemote, + Root, + ServerInfo, + get_delta_filename, + get_delta_filename_path, + get_full_tree_filename, + get_full_tree_filename_path, +) from chia.data_layer.data_store import DataStore - - -def get_full_tree_filename(store_id: bytes32, node_hash: bytes32, generation: int, group_by_store: bool = False) -> str: - if group_by_store: - return f"{store_id}/{node_hash}-full-{generation}-v1.0.dat" - return f"{store_id}-{node_hash}-full-{generation}-v1.0.dat" - - -def get_delta_filename(store_id: bytes32, node_hash: bytes32, generation: int, group_by_store: bool = False) -> str: - if group_by_store: - return f"{store_id}/{node_hash}-delta-{generation}-v1.0.dat" - return f"{store_id}-{node_hash}-delta-{generation}-v1.0.dat" - - -def get_full_tree_filename_path( - foldername: Path, - store_id: bytes32, - node_hash: bytes32, - generation: int, - group_by_store: bool = False, -) -> Path: - if group_by_store: - path = foldername.joinpath(f"{store_id}") - return path.joinpath(f"{node_hash}-full-{generation}-v1.0.dat") - return foldername.joinpath(f"{store_id}-{node_hash}-full-{generation}-v1.0.dat") - - -def get_delta_filename_path( - foldername: Path, - store_id: bytes32, - node_hash: bytes32, - generation: int, - group_by_store: bool = False, -) -> Path: - if group_by_store: - path = foldername.joinpath(f"{store_id}") - return path.joinpath(f"{node_hash}-delta-{generation}-v1.0.dat") - return foldername.joinpath(f"{store_id}-{node_hash}-delta-{generation}-v1.0.dat") +from chia.util.log_exceptions import log_exceptions def is_filename_valid(filename: str, group_by_store: bool = False) -> bool: @@ -87,45 +59,6 @@ def is_filename_valid(filename: str, group_by_store: bool = False) -> bool: return reformatted == filename -async def insert_into_data_store_from_file( - data_store: DataStore, - store_id: bytes32, - root_hash: Optional[bytes32], - filename: Path, -) -> int: - num_inserted = 0 - with open(filename, "rb") as reader: - while True: - chunk = b"" - while len(chunk) < 4: - size_to_read = 4 - len(chunk) - cur_chunk = reader.read(size_to_read) - if cur_chunk is None or cur_chunk == b"": - if size_to_read < 4: - raise Exception("Incomplete read of length.") - break - chunk += cur_chunk - if chunk == b"": - break - - size = int.from_bytes(chunk, byteorder="big") - serialize_nodes_bytes = b"" - while len(serialize_nodes_bytes) < size: - size_to_read = size - len(serialize_nodes_bytes) - cur_chunk = reader.read(size_to_read) - if cur_chunk is None or cur_chunk == b"": - raise Exception("Incomplete read of blob.") - serialize_nodes_bytes += cur_chunk - serialized_node = SerializedNode.from_bytes(serialize_nodes_bytes) - - node_type = NodeType.TERMINAL if serialized_node.is_terminal else NodeType.INTERNAL - await data_store.insert_node(node_type, serialized_node.value1, serialized_node.value2) - num_inserted += 1 - - await data_store.insert_root_with_ancestor_table(store_id=store_id, node_hash=root_hash, status=Status.COMMITTED) - return num_inserted - - @dataclass class WriteFilesResult: result: bool @@ -165,14 +98,8 @@ async def write_files_for_root( pass try: - last_seen_generation = await data_store.get_last_tree_root_by_hash( - store_id, root.node_hash, max_generation=root.generation - ) - if last_seen_generation is None: - with open(filename_diff_tree, mode) as writer: - await data_store.write_tree_to_file(root, node_hash, store_id, True, writer) - else: - open(filename_diff_tree, mode).close() + with open(filename_diff_tree, mode) as writer: + await data_store.write_tree_to_file(root, node_hash, store_id, True, writer) written = True except FileExistsError: pass @@ -250,6 +177,8 @@ async def insert_from_delta_file( if group_files_by_store: client_foldername.joinpath(f"{store_id}").mkdir(parents=True, exist_ok=True) + delta_reader: Optional[DeltaReader] = None + for root_hash in root_hashes: timestamp = int(time.time()) existing_generation += 1 @@ -281,33 +210,34 @@ async def insert_from_delta_file( log.info(f"Successfully downloaded delta file {target_filename_path.name}.") try: - filename_full_tree = get_full_tree_filename_path( - client_foldername, - store_id, - root_hash, - existing_generation, - group_files_by_store, - ) - num_inserted = await insert_into_data_store_from_file( - data_store, - store_id, - None if root_hash == bytes32.zeros else root_hash, - target_filename_path, - ) - log.info( - f"Successfully inserted hash {root_hash} from delta file. " - f"Generation: {existing_generation}. Store id: {store_id}. Nodes inserted: {num_inserted}." - ) - - if target_generation - existing_generation <= maximum_full_file_count - 1: - root = await data_store.get_tree_root(store_id=store_id) - with open(filename_full_tree, "wb") as writer: - await data_store.write_tree_to_file(root, root_hash, store_id, False, writer) - log.info(f"Successfully written full tree filename {filename_full_tree}.") - else: - log.info(f"Skipping full file generation for {existing_generation}") - - await data_store.received_correct_file(store_id, server_info) + with log_exceptions(log=log, message="exception while inserting from delta file"): + filename_full_tree = get_full_tree_filename_path( + client_foldername, + store_id, + root_hash, + existing_generation, + group_files_by_store, + ) + delta_reader = await data_store.insert_into_data_store_from_file( + store_id, + None if root_hash == bytes32.zeros else root_hash, + target_filename_path, + delta_reader=delta_reader, + ) + log.info( + f"Successfully inserted hash {root_hash} from delta file. " + f"Generation: {existing_generation}. Store id: {store_id}." + ) + + if target_generation - existing_generation <= maximum_full_file_count - 1: + root = await data_store.get_tree_root(store_id=store_id) + with open(filename_full_tree, "wb") as writer: + await data_store.write_tree_to_file(root, root_hash, store_id, False, writer) + log.info(f"Successfully written full tree filename {filename_full_tree}.") + else: + log.info(f"Skipping full file generation for {existing_generation}") + + await data_store.received_correct_file(store_id, server_info) except Exception: try: target_filename_path.unlink() @@ -326,7 +256,6 @@ async def insert_from_delta_file( if not filename_exists: # Don't penalize this server if we didn't download the file from it. await data_store.server_misses_file(store_id, server_info, timestamp) - await data_store.rollback_to_generation(store_id, existing_generation - 1) return False return True @@ -386,4 +315,4 @@ async def http_download( new_percentage = f"{progress_byte / size:.0%}" if new_percentage != progress_percentage: progress_percentage = new_percentage - log.debug(f"Downloading delta file {filename}. {progress_percentage} of {size} bytes.") + log.info(f"Downloading delta file {filename}. {progress_percentage} of {size} bytes.") diff --git a/chia/data_layer/util/benchmark.py b/chia/data_layer/util/benchmark.py index e244df6370ea..91d6c553d738 100644 --- a/chia/data_layer/util/benchmark.py +++ b/chia/data_layer/util/benchmark.py @@ -6,15 +6,14 @@ import tempfile import time from pathlib import Path -from typing import Optional from chia_rs.sized_bytes import bytes32 -from chia.data_layer.data_layer_util import Side, TerminalNode, leaf_hash +from chia.data_layer.data_layer_util import Side, Status, leaf_hash from chia.data_layer.data_store import DataStore -async def generate_datastore(num_nodes: int, slow_mode: bool) -> None: +async def generate_datastore(num_nodes: int) -> None: with tempfile.TemporaryDirectory() as temp_directory: temp_directory_path = Path(temp_directory) db_path = temp_directory_path.joinpath("dl_benchmark.sqlite") @@ -23,9 +22,14 @@ async def generate_datastore(num_nodes: int, slow_mode: bool) -> None: if os.path.exists(db_path): os.remove(db_path) - async with DataStore.managed(database=db_path) as data_store: + start_time = time.monotonic() + async with DataStore.managed( + database=db_path, + merkle_blobs_path=temp_directory_path.joinpath("merkle-blobs"), + key_value_blobs_path=temp_directory_path.joinpath("key-value-blobs"), + ) as data_store: store_id = bytes32(b"0" * 32) - await data_store.create_tree(store_id) + await data_store.create_tree(store_id, status=Status.COMMITTED) insert_time = 0.0 insert_count = 0 @@ -37,58 +41,40 @@ async def generate_datastore(num_nodes: int, slow_mode: bool) -> None: for i in range(num_nodes): key = i.to_bytes(4, byteorder="big") value = (2 * i).to_bytes(4, byteorder="big") - seed = leaf_hash(key=key, value=value) - reference_node_hash: Optional[bytes32] = await data_store.get_terminal_node_for_seed(store_id, seed) - side: Optional[Side] = data_store.get_side_for_seed(seed) + seed = leaf_hash(key, value) + node = await data_store.get_terminal_node_for_seed(seed, store_id) - if i == 0: - reference_node_hash = None - side = None if i % 3 == 0: t1 = time.time() - if not slow_mode: - await data_store.insert( - key=key, - value=value, - store_id=store_id, - reference_node_hash=reference_node_hash, - side=side, - ) - else: - await data_store.insert( - key=key, - value=value, - store_id=store_id, - reference_node_hash=reference_node_hash, - side=side, - use_optimized=False, - ) + await data_store.autoinsert( + key=key, + value=value, + store_id=store_id, + status=Status.COMMITTED, + ) t2 = time.time() - insert_time += t2 - t1 - insert_count += 1 + autoinsert_count += 1 elif i % 3 == 1: + assert node is not None + reference_node_hash = node.hash + side_seed = bytes(seed)[0] + side = Side.LEFT if side_seed < 128 else Side.RIGHT t1 = time.time() - if not slow_mode: - await data_store.autoinsert(key=key, value=value, store_id=store_id) - else: - await data_store.autoinsert( - key=key, - value=value, - store_id=store_id, - use_optimized=False, - ) + await data_store.insert( + key=key, + value=value, + store_id=store_id, + reference_node_hash=reference_node_hash, + side=side, + status=Status.COMMITTED, + ) t2 = time.time() - autoinsert_time += t2 - t1 - autoinsert_count += 1 + insert_time += t2 - t1 + insert_count += 1 else: t1 = time.time() - assert reference_node_hash is not None - node = await data_store.get_node(reference_node_hash) - assert isinstance(node, TerminalNode) - if not slow_mode: - await data_store.delete(key=node.key, store_id=store_id) - else: - await data_store.delete(key=node.key, store_id=store_id, use_optimized=False) + assert node is not None + await data_store.delete(key=node.key, store_id=store_id, status=Status.COMMITTED) t2 = time.time() delete_time += t2 - t1 delete_count += 1 @@ -96,13 +82,12 @@ async def generate_datastore(num_nodes: int, slow_mode: bool) -> None: print(f"Average insert time: {insert_time / insert_count}") print(f"Average autoinsert time: {autoinsert_time / autoinsert_count}") print(f"Average delete time: {delete_time / delete_count}") - print(f"Total time for {num_nodes} operations: {insert_time + autoinsert_time + delete_time}") + print(f"Total time for {num_nodes} operations: {insert_time + delete_time + autoinsert_time}") root = await data_store.get_tree_root(store_id=store_id) print(f"Root hash: {root.node_hash}") + finish_time = time.monotonic() + print(f"Total runtime: {finish_time - start_time}") if __name__ == "__main__": - slow_mode = False - if len(sys.argv) > 2 and sys.argv[2] == "slow": - slow_mode = True - asyncio.run(generate_datastore(int(sys.argv[1]), slow_mode)) + asyncio.run(generate_datastore(int(sys.argv[1]))) diff --git a/chia/util/initial-config.yaml b/chia/util/initial-config.yaml index 13bd21a98c7d..6aad76064883 100644 --- a/chia/util/initial-config.yaml +++ b/chia/util/initial-config.yaml @@ -665,6 +665,8 @@ data_layer: port: 9256 database_path: "data_layer/db/data_layer_CHALLENGE.sqlite" + merkle_blobs_path: "data_layer/db/merkle_blobs_CHALLENGE" + key_value_blobs_path: "data_layer/db/key_value_blobs_CHALLENGE" # The location where the server files will be stored. server_files_location: "data_layer/db/server_files_location_CHALLENGE" # The timeout for the client to download a file from a server @@ -707,6 +709,9 @@ data_layer: # Enable to store all .DAT files grouped by store id group_files_by_store: False + # Increasing this number may help sync old clients faster, at the expense of using more RAM memory + merkle_blobs_cache_size: 1 + simulator: # Should the simulator farm a block whenever a transaction is in mempool auto_farm: True