diff --git a/.sqlx/query-270d3344d0fc55c4fa32bc97d77882026371af8105ede628970a807131a13660.json b/.sqlx/query-270d3344d0fc55c4fa32bc97d77882026371af8105ede628970a807131a13660.json deleted file mode 100644 index d2890ca6..00000000 --- a/.sqlx/query-270d3344d0fc55c4fa32bc97d77882026371af8105ede628970a807131a13660.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT p.part_size, p.\"compression\", p.compressed_size, p.segment_id, p.segment_offset\n FROM (SELECT unnest(parts) AS id FROM blobs WHERE id = $1) AS part_id\n JOIN parts AS p ON p.id = part_id.id;", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "part_size", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "compression", - "type_info": "Int2" - }, - { - "ordinal": 2, - "name": "compressed_size", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "segment_id", - "type_info": "Uuid" - }, - { - "ordinal": 4, - "name": "segment_offset", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false - ] - }, - "hash": "270d3344d0fc55c4fa32bc97d77882026371af8105ede628970a807131a13660" -} diff --git a/.sqlx/query-52f93a6666babec678cdf8eb611921189c0f160fcc28a435370355d0d1521dde.json b/.sqlx/query-52f93a6666babec678cdf8eb611921189c0f160fcc28a435370355d0d1521dde.json deleted file mode 100644 index 52d59233..00000000 --- a/.sqlx/query-52f93a6666babec678cdf8eb611921189c0f160fcc28a435370355d0d1521dde.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO blobs\n (id, parts)\n VALUES\n ($1, $2);", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Int8Array" - ] - }, - "nullable": [] - }, - "hash": "52f93a6666babec678cdf8eb611921189c0f160fcc28a435370355d0d1521dde" -} diff --git a/.sqlx/query-7164172dcc7008ff76bb70e2b57f96331170ebe76a013003892950ed78e023ab.json b/.sqlx/query-7164172dcc7008ff76bb70e2b57f96331170ebe76a013003892950ed78e023ab.json deleted file mode 100644 index 5e894266..00000000 --- a/.sqlx/query-7164172dcc7008ff76bb70e2b57f96331170ebe76a013003892950ed78e023ab.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n part_size, \"compression\", compressed_size, segment_id, segment_offset\n FROM parts WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "part_size", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "compression", - "type_info": "Int2" - }, - { - "ordinal": 2, - "name": "compressed_size", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "segment_id", - "type_info": "Uuid" - }, - { - "ordinal": 4, - "name": "segment_offset", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false, - false, - false, - false, - false - ] - }, - "hash": "7164172dcc7008ff76bb70e2b57f96331170ebe76a013003892950ed78e023ab" -} diff --git a/.sqlx/query-940a4f83c63c73e96c0325718904c021084dfc40649711712aaadd1b655f8390.json b/.sqlx/query-940a4f83c63c73e96c0325718904c021084dfc40649711712aaadd1b655f8390.json deleted file mode 100644 index 6e599140..00000000 --- a/.sqlx/query-940a4f83c63c73e96c0325718904c021084dfc40649711712aaadd1b655f8390.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO parts\n (part_size, compression, compressed_size, segment_id, segment_offset)\n VALUES\n ($1, $2, $3, $4, $5)\n RETURNING id;", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int4", - "Int2", - "Int4", - "Uuid", - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "940a4f83c63c73e96c0325718904c021084dfc40649711712aaadd1b655f8390" -} diff --git a/Cargo.lock b/Cargo.lock index c4df20c2..cf0567be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,12 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "anyhow" version = "1.0.98" @@ -49,15 +43,6 @@ dependencies = [ "syn", ] -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -136,29 +121,11 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64ct" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" - [[package]] name = "bitflags" version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" -dependencies = [ - "serde", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] [[package]] name = "bumpalo" @@ -166,12 +133,6 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.10.1" @@ -190,6 +151,8 @@ version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -207,21 +170,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "core-foundation" version = "0.9.4" @@ -238,78 +186,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - -[[package]] -name = "crossbeam-queue" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -321,20 +197,11 @@ dependencies = [ "syn", ] -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" -dependencies = [ - "serde", -] [[package]] name = "encoding_rs" @@ -361,28 +228,6 @@ dependencies = [ "windows-sys 0.60.2", ] -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", -] - -[[package]] -name = "event-listener" -version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -395,29 +240,12 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" -[[package]] -name = "flume" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" -dependencies = [ - "futures-core", - "futures-sink", - "spin", -] - [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foreign-types" version = "0.3.2" @@ -484,17 +312,6 @@ dependencies = [ "futures-util", ] -[[package]] -name = "futures-intrusive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot", -] - [[package]] name = "futures-io" version = "0.3.31" @@ -542,16 +359,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.16" @@ -605,20 +412,6 @@ name = "hashbrown" version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "hashlink" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" -dependencies = [ - "hashbrown", -] [[package]] name = "heck" @@ -626,39 +419,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hkdf" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" -dependencies = [ - "hmac", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "http" version = "1.3.1" @@ -957,22 +717,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] -name = "js-sys" -version = "0.3.77" +name = "jobserver" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ - "once_cell", - "wasm-bindgen", + "getrandom 0.3.3", + "libc", ] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "js-sys" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ - "spin", + "once_cell", + "wasm-bindgen", ] [[package]] @@ -987,16 +748,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" -[[package]] -name = "libsqlite3-sys" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" -dependencies = [ - "pkg-config", - "vcpkg", -] - [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -1031,16 +782,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - [[package]] name = "memchr" version = "2.7.5" @@ -1106,43 +847,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.5", - "smallvec", - "zeroize", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1213,10 +917,10 @@ dependencies = [ ] [[package]] -name = "parking" -version = "2.2.1" +name = "pack1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +checksum = "d6e7cd9bd638dc2c831519a0caa1c006cab771a92b1303403a8322773c5b72d6" [[package]] name = "parking_lot" @@ -1241,15 +945,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -1298,27 +993,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - [[package]] name = "pkg-config" version = "0.3.32" @@ -1438,35 +1112,14 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", + "rand_chacha", + "rand_core", ] [[package]] @@ -1476,16 +1129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", + "rand_core", ] [[package]] @@ -1504,7 +1148,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.1", + "rand", ] [[package]] @@ -1603,26 +1247,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rsa" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" -dependencies = [ - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core 0.6.4", - "signature", - "spki", - "subtle", - "zeroize", -] - [[package]] name = "rustc-demangle" version = "0.1.25" @@ -1649,7 +1273,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1785,58 +1408,28 @@ name = "service" version = "0.1.0" dependencies = [ "anyhow", + "pack1", "proto-codegen", - "sqlx", "tempfile", "tokio", "uuid", + "watto", + "zstd", ] [[package]] -name = "sha1" -version = "0.10.6" +name = "shlex" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] -name = "sha2" -version = "0.10.9" +name = "signal-hook-registry" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" -dependencies = [ - "libc", -] - -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest", - "rand_core 0.6.4", + "libc", ] [[package]] @@ -1856,9 +1449,6 @@ name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -dependencies = [ - "serde", -] [[package]] name = "socket2" @@ -1870,219 +1460,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sqlx" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" -dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", -] - -[[package]] -name = "sqlx-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" -dependencies = [ - "base64", - "bytes", - "crc", - "crossbeam-queue", - "either", - "event-listener", - "futures-core", - "futures-intrusive", - "futures-io", - "futures-util", - "hashbrown", - "hashlink", - "indexmap", - "log", - "memchr", - "once_cell", - "percent-encoding", - "rustls", - "serde", - "serde_json", - "sha2", - "smallvec", - "thiserror", - "tokio", - "tokio-stream", - "tracing", - "url", - "uuid", - "webpki-roots 0.26.11", -] - -[[package]] -name = "sqlx-macros" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" -dependencies = [ - "proc-macro2", - "quote", - "sqlx-core", - "sqlx-macros-core", - "syn", -] - -[[package]] -name = "sqlx-macros-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" -dependencies = [ - "dotenvy", - "either", - "heck", - "hex", - "once_cell", - "proc-macro2", - "quote", - "serde", - "serde_json", - "sha2", - "sqlx-core", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", - "syn", - "tokio", - "url", -] - -[[package]] -name = "sqlx-mysql" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" -dependencies = [ - "atoi", - "base64", - "bitflags", - "byteorder", - "bytes", - "crc", - "digest", - "dotenvy", - "either", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "generic-array", - "hex", - "hkdf", - "hmac", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "percent-encoding", - "rand 0.8.5", - "rsa", - "serde", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "uuid", - "whoami", -] - -[[package]] -name = "sqlx-postgres" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" -dependencies = [ - "atoi", - "base64", - "bitflags", - "byteorder", - "crc", - "dotenvy", - "etcetera", - "futures-channel", - "futures-core", - "futures-util", - "hex", - "hkdf", - "hmac", - "home", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "rand 0.8.5", - "serde", - "serde_json", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "uuid", - "whoami", -] - -[[package]] -name = "sqlx-sqlite" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" -dependencies = [ - "atoi", - "flume", - "futures-channel", - "futures-core", - "futures-executor", - "futures-intrusive", - "futures-util", - "libsqlite3-sys", - "log", - "percent-encoding", - "serde", - "serde_urlencoded", - "sqlx-core", - "thiserror", - "tracing", - "url", - "uuid", -] - [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2098,7 +1475,7 @@ dependencies = [ "bytesize", "futures", "futures-util", - "rand 0.9.1", + "rand", "rand_distr", "reqwest", "serde", @@ -2108,17 +1485,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "stringprep" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" -dependencies = [ - "unicode-bidi", - "unicode-normalization", - "unicode-properties", -] - [[package]] name = "subtle" version = "2.6.1" @@ -2190,26 +1556,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tinystr" version = "0.8.1" @@ -2220,21 +1566,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinyvec" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.46.0" @@ -2408,7 +1739,6 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2440,45 +1770,18 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "typenum" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" - [[package]] name = "unicase" version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" -[[package]] -name = "unicode-bidi" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" - [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-properties" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" - [[package]] name = "untrusted" version = "0.9.0" @@ -2519,12 +1822,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "want" version = "0.3.1" @@ -2549,12 +1846,6 @@ dependencies = [ "wit-bindgen-rt", ] -[[package]] -name = "wasite" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" - [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -2639,6 +1930,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "watto" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfbc1480663d640f8c9f7a1ac70922eea60ac16fea79df883177df3bc7bb8b49" + [[package]] name = "web-sys" version = "0.3.77" @@ -2649,34 +1946,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.1", -] - -[[package]] -name = "webpki-roots" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "whoami" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" -dependencies = [ - "redox_syscall", - "wasite", -] - [[package]] name = "windows-link" version = "0.1.3" @@ -2712,15 +1981,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -2748,21 +2008,6 @@ dependencies = [ "windows-targets 0.53.2", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -2795,12 +2040,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -2813,12 +2052,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -2831,12 +2064,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -2861,12 +2088,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -2879,12 +2100,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -2897,12 +2112,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -2915,12 +2124,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -3057,3 +2260,31 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/service/Cargo.toml b/service/Cargo.toml index 04962d48..bb34bc61 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -5,18 +5,12 @@ edition = "2024" [dependencies] anyhow = "1.0.98" +pack1 = "1.0.0" proto-codegen = { path = "../proto-codegen" } -sqlx = { version = "0.8", features = [ - "derive", - "macros", - "migrate", - "postgres", - "runtime-tokio", - "tls-rustls", - "uuid", -] } -uuid = { version = "1.17.0", features = ["v4"] } tokio = { version = "1.46.0", features = ["full"] } +uuid = { version = "1.17.0", features = ["v4", "v7"] } +watto = "0.2.0" +zstd = "0.13.3" [dev-dependencies] tempfile = "3.20.0" diff --git a/service/build.rs b/service/build.rs deleted file mode 100644 index 3a8149ef..00000000 --- a/service/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=migrations"); -} diff --git a/service/migrations/20250702105701_initial.sql b/service/migrations/20250702105701_initial.sql deleted file mode 100644 index 49b047de..00000000 --- a/service/migrations/20250702105701_initial.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE IF NOT EXISTS parts ( - id bigserial NOT NULL PRIMARY KEY, - part_size int4 NOT NULL, - "compression" int2 NOT NULL, - compressed_size int4 NOT NULL, - segment_id uuid NOT NULL, - segment_offset int4 NOT NULL -); - - -CREATE TABLE IF NOT EXISTS blobs ( - id varchar NOT NULL PRIMARY KEY, - parts int8 ARRAY NOT NULL -); diff --git a/service/src/datamodel.rs b/service/src/datamodel.rs new file mode 100644 index 00000000..66e1a419 --- /dev/null +++ b/service/src/datamodel.rs @@ -0,0 +1,49 @@ +#![allow(unused)] + +use pack1::{U16LE, U32LE, U64LE}; +use watto::Pod; + +pub const PART_MAGIC: [u8; 2] = *b"\xf5P"; +pub const FILE_MAGIC: [u8; 2] = *b"\xf5F"; +pub const PART_VERSION: u16 = 1; +pub const FILE_VERSION: u16 = 1; + +#[derive(Debug)] +#[repr(u8)] +pub enum Compression { + None = 0, + Zstd = 1, +} + +/// Part metadata, like compression and size. +#[derive(Debug)] +#[repr(C)] +pub struct Part { + pub magic: [u8; 2], + pub version: U16LE, + pub part_size: U32LE, + pub compression_algorithm: u8, + pub _padding: [u8; 3], + pub compressed_size: U32LE, +} +unsafe impl Pod for Part {} + +/// File metadata, like how many parts (if any) it consists of. +#[derive(Debug)] +#[repr(C)] +pub struct File { + pub magic: [u8; 2], + pub version: U16LE, + pub num_parts: U32LE, + pub file_size: U64LE, +} +unsafe impl Pod for File {} + +/// The (uncompressed) size and UUID of the file part. +#[derive(Debug)] +#[repr(C)] +pub struct FilePart { + pub part_size: U32LE, + pub part_uuid: [u8; 16], +} +unsafe impl Pod for FilePart {} diff --git a/service/src/db/mod.rs b/service/src/db/mod.rs deleted file mode 100644 index cdba6094..00000000 --- a/service/src/db/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -use sqlx::PgPool; - -pub async fn initialize_db(database_url: &str) -> anyhow::Result { - let db = PgPool::connect(database_url).await?; - - sqlx::migrate!().run(&db).await?; - - Ok(db) -} diff --git a/service/src/lib.rs b/service/src/lib.rs index be481470..6f8cc4b5 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -3,143 +3,166 @@ //! //! It is designed as a library crate to be used by the `server`. +mod datamodel; + use std::fs::OpenOptions; -use std::io::{Read, Seek as _, SeekFrom, Write}; +use std::io::{BufReader, BufWriter, ErrorKind, Read, Write}; +use std::mem; use std::path::{Path, PathBuf}; -use std::sync::Mutex; -use sqlx::PgPool; +use anyhow::Context; use uuid::Uuid; -mod db; - -pub use db::initialize_db; +use watto::Pod; -#[allow(unused)] -#[derive(Debug)] -struct Part { - part_size: i32, - compression: i16, - compressed_size: i32, - segment_id: Uuid, - segment_offset: i32, -} +use crate::datamodel::{ + Compression, FILE_MAGIC, FILE_VERSION, File, FilePart, PART_MAGIC, PART_VERSION, Part, +}; pub struct StorageService { - db: PgPool, - path: PathBuf, - current_segment: Mutex>, + file_path: PathBuf, + part_path: PathBuf, } impl StorageService { - pub fn new(db: PgPool, path: &Path) -> Self { - Self { - path: path.into(), - db, - current_segment: Default::default(), - } + pub fn new(path: &Path) -> anyhow::Result { + let file_path = path.join("files"); + let part_path = path.join("parts"); + + std::fs::create_dir_all(&file_path)?; + std::fs::create_dir_all(&part_path)?; + + Ok(Self { + file_path, + part_path, + }) } - pub async fn assemble_file_from_parts(&self, id: &str, parts: &[i64]) -> anyhow::Result<()> { - sqlx::query!( - r#" - INSERT INTO blobs - (id, parts) - VALUES - ($1, $2);"#, - id, - parts, - ) - .execute(&self.db) - .await?; + pub fn assemble_file_from_parts(&self, key: &str, parts: &[FilePart]) -> anyhow::Result<()> { + let file_size: u64 = parts.iter().map(|part| part.part_size.get() as u64).sum(); + + let file_metadata = File { + magic: FILE_MAGIC, + version: FILE_VERSION.into(), + num_parts: (parts.len() as u32).into(), + file_size: file_size.into(), + }; + + let file_path = self.file_path.join(format!("{key}.bin")); + let file_file = OpenOptions::new() + .write(true) + .create_new(true) + .open(file_path)?; + + let mut writer = BufWriter::new(file_file); + + writer.write_all(file_metadata.as_bytes())?; + writer.write_all(parts.as_bytes())?; + + let part_file = writer.into_inner()?; + part_file.sync_data()?; + drop(part_file); Ok(()) } - async fn get_file_parts(&self, id: &str) -> anyhow::Result> { - let parts = sqlx::query_as!( - Part, - r#" - SELECT p.part_size, p."compression", p.compressed_size, p.segment_id, p.segment_offset - FROM (SELECT unnest(parts) AS id FROM blobs WHERE id = $1) AS part_id - JOIN parts AS p ON p.id = part_id.id;"#, - id, - ) - .fetch_all(&self.db) - .await?; - - Ok(parts) - } + pub fn get_file(&self, key: &str) -> anyhow::Result>> { + let file_path = self.file_path.join(format!("{key}.bin")); + let file_file = match OpenOptions::new().read(true).open(file_path) { + Ok(file_file) => file_file, + Err(err) if err.kind() == ErrorKind::NotFound => { + return Ok(None); + } + err => err?, + }; - pub async fn put_part(&self, contents: &[u8]) -> anyhow::Result { - let part_size = contents.len(); + let mut reader = BufReader::new(file_file); - let segment = { - let mut segment = self.current_segment.lock().unwrap(); - *segment.get_or_insert_with(Uuid::new_v4) - }; + let mut metadata_buf = vec![0; mem::size_of::()]; + reader.read_exact(&mut metadata_buf)?; + let file_metadata = File::ref_from_bytes(&metadata_buf).context("reading File metadata")?; + + let mut parts_buf = + vec![0; mem::size_of::() * file_metadata.num_parts.get() as usize]; + reader.read_exact(&mut parts_buf)?; + let parts = + FilePart::slice_from_bytes(&parts_buf).context("reading File Parts metadata")?; + + let mut contents = Vec::with_capacity(file_metadata.file_size.get() as usize); + for part in parts { + let part_contents = self + .get_part(Uuid::from_bytes(part.part_uuid))? + .context("part not found")?; + contents.extend_from_slice(&part_contents); + } - let segment_path = self.path.join(format!("{segment}.bin")); - let mut segment_file = OpenOptions::new() - .append(true) - .create(true) - .open(segment_path)?; - - let segment_offset = segment_file.seek(SeekFrom::End(0))?; - segment_file.write_all(contents)?; - segment_file.sync_data()?; - drop(segment_file); - - let id = sqlx::query!( - r#" - INSERT INTO parts - (part_size, compression, compressed_size, segment_id, segment_offset) - VALUES - ($1, $2, $3, $4, $5) - RETURNING id;"#, - part_size as i32, - 0, - part_size as i32, - segment, - segment_offset as i32, - ) - .fetch_one(&self.db) - .await?; - - Ok(id.id) + Ok(Some(contents)) } - pub async fn get_part(&self, id: i64) -> anyhow::Result>> { - let Some(part) = sqlx::query_as!( - Part, - r#" - SELECT - part_size, "compression", compressed_size, segment_id, segment_offset - FROM parts WHERE id = $1"#, - id - ) - .fetch_optional(&self.db) - .await? - else { - return Ok(None); + pub fn put_part(&self, contents: &[u8]) -> anyhow::Result { + let part_size = contents.len() as u32; + + let compressed = zstd::bulk::compress(contents, 0)?; + let compressed_size = compressed.len() as u32; + + let part_metadata = Part { + magic: PART_MAGIC, + version: PART_VERSION.into(), + part_size: part_size.into(), + compression_algorithm: Compression::Zstd as u8, + _padding: [0; 3], + compressed_size: compressed_size.into(), }; - let contents = self.get_part_from_storage(&part).await?; - Ok(Some(contents)) + let part_uuid = Uuid::new_v4(); + let part_path = self.part_path.join(format!("{part_uuid}.bin")); + let part_file = OpenOptions::new() + .write(true) + .create_new(true) + .open(part_path)?; + + let mut writer = BufWriter::new(part_file); + + writer.write_all(part_metadata.as_bytes())?; + writer.write_all(&compressed)?; + + let part_file = writer.into_inner()?; + part_file.sync_data()?; + drop(part_file); + + Ok(FilePart { + part_size: part_size.into(), + part_uuid: part_uuid.into_bytes(), + }) } - async fn get_part_from_storage(&self, part: &Part) -> anyhow::Result> { - let segment_path = self.path.join(format!("{}.bin", part.segment_id)); - let mut segment_file = OpenOptions::new().read(true).open(segment_path)?; + pub fn get_part(&self, part_uuid: Uuid) -> anyhow::Result>> { + let part_path = self.part_path.join(format!("{part_uuid}.bin")); + let part_file = match OpenOptions::new().read(true).open(part_path) { + Ok(part_file) => part_file, + Err(err) if err.kind() == ErrorKind::NotFound => { + return Ok(None); + } + err => err?, + }; - segment_file.seek(SeekFrom::Start(part.segment_offset as u64))?; + let mut reader = BufReader::new(part_file); - let mut buf = vec![0; part.part_size as usize]; - segment_file.read_exact(&mut buf)?; + let mut metadata_buf = vec![0; mem::size_of::()]; + reader.read_exact(&mut metadata_buf)?; + let part_metadata = Part::ref_from_bytes(&metadata_buf).context("reading Part metadata")?; - drop(segment_file); + // TODO: verify magic, version, etc… + let contents = match part_metadata.compression_algorithm { + 1 /* Zstd */ => zstd::decode_all(reader)?, + _ => { + let mut buf = Vec::with_capacity(part_metadata.part_size.get() as usize); + reader.read_to_end(&mut buf)?; + buf + } + }; - Ok(buf) + Ok(Some(contents)) } } @@ -147,38 +170,35 @@ impl StorageService { mod tests { use super::*; - #[sqlx::test] - async fn stores_in_segments(db: PgPool) { + #[test] + fn stores_parts() { let tempdir = tempfile::tempdir().unwrap(); - let service = StorageService::new(db, tempdir.path()); + let service = StorageService::new(tempdir.path()).unwrap(); - let blob: &[u8] = b"oh hai!"; - let id = service.put_part(blob).await.unwrap(); + let file_part = service.put_part(b"oh hai!").unwrap(); + + let read_part = service + .get_part(Uuid::from_bytes(file_part.part_uuid)) + .unwrap() + .unwrap(); - let got_blob = service.get_part(id).await.unwrap(); - assert_eq!(got_blob.unwrap(), blob); + assert_eq!(read_part, b"oh hai!"); } - #[sqlx::test] - async fn stores_blob_as_parts(db: PgPool) { + #[test] + fn assembles_file_from_parts() { let tempdir = tempfile::tempdir().unwrap(); - let service = StorageService::new(db, tempdir.path()); + let service = StorageService::new(tempdir.path()).unwrap(); + + let part1 = service.put_part(b"oh ").unwrap(); + let part2 = service.put_part(b"hai!").unwrap(); - let part1 = service.put_part(b"oh ").await.unwrap(); - let part2 = service.put_part(b"hai!").await.unwrap(); service - .assemble_file_from_parts("some/file/id", &[part1, part2]) - .await + .assemble_file_from_parts("the_file_key", &[part1, part2]) .unwrap(); - let parts = service.get_file_parts("some/file/id").await.unwrap(); - - let mut contents = vec![]; - for part in parts { - let part = service.get_part_from_storage(&part).await.unwrap(); - contents.extend_from_slice(&part); - } + let file_contents = service.get_file("the_file_key").unwrap().unwrap(); - assert_eq!(contents, b"oh hai!"); + assert_eq!(file_contents, b"oh hai!"); } }