diff --git a/Cargo.lock b/Cargo.lock index 813797a24..c37c98f5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -788,9 +788,9 @@ checksum = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "aptos-abstract-gas-usage" @@ -3233,6 +3233,80 @@ dependencies = [ "url", ] +[[package]] +name = "aptos-indexer-processor-sdk" +version = "0.1.0" +source = "git+https://github.com/larryl3u/aptos-indexer-processor-sdk?rev=bbcc42d5a03cffe861b89c080ef7de918f37b4d0#bbcc42d5a03cffe861b89c080ef7de918f37b4d0" +dependencies = [ + "ahash 0.8.11", + "anyhow", + "aptos-indexer-transaction-stream", + "aptos-protos 1.3.1", + "aptos-system-utils 0.1.0 (git+https://github.com/aptos-labs/aptos-core.git?rev=2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686)", + "async-trait", + "autometrics", + "axum 0.7.9", + "backtrace", + "bcs 0.1.4", + "bigdecimal", + "chrono", + "clap 4.5.21", + "derive_builder 0.20.2", + "diesel", + "diesel-async 0.5.2", + "diesel_migrations", + "field_count", + "futures", + "futures-util", + "hex", + "indexmap 2.6.0", + "instrumented-channel", + "kanal", + "mockall", + "native-tls", + "num_cpus", + "once_cell", + "petgraph 0.6.5", + "postgres-native-tls", + "prometheus", + "prometheus-client", + "serde", + "serde_json", + "serde_yaml 0.8.26", + "sha2 0.9.9", + "tempfile", + "thiserror 1.0.69", + "tiny-keccak", + "tokio", + "tokio-postgres", + "toml 0.7.8", + "tracing", + "tracing-subscriber 0.3.18", + "url", +] + +[[package]] +name = "aptos-indexer-transaction-stream" +version = "0.1.0" +source = "git+https://github.com/larryl3u/aptos-indexer-processor-sdk?rev=bbcc42d5a03cffe861b89c080ef7de918f37b4d0#bbcc42d5a03cffe861b89c080ef7de918f37b4d0" +dependencies = [ + "anyhow", + "aptos-moving-average 0.1.0 (git+https://github.com/larryl3u/aptos-indexer-processor-sdk?rev=bbcc42d5a03cffe861b89c080ef7de918f37b4d0)", + "aptos-protos 1.3.1", + "aptos-transaction-filter", + "chrono", + "futures-util", + "once_cell", + "prometheus", + "prost 0.13.5", + "sample", + "serde", + "tokio", + "tonic 0.12.3", + "tracing", + "url", +] + [[package]] name = "aptos-infallible" version = "0.1.0" @@ -3807,7 +3881,7 @@ dependencies = [ [[package]] name = "aptos-moving-average" version = "0.1.0" -source = "git+https://github.com/movementlabsxyz/aptos-indexer-processors?rev=20be1190105908fd4fea4e78c330997658e9428e#20be1190105908fd4fea4e78c330997658e9428e" +source = "git+https://github.com/movementlabsxyz/aptos-indexer-processors?rev=77a36245400250e7d8a854360194288d078681bc#77a36245400250e7d8a854360194288d078681bc" dependencies = [ "chrono", ] @@ -3815,7 +3889,7 @@ dependencies = [ [[package]] name = "aptos-moving-average" version = "0.1.0" -source = "git+https://github.com/movementlabsxyz/aptos-indexer-processors?rev=77a36245400250e7d8a854360194288d078681bc#77a36245400250e7d8a854360194288d078681bc" +source = "git+https://github.com/larryl3u/aptos-indexer-processor-sdk?rev=bbcc42d5a03cffe861b89c080ef7de918f37b4d0#bbcc42d5a03cffe861b89c080ef7de918f37b4d0" dependencies = [ "chrono", ] @@ -4393,6 +4467,19 @@ dependencies = [ "regex", ] +[[package]] +name = "aptos-profiler" +version = "0.1.0" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686#2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686" +dependencies = [ + "anyhow", + "backtrace", + "jemalloc-sys", + "jemallocator", + "pprof", + "regex", +] + [[package]] name = "aptos-profiler" version = "0.1.0" @@ -4462,6 +4549,17 @@ dependencies = [ "tonic 0.11.0", ] +[[package]] +name = "aptos-protos" +version = "1.3.1" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686#2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686" +dependencies = [ + "pbjson", + "prost 0.13.5", + "serde", + "tonic 0.12.3", +] + [[package]] name = "aptos-proxy" version = "0.1.0" @@ -5264,6 +5362,26 @@ dependencies = [ "url", ] +[[package]] +name = "aptos-system-utils" +version = "0.1.0" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686#2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686" +dependencies = [ + "anyhow", + "aptos-profiler 0.1.0 (git+https://github.com/aptos-labs/aptos-core.git?rev=2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686)", + "async-mutex", + "http 0.2.12", + "hyper 0.14.31", + "lazy_static", + "mime", + "pprof", + "regex", + "rstack-self", + "tokio", + "tracing", + "url", +] + [[package]] name = "aptos-system-utils" version = "0.1.0" @@ -5522,6 +5640,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "aptos-transaction-filter" +version = "0.1.0" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686#2dd9c73b27fdcbe78c7391fd43c9a5d00b93e686" +dependencies = [ + "anyhow", + "aptos-protos 1.3.1", + "derivative", + "derive_builder 0.20.2", + "memchr", + "once_cell", + "prost 0.13.5", + "serde", + "serde_json", + "serde_yaml 0.8.26", + "thiserror 1.0.69", +] + [[package]] name = "aptos-types" version = "0.0.3" @@ -6397,6 +6533,39 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "autometrics" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10eaae539e7319a3813dc8cd53776a7128bdd6d82067275c12586f5a0fce9137" +dependencies = [ + "autometrics-macros", + "cfg_aliases 0.1.1", + "http 1.1.0", + "linkme", + "metrics-exporter-prometheus", + "once_cell", + "opentelemetry-prometheus", + "opentelemetry_sdk", + "prometheus", + "prometheus-client", + "spez", + "thiserror 1.0.69", +] + +[[package]] +name = "autometrics-macros" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf7c9ebfee6425011c65788c746adf80fac99ba38957ba1cdb824b593cfc993" +dependencies = [ + "percent-encoding", + "proc-macro2", + "quote", + "regex", + "syn 2.0.87", +] + [[package]] name = "aws-config" version = "1.5.10" @@ -6847,6 +7016,8 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", + "hyper 1.5.0", + "hyper-util", "itoa", "matchit", "memchr", @@ -6855,10 +7026,15 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", "sync_wrapper 1.0.1", + "tokio", "tower 0.5.1", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -6896,6 +7072,7 @@ dependencies = [ "sync_wrapper 1.0.1", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -6997,6 +7174,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "bcs" +version = "0.1.4" +source = "git+https://github.com/aptos-labs/bcs.git?rev=d31fab9d81748e2594be5cd5cdf845786a30562d#d31fab9d81748e2594be5cd5cdf845786a30562d" +dependencies = [ + "serde", + "thiserror 1.0.69", +] + [[package]] name = "bcs" version = "0.1.6" @@ -7367,7 +7553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2506947f73ad44e344215ccd6403ac2ae18cd8e046e581a441bf8d199f257f03" dependencies = [ "borsh-derive 1.5.3", - "cfg_aliases", + "cfg_aliases 0.2.1", ] [[package]] @@ -7669,7 +7855,7 @@ version = "0.7.0" source = "git+https://github.com/eigerco/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ "bytes 1.8.0", - "prost 0.13.3", + "prost 0.13.5", "prost-build", "prost-types 0.13.3", "protox", @@ -7687,7 +7873,7 @@ dependencies = [ "celestia-types", "http 1.1.0", "jsonrpsee", - "prost 0.13.3", + "prost 0.13.5", "serde", "thiserror 1.0.69", "tracing", @@ -7712,7 +7898,7 @@ dependencies = [ "multiaddr", "multihash", "nmt-rs", - "prost 0.13.3", + "prost 0.13.5", "ruint", "serde", "serde_repr", @@ -7744,6 +7930,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "cfg_aliases" version = "0.2.1" @@ -8208,6 +8400,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -8770,6 +8972,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "delegate" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e018fccbeeb50ff26562ece792ed06659b9c2dae79ece77c4456bb10d9bf79b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "der" version = "0.5.1" @@ -8846,7 +9059,16 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" dependencies = [ - "derive_builder_macro", + "derive_builder_macro 0.12.0", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro 0.20.2", ] [[package]] @@ -8861,16 +9083,38 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.10", + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "derive_builder_macro" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" dependencies = [ - "derive_builder_core", + "derive_builder_core 0.12.0", "syn 1.0.109", ] +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core 0.20.2", + "syn 2.0.87", +] + [[package]] name = "derive_more" version = "0.99.18" @@ -8946,6 +9190,20 @@ dependencies = [ "tokio-postgres", ] +[[package]] +name = "diesel-async" +version = "0.5.2" +source = "git+https://github.com/weiznich/diesel_async.git?rev=e3beac66cd41ab53d78a10328bb72f272103e5d1#e3beac66cd41ab53d78a10328bb72f272103e5d1" +dependencies = [ + "async-trait", + "bb8", + "diesel", + "futures-util", + "scoped-futures", + "tokio", + "tokio-postgres", +] + [[package]] name = "diesel_derives" version = "2.2.3" @@ -9104,6 +9362,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "dsl_auto_type" version = "0.1.2" @@ -9118,6 +9382,12 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "dtoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04" + [[package]] name = "dunce" version = "1.0.5" @@ -9681,6 +9951,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" + [[package]] name = "funty" version = "1.1.0" @@ -10109,7 +10385,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0edfdfb507593d47605b3bb2fb36628b391e3d397e520b85852dea2412c8e2d1" dependencies = [ - "prost 0.13.3", + "prost 0.13.5", "prost-types 0.13.3", "tonic 0.12.3", ] @@ -11201,6 +11477,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "instrumented-channel" +version = "0.1.0" +source = "git+https://github.com/larryl3u/aptos-indexer-processor-sdk?rev=bbcc42d5a03cffe861b89c080ef7de918f37b4d0#bbcc42d5a03cffe861b89c080ef7de918f37b4d0" +dependencies = [ + "delegate", + "derive_builder 0.20.2", + "kanal", + "once_cell", + "prometheus", + "prometheus-client", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -11857,6 +12146,26 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linkme" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b1703c00b2a6a70738920544aa51652532cacddfec2e162d2e29eae01e665c" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d55ca5d5a14363da83bf3c33874b8feaa34653e760d5216d7ef9829c88001a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "linux-raw-sys" version = "0.1.4" @@ -11994,6 +12303,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "maplit" version = "1.0.2" @@ -12048,6 +12366,8 @@ dependencies = [ "movement-da-util", "movement-signer", "movement-signer-loader", + "poem", + "prometheus", "rand 0.7.3", "serde", "serde_derive", @@ -12166,7 +12486,7 @@ dependencies = [ "once_cell", "poem", "poem-openapi", - "prost 0.13.3", + "prost 0.13.5", "rand 0.7.3", "schemars", "serde", @@ -12389,6 +12709,57 @@ dependencies = [ "zeroize", ] +[[package]] +name = "metrics" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +dependencies = [ + "ahash 0.8.11", + "metrics-macros", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" +dependencies = [ + "base64 0.21.7", + "indexmap 1.9.3", + "metrics", + "metrics-util", + "quanta", + "thiserror 1.0.69", +] + +[[package]] +name = "metrics-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "metrics-util" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "111cb375987443c3de8d503580b536f77dc8416d32db62d9456db5d93bd7ac47" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.13.2", + "metrics", + "num_cpus", + "quanta", + "sketches-ddsketch", +] + [[package]] name = "miette" version = "7.2.0" @@ -12510,6 +12881,33 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" +[[package]] +name = "mockall" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43766c2b5203b10de348ffe19f7e54564b64f3d6018ff7648d1e2d6d3a0f0a48" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cbce79ec385a1d4f54baa90a76401eb15d9cab93685f62e7e9f942aa00ae2" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "more-asserts" version = "0.3.1" @@ -13888,7 +14286,7 @@ dependencies = [ "clap_complete", "dashmap 5.5.3", "diesel", - "diesel-async", + "diesel-async 0.5.1", "dirs 5.0.1", "futures", "hex", @@ -13911,7 +14309,7 @@ dependencies = [ "move-vm-runtime 0.1.0 (git+https://github.com/movementlabsxyz/aptos-core?rev=1d1cdbbd7fabb80dcb95ba5e23213faa072fab67)", "pathsearch", "poem", - "processor 1.0.0 (git+https://github.com/movementlabsxyz/aptos-indexer-processors?rev=77a36245400250e7d8a854360194288d078681bc)", + "processor 1.0.0", "rand 0.7.3", "reqwest 0.11.27", "self_update", @@ -13971,7 +14369,7 @@ dependencies = [ "movement-signer-loader", "movement-tracing", "movement-types", - "prost 0.13.3", + "prost 0.13.5", "serde", "serde_json", "tempfile", @@ -14166,7 +14564,7 @@ dependencies = [ "movement-da-light-node-proto", "movement-da-util", "movement-types", - "prost 0.13.3", + "prost 0.13.5", "rand 0.7.3", "serde_json", "thiserror 1.0.69", @@ -14181,7 +14579,7 @@ name = "movement-da-light-node-proto" version = "0.3.4" dependencies = [ "buildtime", - "prost 0.13.3", + "prost 0.13.5", "tonic 0.12.3", "tonic-build", ] @@ -14247,7 +14645,7 @@ dependencies = [ "movement-da-light-node-setup", "movement-da-util", "movement-signer", - "prost 0.13.3", + "prost 0.13.5", "rand 0.7.3", "serde_json", "thiserror 1.0.69", @@ -14324,7 +14722,7 @@ dependencies = [ "movement-types", "once_cell", "poem", - "prost 0.13.3", + "prost 0.13.5", "rand 0.7.3", "rand_core 0.9.0", "rocksdb", @@ -14345,7 +14743,7 @@ name = "movement-da-sequencer-proto" version = "0.3.4" dependencies = [ "buildtime", - "prost 0.13.3", + "prost 0.13.5", "tonic 0.12.3", "tonic-build", ] @@ -14376,7 +14774,7 @@ dependencies = [ "movement-signer-loader", "movement-signer-local", "movement-types", - "prost 0.13.3", + "prost 0.13.5", "rand 0.8.5", "serde", "serde_derive", @@ -14450,7 +14848,7 @@ dependencies = [ "movement-signer-loader", "movement-tracing", "movement-types", - "prost 0.13.3", + "prost 0.13.5", "rocksdb", "serde_json", "sha2 0.10.8", @@ -14470,13 +14868,14 @@ name = "movement-indexer-service" version = "0.3.4" dependencies = [ "anyhow", + "aptos-indexer-processor-sdk", "clap 4.5.21", "dot-movement", "futures", "maptos-execution-util", "num_cpus", "poem", - "processor 1.0.0 (git+https://github.com/movementlabsxyz/aptos-indexer-processors?rev=20be1190105908fd4fea4e78c330997658e9428e)", + "processor 0.1.0", "reqwest 0.12.9", "serde_json", "serde_yaml 0.9.34+deprecated", @@ -14786,7 +15185,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -14840,7 +15239,7 @@ checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags 2.6.0", "cfg-if", - "cfg_aliases", + "cfg_aliases 0.2.1", "libc", ] @@ -15163,6 +15562,52 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +dependencies = [ + "futures-core", + "futures-sink", + "indexmap 2.6.0", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror 1.0.69", + "urlencoding", +] + +[[package]] +name = "opentelemetry-prometheus" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8f082da115b0dcb250829e3ed0b8792b8f963a1ad42466e48422fbe6a079bd" +dependencies = [ + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "prometheus", + "protobuf", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "ordered-float 4.6.0", + "thiserror 1.0.69", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -15187,6 +15632,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ouroboros" version = "0.15.6" @@ -16081,6 +16535,32 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "predicates" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" +dependencies = [ + "anstyle", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" + +[[package]] +name = "predicates-tree" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "pretty" version = "0.10.0" @@ -16217,24 +16697,24 @@ dependencies = [ [[package]] name = "processor" -version = "1.0.0" -source = "git+https://github.com/movementlabsxyz/aptos-indexer-processors?rev=20be1190105908fd4fea4e78c330997658e9428e#20be1190105908fd4fea4e78c330997658e9428e" +version = "0.1.0" +source = "git+https://github.com/larryl3u/aptos-indexer-processors-v2?rev=2e5dd83639886fe9e44af97726d79895a3b4f650#2e5dd83639886fe9e44af97726d79895a3b4f650" dependencies = [ "ahash 0.8.11", "allocative", "allocative_derive", "anyhow", - "aptos-moving-average 0.1.0 (git+https://github.com/movementlabsxyz/aptos-indexer-processors?rev=20be1190105908fd4fea4e78c330997658e9428e)", - "aptos-protos 1.3.0 (git+https://github.com/movementlabsxyz/aptos-core?rev=338f9a1bcc06f62ce4a4994f1642b9a61b631ee0)", + "aptos-indexer-processor-sdk", "async-trait", - "bcs 0.1.6 (git+https://github.com/movementlabsxyz/bcs.git?rev=bc16d2d39cabafaabd76173dd1b04b2aa170cf0c)", + "bcs 0.1.4", "bigdecimal", "bitflags 2.6.0", "canonical_json", "chrono", "clap 4.5.21", + "const_format", "diesel", - "diesel-async", + "diesel-async 0.5.2", "diesel_migrations", "enum_dispatch", "field_count", @@ -16247,28 +16727,27 @@ dependencies = [ "hyper 0.14.31", "itertools 0.12.1", "jemallocator", - "kanal", "lazy_static", + "log", "native-tls", - "num", "num_cpus", "once_cell", "parquet", "parquet_derive", "postgres-native-tls", "prometheus", - "prost 0.12.6", + "prost 0.13.5", + "rayon", "regex", "serde", "serde_json", - "server-framework 1.0.0 (git+https://github.com/movementlabsxyz/aptos-indexer-processors?rev=20be1190105908fd4fea4e78c330997658e9428e)", - "sha2 0.9.9", - "sha3 0.9.1", - "strum 0.26.3", + "sha2 0.10.8", + "sha3 0.10.8", + "strum 0.24.1", "tiny-keccak", "tokio", "tokio-postgres", - "tonic 0.11.0", + "tonic 0.12.3", "tracing", "unescape", "url", @@ -16293,7 +16772,7 @@ dependencies = [ "chrono", "clap 4.5.21", "diesel", - "diesel-async", + "diesel-async 0.5.1", "diesel_migrations", "enum_dispatch", "field_count", @@ -16363,6 +16842,29 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "prometheus-client" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504ee9ff529add891127c4827eb481bd69dc0ebc72e9a682e187db4caa60c3ca" +dependencies = [ + "dtoa", + "itoa", + "parking_lot 0.12.3", + "prometheus-client-derive-encode", +] + +[[package]] +name = "prometheus-client-derive-encode" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "proptest" version = "1.5.0" @@ -16416,12 +16918,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes 1.8.0", - "prost-derive 0.13.3", + "prost-derive 0.13.5", ] [[package]] @@ -16438,7 +16940,7 @@ dependencies = [ "once_cell", "petgraph 0.6.5", "prettyplease", - "prost 0.13.3", + "prost 0.13.5", "prost-types 0.13.3", "regex", "syn 2.0.87", @@ -16473,9 +16975,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", "itertools 0.13.0", @@ -16493,7 +16995,7 @@ dependencies = [ "logos", "miette", "once_cell", - "prost 0.13.3", + "prost 0.13.5", "prost-types 0.13.3", ] @@ -16521,7 +17023,7 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" dependencies = [ - "prost 0.13.3", + "prost 0.13.5", ] [[package]] @@ -16557,7 +17059,7 @@ checksum = "6f352af331bf637b8ecc720f7c87bf903d2571fa2e14a66e9b2558846864b54a" dependencies = [ "bytes 1.8.0", "miette", - "prost 0.13.3", + "prost 0.13.5", "prost-reflect", "prost-types 0.13.3", "protox-parse", @@ -16612,6 +17114,22 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "quanta" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -16712,7 +17230,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" dependencies = [ - "cfg_aliases", + "cfg_aliases 0.2.1", "libc", "once_cell", "socket2", @@ -16867,6 +17385,15 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "rayon" version = "1.10.0" @@ -17548,7 +18075,7 @@ dependencies = [ "openssl-probe", "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.11.1", ] [[package]] @@ -17561,7 +18088,19 @@ dependencies = [ "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", ] [[package]] @@ -17597,7 +18136,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afbb878bdfdf63a336a5e63561b1835e7a8c91524f51621db870169eac84b490" dependencies = [ - "core-foundation", + "core-foundation 0.9.4", "core-foundation-sys", "jni", "log", @@ -17606,7 +18145,7 @@ dependencies = [ "rustls-native-certs 0.7.3", "rustls-platform-verifier-android", "rustls-webpki 0.102.8", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "webpki-roots 0.26.6", "winapi 0.3.9", @@ -17682,6 +18221,14 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sample" +version = "0.1.0" +source = "git+https://github.com/larryl3u/aptos-indexer-processor-sdk?rev=bbcc42d5a03cffe861b89c080ef7de918f37b4d0#bbcc42d5a03cffe861b89c080ef7de918f37b4d0" +dependencies = [ + "tracing", +] + [[package]] name = "schannel" version = "0.1.26" @@ -17814,18 +18361,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.6.0", - "core-foundation", + "core-foundation 0.9.4", "core-foundation-sys", "libc", "num-bigint 0.4.6", "security-framework-sys", ] +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags 2.6.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -18426,6 +18986,12 @@ dependencies = [ "walkdir", ] +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" + [[package]] name = "slab" version = "0.4.9" @@ -18510,6 +19076,17 @@ dependencies = [ "sha1", ] +[[package]] +name = "spez" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87e960f4dca2788eeb86bbdde8dd246be8948790b7618d656e68f9b720a86e8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "spin" version = "0.5.2" @@ -18650,6 +19227,9 @@ name = "strum" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros 0.24.3", +] [[package]] name = "strum" @@ -18888,7 +19468,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys 0.5.0", ] @@ -18899,7 +19479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.6.0", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys 0.6.0", ] @@ -18983,7 +19563,7 @@ dependencies = [ "k256", "num-traits", "once_cell", - "prost 0.13.3", + "prost 0.13.5", "ripemd", "serde", "serde_bytes", @@ -19006,7 +19586,7 @@ checksum = "9ae9e1705aa0fa5ecb2c6aa7fb78c2313c4a31158ea5f02048bf318f849352eb" dependencies = [ "bytes 1.8.0", "flex-error", - "prost 0.13.3", + "prost 0.13.5", "serde", "serde_bytes", "subtle-encoding", @@ -19044,6 +19624,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + [[package]] name = "textwrap" version = "0.11.0" @@ -19432,6 +20018,18 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.19.15", +] + [[package]] name = "toml" version = "0.8.19" @@ -19460,6 +20058,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap 2.6.0", + "serde", + "serde_spanned", "toml_datetime", "winnow 0.5.40", ] @@ -19575,7 +20175,8 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project 1.1.7", - "prost 0.13.3", + "prost 0.13.5", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "socket2", "tokio", @@ -19586,6 +20187,7 @@ dependencies = [ "tower-service", "tracing", "webpki-roots 0.26.6", + "zstd 0.13.2", ] [[package]] @@ -19621,7 +20223,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "878d81f52e7fcfd80026b7fdb6a9b578b3c3653ba987f87f0dce4b64043cba27" dependencies = [ - "prost 0.13.3", + "prost 0.13.5", "prost-types 0.13.3", "tokio", "tokio-stream", @@ -19678,8 +20280,10 @@ dependencies = [ "futures-util", "pin-project-lite", "sync_wrapper 0.1.2", + "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -20263,7 +20867,7 @@ checksum = "a769a71e45deef489beed23167f79ee75d41f482b5e3d96ddab833f24fd07e51" dependencies = [ "async-trait", "bytes 1.8.0", - "derive_builder", + "derive_builder 0.12.0", "http 1.1.0", "reqwest 0.12.9", "rustify", @@ -20534,7 +21138,7 @@ dependencies = [ name = "whitelist" version = "0.3.4" dependencies = [ - "prost 0.13.3", + "prost 0.13.5", "thiserror 1.0.69", "tonic 0.12.3", ] diff --git a/Cargo.toml b/Cargo.toml index 71e82c390..d50bf0c93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -202,7 +202,8 @@ move-package = { git = "https://github.com/movementlabsxyz/aptos-core", rev = "1 movement = { git = "https://github.com/movementlabsxyz/aptos-core", rev = "1d1cdbbd7fabb80dcb95ba5e23213faa072fab67" } # Indexer -processor = { git = "https://github.com/movementlabsxyz/aptos-indexer-processors", rev = "20be1190105908fd4fea4e78c330997658e9428e" } +processor = { git = "https://github.com/larryl3u/aptos-indexer-processors-v2", rev = "2e5dd83639886fe9e44af97726d79895a3b4f650" } +aptos-indexer-processor-sdk = { git = "https://github.com/larryl3u/aptos-indexer-processor-sdk", rev = "bbcc42d5a03cffe861b89c080ef7de918f37b4d0" } server-framework = { git = "https://github.com/movementlabsxyz/aptos-indexer-processors", rev = "20be1190105908fd4fea4e78c330997658e9428e" } bcs = { git = "https://github.com/movementlabsxyz/bcs.git", rev = "bc16d2d39cabafaabd76173dd1b04b2aa170cf0c" } @@ -261,7 +262,7 @@ alloy-transport-http = { git = "https://github.com/alloy-rs/alloy.git", rev = "8 ] } alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy.git", rev = "83343b172585fe4e040fb104b4d1421f58cbf9a2" } -anyhow = "1.0" +anyhow = "^1.0.98" async-stream = "0.3.0" async-trait = "0.1.71" async-recursion = "1.1.1" diff --git a/docker/compose/movement-indexer/README.md b/docker/compose/movement-indexer/README.md index f767d890a..5724c3d4d 100644 --- a/docker/compose/movement-indexer/README.md +++ b/docker/compose/movement-indexer/README.md @@ -7,8 +7,6 @@ rm -rf ~/.movement/* \ && docker compose \ --env-file docker/compose/movement-indexer/.env \ -f docker/compose/movement-full-node/docker-compose.yml \ - -f docker/compose/movement-full-node/docker-compose.setup-local.yml \ - -f docker/compose/movement-full-node/docker-compose.celestia-local.yml \ -f docker/compose/movement-indexer/docker-compose.local-development.indexer.yml \ up ``` diff --git a/flake.nix b/flake.nix index 2f4f93af1..3da92a057 100644 --- a/flake.nix +++ b/flake.nix @@ -158,7 +158,7 @@ # export PKG_CONFIG_PATH=$PKG_CONFIG_PATH_FOR_TARGET # Export linker flags if on Darwin (macOS) - if [[ "$(${pkgs.stdenv.hostPlatform.system})" =~ "darwin" ]]; then + if [[ "${pkgs.stdenv.hostPlatform.system}" =~ "darwin" ]]; then export LDFLAGS="-L/opt/homebrew/opt/zlib/lib" export CPPFLAGS="-I/opt/homebrew/opt/zlib/include" fi diff --git a/networks/movement/indexer/Cargo.toml b/networks/movement/indexer/Cargo.toml index 6cfa13ec6..cd9fd05ce 100644 --- a/networks/movement/indexer/Cargo.toml +++ b/networks/movement/indexer/Cargo.toml @@ -21,6 +21,7 @@ futures = { workspace = true } num_cpus = { workspace = true } poem = { workspace = true } processor = { workspace = true } +aptos-indexer-processor-sdk = { workspace = true } server-framework = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/networks/movement/indexer/src/main.rs b/networks/movement/indexer/src/main.rs index d59fea36e..f959a55f6 100644 --- a/networks/movement/indexer/src/main.rs +++ b/networks/movement/indexer/src/main.rs @@ -1,21 +1,15 @@ -use processor::IndexerGrpcProcessorConfig; -use server_framework::RunnableConfig; -use std::io::Write; +use aptos_indexer_processor_sdk::server_framework::RunnableConfig; +use processor::config::indexer_processor_config::IndexerProcessorConfig; use tokio::task::JoinSet; use tokio::time::Duration; mod service; const RUNTIME_WORKER_MULTIPLIER: usize = 2; +const DEFAULT_PROCESSOR_NAMES: &[&str] = &["default_processor"]; fn main() -> Result<(), anyhow::Error> { - use tracing_subscriber::EnvFilter; - - tracing_subscriber::fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), - ) - .init(); + init_logger(None); let dot_movement = dot_movement::DotMovement::try_from_env()?; let maptos_config = @@ -27,93 +21,34 @@ fn main() -> Result<(), anyhow::Error> { maptos_config.indexer.maptos_indexer_grpc_healthcheck_port ); - let default_indexer_config = build_processor_conf("default_processor", &maptos_config)?; - let usertx_indexer_config = build_processor_conf("user_transaction_processor", &maptos_config)?; - let accounttx_indexer_config = - build_processor_conf("account_transactions_processor", &maptos_config)?; - let coin_indexer_config = build_processor_conf("coin_processor", &maptos_config)?; - let event_indexer_config = build_processor_conf("events_processor", &maptos_config)?; - let fungible_indexer_config = build_processor_conf("fungible_asset_processor", &maptos_config)?; - let txmeta_indexer_config = - build_processor_conf("transaction_metadata_processor", &maptos_config)?; - - // Token processor - let activate_tokes: bool = std::env::var("ACTIVATE_TOKEN_INDEXING") - .map(|t| t.parse().unwrap_or(true)) - .unwrap_or(true); - let token_configs = if activate_tokes { - let token_indexer_config = build_processor_conf( - "token_processor - nft_points_contract: null", - &maptos_config, - )?; - - let tokenv2_indexer_config = build_processor_conf( - "token_v2_processor - query_retries: 5", - &maptos_config, - )?; - Some((token_indexer_config, tokenv2_indexer_config)) - } else { - None - }; - - // ANS processor. - let ans_indexer_config = build_processor_conf( - "ans_processor - ans_v1_primary_names_table_handle: temp - ans_v1_name_records_table_handle: temp - ans_v2_contract_address: 0x67bf15b3eed0fc62deea9630bbbd1d48842550655140f913699a1ca7e6f727d8", - &maptos_config, - )?; + let processor_configs = get_processor_configs(&maptos_config)?; + let runtime = get_movement_runtime_builder(); - let num_cpus = num_cpus::get(); - let worker_threads = (num_cpus * RUNTIME_WORKER_MULTIPLIER).max(16); - println!( - "[Processor] Starting processor tokio runtime: num_cpus={}, worker_threads={}", - num_cpus, worker_threads - ); + let ret: Result<(), anyhow::Error> = runtime.block_on({ + async move { + let mut set = JoinSet::new(); + // TODO: use generic health check server. - let mut builder = tokio::runtime::Builder::new_multi_thread(); - let ret: Result<(), anyhow::Error> = builder - .disable_lifo_slot() - .enable_all() - .worker_threads(worker_threads) - .build() - .unwrap() - .block_on({ - async move { - // Test the Grpc connection. - // The gRpc connection can fail because the Movement-node is started but the port is still not open. - // If the connection fail wait and retry. - test_grpc_connection(&maptos_config).await?; - - let mut set = JoinSet::new(); - set.spawn(async move { crate::service::run_service(health_check_url).await }); - set.spawn(async move { default_indexer_config.run().await }); - //wait all the migration is done. - tokio::time::sleep(Duration::from_secs(12)).await; - set.spawn(async move { usertx_indexer_config.run().await }); - set.spawn(async move { accounttx_indexer_config.run().await }); - set.spawn(async move { coin_indexer_config.run().await }); - set.spawn(async move { event_indexer_config.run().await }); - set.spawn(async move { fungible_indexer_config.run().await }); - set.spawn(async move { txmeta_indexer_config.run().await }); - if let Some((token_indexer_config, tokenv2_indexer_config)) = token_configs { - set.spawn(async move { token_indexer_config.run().await }); - set.spawn(async move { tokenv2_indexer_config.run().await }); - } - set.spawn(async move { ans_indexer_config.run().await }); - - while let Some(res) = set.join_next().await { - tracing::error!("An Error occurs during indexer execution: {res:?}"); - // If a processor break to avoid data inconsistency between processor - break; - } - set.shutdown().await; - Err(anyhow::anyhow!("At least One indexer processor failed. Exit")) + let mut port_offset = 0; + for mut config in processor_configs { + let processor_name = config.server_config.processor_config.name().to_string(); + config.server_config.processor_config.health_check_port += port_offset; + + set.spawn(async move { config.run().await }); + port_offset += 1; + } + //wait all the migration is done. + tokio::time::sleep(Duration::from_secs(12)).await; + + while let Some(res) = set.join_next().await { + tracing::error!("An Error occurs during indexer execution: {res:?}"); + // If a processor break to avoid data inconsistency between processor + break; } - }); + set.shutdown().await; + Err(anyhow::anyhow!("At least One indexer processor failed. Exit")) + } + }); if let Err(err) = ret { tracing::error!("Indexer execution failed: {err}"); std::process::exit(1); @@ -122,111 +57,94 @@ fn main() -> Result<(), anyhow::Error> { } } -fn build_processor_conf( +fn build_processor_conf_v2( processor_name: &str, maptos_config: &maptos_execution_util::config::Config, -) -> Result { - let indexer_grpc_data_service_address = build_grpc_url(maptos_config); - - let default_sleep_time_between_request: u64 = std::env::var("SLEEP_TIME_BETWEEN_REQUEST_MS") - .map(|t| t.parse().unwrap_or(10)) - .unwrap_or(10); - - //create config file - let mut indexer_config_content = format!( - "processor_config: - type: {} -postgres_connection_string: {} -indexer_grpc_data_service_address: {} -indexer_grpc_http2_ping_interval_in_secs: {} -indexer_grpc_http2_ping_timeout_in_secs: {} -auth_token: \"{}\" -default_sleep_time_between_request: {}", +) -> Result { + let indexer_processor_raw = format!( + r#" +health_check_port: 8085 +server_config: + processor_config: + type: {} + channel_size: 100 + transaction_stream_config: + indexer_grpc_data_service_address: "{}" + auth_token: "{}" + request_name_header: "{}" + processor_mode: + type: "default" + db_config: + type: postgres_config + connection_string: {} + "#, processor_name, - maptos_config.indexer_processor.postgres_connection_string, - indexer_grpc_data_service_address, - maptos_config.indexer.maptos_indexer_grpc_inactivity_timeout, - maptos_config.indexer.maptos_indexer_grpc_inactivity_ping_interval, + build_grpc_url(maptos_config), maptos_config.indexer_processor.indexer_processor_auth_token, - default_sleep_time_between_request, + "".to_string(), + maptos_config.indexer_processor.postgres_connection_string, ); - - // If the starting version is not defined, don't put a default value in the conf. - if let Ok(start_version) = std::env::var("INDEXER_STARTING_VERSION") { - if let Ok(start_version) = start_version.parse::() { - indexer_config_content.push('\n'); - indexer_config_content.push_str(&format!("starting_version: {}", start_version)); - } - } - - //let indexer_config_path = dot_movement.get_path().join("indexer_config.yaml"); - let mut output_file = tempfile::NamedTempFile::new()?; - write!(output_file, "{}", indexer_config_content)?; - - let indexer_config = - server_framework::load::(&output_file.path().to_path_buf())?; - - // Leave here for debug purpose. Will be removed later. - // Use to print the generated config, to have an example when activating a new processor. - // indexer_config.processor_config = ProcessorConfig::TokenV2Processor(TokenV2ProcessorConfig { - // query_retries: 5, - // query_retry_delay_ms: 100, - // }); - - // let yaml = serde_yaml::to_string(&indexer_config)?; - // println!("{yaml}",); - - Ok(indexer_config) + let config = serde_yaml::from_str::(&indexer_processor_raw)?; + // TODO: Fix the starting version. + Ok(config) } -use reqwest::Client as HttpClient; - -async fn test_grpc_connection( - maptos_config: &maptos_execution_util::config::Config, -) -> Result<(), anyhow::Error> { - let indexer_grpc_data_service_address = build_grpc_url(maptos_config); - - let client = HttpClient::builder() - .http2_prior_knowledge() // Enforce HTTP/2 for gRpc - .timeout(Duration::from_secs(10)) - .build()?; - - let mut retry = 0; - while retry < 5 { - let response = client - .get(&indexer_grpc_data_service_address) - .header("Content-Type", "application/grpc") - .send() - .await; - - match response { - Ok(resp) => { - let status = resp.status(); - let body = resp.text().await?; - println!("Received response: {} {:?}", status, body); - if status.is_success() { - break; - } else { - tracing::info!("GRpc server return a bad status: {:?}. Retrying...", status); - tokio::time::sleep(Duration::from_secs(1)).await; // Wait before retrying - } - } - Err(err) => { - tracing::info!("Failed to connect to the gRp server: {:?}. Retrying...", err); - tokio::time::sleep(Duration::from_secs(1)).await; // Wait before retrying - } - } - retry += 1; - } - - if retry == 5 { - Err(anyhow::anyhow!( - "Faild to connect to the Grpc server : {indexer_grpc_data_service_address}" - )) - } else { - Ok(()) - } -} +// fn build_processor_conf( +// processor_name: &str, +// maptos_config: &maptos_execution_util::config::Config, +// ) -> Result { +// let indexer_grpc_data_service_address = build_grpc_url(maptos_config); + +// let default_sleep_time_between_request: u64 = std::env::var("SLEEP_TIME_BETWEEN_REQUEST_MS") +// .map(|t| t.parse().unwrap_or(10)) +// .unwrap_or(10); + +// //create config file +// let mut indexer_config_content = format!( +// "processor_config: +// type: {} +// postgres_connection_string: {} +// indexer_grpc_data_service_address: {} +// indexer_grpc_http2_ping_interval_in_secs: {} +// indexer_grpc_http2_ping_timeout_in_secs: {} +// auth_token: \"{}\" +// default_sleep_time_between_request: {}", +// processor_name, +// maptos_config.indexer_processor.postgres_connection_string, +// indexer_grpc_data_service_address, +// maptos_config.indexer.maptos_indexer_grpc_inactivity_timeout, +// maptos_config.indexer.maptos_indexer_grpc_inactivity_ping_interval, +// maptos_config.indexer_processor.indexer_processor_auth_token, +// default_sleep_time_between_request, +// ); + +// // If the starting version is not defined, don't put a default value in the conf. +// if let Ok(start_version) = std::env::var("INDEXER_STARTING_VERSION") { +// if let Ok(start_version) = start_version.parse::() { +// indexer_config_content.push('\n'); +// indexer_config_content.push_str(&format!("starting_version: {}", start_version)); +// } +// } + +// //let indexer_config_path = dot_movement.get_path().join("indexer_config.yaml"); +// let mut output_file = tempfile::NamedTempFile::new()?; +// write!(output_file, "{}", indexer_config_content)?; + +// let indexer_config = +// server_framework::load::(&output_file.path().to_path_buf())?; + +// // Leave here for debug purpose. Will be removed later. +// // Use to print the generated config, to have an example when activating a new processor. +// // indexer_config.processor_config = ProcessorConfig::TokenV2Processor(TokenV2ProcessorConfig { +// // query_retries: 5, +// // query_retry_delay_ms: 100, +// // }); + +// // let yaml = serde_yaml::to_string(&indexer_config)?; +// // println!("{yaml}",); + +// Ok(indexer_config) +// } fn build_grpc_url(maptos_config: &maptos_execution_util::config::Config) -> String { let indexer_grpc_data_service_address = format!( @@ -240,3 +158,51 @@ fn build_grpc_url(maptos_config: &maptos_execution_util::config::Config) -> Stri ); indexer_grpc_data_service_address } + +fn init_logger(log_level: Option) { + use tracing_subscriber::EnvFilter; + let log_level = log_level.unwrap_or("info".to_string()); + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(log_level)), + ) + .init(); +} + +fn get_movement_runtime_builder() -> tokio::runtime::Runtime { + let num_cpus = num_cpus::get(); + let worker_threads = (num_cpus * RUNTIME_WORKER_MULTIPLIER).max(16); + println!( + "[Processor] Starting processor tokio runtime: num_cpus={}, worker_threads={}", + num_cpus, worker_threads + ); + + let mut builder = tokio::runtime::Builder::new_multi_thread(); + builder + .disable_lifo_slot() + .enable_all() + .worker_threads(worker_threads) + .build() + .unwrap() +} + +fn get_processor_configs( + maptos_config: &maptos_execution_util::config::Config, +) -> Result, anyhow::Error> { + let mut configs = vec![]; + if maptos_config.indexer_processor.processor_names.is_empty() { + for processor_name in DEFAULT_PROCESSOR_NAMES.iter() { + let config = build_processor_conf_v2(processor_name, maptos_config)?; + configs.push(config); + } + } else { + for processor_name in maptos_config.indexer_processor.processor_names.iter() { + let config = build_processor_conf_v2(processor_name, maptos_config)?; + configs.push(config); + } + } + + tracing::info!("Total number of processor configs: {}", configs.len()); + + Ok(configs) +} diff --git a/protocol-units/execution/maptos/util/Cargo.toml b/protocol-units/execution/maptos/util/Cargo.toml index 3e273591a..6e0839ccc 100644 --- a/protocol-units/execution/maptos/util/Cargo.toml +++ b/protocol-units/execution/maptos/util/Cargo.toml @@ -29,7 +29,11 @@ hex = { workspace = true } tokio = { workspace = true } url = { workspace = true } tracing = { workspace = true } +poem = { workspace = true } aptos-sdk = { workspace = true } movement-signer-loader = { workspace = true } movement-signer = { workspace = true } + +prometheus = { workspace = true } +reqwest = { workspace = true } diff --git a/protocol-units/execution/maptos/util/src/config/common.rs b/protocol-units/execution/maptos/util/src/config/common.rs index 1172ec0a2..f41cea8a1 100644 --- a/protocol-units/execution/maptos/util/src/config/common.rs +++ b/protocol-units/execution/maptos/util/src/config/common.rs @@ -264,3 +264,31 @@ env_default!( u64, 1024 * 1024 //1Mo ); + +// Monitoring metrics. +env_default!( + default_ecosystem_metrics_listen_hostname, + "MAPTOS_ECOSYSTEM_METRICS_LISTEN_HOSTNAME", + String, + "0.0.0.0".to_string() +); +env_default!( + default_ecosystem_metrics_listen_port, + "MAPTOS_ECOSYSTEM_METRICS_LISTEN_PORT", + u16, + 13122 +); + +// Health check. +env_default!( + default_ecosystem_health_check_listen_hostname, + "MAPTOS_ECOSYSTEM_HEALTH_CHECK_LISTEN_HOSTNAME", + String, + "0.0.0.0".to_string() +); +env_default!( + default_ecosystem_health_check_listen_port, + "MAPTOS_ECOSYSTEM_HEALTH_CHECK_LISTEN_PORT", + u16, + 13222 +); diff --git a/protocol-units/execution/maptos/util/src/config/health_check.rs b/protocol-units/execution/maptos/util/src/config/health_check.rs new file mode 100644 index 000000000..97b3b6e16 --- /dev/null +++ b/protocol-units/execution/maptos/util/src/config/health_check.rs @@ -0,0 +1,73 @@ +use super::common::{ + default_ecosystem_health_check_listen_hostname, default_ecosystem_health_check_listen_port, + default_maptos_indexer_grpc_listen_hostname, default_maptos_indexer_grpc_listen_port, +}; +use poem::http::StatusCode; +use poem::listener::TcpListener; +use poem::web::Data; +use poem::EndpointExt; +use poem::{get, handler, Response, Route, Server}; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc::{channel, Receiver}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HealthCheckConfig { + #[serde(default = "default_ecosystem_health_check_listen_hostname")] + pub listen_hostname: String, + #[serde(default = "default_ecosystem_health_check_listen_port")] + pub listen_port: u16, + #[serde(default = "default_maptos_indexer_grpc_listen_hostname")] + pub grpc_hostname: String, + #[serde(default = "default_maptos_indexer_grpc_listen_port")] + pub grpc_port: u16, +} + +#[handler] +async fn health_handler() -> Response { + // Check gRPC indexer connection + let grpc_url = format!("http://{}:{}", + default_maptos_indexer_grpc_listen_hostname(), + default_maptos_indexer_grpc_listen_port() + ); + + match check_grpc_connection(&grpc_url).await { + Ok(_) => Response::builder() + .status(StatusCode::OK) + .body("{\"status\": \"healthy\", \"grpc\": \"connected\"}"), + Err(e) => Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(format!("{{\"status\": \"unhealthy\", \"grpc\": \"failed\", \"error\": \"{}\"}}", e)) + } +} + +async fn check_grpc_connection(url: &str) -> Result<(), String> { + let client = reqwest::Client::builder() + .http2_prior_knowledge() + .timeout(std::time::Duration::from_secs(5)) + .build() + .map_err(|e| format!("Failed to create client: {}", e))?; + + let response = client + .get(url) + .header("Content-Type", "application/grpc") + .send() + .await + .map_err(|e| format!("Connection failed: {}", e))?; + + if response.status().is_success() || response.status() == 405 { + Ok(()) // 405 Method Not Allowed is normal for gRPC GET requests + } else { + Err(format!("gRPC server returned status: {}", response.status())) + } +} + +impl HealthCheckConfig { + pub async fn start_health_check_server(&self) -> Result<(), anyhow::Error> { + let app = Route::new().at("/health", get(health_handler)); + + let server = Server::new(TcpListener::bind(format!("{}:{}", self.listen_hostname, self.listen_port))); + server.run(app).await?; + + Ok(()) + } +} \ No newline at end of file diff --git a/protocol-units/execution/maptos/util/src/config/indexer_processor.rs b/protocol-units/execution/maptos/util/src/config/indexer_processor.rs index 73f67cb1e..ecbd85be7 100644 --- a/protocol-units/execution/maptos/util/src/config/indexer_processor.rs +++ b/protocol-units/execution/maptos/util/src/config/indexer_processor.rs @@ -8,6 +8,12 @@ pub struct Config { #[serde(default = "default_indexer_processor_auth_token")] pub indexer_processor_auth_token: String, + + /// The list of the processor names. + /// If not provided, all processors will run for backward compatibility. + /// The list of processors is defined in the indexer side. + #[serde(default)] + pub processor_names: Vec, } impl Default for Config { @@ -15,6 +21,7 @@ impl Default for Config { Self { postgres_connection_string: default_postgres_connection_string(), indexer_processor_auth_token: default_indexer_processor_auth_token(), + processor_names: vec![], } } } diff --git a/protocol-units/execution/maptos/util/src/config/metrics.rs b/protocol-units/execution/maptos/util/src/config/metrics.rs new file mode 100644 index 000000000..b5c46eb83 --- /dev/null +++ b/protocol-units/execution/maptos/util/src/config/metrics.rs @@ -0,0 +1,86 @@ +use super::common::{ + default_ecosystem_metrics_listen_hostname, default_ecosystem_metrics_listen_port, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tokio::task::JoinHandle; +use prometheus::{Encoder, TextEncoder, Registry, gather}; +use poem::{listener::TcpListener, get, handler, Route, Server, EndpointExt, IntoResponse}; +use poem::http::StatusCode; + +// Metrics configuration that allows to add to existing services. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MetricsConfig { + #[serde(default = "default_ecosystem_metrics_listen_hostname")] + pub listen_hostname: String, + #[serde(default = "default_ecosystem_metrics_listen_port")] + pub listen_port: u16, +} + +impl MetricsConfig { + pub async fn start_metrics_server(&self) -> Result, anyhow::Error> { + let bind_address = format!("{}:{}", self.listen_hostname, self.listen_port); + + // Set up the metrics endpoint route + let metrics_route = Route::new() + .at("/metrics", get(metrics_handler)); + + // Start the metrics server + let server_handle = tokio::spawn(async move { + let listener = TcpListener::bind(&bind_address); + + tracing::info!("Starting Prometheus metrics server on http://{}/metrics", bind_address); + + if let Err(e) = Server::new(listener).run(metrics_route).await { + tracing::error!("Metrics server error: {}", e); + } + }); + + // Give the server a moment to start + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + Ok(server_handle) + } + + /// Get the metrics endpoint URL for this configuration + pub fn metrics_endpoint(&self) -> String { + format!("http://{}:{}/metrics", self.listen_hostname, self.listen_port) + } +} + +/// Handler for the /metrics endpoint that serves Prometheus-formatted metrics +#[handler] +async fn metrics_handler() -> impl IntoResponse { + // Collect metrics from the global registry (includes all registered metrics) + let global_metrics = gather(); + + // Encode metrics in Prometheus text format + let encoder = TextEncoder::new(); + let mut buffer = vec![]; + + match encoder.encode(&global_metrics, &mut buffer) { + Ok(_) => { + // Convert to string and return with proper content type + match String::from_utf8(buffer) { + Ok(metrics_text) => { + poem::Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/plain; version=0.0.4; charset=utf-8") + .body(metrics_text) + } + Err(e) => { + tracing::error!("Failed to convert metrics to string: {}", e); + poem::Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body("Error encoding metrics") + } + } + } + Err(e) => { + tracing::error!("Failed to encode metrics: {}", e); + poem::Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body("Error encoding metrics") + } + } +} diff --git a/protocol-units/execution/maptos/util/src/config/mod.rs b/protocol-units/execution/maptos/util/src/config/mod.rs index 89e238d02..6043f5bb0 100644 --- a/protocol-units/execution/maptos/util/src/config/mod.rs +++ b/protocol-units/execution/maptos/util/src/config/mod.rs @@ -4,10 +4,12 @@ pub mod common; pub mod da_sequencer; pub mod faucet; pub mod fin; +pub mod health_check; pub mod indexer; pub mod indexer_processor; pub mod load_shedding; pub mod mempool; +pub mod metrics; use serde::{Deserialize, Serialize};