diff --git a/Cargo.lock b/Cargo.lock index f4a338367..d940521f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "amispec" version = "0.1.0" @@ -159,6 +165,28 @@ dependencies = [ "serde", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "astral-tokio-tar" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1abb2bfba199d9ec4759b797115ba6ae435bdd920ce99783bb53aeff57ba919b" +dependencies = [ + "filetime", + "futures-core", + "libc", + "portable-atomic", + "rustc-hash 2.1.1", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "async-channel" version = "2.3.1" @@ -171,6 +199,20 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compression" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8" +dependencies = [ + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd", + "zstd-safe", +] + [[package]] name = "async-fs" version = "2.1.2" @@ -335,6 +377,7 @@ dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -421,6 +464,41 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-s3" +version = "1.85.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5c82dae9304e7ced2ff6cca43dceb2d6de534c95a506ff0f168a7463c9a885d" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + [[package]] name = "aws-sdk-ssm" version = "1.74.0" @@ -475,6 +553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3503af839bd8751d0bdc5a46b9cac93a003a353e635b0c12cf2376b5b53e41ea" dependencies = [ "aws-credential-types", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", @@ -501,12 +580,44 @@ dependencies = [ "tokio", ] +[[package]] +name = "aws-smithy-checksums" +version = "0.63.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f77a921dbd2c78ebe70726799787c1d110a2245dd65e39b20923dfdfb2deee" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + [[package]] name = "aws-smithy-http" version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99335bec6cdc50a346fda1437f9fefe33abf8c99060739a546a16457f2862ca9" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -947,6 +1058,44 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "casi" +version = "0.1.0" +dependencies = [ + "astral-tokio-tar", + "async-compression", + "async-trait", + "aws-config", + "aws-sdk-s3", + "aws-smithy-types", + "aws-types", + "bon", + "chrono", + "clap", + "futures", + "hex", + "indicatif", + "oci-spec", + "olpc-cjson", + "owo-colors", + "parking_lot", + "rand 0.8.5", + "semver", + "serde", + "serde_json", + "sha2", + "snafu", + "tempfile", + "tokio", + "tracing", + "tracing-indicatif", + "tracing-subscriber", + "url", + "uuid", + "walkdir", + "zstd", +] + [[package]] name = "cc" version = "1.2.21" @@ -1124,6 +1273,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const_format" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1174,6 +1343,19 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc-fast" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf62af4cc77d8fe1c22dde4e721d87f2f54056139d8c412e1366b740305f56f" +dependencies = [ + "crc", + "digest", + "libc", + "rand 0.9.1", + "regex", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -1296,6 +1478,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.101", +] + [[package]] name = "digest" version = "0.10.7" @@ -1454,6 +1667,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1614,6 +1833,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getset" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "gimli" version = "0.31.1" @@ -1635,8 +1866,8 @@ dependencies = [ "aho-corasick", "bstr", "log", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", ] [[package]] @@ -1755,6 +1986,11 @@ name = "hashbrown" version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -1770,9 +2006,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.9" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -2191,6 +2427,7 @@ dependencies = [ "number_prefix", "portable-atomic", "unicode-width 0.2.0", + "vt100", "web-time", ] @@ -2243,6 +2480,23 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -2521,6 +2775,15 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.3", +] + [[package]] name = "lzma-rs" version = "0.3.0" @@ -2537,6 +2800,25 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -2615,6 +2897,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -2651,9 +2943,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ "hermit-abi", "libc", @@ -2701,6 +2993,23 @@ dependencies = [ "which 6.0.3", ] +[[package]] +name = "oci-spec" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e9beda9d92fac7bf4904c34c83340ef1024159faee67179a04e0277523da33" +dependencies = [ + "const_format", + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror 2.0.12", +] + [[package]] name = "olpc-cjson" version = "0.1.4" @@ -2749,6 +3058,22 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "owo-colors" +version = "4.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" +dependencies = [ + "supports-color 2.1.0", + "supports-color 3.0.2", +] + [[package]] name = "papergrid" version = "0.11.0" @@ -3024,6 +3349,28 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -3309,8 +3656,17 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -3321,7 +3677,7 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.5", ] [[package]] @@ -3330,6 +3686,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -3837,6 +4199,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shared_child" version = "1.0.2" @@ -3983,6 +4354,25 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "supports-color" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89" +dependencies = [ + "is-terminal", + "is_ci", +] + +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + [[package]] name = "syn" version = "1.0.109" @@ -4261,6 +4651,15 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "time" version = "0.3.41" @@ -4642,6 +5041,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-indicatif" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8201ca430e0cd893ef978226fd3516c06d9c494181c8bf4e5b32e30ed4b40aa1" +dependencies = [ + "indicatif", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -4874,6 +5315,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unplug" version = "0.1.0" @@ -4967,6 +5414,12 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "version_check" version = "0.9.5" @@ -4979,6 +5432,39 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -5218,9 +5704,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-registry" @@ -5230,7 +5716,7 @@ checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", "windows-strings 0.3.1", - "windows-targets 0.53.0", + "windows-targets 0.53.3", ] [[package]] @@ -5296,10 +5782,11 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.0" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ + "windows-link", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", diff --git a/Cargo.toml b/Cargo.toml index aa8c9d9f9..326986a89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "tools/bottlerocket-variant", "tools/buildsys", "tools/buildsys-config", + "tools/casi", "tools/include-env-compressed", "tools/include-env-compressed/include-env-compressed-macro", "tools/oci-cli-wrapper", @@ -89,6 +90,8 @@ unplug = { version = "0.1", path = "tools/unplug", artifact = [ "bin:unplug" ] } update-metadata = { version = "0.1", path = "tools/update-metadata" } anyhow = "1" +astral-tokio-tar = "0.5" +async-compression = "0.4" async-recursion = "1" async-stream = "0.3" async-trait = "0.1" @@ -100,6 +103,7 @@ aws-sdk-ec2 = { version = "1", default-features = false, features = ["default-ht aws-sdk-kms = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] } aws-sdk-ssm = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] } aws-sdk-sts = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] } +aws-sdk-s3 = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] } aws-smithy-types = "1" aws-types = "1" base64 = "0.22" @@ -131,10 +135,13 @@ lzma-rs = "0.3" maplit = "1" nix = "0.29" nonzero_ext = "0.3" -num_cpus = "1" +num_cpus = "1.17" num-traits = "0.2" +oci-spec = "0.8" once_cell = "1.21" olpc-cjson = "0.1" +owo-colors = "4.2" +parking_lot = "0.12" proc-macro2 = "1" quote = "1" rand = { version = "0.8", default-features = false } @@ -166,6 +173,8 @@ tough = "0.21" tough-kms = "0.13" tough-ssm = "0.16" tracing = "0.1" +tracing-indicatif = "=0.3.9" +tracing-subscriber = "0.3" tuftool = { version = "0.14", artifact = [ "bin:tuftool" ] } uds = "0.4.1" unescape = "0.1" diff --git a/deny.toml b/deny.toml index ba6c4b637..4c3bb961f 100644 --- a/deny.toml +++ b/deny.toml @@ -62,6 +62,8 @@ wildcards = "deny" skip = [ # several dependencies are using multiple versions of base64 { name = "base64" }, + # owo-colors uses two versions of supports-color by itself + { name = "supports-color" } ] skip-tree = [ diff --git a/tools/casi/Cargo.toml b/tools/casi/Cargo.toml new file mode 100644 index 000000000..1a36adb62 --- /dev/null +++ b/tools/casi/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "casi" +version = "0.1.0" +authors = ["Jarrett Tierney "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false + +[dependencies] +astral-tokio-tar.workspace = true +async-compression = { workspace = true, features = ["tokio", "zstd"] } +async-trait.workspace = true +aws-config.workspace = true +aws-sdk-s3.workspace = true +aws-smithy-types.workspace = true +aws-types.workspace = true +bon.workspace = true +chrono = { workspace = true, features = ["serde", "now"] } +clap = { workspace = true, features = ["derive"] } +futures.workspace = true +hex.workspace = true +indicatif.workspace = true +oci-spec.workspace = true +olpc-cjson.workspace = true +owo-colors = { workspace = true, features = ["supports-colors"] } +parking_lot = { workspace = true, features = ["send_guard", "arc_lock"] } +rand.workspace = true +semver = { workspace = true, features = ["serde"] } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +sha2.workspace = true +snafu.workspace = true +tempfile.workspace = true +tokio = { workspace = true, features = ["full"] } +tracing.workspace = true +tracing-indicatif.workspace = true +tracing-subscriber = { workspace = true, features = [ + "env-filter", + "fmt", + "registry", +] } +url = { workspace = true, features = ["serde"] } +uuid = { workspace = true, features = ["v4"] } +walkdir.workspace = true +zstd.workspace = true diff --git a/tools/casi/src/constants.rs b/tools/casi/src/constants.rs new file mode 100644 index 000000000..d40f421cb --- /dev/null +++ b/tools/casi/src/constants.rs @@ -0,0 +1,48 @@ +//! Constants used throughout the casi crate. +//! +//! This module centralizes magic numbers and commonly used values to improve +//! code maintainability and reduce the likelihood of errors. + +/// Default buffer size for I/O operations (64KB) +pub const DEFAULT_BUFFER_SIZE: usize = 64 * 1024; + +/// Buffer size for hashing operations (128MB) +/// Uses 8KB chunks which align with standard memory page sizes for optimal +/// performance when reading from files and network streams. +pub const HASH_BUFFER_SIZE: usize = 128 * 1024 * 1024; + +/// Maximum number of multipart upload parts for S3 +pub const MAX_MULTIPART_PARTS: usize = 10_000; + +/// Minimum size for S3 multipart upload parts (5MB) +pub const MIN_MULTIPART_PART_SIZE: usize = 5 * 1024 * 1024; + +/// Default compression level for Zstd +pub const DEFAULT_COMPRESSION_LEVEL: i32 = 3; + +/// Maximum file name length for cross-platform compatibility +pub const MAX_FILENAME_LENGTH: usize = 255; + +/// Default timeout for network operations (30 seconds) +pub const DEFAULT_NETWORK_TIMEOUT_SECS: u64 = 30; + +/// Maximum retry attempts for transient failures +pub const MAX_RETRY_ATTEMPTS: usize = 3; + +/// Default page size for listing operations +pub const DEFAULT_PAGE_SIZE: usize = 100; + +/// Maximum artifacts to display in table format +pub const MAX_TABLE_DISPLAY_ARTIFACTS: usize = 1000; + +/// Hash truncation length for display purposes +pub const HASH_DISPLAY_LENGTH: usize = 16; + +/// Schema version for OCI manifests +pub const OCI_MANIFEST_SCHEMA_VERSION: u32 = 2; + +/// Default file permissions for created files (0o644) +pub const DEFAULT_FILE_PERMISSIONS: u32 = 0o644; + +/// Default directory permissions for created directories (0o755) +pub const DEFAULT_DIR_PERMISSIONS: u32 = 0o755; diff --git a/tools/casi/src/error.rs b/tools/casi/src/error.rs new file mode 100644 index 000000000..71a0f0da9 --- /dev/null +++ b/tools/casi/src/error.rs @@ -0,0 +1,20 @@ +use snafu::Snafu; + +pub type Result = std::result::Result; + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum Error { + #[snafu(display( + "Failed to read data for hashing: {source} - check if the file exists and is readable" + ))] + HashingReadError { source: std::io::Error }, + + #[snafu(display("Failed to initialize logging system: {source}"))] + LogInit { + source: Box, + }, + + #[snafu(display("Failed to parse log directive: {directive}"))] + LogDirectiveParse { directive: String }, +} diff --git a/tools/casi/src/io.rs b/tools/casi/src/io.rs new file mode 100644 index 000000000..3e2b8b387 --- /dev/null +++ b/tools/casi/src/io.rs @@ -0,0 +1,406 @@ +//! I/O utilities for content hashing and streaming operations. +//! +//! This module provides utilities for computing SHA-256 hashes of streaming +//! data and thread-safe wrappers for async readers and writers. All hashing +//! operations use the SHA-256 algorithm for content-addressable storage. +use crate::error::{self, Result}; +use async_compression::tokio::{bufread as zstd_read, write as zstd_write}; +use parking_lot::Mutex; +use sha2::{Digest, Sha256}; +use snafu::ResultExt; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Poll; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncBufRead, AsyncRead, AsyncWrite}; +use tracing::{instrument, trace}; +use tracing_indicatif::span_ext::IndicatifSpanExt; + +/// Common trait for IO components that support progress tracking +trait ProgressTracker { + /// Update the progress by the specified number of bytes + fn update_progress(&mut self, bytes: u64); +} + +struct Progress { + span: Option, + bytes: usize, + start: Instant, +} + +/// Computes SHA-256 hash and size of data from an async reader. +/// +/// Reads all data from the provided reader in 8KB chunks, computing the SHA-256 hash +/// and tracking the total number of bytes processed. Returns the hex-encoded hash +/// and byte count for content-addressable storage operations. +#[instrument(level = "trace", skip(reader))] +pub async fn compute_sha256_hash(reader: &mut R) -> Result<(String, u64)> { + let mut hasher = AsyncSha256::new(); + tokio::io::copy(reader, &mut hasher) + .await + .context(error::HashingReadSnafu)?; + let size = hasher.size(); + let hash = hasher.to_hash(); + trace!("SHA-256 hash computation completed, hash: {}", &hash[..16]); + + Ok((hash, size)) +} + +/// Thread-safe wrapper for async writers with Send + Sync guarantees. +/// +/// Provides a cloneable wrapper around async writers that can be safely +/// shared across threads. Uses Arc> for interior mutability +/// while maintaining async compatibility. +#[derive(Clone)] +pub struct Writer { + /// Thread-safe interior containing the async writer + inner: Arc>>, +} + +unsafe impl Send for Writer {} +unsafe impl Sync for Writer {} + +/// Internal wrapper for pinned async I/O components with progress tracking. +struct InnerIO { + io_component: Pin>, + progress: Progress, +} + +impl ProgressTracker for InnerIO { + fn update_progress(&mut self, bytes: u64) { + self.progress.bytes += bytes as usize; + if let Some(span) = self.progress.span.as_mut() { + span.pb_inc(bytes); + } + } +} + +impl Writer { + /// Creates a new thread-safe writer wrapper. + #[allow(clippy::arc_with_non_send_sync)] // Clippy does not detect the send_guard feature on parking_lot + pub fn new(writer: impl tokio::io::AsyncWrite + 'static, span: Option) -> Self { + Self { + inner: Arc::new(Mutex::new(InnerIO { + io_component: Box::pin(writer), + progress: Progress { + bytes: 0, + start: Instant::now(), + span, + }, + })), + } + } + + /// Creates a new thread-safe writer with zstd encoding + #[allow(clippy::arc_with_non_send_sync)] // Clippy does not detect the send_guard feature on parking_lot + pub fn with_encode( + writer: impl tokio::io::AsyncWrite + 'static, + span: Option, + ) -> Self { + Self { + inner: Arc::new(Mutex::new(InnerIO { + io_component: Box::pin(zstd_write::ZstdEncoder::new(writer)), + progress: Progress { + bytes: 0, + start: Instant::now(), + span, + }, + })), + } + } + + /// Creates a new thread-safe writer with zstd decoding + #[allow(clippy::arc_with_non_send_sync)] // Clippy does not detect the send_guard feature on parking_lot + pub fn with_decode( + writer: impl tokio::io::AsyncWrite + 'static, + span: Option, + ) -> Self { + Self { + inner: Arc::new(Mutex::new(InnerIO { + io_component: Box::pin(zstd_write::ZstdDecoder::new(writer)), + progress: Progress { + bytes: 0, + start: Instant::now(), + span, + }, + })), + } + } + + pub fn progress_bytes(&self) -> usize { + self.inner.lock().progress.bytes + } + + pub fn elapsed(&self) -> Duration { + self.inner.lock().progress.start.elapsed() + } +} + +impl tokio::io::AsyncWrite for Writer { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + let mut this = self.get_mut().inner.lock(); + match this.io_component.as_mut().poll_write(cx, buf) { + Poll::Ready(Ok(size)) => { + this.update_progress(size as u64); + Poll::Ready(Ok(size)) + } + Poll::Ready(Err(e)) => Poll::Ready(Err(e)), + Poll::Pending => { + cx.waker().wake_by_ref(); + Poll::Pending + } + } + } + + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.get_mut() + .inner + .lock() + .io_component + .as_mut() + .poll_flush(cx) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.get_mut() + .inner + .lock() + .io_component + .as_mut() + .poll_shutdown(cx) + } +} + +/// Thread-safe wrapper for async readers with Send + Sync guarantees. +/// +/// Provides a cloneable wrapper around async readers that can be safely +/// shared across threads. Uses Arc> for interior mutability +/// while maintaining async compatibility for storage backend operations. +#[derive(Clone)] +pub struct Reader { + /// Thread-safe interior containing the async reader + inner: Arc>>, +} + +unsafe impl Send for Reader {} +unsafe impl Sync for Reader {} + +impl Reader { + /// Creates a new thread-safe reader wrapper. + #[allow(clippy::arc_with_non_send_sync)] // Clippy does not detect the send_guard feature on parking_lot + pub fn new(reader: impl AsyncRead + 'static) -> Self { + Self { + inner: Arc::new(Mutex::new(InnerIO { + io_component: Box::pin(reader), + progress: Progress { + bytes: 0, + start: Instant::now(), + span: None, + }, + })), + } + } + + /// Creates a new thread-safe reader wrapper with zstd decoding + #[allow(clippy::arc_with_non_send_sync)] // Clippy does not detect the send_guard feature on parking_lot + pub fn with_decode(reader: impl AsyncBufRead + 'static) -> Self { + Self { + inner: Arc::new(Mutex::new(InnerIO { + io_component: Box::pin(zstd_read::ZstdDecoder::new(reader)), + progress: Progress { + bytes: 0, + start: Instant::now(), + span: None, + }, + })), + } + } + + /// Creates a new thread-safe reader wrapper with zstd encoding + #[allow(clippy::arc_with_non_send_sync)] // Clippy does not detect the send_guard feature on parking_lot + pub fn with_encode(reader: impl AsyncBufRead + 'static) -> Self { + Self { + inner: Arc::new(Mutex::new(InnerIO { + io_component: Box::pin(zstd_read::ZstdEncoder::new(reader)), + progress: Progress { + bytes: 0, + start: Instant::now(), + span: None, + }, + })), + } + } + + pub fn set_span(&self, span: tracing::Span) { + self.inner.lock().progress.span = Some(span); + } +} + +impl tokio::io::AsyncRead for Reader { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let mut this = self.get_mut().inner.lock(); + match this.io_component.as_mut().poll_read(cx, buf) { + Poll::Ready(Ok(_)) => { + if buf.remaining() == 0 { + this.update_progress(buf.filled().len() as u64); + } + Poll::Ready(Ok(())) + } + Poll::Ready(Err(e)) => Poll::Ready(Err(e)), + Poll::Pending => { + cx.waker().wake_by_ref(); + Poll::Pending + } + } + } +} + +struct AsyncSha256 { + digest: Sha256, + size: u64, +} + +impl AsyncSha256 { + fn new() -> Self { + Self { + digest: Sha256::new(), + size: 0, + } + } + + fn to_hash(&self) -> String { + let result = self.digest.clone().finalize(); + hex::encode(result.as_slice()) + } + + fn size(&self) -> u64 { + self.size + } +} + +impl AsyncWrite for AsyncSha256 { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> Poll> { + let this = self.get_mut(); + this.digest.update(buf); + this.size += buf.len() as u64; + Poll::Ready(Ok(buf.len())) + } + + fn poll_flush( + self: Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::constants::HASH_BUFFER_SIZE; + use std::io::Cursor; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn test_compute_sha256_hash_empty_input() { + // Arrange + let mut reader = Cursor::new(b""); + + // Act + let (hash, size) = compute_sha256_hash(&mut reader).await.unwrap(); + + // Assert + assert_eq!(size, 0); + assert_eq!( + hash, + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + } + + #[tokio::test] + async fn test_compute_sha256_hash_known_input() { + // Arrange + let test_data = b"hello world"; + let mut reader = Cursor::new(test_data); + + // Act + let (hash, size) = compute_sha256_hash(&mut reader).await.unwrap(); + + // Assert + assert_eq!(size, test_data.len() as u64); + assert_eq!( + hash, + "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ); + } + + #[tokio::test] + async fn test_compute_sha256_hash_large_input() { + // Arrange - Create data larger than buffer size + let test_data = vec![0u8; HASH_BUFFER_SIZE * 2 + 100]; + let mut reader = Cursor::new(&test_data); + + // Act + let (hash, size) = compute_sha256_hash(&mut reader).await.unwrap(); + + // Assert + assert_eq!(size, test_data.len() as u64); + // Hash of all zeros with this length + assert!(!hash.is_empty()); + assert_eq!(hash.len(), 64); // SHA-256 produces 64 hex characters + } + + #[tokio::test] + async fn test_reader_wrapper_functionality() { + // Arrange + let test_data = b"test data for reader"; + let cursor = Cursor::new(test_data); + let mut reader = Reader::new(cursor); + + // Act + let mut buffer = Vec::new(); + reader.read_to_end(&mut buffer).await.unwrap(); + + // Assert + assert_eq!(buffer, test_data); + } + + #[tokio::test] + async fn test_reader_wrapper_clone() { + // Arrange + let test_data = b"cloneable data"; + let cursor = Cursor::new(test_data); + let reader = Reader::new(cursor); + + // Act + let cloned_reader = reader.clone(); + + // Assert - Both readers should share the same Arc (this tests the Clone implementation) + assert!(Arc::ptr_eq(&reader.inner, &cloned_reader.inner)); + } +} diff --git a/tools/casi/src/lib.rs b/tools/casi/src/lib.rs new file mode 100644 index 000000000..ea2d3c5e7 --- /dev/null +++ b/tools/casi/src/lib.rs @@ -0,0 +1,7 @@ +// Re-export main types for convenience +pub use error::{Error, Result}; + +pub mod constants; +pub mod error; +pub mod io; +pub mod logging; diff --git a/tools/casi/src/logging/context.rs b/tools/casi/src/logging/context.rs new file mode 100644 index 000000000..829768067 --- /dev/null +++ b/tools/casi/src/logging/context.rs @@ -0,0 +1,490 @@ +//! Enhanced trace context system for operation correlation and structured logging. +//! +//! This module provides the core types for the enhanced tracing system, +//! including trace context, performance metrics, and error context. +//! +//! The TraceContext approach is the preferred and recommended method for +//! logging and tracing in cassi. It provides a comprehensive, structured way to +//! track operations, maintain context across async boundaries, and collect +//! performance metrics. +use std::collections::HashMap; +use std::time::{Duration, Instant, SystemTime}; + +use serde::{Deserialize, Serialize}; +use tokio::sync::OnceCell; +use tracing::Span; +use uuid::Uuid; + +static SESSION_ID: OnceCell = OnceCell::const_new(); + +/// Enhanced trace context for operation correlation and structured logging. +/// +/// TraceContext provides a consistent way to track related operations +/// and maintain context across async boundaries. This is the recommended +/// approach for all logging and tracing in cassi. +/// +/// Use the trace_context! macro to create instances with minimal boilerplate. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TraceContext { + /// Unique identifier for this operation session + pub session_id: String, + /// Correlation ID for related operations + pub correlation_id: String, + /// Operation type (create, store, fetch, etc.) + pub operation: String, + /// Target artifact ID (if applicable) + pub artifact_id: Option, + /// User-provided labels for filtering and categorization + pub labels: HashMap, + /// Timestamp when context was created + pub created_at: SystemTime, +} + +impl TraceContext { + pub fn record(&self, span: &Span) { + span.record("session_id", self.session_id.clone()); + span.record("correlation_id", self.correlation_id.clone()); + if let Some(artifact_id) = self.artifact_id.as_ref() { + span.record("artifact_id", artifact_id.clone()); + } + for (key, value) in self.labels.iter() { + span.record(key.as_str(), value.clone()); + } + } + + /// Create a new trace context for an operation. + pub async fn new(operation: impl Into) -> Self { + let operation_str = operation.into(); + let session_id = SESSION_ID + .get_or_init(async || Uuid::new_v4()) + .await + .to_string(); + let ctx_id = Uuid::new_v4().to_string(); + let correlation_id = format!("{}-{}", operation_str, &ctx_id[..8]); + + Self { + session_id, + correlation_id, + operation: operation_str, + artifact_id: None, + labels: HashMap::new(), + created_at: SystemTime::now(), + } + } + + /// Create a child context for a related operation. + pub fn child(&self, operation: impl Into) -> Self { + let operation_str = operation.into(); + let child_id = Uuid::new_v4().to_string(); + let correlation_id = format!("{}-{}", operation_str, &child_id[..8]); + + Self { + session_id: self.session_id.clone(), + correlation_id, + operation: operation_str, + artifact_id: self.artifact_id.clone(), + labels: self.labels.clone(), + created_at: SystemTime::now(), + } + } + + /// Set the artifact ID for this context. + pub fn with_artifact_id(mut self, artifact_id: impl Into) -> Self { + self.artifact_id = Some(artifact_id.into()); + self + } + + /// Add a label to this context. + pub fn with_label(mut self, key: impl Into, value: impl Into) -> Self { + self.labels.insert(key.into(), value.into()); + self + } + + /// Add multiple labels to this context. + pub fn with_labels(mut self, labels: HashMap) -> Self { + self.labels.extend(labels); + self + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Throughput(f64); + +impl From for Throughput { + fn from(value: f64) -> Self { + Self(value) + } +} + +impl std::fmt::Display for Throughput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let round: u64 = self.0.ceil() as u64; + if round > 1024 * 1024 * 1024 { + f.write_fmt(format_args!("{}gb/s", round / (1024 * 1024 * 1024))) + } else if round > 1024 * 1024 { + f.write_fmt(format_args!("{}mb/s", round / (1024 * 1024))) + } else if round > 1024 { + f.write_fmt(format_args!("{}kb/s", round / 1024)) + } else { + f.write_fmt(format_args!("{round}b/s")) + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ByteCount(u64); + +pub fn byte_count(value: impl Into) -> String { + ByteCount(value.into()).to_string() +} + +impl From for ByteCount { + fn from(value: u64) -> Self { + Self(value) + } +} + +impl std::fmt::Display for ByteCount { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.0 > 1024 * 1024 * 1024 { + f.write_fmt(format_args!("{}gb", self.0 / (1024 * 1024 * 1024))) + } else if self.0 > 1024 * 1024 { + f.write_fmt(format_args!("{}mb", self.0 / (1024 * 1024))) + } else if self.0 > 1024 { + f.write_fmt(format_args!("{}kb", self.0 / 1024)) + } else { + f.write_fmt(format_args!("{}b", self.0)) + } + } +} + +/// Performance metrics collected during operations. +/// +/// PerformanceMetrics provides standardized performance tracking +/// across all CAS operations. It integrates with the TraceContext approach +/// to provide comprehensive operational insights. +/// +/// Use the trace_metrics! macro to create instances with minimal boilerplate. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PerformanceMetrics { + /// Operation start time (serialized as duration since epoch) + #[serde(skip, default = "Instant::now")] + pub start_time: Instant, + /// Operation duration (set when operation completes) + pub duration: Option, + /// Total bytes processed + pub bytes_processed: ByteCount, + /// Number of files processed + pub files_processed: u32, + /// Throughput in bytes per second + pub throughput: Option, + /// Cache hit ratio (0.0 to 1.0) + pub cache_hit_ratio: Option, + /// Number of network requests made + pub network_requests: u32, + /// Total network bytes transferred + pub network_bytes: ByteCount, +} + +impl PerformanceMetrics { + pub fn record(&self, span: &Span) { + if let Some(duration) = self.duration.as_ref() { + span.record("duration_ms", duration.as_millis()); + } + if self.bytes_processed.0 > 0 { + span.record("bytes_processed", self.bytes_processed.to_string()); + } + if self.files_processed > 0 { + span.record("files_processed", self.files_processed); + } + if let Some(throughput) = self.throughput.as_ref() { + span.record("throughput_bps", throughput.to_string()); + } + if let Some(cache_hit) = self.cache_hit_ratio { + span.record("cache_hit_ratio", cache_hit); + } + if self.network_requests > 0 { + span.record("network_requests", self.network_requests); + } + if self.network_bytes.0 > 0 { + span.record("network_bytes", self.network_bytes.to_string()); + } + } + /// Create new performance metrics with current timestamp. + pub fn new() -> Self { + Self { + start_time: Instant::now(), + duration: None, + bytes_processed: 0.into(), + files_processed: 0, + throughput: None, + cache_hit_ratio: None, + network_requests: 0, + network_bytes: 0.into(), + } + } + + /// Mark the operation as complete and calculate final metrics. + pub fn complete(&mut self) { + let duration = self.start_time.elapsed(); + self.duration = Some(duration); + + // Calculate throughput if we have duration and bytes + if duration.as_secs_f64() > 0.0 && self.bytes_processed.0 > 0 { + self.throughput = Some(Throughput( + self.bytes_processed.0 as f64 / duration.as_secs_f64(), + )); + } + } + + /// Add bytes to the processed count. + pub fn add_bytes(&mut self, bytes: u64) { + self.bytes_processed.0 += bytes; + } + + /// Increment the file count. + pub fn add_file(&mut self) { + self.files_processed += 1; + } + + /// Add a network request to the metrics. + pub fn add_network_request(&mut self, bytes: u64) { + self.network_requests += 1; + self.network_bytes.0 += bytes; + } + + /// Set the cache hit ratio. + pub fn set_cache_hit_ratio(&mut self, ratio: f64) { + self.cache_hit_ratio = Some(ratio.clamp(0.0, 1.0)); + } + + /// Get the current duration since start. + pub fn current_duration(&self) -> Duration { + self.start_time.elapsed() + } + + /// Get the current throughput based on elapsed time. + pub fn current_throughput(&self) -> Option { + let elapsed = self.start_time.elapsed().as_secs_f64(); + if elapsed > 0.0 && self.bytes_processed.0 > 0 { + Some(self.bytes_processed.0 as f64 / elapsed) + } else { + None + } + } +} + +impl Default for PerformanceMetrics { + fn default() -> Self { + Self::new() + } +} + +/// Rich error context for debugging and analysis. +/// +/// ErrorContext provides structured error information that helps +/// with debugging and error pattern analysis. When combined with TraceContext +/// in the trace_error! macro, it creates comprehensive error logs with full context. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ErrorContext { + /// Error category for classification + pub category: ErrorCategory, + /// Error code for programmatic handling + pub error_code: String, + /// Human-readable error message + pub message: String, + /// Stack trace (if available) + pub stack_trace: Option, + /// Recovery attempts made + pub recovery_attempts: Vec, + /// Related operations that might have contributed + pub related_operations: Vec, + /// Additional context fields + pub context: HashMap, +} + +impl ErrorContext { + /// Create a new error context. + pub fn new( + category: ErrorCategory, + error_code: impl Into, + message: impl Into, + ) -> Self { + Self { + category, + error_code: error_code.into(), + message: message.into(), + stack_trace: None, + recovery_attempts: Vec::new(), + related_operations: Vec::new(), + context: HashMap::new(), + } + } + + /// Add a recovery attempt to the error context. + pub fn add_recovery_attempt(&mut self, attempt: RecoveryAttempt) { + self.recovery_attempts.push(attempt); + } + + /// Add a related operation ID. + pub fn add_related_operation(&mut self, operation_id: impl Into) { + self.related_operations.push(operation_id.into()); + } + + /// Add context information. + pub fn add_context(&mut self, key: impl Into, value: impl Into) { + self.context.insert(key.into(), value.into()); + } + + /// Set the stack trace. + pub fn with_stack_trace(mut self, stack_trace: impl Into) -> Self { + self.stack_trace = Some(stack_trace.into()); + self + } +} + +/// Error categories for classification and filtering. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum ErrorCategory { + /// Network-related errors (timeouts, connection failures) + Network, + /// File system errors (permissions, disk space, I/O) + FileSystem, + /// Authentication and authorization errors + Authentication, + /// Input validation and format errors + Validation, + /// Internal application errors and bugs + Internal, + /// Configuration and setup errors + Configuration, + /// Resource exhaustion (memory, disk, limits) + Resource, +} + +impl ErrorCategory { + /// Get a human-readable description of the error category. + pub fn description(&self) -> &'static str { + match self { + ErrorCategory::Network => "Network communication error", + ErrorCategory::FileSystem => "File system operation error", + ErrorCategory::Authentication => "Authentication or authorization error", + ErrorCategory::Validation => "Input validation error", + ErrorCategory::Internal => "Internal application error", + ErrorCategory::Configuration => "Configuration error", + ErrorCategory::Resource => "Resource exhaustion error", + } + } +} + +/// Information about a recovery attempt. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecoveryAttempt { + /// Type of recovery attempted + pub recovery_type: String, + /// Timestamp of the attempt + pub attempted_at: SystemTime, + /// Whether the recovery was successful + pub successful: bool, + /// Additional details about the attempt + pub details: Option, +} + +impl RecoveryAttempt { + /// Create a new recovery attempt record. + pub fn new(recovery_type: impl Into, successful: bool) -> Self { + Self { + recovery_type: recovery_type.into(), + attempted_at: SystemTime::now(), + successful, + details: None, + } + } + + /// Add details to the recovery attempt. + pub fn with_details(mut self, details: impl Into) -> Self { + self.details = Some(details.into()); + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_trace_context_creation() { + let ctx = TraceContext::new("store").await; + + assert_eq!(ctx.operation, "store"); + assert!(ctx.artifact_id.is_none()); + assert!(ctx.labels.is_empty()); + assert!(!ctx.session_id.is_empty()); + assert!(!ctx.correlation_id.is_empty()); + } + + #[tokio::test] + async fn test_trace_context_child() { + let parent = TraceContext::new("store") + .await + .with_artifact_id("test-artifact") + .with_label("env", "test"); + + let child = parent.child("compress"); + + assert_eq!(child.session_id, parent.session_id); + assert_ne!(child.correlation_id, parent.correlation_id); + assert_eq!(child.operation, "compress"); + assert_eq!(child.artifact_id, parent.artifact_id); + assert_eq!(child.labels, parent.labels); + } + + #[test] + fn test_performance_metrics() { + let mut metrics = PerformanceMetrics::new(); + + metrics.add_bytes(1024); + metrics.add_file(); + metrics.add_network_request(512); + + assert_eq!(metrics.bytes_processed.0, 1024); + assert_eq!(metrics.files_processed, 1); + assert_eq!(metrics.network_requests, 1); + assert_eq!(metrics.network_bytes.0, 512); + + // Test completion + std::thread::sleep(std::time::Duration::from_millis(10)); + metrics.complete(); + + assert!(metrics.duration.is_some()); + assert!(metrics.throughput.is_some()); + } + + #[test] + fn test_error_context() { + let mut error_ctx = + ErrorContext::new(ErrorCategory::Network, "TIMEOUT", "Connection timed out"); + + error_ctx.add_recovery_attempt(RecoveryAttempt::new("retry", false)); + error_ctx.add_related_operation("store-abc123"); + error_ctx.add_context("endpoint", "s3.amazonaws.com"); + + assert_eq!(error_ctx.category, ErrorCategory::Network); + assert_eq!(error_ctx.error_code, "TIMEOUT"); + assert_eq!(error_ctx.recovery_attempts.len(), 1); + assert_eq!(error_ctx.related_operations.len(), 1); + assert_eq!(error_ctx.context.len(), 1); + } + + #[test] + fn test_error_category_description() { + assert_eq!( + ErrorCategory::Network.description(), + "Network communication error" + ); + assert_eq!( + ErrorCategory::FileSystem.description(), + "File system operation error" + ); + } +} diff --git a/tools/casi/src/logging/coordinator.rs b/tools/casi/src/logging/coordinator.rs new file mode 100644 index 000000000..56da74e1b --- /dev/null +++ b/tools/casi/src/logging/coordinator.rs @@ -0,0 +1,202 @@ +//! LogCoordinator implementation for coordinating logging display. +//! +//! The LogCoordinator is responsible for initializing the tracing subscriber system +//! and setting up logging for CAS operations. + +// Standard library imports +use std::sync::Arc; +use std::sync::LazyLock; + +use indicatif::ProgressStyle; +// External crate imports +use snafu::ResultExt; +use tracing_indicatif::IndicatifLayer; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::fmt; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; + +// Internal imports +use crate::error::LogDirectiveParseSnafu; +use crate::error::LogInitSnafu; +use crate::error::Result; +use crate::logging::CasFormatter; +use crate::logging::LogVerbosity; + +pub static SPINNER_STYLE: LazyLock = LazyLock::new(|| { + ProgressStyle::with_template("[{elapsed_precise}] {span_child_prefix} {cmd} {span_name} {msg} {spinner:.green} {span_fields}").unwrap() + .tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏", "✔"]) +}); +pub static COUNTER_STYLE: LazyLock = LazyLock::new(|| { + ProgressStyle::with_template("[{elapsed_precise}] {span_child_prefix} {cmd} {span_name} {msg} {spinner:.green} {bar:40.cyan/blue} {pos}/{len} {span_fields}").unwrap() + .tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏", "✔"]) + .progress_chars("##-") +}); + +/// Central coordinator for logging initialization. +/// +/// The LogCoordinator implements: +/// - Tracing subscriber setup with verbosity filtering +#[derive(Clone)] +pub struct LogCoordinator { + inner: Arc, +} + +struct Inner { + verbosity: LogVerbosity, + quiet: bool, +} + +impl LogCoordinator { + /// Initialize the logging system with specified verbosity and quiet mode. + /// + /// This sets up the tracing subscriber with: + /// - Appropriate log level filtering based on verbosity + /// - AWS SDK log filtering to reduce noise + /// - Console output coordination + /// + /// Returns `Error::LogInit` if the tracing subscriber fails to initialize. + pub async fn init(verbosity: LogVerbosity, quiet: bool) -> Result { + // Create environment filter with appropriate verbosity + let mut filter = EnvFilter::new(verbosity.as_filter_string()); + + // Filter noisy AWS SDK logs to TRACE level only + // Using parse_directives to avoid potential parsing errors + let directives = [ + "aws_sdk_s3=warn", + "aws_smithy_runtime=warn", + "aws_smithy_http=warn", + "aws_config=warn", + "hyper=warn", + "reqwest=warn", + ]; + + for directive in &directives { + match directive.parse() { + Ok(parsed) => filter = filter.add_directive(parsed), + Err(_) => { + return LogDirectiveParseSnafu { + directive: directive.to_string(), + } + .fail(); + } + } + } + + // Allow environment override + if let Ok(env_filter) = std::env::var("RUST_LOG") { + if !env_filter.is_empty() { + filter = EnvFilter::new(env_filter); + } + } + + // Create the tracing subscriber with CasFormatter for enhanced CAS operation logging + // In quiet mode, we only want to show errors, so we use the special Error level filter + let formatter = CasFormatter::new(quiet); + let indicatif_layer = IndicatifLayer::new() + .with_progress_style(ProgressStyle::with_template("[{elapsed_precise}] {span_child_prefix} {cmd} {span_name} {msg} {spinner:.green} {bar:40.cyan/blue} {binary_bytes}/{binary_total_bytes} {span_fields}").unwrap() + .tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏", "✔"]) + .progress_chars("##-") + ).with_span_field_formatter(formatter.clone()); + + let subscriber = tracing_subscriber::registry() + .with(filter) + .with( + fmt::layer() + .with_writer(indicatif_layer.get_stderr_writer()) + .with_target(verbosity != LogVerbosity::Info) + .with_thread_ids(verbosity == LogVerbosity::Trace) + .with_thread_names(verbosity == LogVerbosity::Trace) + .with_file(verbosity == LogVerbosity::Trace) + .with_line_number(verbosity == LogVerbosity::Trace) + .fmt_fields(formatter.clone()) + .event_format(formatter.clone()), + ) + .with(indicatif_layer); + + // Initialize the global subscriber + subscriber + .try_init() + .map_err(|e| Box::new(e) as Box) + .context(LogInitSnafu)?; + + let inner = Arc::new(Inner { verbosity, quiet }); + + Ok(LogCoordinator { inner }) + } + + /// Get the current verbosity level. + pub fn verbosity(&self) -> LogVerbosity { + self.inner.verbosity + } + + /// Check if quiet mode is enabled. + pub fn is_quiet(&self) -> bool { + self.inner.quiet + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::sync::OnceCell; + + const INIT: OnceCell = OnceCell::const_new(); + + fn init_test_logging() { + let _ = INIT.get_or_init(async || { + LogCoordinator::init(LogVerbosity::Info, false) + .await + .unwrap() + }); + } + + #[tokio::test] + async fn test_log_manager_init() { + // Test that we can create a LogCoordinator even if tracing is already initialized + let log_coordinator = LogCoordinator::init(LogVerbosity::Info, false).await; + // The first call might succeed, subsequent calls will fail due to global subscriber + // but that's expected behavior + let _ = log_coordinator; + } + + #[tokio::test] + async fn test_log_manager_quiet_mode() { + init_test_logging(); + + // Create a LogCoordinator directly without initializing tracing again + let inner = Arc::new(Inner { + verbosity: LogVerbosity::Info, + quiet: true, + }); + let log_manager = LogCoordinator { inner }; + + assert!(log_manager.is_quiet()); + } + + #[tokio::test] + async fn test_log_manager_verbosity() { + init_test_logging(); + + // Create a LogCoordinator directly without initializing tracing again + let inner = Arc::new(Inner { + verbosity: LogVerbosity::Debug, + quiet: false, + }); + let log_manager = LogCoordinator { inner }; + + assert_eq!(log_manager.verbosity(), LogVerbosity::Debug); + } + + #[test] + fn test_log_verbosity_filter_strings() { + assert_eq!(LogVerbosity::Info.as_filter_string(), "info"); + assert_eq!(LogVerbosity::Debug.as_filter_string(), "debug"); + assert_eq!(LogVerbosity::Trace.as_filter_string(), "trace"); + } + + #[test] + fn test_log_verbosity_default() { + assert_eq!(LogVerbosity::default(), LogVerbosity::Info); + } +} diff --git a/tools/casi/src/logging/formatter.rs b/tools/casi/src/logging/formatter.rs new file mode 100644 index 000000000..4cf962c13 --- /dev/null +++ b/tools/casi/src/logging/formatter.rs @@ -0,0 +1,177 @@ +//! Custom tracing formatter for CAS operations. +//! +//! The CasFormatter implements formatting for casi logging events, +//! with hierarchical span display, colored output based on terminal capabilities, +//! and field-specific formatting for CAS operations and metadata. It works +//! together with CasVisitor to create a consistent logging experience. + +use super::visitor::CasVisitor; +use owo_colors::{OwoColorize, Stream}; +use std::fmt; +use tracing::{Event, Subscriber}; +use tracing_subscriber::fmt::format::{FormatFields, Writer}; +use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormattedFields}; +use tracing_subscriber::registry::LookupSpan; + +/// Custom tracing formatter for CAS operations. +/// +/// The CasFormatter implements: +/// - Hierarchical span display with indentation +/// - Color differentiation for log levels with terminal capability detection +/// - Field-specific formatting for CAS operation metadata +/// - Standardized timestamp and field formatting +#[derive(Clone, Default)] +pub struct CasFormatter { + /// Whether quiet mode is enabled + quiet: bool, +} + +impl CasFormatter { + /// Create a new CasFormatter with the specified quiet mode setting + pub fn new(quiet: bool) -> Self { + Self { quiet } + } + + /// Format the log level with color differentiation. + fn format_level(&self, level: &tracing::Level, writer: &mut Writer<'_>) -> fmt::Result { + match *level { + tracing::Level::ERROR => { + let colored = "ERROR".if_supports_color(Stream::Stderr, |text| text.red()); + write!(writer, "{colored}") + } + tracing::Level::WARN => { + let colored = "WARN ".if_supports_color(Stream::Stderr, |text| text.yellow()); + write!(writer, "{colored}") + } + tracing::Level::INFO => { + let colored = "INFO ".if_supports_color(Stream::Stderr, |text| text.green()); + write!(writer, "{colored}") + } + tracing::Level::DEBUG => { + let colored = "DEBUG".if_supports_color(Stream::Stderr, |text| text.blue()); + write!(writer, "{colored}") + } + tracing::Level::TRACE => { + let colored = "TRACE".if_supports_color(Stream::Stderr, |text| text.cyan()); + write!(writer, "{colored}") + } + } + } + + /// Format the timestamp in a simplified, more readable format. + fn format_timestamp(&self, writer: &mut Writer<'_>) -> fmt::Result { + let now = chrono::Utc::now(); + let timestamp = now.format("%H:%M:%S%.3f UTC"); + write!( + writer, + "{}", + timestamp + .to_string() + .if_supports_color(Stream::Stderr, |text| text.dimmed()) + ) + } + + /// Format span information with hierarchy and context. + fn format_span_context( + &self, + ctx: &FmtContext<'_, S, N>, + writer: &mut Writer<'_>, + ) -> fmt::Result + where + S: Subscriber + for<'a> LookupSpan<'a>, + N: for<'a> FormatFields<'a> + 'static, + { + if let Some(scope) = ctx.lookup_current() { + let span_name = scope.name(); + write!( + writer, + "{} ", + span_name.if_supports_color(Stream::Stderr, |text| text.bold().cyan().to_string()) + )?; + } + + Ok(()) + } + + /// Format the target (module path) if enabled. + fn format_target( + &self, + target: &str, + writer: &mut Writer<'_>, + show_target: bool, + ) -> fmt::Result { + if show_target && !target.is_empty() { + let colored_target = target.if_supports_color(Stream::Stderr, |text| text.dimmed()); + write!(writer, "{colored_target}: ")?; + } + Ok(()) + } +} + +impl FormatEvent for CasFormatter +where + S: Subscriber + for<'a> LookupSpan<'a>, + N: for<'a> FormatFields<'a> + 'static, +{ + fn format_event( + &self, + ctx: &FmtContext<'_, S, N>, + mut writer: Writer<'_>, + event: &Event<'_>, + ) -> fmt::Result { + // In quiet mode, only process ERROR level events + // and skip all other events to make quiet mode truly minimal + if self.quiet && !matches!(*event.metadata().level(), tracing::Level::ERROR) { + return Ok(()); + } + + // Format timestamp + self.format_timestamp(&mut writer)?; + write!(writer, " ")?; + + // Format log level + self.format_level(event.metadata().level(), &mut writer)?; + write!(writer, " ")?; + + // Format span context + self.format_span_context(ctx, &mut writer)?; + + // Format target (module path) - only show for DEBUG and TRACE levels + let show_target = matches!( + *event.metadata().level(), + tracing::Level::DEBUG | tracing::Level::TRACE + ); + self.format_target(event.metadata().target(), &mut writer, show_target)?; + + let mut visitor = CasVisitor::new(writer); + event.record(&mut visitor); + + // We want to print out the parent spans fields with the vent + let span = event + .parent() + .and_then(|id| ctx.span(id)) + .or_else(|| ctx.lookup_current()); + if let Some(span) = span { + let ext = span.extensions(); + let fields = ext.get::>().unwrap(); + write!(visitor.writer(), " {fields}")?; + } + + // Add newline + writeln!(visitor.writer())?; + + Ok(()) + } +} + +impl<'writer> FormatFields<'writer> for CasFormatter { + fn format_fields( + &self, + writer: Writer<'writer>, + fields: R, + ) -> fmt::Result { + let mut visitor = CasVisitor::new(writer); + fields.record(&mut visitor); + Ok(()) + } +} diff --git a/tools/casi/src/logging/macros.rs b/tools/casi/src/logging/macros.rs new file mode 100644 index 000000000..3e6acca1a --- /dev/null +++ b/tools/casi/src/logging/macros.rs @@ -0,0 +1,201 @@ +#[macro_export] +macro_rules! operation { + ($operation: literal ($($field: ident = $value: expr),*) $fn_name: ident ($self: ident, $($arg_n: expr),*)) => {{ + use tracing::Instrument; + + async move { + let span = tracing::Span::current(); + let ctx = $crate::logging::TraceContext::new($operation).await; + let mut metrics = $crate::logging::PerformanceMetrics::new(); + ctx.record(&span); + tracing::info!("{} started", $operation); + match $self + .$fn_name(&mut metrics, $($arg_n),*) + .await { + Ok(result) => { + metrics.complete(); + metrics.record(&span); + tracing::info!( + "{} completed successfully", + $operation, + ); + Ok(result) + }, + Err(e) => { + metrics.complete(); + metrics.record(&span); + tracing::error!( + "{} failed: {}", + $operation, + e + ); + Err(e) + } + } + }.instrument(tracing::info_span!( + $operation, + operation = $operation, + session_id = tracing::field::Empty, + correlation_id = tracing::field::Empty, + artifact_id = tracing::field::Empty, + duration_ms = tracing::field::Empty, + bytes_processed = tracing::field::Empty, + files_processed = tracing::field::Empty, + throughput_bps = tracing::field::Empty, + cache_hit_ratio = tracing::field::Empty, + network_requests = tracing::field::Empty, + network_bytes = tracing::field::Empty, + $($field = $value),* + )) + .await + }}; + + ($operation: literal ($($field: ident = $value: expr),*) $fn_name: ident ($self: ident, $($arg_n: expr),*) where artifact_id = $artifact_id: expr) => {{ + use tracing::Instrument; + + async move { + let span = tracing::Span::current(); + let ctx = $crate::logging::TraceContext::new($operation).await; + let mut metrics = $crate::logging::PerformanceMetrics::new(); + ctx.record(&span); + tracing::info!("{} started", $operation); + match $self + .$fn_name(&mut metrics, $($arg_n),*) + .await { + Ok(result) => { + metrics.complete(); + metrics.record(&span); + tracing::info!( + "{} completed successfully", + $operation, + ); + Ok(result) + }, + Err(e) => { + metrics.complete(); + metrics.record(&span); + tracing::error!( + "{} failed: {}", + $operation, + e + ); + Err(e) + } + } + }.instrument(tracing::info_span!( + $operation, + operation = $operation, + session_id = tracing::field::Empty, + correlation_id = tracing::field::Empty, + artifact_id = $artifact_id, + duration_ms = tracing::field::Empty, + bytes_processed = tracing::field::Empty, + files_processed = tracing::field::Empty, + throughput_bps = tracing::field::Empty, + cache_hit_ratio = tracing::field::Empty, + network_requests = tracing::field::Empty, + network_bytes = tracing::field::Empty, + $($field = $value),* + )) + .await + }}; +} + +#[macro_export] +macro_rules! inline_operation { + ($operation: literal ($($field: ident = $value: expr),*) $fn: ident) => {{ + use tracing::Instrument; + + async move { + let span = tracing::Span::current(); + let ctx = $crate::logging::TraceContext::new($operation).await; + let mut metrics = $crate::logging::PerformanceMetrics::new(); + ctx.record(&span); + tracing::info!("{} started", $operation); + match $fn(&mut metrics) + .await { + Ok(result) => { + metrics.complete(); + metrics.record(&span); + tracing::info!( + "{} completed successfully", + $operation, + ); + Ok(result) + }, + Err(e) => { + metrics.complete(); + metrics.record(&span); + tracing::error!( + "{} failed: {}", + $operation, + e + ); + Err(e) + } + } + }.instrument(tracing::info_span!( + $operation, + operation = $operation, + session_id = tracing::field::Empty, + correlation_id = tracing::field::Empty, + artifact_id = tracing::field::Empty, + duration_ms = tracing::field::Empty, + bytes_processed = tracing::field::Empty, + files_processed = tracing::field::Empty, + throughput_bps = tracing::field::Empty, + cache_hit_ratio = tracing::field::Empty, + network_requests = tracing::field::Empty, + network_bytes = tracing::field::Empty, + $($field = $value),* + )) + }}; + + ($operation: literal ($($field: ident = $value: expr),*) $fn: ident; where artifact_id = $artifact_id: expr) => {{ + use tracing::Instrument; + + async move { + let span = tracing::Span::current(); + let ctx = $crate::logging::TraceContext::new($operation).await; + let mut metrics = $crate::logging::PerformanceMetrics::new(); + ctx.record(&span); + tracing::info!("{} started", $operation); + match $fn(&mut metrics) + .await { + Ok(result) => { + metrics.complete(); + metrics.record(&span); + tracing::info!( + "{} completed successfully", + $operation, + ); + Ok(result) + }, + Err(e) => { + metrics.complete(); + metrics.record(&span); + tracing::error!( + "{} failed: {}", + $operation, + e + ); + Err(e) + } + } + }.instrument(tracing::info_span!( + $operation, + operation = $operation, + session_id = tracing::field::Empty, + correlation_id = tracing::field::Empty, + artifact_id = $artifact_id, + duration_ms = tracing::field::Empty, + bytes_processed = tracing::field::Empty, + files_processed = tracing::field::Empty, + throughput_bps = tracing::field::Empty, + cache_hit_ratio = tracing::field::Empty, + network_requests = tracing::field::Empty, + network_bytes = tracing::field::Empty, + $($field = $value),* + )) + }}; +} diff --git a/tools/casi/src/logging/mod.rs b/tools/casi/src/logging/mod.rs new file mode 100644 index 000000000..568b5470a --- /dev/null +++ b/tools/casi/src/logging/mod.rs @@ -0,0 +1,201 @@ +//! Console logging system for casi. +//! +//! This module implements structured logging +//! for CAS operations using tracing and owo-colors crates. +//! The system provides clear feedback for operations +//! while separating data output from logging information. +//! +//! ## Core Components +//! +//! - [`LogCoordinator`] - Central coordinator for logging initialization +//! - [`LogVerbosity`] - Verbosity level configuration for different logging targets +//! - [`TraceContext`] - Enhanced trace context for operation correlation +//! - [`TraceEvent`] - Structured event types for consistent logging +//! +//! ## Usage +//! +//! ```rust +//! use casi::logging::{LogCoordinator, LogVerbosity}; +//! +//! # async fn example() -> Result<(), Box> { +//! // Initialize logging system +//! let log_manager = LogCoordinator::init(LogVerbosity::Info, false).await?; +//! +//! // Check verbosity level +//! println!("Verbosity: {:?}", log_manager.verbosity()); +//! println!("Quiet mode: {}", log_manager.is_quiet()); +//! +//! println!("✓ Logging system initialized"); +//! # Ok(()) +//! # } +//! ``` + +pub mod context; +pub mod coordinator; +pub mod formatter; +pub mod macros; +pub mod visitor; + +// Re-export main types for convenience +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +pub use context::{ + ByteCount, ErrorCategory, ErrorContext, PerformanceMetrics, Throughput, TraceContext, + byte_count, +}; +pub use coordinator::LogCoordinator; +pub use formatter::CasFormatter; + +/// Verbosity levels for logging output. +/// +/// Controls the level of detail in log messages and which events are displayed. +/// Higher verbosity levels include all messages from lower levels. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] +pub enum LogVerbosity { + /// Only errors and critical information + Error, + /// Warnings and errors + Warn, + /// Standard informational messages (default) + #[default] + Info, + /// Detailed debugging information + Debug, + /// Comprehensive trace information including internal operations + Trace, + /// All events including internal operations and memory allocations + All, +} + +impl LogVerbosity { + /// Convert to tracing filter string for subscriber configuration + pub fn as_filter_string(&self) -> &'static str { + match self { + LogVerbosity::Error => "error", + LogVerbosity::Warn => "warn", + LogVerbosity::Info => "info", + LogVerbosity::Debug => "debug", + LogVerbosity::Trace => "trace", + LogVerbosity::All => "trace", // Use trace as the highest tracing level + } + } + + /// Get a human-readable description of the verbosity level + pub fn description(&self) -> &'static str { + match self { + LogVerbosity::Error => "Only critical errors and failures", + LogVerbosity::Warn => "Warnings and errors", + LogVerbosity::Info => "Standard operational information", + LogVerbosity::Debug => "Detailed debugging information", + LogVerbosity::Trace => "Comprehensive trace information", + LogVerbosity::All => "All events including internal operations", + } + } + + /// Check if this verbosity level includes another level + pub fn includes(&self, other: LogVerbosity) -> bool { + use LogVerbosity::*; + match self { + Error => matches!(other, Error), + Warn => matches!(other, Error | Warn), + Info => matches!(other, Error | Warn | Info), + Debug => matches!(other, Error | Warn | Info | Debug), + Trace => matches!(other, Error | Warn | Info | Debug | Trace), + All => true, + } + } +} + +/// Result of an operation. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OperationResult { + /// Whether the operation was successful + pub success: bool, + /// Result message or summary + pub message: Option, + /// Output data from the operation + pub data: HashMap, + /// Warnings encountered during the operation + pub warnings: Vec, +} + +impl OperationResult { + /// Create a successful result. + pub fn success() -> Self { + Self { + success: true, + message: None, + data: HashMap::new(), + warnings: Vec::new(), + } + } + + /// Create a successful result with a message. + pub fn success_with_message(message: impl Into) -> Self { + Self { + success: true, + message: Some(message.into()), + data: HashMap::new(), + warnings: Vec::new(), + } + } + + /// Create a failure result. + pub fn failure(message: impl Into) -> Self { + Self { + success: false, + message: Some(message.into()), + data: HashMap::new(), + warnings: Vec::new(), + } + } + + /// Add data to the result. + pub fn with_data(mut self, key: impl Into, value: serde_json::Value) -> Self { + self.data.insert(key.into(), value); + self + } + + /// Add a warning to the result. + pub fn with_warning(mut self, warning: impl Into) -> Self { + self.warnings.push(warning.into()); + self + } +} + +#[macro_export] +macro_rules! success { + (fields($($field: ident = $value: expr),*), $message: literal, $($arg: expr),*) => { + tracing::info!( + $($field = $value),*, + "{}", + format!($message, $($arg),*).if_supports_color(owo_colors::Stream::Stderr, |text| text.bold().bright_green().to_string()) + ) + }; + (fields($($field: ident = $value: expr),*), $message: literal) => { + tracing::info!( + $($field = $value),*, + "{}", + format!($message).if_supports_color(owo_colors::Stream::Stderr, |text| text.bold().bright_green().to_string()) + ) + }; +} + +#[macro_export] +macro_rules! failure { + (fields($($field: ident = $value: expr),*), $message: literal, $($arg: expr),*) => { + tracing::error!( + $($field = $value),*, + "{}", + format!($message, $($arg),*).if_supports_color(owo_colors::Stream::Stderr, |text| text.bold().bright_red().to_string()) + ) + }; + (fields($($field: ident = $value: expr),*), $message: literal) => { + tracing::error!( + $($field = $value),*, + "{}", + format!($message).if_supports_color(owo_colors::Stream::Stderr, |text| text.bold().bright_red().to_string()) + ) + }; +} diff --git a/tools/casi/src/logging/visitor.rs b/tools/casi/src/logging/visitor.rs new file mode 100644 index 000000000..eb37ad61a --- /dev/null +++ b/tools/casi/src/logging/visitor.rs @@ -0,0 +1,330 @@ +//! Custom field visitor for CAS operation logging. +//! +//! The CasVisitor implements field formatting for tracing events, +//! with specific handling for artifact IDs, operation context, and metadata +//! used in CAS operations. It formats different field types with distinct +//! visual representations to improve log readability. + +use std::{collections::HashSet, fmt}; + +use owo_colors::{OwoColorize, Stream}; +use tracing::field::{Field, Visit}; +use tracing_subscriber::fmt::format::Writer; + +/// Custom field visitor for structured field display in CAS operations. +/// +/// The CasVisitor implements: +/// - Specific handling for artifact IDs and operation context +/// - Standardized formatting for timestamps and operation metadata +/// - Color differentiation for field names and values based on field type +/// - Hierarchical display of nested operation information +pub struct CasVisitor<'writer> { + writer: Writer<'writer>, + is_first_field: bool, + result: fmt::Result, + written: HashSet, +} + +impl<'writer> CasVisitor<'writer> { + /// Create a new CasVisitor with the given writer. + pub fn new(writer: Writer<'writer>) -> Self { + Self { + writer, + is_first_field: true, + result: Ok(()), + written: HashSet::new(), + } + } + + /// Get a mutable reference to the writer. + pub fn writer(&mut self) -> &mut Writer<'writer> { + &mut self.writer + } + + /// Write a field separator if this is not the first field. + fn write_separator(&mut self) { + if self.result.is_ok() { + if self.is_first_field { + self.is_first_field = false; + } else { + self.result = write!(self.writer, " "); + } + } + } + + /// Format a field name with appropriate styling. + fn format_field_name(&mut self, name: &str) { + if self.result.is_ok() { + self.result = write!( + self.writer, + "\n {}=", + name.if_supports_color(Stream::Stderr, |text| text.bold().blue().to_string()) + ); + } + } + + /// Format a field value with special handling for known CAS field types. + /// Returns whether the field was actually displayed (false if skipped). + fn format_field_value(&mut self, field: &Field, value: &dyn fmt::Display) { + if self.result.is_ok() { + if self.written.contains(field.name()) { + return; + } + self.written.insert(field.name().to_string()); + match field.name() { + // Special formatting for artifact IDs + "artifact_id" | "id" => { + // Clean up Option wrapper: convert 'Some("test_app:test")' to just 'test_app:test' + let value_str = format!("{value}"); + + // Skip printing artifact_id if it's None + if value_str == "None" { + // We do nothing, effectively skipping this field completely + return; + } + + // We need to write a separator since we're going to output this field + self.write_separator(); + self.format_field_name(field.name()); + + let cleaned_value = + if value_str.starts_with("Some(") && value_str.ends_with(")") { + // Extract content between quotes within Some() + if let Some(start) = value_str.find('"') { + if let Some(end) = value_str[start + 1..].find('"') { + &value_str[start + 1..start + 1 + end] + } else { + &value_str + } + } else { + // If no quotes found, try extracting what's between Some( and ) + &value_str[5..value_str.len() - 1] + } + } else { + &value_str + }; + + self.result = write!( + self.writer, + "{}", + cleaned_value.if_supports_color(Stream::Stderr, |text| text + .green() + .bold() + .to_string()) + ); + + // Return early as we've handled everything + } + // Special formatting for file paths + "path" | "file_path" | "source_path" | "dest_path" => { + self.write_separator(); + self.format_field_name(field.name()); + self.result = write!( + self.writer, + "\"{}\"", + value.if_supports_color(Stream::Stderr, |text| text.cyan().to_string()) + ); + } + // Special formatting for hashes + "hash" | "sha256" | "digest" => { + self.write_separator(); + self.format_field_name(field.name()); + let hash_str = format!("{value}"); + let truncated = if hash_str.len() > 12 { + format!("{}...", &hash_str[..12]) + } else { + hash_str + }; + let colored_value = + truncated.if_supports_color(Stream::Stderr, |text| text.yellow()); + self.result = write!(self.writer, "{colored_value}"); + } + // Special formatting for sizes and counts + "size" | "bytes" | "length" | "count" => { + self.write_separator(); + self.format_field_name(field.name()); + self.result = write!( + self.writer, + "{}", + value.if_supports_color(Stream::Stderr, |text| text.magenta().to_string()) + ); + } + // Special formatting for operation types + "operation" | "op_type" | "backend" => { + self.write_separator(); + self.format_field_name(field.name()); + self.result = write!( + self.writer, + "{}", + value.if_supports_color(Stream::Stderr, |text| text + .bright_blue() + .to_string()) + ); + } + // Special formatting for durations and timing + "elapsed" | "duration" | "timeout" => { + self.write_separator(); + self.format_field_name(field.name()); + self.result = write!( + self.writer, + "{}", + value.if_supports_color(Stream::Stderr, |text| text + .bright_cyan() + .to_string()) + ); + } + // Special formatting for error-related fields + "error" | "err" | "failure" => { + self.write_separator(); + self.format_field_name(field.name()); + self.result = write!( + self.writer, + "\"{}\"", + value.if_supports_color(Stream::Stderr, |text| text.red().to_string()) + ); + } + // Special formatting for status fields + "status" | "state" => { + self.write_separator(); + self.format_field_name(field.name()); + let status_str = format!("{value}"); + match status_str.to_lowercase().as_str() { + "success" | "ok" | "complete" | "finished" => { + let colored = + status_str.if_supports_color(Stream::Stderr, |text| text.green()); + self.result = write!(self.writer, "{colored}"); + } + "error" | "failed" | "failure" => { + let colored = + status_str.if_supports_color(Stream::Stderr, |text| text.red()); + self.result = write!(self.writer, "{colored}"); + } + "warning" | "warn" => { + let colored = + status_str.if_supports_color(Stream::Stderr, |text| text.yellow()); + self.result = write!(self.writer, "{colored}"); + } + "pending" | "in_progress" | "running" => { + let colored = + status_str.if_supports_color(Stream::Stderr, |text| text.blue()); + self.result = write!(self.writer, "{colored}"); + } + _ => { + let colored = + status_str.if_supports_color(Stream::Stderr, |text| text.white()); + self.result = write!(self.writer, "{colored}"); + } + } + } + // Default formatting for other fields + _ => { + self.write_separator(); + self.format_field_name(field.name()); + let value_str = format!("{value}"); + // Check if the value looks like a string that should be quoted + if value_str.contains(' ') + || value_str.contains('\t') + || value_str.contains('\n') + { + self.result = write!(self.writer, "\"{value_str}\""); + } else { + self.result = write!(self.writer, "{value_str}"); + } + } + } + } + } + + /// Handle the special "message" field which contains the main log message. + fn handle_message_field(&mut self, value: &dyn fmt::Display) { + if self.result.is_ok() { + // The message field is the main content, so we don't format it as key=value + // Instead, we just write it directly + self.result = write!(self.writer, "{value}"); + } + } +} + +impl<'writer> Visit for CasVisitor<'writer> { + fn record_f64(&mut self, field: &Field, value: f64) { + if field.name() == "message" { + self.write_separator(); + self.handle_message_field(&value); + } else { + // Note: Don't call write_separator() here as format_field_value + // handles artifact_id specially and may skip writing anything + self.format_field_value(field, &value); + } + } + + fn record_i64(&mut self, field: &Field, value: i64) { + if field.name() == "message" { + self.write_separator(); + self.handle_message_field(&value); + } else { + // Note: Don't call write_separator() here as format_field_value + // handles artifact_id specially and may skip writing anything + self.format_field_value(field, &value); + } + } + + fn record_u64(&mut self, field: &Field, value: u64) { + if field.name() == "message" { + self.write_separator(); + self.handle_message_field(&value); + } else { + // Note: Don't call write_separator() here as format_field_value + // handles artifact_id specially and may skip writing anything + self.format_field_value(field, &value); + } + } + + fn record_bool(&mut self, field: &Field, value: bool) { + if field.name() == "message" { + self.write_separator(); + self.handle_message_field(&value); + } else { + // For boolean values, handle specially since we don't call format_field_value + if field.name() == "artifact_id" || field.name() == "id" { + // Skip if it's an artifact_id field (though this shouldn't happen for booleans) + return; + } + + self.write_separator(); + self.format_field_name(field.name()); + if self.result.is_ok() { + if value { + let colored_value = + "true".if_supports_color(Stream::Stderr, |text| text.green()); + self.result = write!(self.writer, "{colored_value}"); + } else { + let colored_value = + "false".if_supports_color(Stream::Stderr, |text| text.red()); + self.result = write!(self.writer, "{colored_value}"); + } + } + } + } + + fn record_str(&mut self, field: &Field, value: &str) { + if field.name() == "message" { + self.write_separator(); + self.handle_message_field(&value); + } else { + // Note: Don't call write_separator() here as format_field_value + // handles artifact_id specially and may skip writing anything + self.format_field_value(field, &value); + } + } + + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + if field.name() == "message" { + self.write_separator(); + self.handle_message_field(&format_args!("{value:?}")); + } else { + // Note: Don't call write_separator() here as format_field_value + // handles artifact_id specially and may skip writing anything + self.format_field_value(field, &format_args!("{value:?}")); + } + } +}