Merge branch 'main' into redis_reconnect

Author: MarcusSorealheis

.bazelrc

@@ -88,11 +88,13 @@ build --@rules_rust//:extra_rustc_flag=-Wtrivial_casts
 build --@rules_rust//:extra_rustc_flag=-Wtrivial_numeric_casts
 build --@rules_rust//:extra_rustc_flag=-Dunconditional_recursion
 build --@rules_rust//:extra_rustc_flag=-Dunexpected_cfgs
+build --@rules_rust//:extra_rustc_flag=-Dunknown_lints
 build --@rules_rust//:extra_rustc_flag=-Dunnameable_test_items
 build --@rules_rust//:extra_rustc_flag=-Wunreachable_pub
 build --@rules_rust//:extra_rustc_flag=-Dunsafe_op_in_unsafe_fn
 build --@rules_rust//:extra_rustc_flag=-Dunstable_syntax_pre_expansion
 build --@rules_rust//:extra_rustc_flag=-Wunused_import_braces
+build --@rules_rust//:extra_rustc_flag=-Dunused_imports
 build --@rules_rust//:extra_rustc_flag=-Wunused_lifetimes
 build --@rules_rust//:extra_rustc_flag=-Wunused_qualifications
 build --@rules_rust//:extra_rustc_flag=-Wvariant_size_differences
@@ -103,25 +105,40 @@ build --@rules_rust//:clippy_flag=-Wclippy::pedantic
 build --@rules_rust//:clippy_flag=-Dclippy::alloc_instead_of_core
 build --@rules_rust//:clippy_flag=-Dclippy::as_underscore
 build --@rules_rust//:clippy_flag=-Dclippy::await_holding_lock
+build --@rules_rust//:clippy_flag=-Dclippy::bind_instead_of_map
+build --@rules_rust//:clippy_flag=-Dclippy::collapsible_if
 build --@rules_rust//:clippy_flag=-Wclippy::dbg_macro
 build --@rules_rust//:clippy_flag=-Wclippy::decimal_literal_representation
+build --@rules_rust//:clippy_flag=-Dclippy::disallowed_methods
+build --@rules_rust//:clippy_flag=-Dclippy::doc_markdown
 build --@rules_rust//:clippy_flag=-Dclippy::elidable_lifetime_names
 build --@rules_rust//:clippy_flag=-Dclippy::explicit_into_iter_loop
 build --@rules_rust//:clippy_flag=-Dclippy::future_not_send
 build --@rules_rust//:clippy_flag=-Aclippy::get_unwrap
+build --@rules_rust//:clippy_flag=-Dclippy::implicit_clone
+build --@rules_rust//:clippy_flag=-Dclippy::implicit_hasher
+build --@rules_rust//:clippy_flag=-Dclippy::manual_is_variant_and
+build --@rules_rust//:clippy_flag=-Dclippy::map_unwrap_or
 build --@rules_rust//:clippy_flag=-Dclippy::missing_const_for_fn
 build --@rules_rust//:clippy_flag=-Aclippy::missing_docs_in_private_items
+build --@rules_rust//:clippy_flag=-Dclippy::or_fun_call
 build --@rules_rust//:clippy_flag=-Wclippy::print_stdout
+build --@rules_rust//:clippy_flag=-Dclippy::ptr_arg
 build --@rules_rust//:clippy_flag=-Dclippy::redundant_closure_for_method_calls
 build --@rules_rust//:clippy_flag=-Dclippy::semicolon_if_nothing_returned
+build --@rules_rust//:clippy_flag=-Dclippy::single_char_pattern
 build --@rules_rust//:clippy_flag=-Dclippy::std_instead_of_core
 build --@rules_rust//:clippy_flag=-Dclippy::string_lit_as_bytes
 build --@rules_rust//:clippy_flag=-Dclippy::todo
 build --@rules_rust//:clippy_flag=-Aclippy::too_long_first_doc_paragraph
+build --@rules_rust//:clippy_flag=-Dclippy::unchecked_duration_subtraction
 build --@rules_rust//:clippy_flag=-Wclippy::unimplemented
+build --@rules_rust//:clippy_flag=-Dclippy::unnecessary_semicolon
 build --@rules_rust//:clippy_flag=-Aclippy::unwrap_in_result
 build --@rules_rust//:clippy_flag=-Aclippy::unwrap_used
 build --@rules_rust//:clippy_flag=-Wclippy::use_debug
+build --@rules_rust//:clippy_flag=-Dclippy::used_underscore_binding
+build --@rules_rust//:clippy_flag=-Dclippy::useless_format
 build --@rules_rust//:clippy_flag=-Dclippy::cast_possible_truncation
 build --@rules_rust//:clippy_flag=-Aclippy::cast_possible_wrap
 build --@rules_rust//:clippy_flag=-Aclippy::cast_precision_loss

Cargo.toml

@@ -114,9 +114,11 @@ overlapping-range-endpoints = "deny"
 suspicious-double-ref-op = "deny"
 unconditional-recursion = "deny"
 unexpected-cfgs = "deny"
+unknown-lints = "deny"
 unnameable-test-items = "deny"
 unsafe-op-in-unsafe-fn = "deny"
 unstable-syntax-pre-expansion = "deny"
+unused-imports = "deny"
 
 keyword-idents = "warn"
 let-underscore = "warn"
@@ -147,14 +149,29 @@ pedantic = { level = "warn", priority = -1 }
 alloc-instead-of-core = "deny"
 as-underscore = "deny"
 await-holding-lock = "deny"
+bind-instead-of-map = "deny"
+collapsible-if = "deny"
+disallowed-methods = "deny"
+doc-markdown = "deny"
 elidable-lifetime-names = "deny"
 explicit-into-iter-loop = "deny"
 future-not-send = "deny"
+implicit-clone = "deny"
+implicit-hasher = "deny"
+manual-is-variant-and = "deny"
+map-unwrap-or = "deny"
+or-fun-call = "deny"
+ptr-arg = "deny"
 redundant-closure-for-method-calls = "deny"
 semicolon-if-nothing-returned = "deny"
+single-char-pattern = "deny"
 std-instead-of-core = "deny"
 string-lit-as-bytes = "deny"
 todo = "deny"
+unchecked-duration-subtraction = "deny"
+unnecessary-semicolon = "deny"
+used-underscore-binding = "deny"
+useless-format = "deny"
 
 # Restriction Warnings with default priority
 dbg-macro = "warn"

deployment-examples/metrics/docker-compose.yaml

@@ -47,7 +47,7 @@ services:
 
   # Grafana for visualization
   grafana:
-    image: grafana/grafana:10.3.0
+    image: grafana/grafana:12.4.0
     container_name: grafana
     restart: unless-stopped
     ports:

nativelink-config/generate-stores-config/src/main.rs

@@ -22,23 +22,31 @@ fn main() {
     for block in json_start.captures_iter(&stores_rs) {
         let start_marker = block.get(0).unwrap().end();
         let end_match = block_end.find(&stores_rs[start_marker..]).unwrap();
-        let end_marker =end_match.start();
-        let contents = &stores_rs[start_marker..(start_marker+end_marker)].split("\n").map(|line| line.replacen("///", "", 1)).collect::<Vec<String>>().join("\n");
+        let end_marker = end_match.start();
+        let contents = &stores_rs[start_marker..(start_marker + end_marker)]
+            .split("\n")
+            .map(|line| line.replacen("///", "", 1))
+            .collect::<Vec<String>>()
+            .join("\n");
         blocks.push(contents.trim().to_string());
     }
 
-    let mut output = String::from("// Generated by generate-stores-config from stores.rs comments for testing
+    let mut output = String::from(
+        "// Generated by generate-stores-config from stores.rs comments for testing
 {
   servers: [],
   stores: [
-");
+",
+    );
 
     for (index, contents) in blocks.iter().enumerate() {
-        let more_output = format!(r#"    {{
+        let more_output = format!(
+            r#"    {{
       name: "{index}",
       {contents}
     }},
-"#);
+"#
+        );
         output.push_str(&more_output);
     }
     output.push_str("]}\n");

nativelink-config/src/cas_server.rs

@@ -732,7 +732,7 @@ pub struct LocalWorkerConfig {
 
     /// Maximum time allowed for uploading action results to CAS after execution
     /// completes. If upload takes longer than this, the action fails with
-    /// DeadlineExceeded and may be retried by the scheduler. Value in seconds.
+    /// `DeadlineExceeded` and may be retried by the scheduler. Value in seconds.
     ///
     /// Default: 600 (seconds / 10 mins)
     #[serde(default, deserialize_with = "convert_duration_with_shellexpand")]

nativelink-config/src/schedulers.rs

@@ -128,7 +128,7 @@ pub struct SimpleSpec {
     /// any worker update before being timed out and re-queued.
     /// This applies regardless of worker keepalive status, catching cases
     /// where a worker is alive (sending keepalives) but stuck on a specific
-    /// action. Set to 0 to disable (relies only on worker_timeout_s).
+    /// action. Set to 0 to disable (relies only on `worker_timeout_s`).
     ///
     /// Default: 0 (disabled)
     #[serde(default, deserialize_with = "convert_duration_with_shellexpand")]

nativelink-config/src/stores.rs

@@ -616,7 +616,7 @@ pub struct FilesystemSpec {
     pub block_size: u64,
 
     /// Maximum number of concurrent write operations allowed.
-    /// Each write involves streaming data to a temp file and calling sync_all(),
+    /// Each write involves streaming data to a temp file and calling `sync_all()`,
     /// which can saturate disk I/O when many writes happen simultaneously.
     /// Limiting concurrency prevents disk saturation from blocking the async
     /// runtime.
@@ -1164,14 +1164,14 @@ pub struct RedisSpec {
     #[serde(deserialize_with = "convert_vec_string_with_shellexpand")]
     pub addresses: Vec<String>,
 
-    /// DEPRECATED: use command_timeout_ms
+    /// DEPRECATED: use `command_timeout_ms`
     /// The response timeout for the Redis connection in seconds.
     ///
     /// Default: 10
     #[serde(default)]
     pub response_timeout_s: u64,
 
-    /// DEPRECATED: use connection_timeout_ms
+    /// DEPRECATED: use `connection_timeout_ms`
     ///
     /// The connection timeout for the Redis connection in seconds.
     ///

nativelink-macro/src/lib.rs

@@ -13,32 +13,71 @@
 // limitations under the License.
 
 use proc_macro::TokenStream;
-use quote::quote;
+use proc_macro2::TokenTree;
+use quote::{format_ident, quote};
 use syn::{ItemFn, parse_macro_input};
 
+// Helper function for debugging. Add prettyplease as dependency
+//
+// fn unparse(input: proc_macro2::TokenStream) -> String {
+//     let item = syn::parse2(input).unwrap();
+//     let file = syn::File {
+//         attrs: vec![],
+//         items: vec![item],
+//         shebang: None,
+//     };
+
+//     prettyplease::unparse(&file)
+// }
+
+// Either use this as-is or as `#[nativelink_test("foo")]` where foo is the path for nativelink-util
+// Mostly used inside nativelink-util as `#[nativelink_test("crate")]`
+// If you start it with an ident instead, e.g. `#[nativelink_test(flavor = "multi_thread")]` we feed it into tokio::test
 #[proc_macro_attribute]
 pub fn nativelink_test(attr: TokenStream, item: TokenStream) -> TokenStream {
     let attr = proc_macro2::TokenStream::from(attr);
     let input_fn = parse_macro_input!(item as ItemFn);
+    let mut maybe_crate_ident: Option<proc_macro2::TokenStream> = None;
+    let mut maybe_tokio_attrs: Option<proc_macro2::TokenStream> = None;
+
+    for a in attr.clone() {
+        assert!(maybe_crate_ident.is_none());
+
+        match a {
+            TokenTree::Literal(l) => {
+                let s = format_ident!("{}", l.to_string().replace('"', ""));
+                maybe_crate_ident = Some(quote! {#s});
+            }
+            TokenTree::Ident(_) => {
+                maybe_tokio_attrs = Some(attr);
+                break;
+            }
+            _ => {
+                panic!("unsupported tokentree: {a:?}");
+            }
+        }
+    }
 
     let fn_name = &input_fn.sig.ident;
     let fn_block = &input_fn.block;
     let fn_inputs = &input_fn.sig.inputs;
     let fn_output = &input_fn.sig.output;
     let fn_attr = &input_fn.attrs;
+    let crate_ident = maybe_crate_ident.unwrap_or_else(|| quote!(::nativelink_util));
+    let tokio_attrs = maybe_tokio_attrs.unwrap_or_else(|| quote!());
 
     let expanded = quote! {
         #(#fn_attr)*
         #[expect(
             clippy::disallowed_methods,
             reason = "`tokio::test` uses `tokio::runtime::Runtime::block_on`"
         )]
-        #[tokio::test(#attr)]
+        #[tokio::test(#tokio_attrs)]
         #[::tracing_test::traced_test]
         async fn #fn_name(#fn_inputs) #fn_output {
-            ::nativelink_util::__tracing::error_span!(stringify!(#fn_name))
+            #crate_ident::__tracing::error_span!(stringify!(#fn_name))
                 .in_scope(|| async move {
-                    ::nativelink_util::common::reseed_rng_for_test().unwrap();
+                    #crate_ident::common::reseed_rng_for_test().unwrap();
                     let res = #fn_block;
                     logs_assert(|lines: &[&str]| {
                         for line in lines {

nativelink-proto/gen_lib_rs_tool.py

@@ -50,6 +50,8 @@
     clippy::missing_const_for_fn,
     clippy::similar_names,
     clippy::std_instead_of_core,
+    clippy::use_self,
+    rustdoc::broken_intra_doc_links,
     rustdoc::invalid_html_tags
 )]
 """

nativelink-proto/genproto/lib.rs

@@ -30,6 +30,8 @@
     clippy::missing_const_for_fn,
     clippy::similar_names,
     clippy::std_instead_of_core,
+    clippy::use_self,
+    rustdoc::broken_intra_doc_links,
     rustdoc::invalid_html_tags
 )]