diff --git a/crates/cargo-codspeed/src/build.rs b/crates/cargo-codspeed/src/build.rs
index ab099b6d..6bf9c117 100644
--- a/crates/cargo-codspeed/src/build.rs
+++ b/crates/cargo-codspeed/src/build.rs
@@ -145,8 +145,9 @@ impl BuildOptions<'_> {
                 // Use --config to set rustflags
                 // Our rust integration has an msrv of 1.74, --config is available since 1.63
                 // https://doc.rust-lang.org/nightly/cargo/CHANGELOG.html#cargo-163-2022-08-11
+                // Note: We have to use `target.cfg(all())` since `build` has a lower precedence.
                 let config_value = format!(
-                    "build.rustflags=[{}]",
+                    "target.'cfg(all())'.rustflags=[{}]",
                     flags.into_iter().map(|f| format!("\"{f}\"")).join(",")
                 );
                 cargo.arg("--config").arg(config_value);
diff --git a/crates/cargo-codspeed/tests/cargo_config.in/.cargo/config.toml b/crates/cargo-codspeed/tests/cargo_config.in/.cargo/config.toml
index 7b4d0356..d1905f4b 100644
--- a/crates/cargo-codspeed/tests/cargo_config.in/.cargo/config.toml
+++ b/crates/cargo-codspeed/tests/cargo_config.in/.cargo/config.toml
@@ -1,2 +1,10 @@
+# Note: This does nothing since `target.<cfg>.rustflags` takes precedence over `build.rustflags`
+# See: https://doc.rust-lang.org/nightly/cargo/reference/config.html#buildrustflags
 [build]
+rustflags = ["--cfg", "this_does_nothing"]
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["--cfg", "target_feature_flag"]
+
+[target.'cfg(all())']
 rustflags = ["--cfg", "custom_feature_flag"]
diff --git a/crates/cargo-codspeed/tests/cargo_config.in/benches/cargo_config_bench.rs b/crates/cargo-codspeed/tests/cargo_config.in/benches/cargo_config_bench.rs
index d763db64..36c2478c 100644
--- a/crates/cargo-codspeed/tests/cargo_config.in/benches/cargo_config_bench.rs
+++ b/crates/cargo-codspeed/tests/cargo_config.in/benches/cargo_config_bench.rs
@@ -8,6 +8,11 @@ fn bench_failing_without_custom_flag(c: &mut Criterion) {
             compile_error!(
                 "custom_feature_flag is not enabled - .cargo/config.toml rustflags not applied"
             );
+
+            #[cfg(not(target_feature_flag))]
+            compile_error!(
+                "target_feature_flag is not enabled - .cargo/config.toml rustflags not applied"
+            );
         })
     });
 }