Skip to content

Drop const_concat! macro and str-focused const combine functions #5315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
davidhewitt merged 2 commits into from
Aug 13, 2025
Merged

Drop const_concat! macro and str-focused const combine functions #5315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8b3c986
Drop const_concat! macro and str-focused const combine functions
Tpt Aug 13, 2025
3e80b66
Makes combine not `pub`
Tpt Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions pyo3-macros-backend/src/introspection.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ impl IntrospectionNode<'_> {
const _: () = {
#[used]
#[no_mangle]
static #static_name: &'static str = #content;
static #static_name: &'static [u8] = #content;
};
}
}
Expand Down Expand Up @@ -485,11 +485,16 @@ impl ConcatenationBuilder {

if let [ConcatenationBuilderElement::String(string)] = elements.as_slice() {
// We avoid the const_concat! macro if there is only a single string
return string.to_token_stream();
return quote! { #string.as_bytes() };
}

quote! {
#pyo3_crate_path::impl_::concat::const_concat!(#(#elements , )*)
{
const PIECES: &[&[u8]] = &[#(#elements.as_bytes() , )*];
&#pyo3_crate_path::impl_::concat::combine_to_array::<{
#pyo3_crate_path::impl_::concat::combined_len(PIECES)
}>(PIECES)
}
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions pyo3-macros-backend/src/pyclass.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2396,8 +2396,8 @@ impl<'a> PyClassImplsBuilder<'a> {
#cls,
{ impl_::pyclass::HasNewTextSignature::<#cls>::VALUE }
>::DOC_PIECES;
const LEN: usize = impl_::concat::combined_len_bytes(DOC_PIECES);
const DOC: &'static [u8] = &impl_::concat::combine_bytes_to_array::<LEN>(DOC_PIECES);
const LEN: usize = impl_::concat::combined_len(DOC_PIECES);
const DOC: &'static [u8] = &impl_::concat::combine_to_array::<LEN>(DOC_PIECES);
impl_::pyclass::doc::doc_bytes_as_cstr(DOC)
};

Expand Down
139 changes: 11 additions & 128 deletions src/impl_/concat.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,86 +1,8 @@
/// `concat!` but working with constants
#[macro_export]
#[doc(hidden)]
macro_rules! const_concat {
($e:expr) => {{
$e
}};
($l:expr, $($r:expr),+ $(,)?) => {{
const PIECES: &[&str] = &[$l, $($r),+];
const RAW_BYTES: &[u8] = &$crate::impl_::concat::combine_to_array::<{
$crate::impl_::concat::combined_len(PIECES)
}>(PIECES);
// Safety: `RAW_BYTES` is combined from valid &str pieces
unsafe { ::std::str::from_utf8_unchecked(RAW_BYTES) }
}}
}

pub use const_concat;

/// Calculates the total byte length of all string pieces in the array.
///
/// This is a useful utility in order to determine the size needed for the constant
/// `combine` function.
pub const fn combined_len(pieces: &[&str]) -> usize {
let mut len = 0;
let mut pieces_idx = 0;
while pieces_idx < pieces.len() {
len += pieces[pieces_idx].len();
pieces_idx += 1;
}
len
}

/// Combines all string pieces into a single byte array.
///
/// `out` should be a buffer at the correct size of `combined_len(pieces)`, else this will panic.
#[cfg(mut_ref_in_const_fn)] // requires MSRV 1.83
#[allow(clippy::incompatible_msrv)] // `split_at_mut` also requires MSRV 1.83
pub const fn combine(pieces: &[&str], mut out: &mut [u8]) {
let mut pieces_idx = 0;
while pieces_idx < pieces.len() {
let piece = pieces[pieces_idx].as_bytes();
slice_copy_from_slice(out, piece);
// using split_at_mut because range indexing not yet supported in const fn
out = out.split_at_mut(piece.len()).1;
pieces_idx += 1;
}
// should be no trailing buffer
assert!(out.is_empty(), "output buffer too large");
}

/// Wrapper around combine which has a const generic parameter, this is going to be more codegen
/// at compile time (?)
///
/// Unfortunately the `&mut [u8]` buffer needs MSRV 1.83
pub const fn combine_to_array<const LEN: usize>(pieces: &[&str]) -> [u8; LEN] {
let mut out: [u8; LEN] = [0u8; LEN];
#[cfg(mut_ref_in_const_fn)]
combine(pieces, &mut out);
#[cfg(not(mut_ref_in_const_fn))] // inlined here for higher code
{
let mut out_idx = 0;
let mut pieces_idx = 0;
while pieces_idx < pieces.len() {
let piece = pieces[pieces_idx].as_bytes();
let mut piece_idx = 0;
while piece_idx < piece.len() {
out[out_idx] = piece[piece_idx];
out_idx += 1;
piece_idx += 1;
}
pieces_idx += 1;
}
assert!(out_idx == out.len(), "output buffer too large");
}
out
}

/// Calculates the total byte length of all byte pieces in the array.
///
/// This is a useful utility in order to determine the size needed for the constant
/// `combine_bytes` function.
pub const fn combined_len_bytes(pieces: &[&[u8]]) -> usize {
/// `combine` function.
pub const fn combined_len(pieces: &[&[u8]]) -> usize {
let mut len = 0;
let mut pieces_idx = 0;
while pieces_idx < pieces.len() {
Expand All @@ -95,7 +17,7 @@ pub const fn combined_len_bytes(pieces: &[&[u8]]) -> usize {
/// `out` should be a buffer at the correct size of `combined_len(pieces)`, else this will panic.
#[cfg(mut_ref_in_const_fn)] // requires MSRV 1.83
#[allow(clippy::incompatible_msrv)] // `split_at_mut` also requires MSRV 1.83
pub const fn combine_bytes(pieces: &[&[u8]], mut out: &mut [u8]) {
Copy link
Contributor Author

@Tpt Tpt Aug 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we need combine to be pub

const fn combine(pieces: &[&[u8]], mut out: &mut [u8]) {
let mut pieces_idx = 0;
while pieces_idx < pieces.len() {
let piece = pieces[pieces_idx];
Expand All @@ -108,14 +30,14 @@ pub const fn combine_bytes(pieces: &[&[u8]], mut out: &mut [u8]) {
assert!(out.is_empty(), "output buffer too large");
}

/// Wrapper around `combine_bytes` which has a const generic parameter, this is going to be more codegen
/// Wrapper around `combine` which has a const generic parameter, this is going to be more codegen
/// at compile time (?)
///
/// Unfortunately the `&mut [u8]` buffer needs MSRV 1.83
pub const fn combine_bytes_to_array<const LEN: usize>(pieces: &[&[u8]]) -> [u8; LEN] {
pub const fn combine_to_array<const LEN: usize>(pieces: &[&[u8]]) -> [u8; LEN] {
let mut out: [u8; LEN] = [0u8; LEN];
#[cfg(mut_ref_in_const_fn)]
combine_bytes(pieces, &mut out);
combine(pieces, &mut out);
#[cfg(not(mut_ref_in_const_fn))] // inlined here for higher code
{
let mut out_idx = 0;
Expand Down Expand Up @@ -151,71 +73,32 @@ mod tests {

#[test]
fn test_combined_len() {
let pieces = ["foo", "bar", "baz"];
let pieces: [&[u8]; 3] = [b"foo", b"bar", b"baz"];
assert_eq!(combined_len(&pieces), 9);
let empty: [&str; 0] = [];
let empty: [&[u8]; 0] = [];
assert_eq!(combined_len(&empty), 0);
}

#[test]
fn test_combine_to_array() {
let pieces = ["foo", "bar"];
let combined = combine_to_array::<6>(&pieces);
assert_eq!(&combined, b"foobar");
}

#[test]
fn test_const_concat_macro() {
const RESULT: &str = const_concat!("foo", "bar", "baz");
assert_eq!(RESULT, "foobarbaz");
const SINGLE: &str = const_concat!("abc");
assert_eq!(SINGLE, "abc");
}

#[test]
fn test_combined_len_bytes() {
let pieces: [&[u8]; 3] = [b"foo", b"bar", b"baz"];
assert_eq!(combined_len_bytes(&pieces), 9);
let empty: [&[u8]; 0] = [];
assert_eq!(combined_len_bytes(&empty), 0);
}

#[test]
fn test_combine_bytes_to_array() {
let pieces: [&[u8]; 2] = [b"foo", b"bar"];
let combined = combine_bytes_to_array::<6>(&pieces);
let combined = combine_to_array::<6>(&pieces);
assert_eq!(&combined, b"foobar");
}

#[test]
#[should_panic(expected = "index out of bounds")]
fn test_combine_to_array_buffer_too_small() {
let pieces = ["foo", "bar"];
let pieces: [&[u8]; 2] = [b"foo", b"bar"];
// Intentionally wrong length
let _ = combine_to_array::<5>(&pieces);
}

#[test]
#[should_panic(expected = "output buffer too large")]
fn test_combine_to_array_buffer_too_big() {
let pieces = ["foo", "bar"];
// Intentionally wrong length
let _ = combine_to_array::<10>(&pieces);
}

#[test]
#[should_panic(expected = "index out of bounds")]
fn test_combine_bytes_to_array_buffer_too_small() {
let pieces: [&[u8]; 2] = [b"foo", b"bar"];
// Intentionally wrong length
let _ = combine_bytes_to_array::<5>(&pieces);
}

#[test]
#[should_panic(expected = "output buffer too large")]
fn test_combine_bytes_to_array_buffer_too_big() {
let pieces: [&[u8]; 2] = [b"foo", b"bar"];
// Intentionally wrong length
let _ = combine_bytes_to_array::<10>(&pieces);
let _ = combine_to_array::<10>(&pieces);
}
}
6 changes: 3 additions & 3 deletions src/impl_/pyclass/doc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ mod tests {
#[test]
#[cfg(feature = "macros")]
fn test_doc_generator() {
use crate::impl_::concat::{combine_bytes_to_array, combined_len_bytes};
use crate::impl_::concat::{combine_to_array, combined_len};

/// A dummy class with signature.
#[crate::pyclass(crate = "crate")]
Expand All @@ -86,15 +86,15 @@ mod tests {
// simulate what the macro is doing
const PIECES: &[&[u8]] = PyClassDocGenerator::<MyClass, true>::DOC_PIECES;
assert_eq!(
&combine_bytes_to_array::<{ combined_len_bytes(PIECES) }>(PIECES),
&combine_to_array::<{ combined_len(PIECES) }>(PIECES),
b"MyClass(x, y)\n--\n\nA dummy class with signature.\0"
);

// simulate if the macro detected no text signature
const PIECES_WITHOUT_SIGNATURE: &[&[u8]] =
PyClassDocGenerator::<MyClass, false>::DOC_PIECES;
assert_eq!(
&combine_bytes_to_array::<{ combined_len_bytes(PIECES_WITHOUT_SIGNATURE) }>(
&combine_to_array::<{ combined_len(PIECES_WITHOUT_SIGNATURE) }>(
PIECES_WITHOUT_SIGNATURE
),
b"A dummy class with signature.\0"
Expand Down
Loading