v2.0: Track position where normalization failed, bump MSRV, edition 2021

Author: clarfonthey

.travis.yml

@@ -1,7 +1,7 @@
 [package]
 
 name = "unicode-normalization"
-version = "0.1.24"
+version = "0.2.0"
 authors = [
     "kwantam <[email protected]>",
     "Manish Goregaokar <[email protected]>",
@@ -27,17 +27,16 @@ Decomposition and Recomposition, as described in
 Unicode Standard Annex #15.
 """
 
-rust-version = "1.36"
+rust-version = "1.84"
 
-edition = "2018"
+edition = "2021"
 
 exclude = ["target/*", "Cargo.lock", "scripts/tmp", "*.txt", "tests/*"]
 
 [dependencies.tinyvec]
 version = "1"
 features = ["alloc"]
 
-
 [features]
 default = ["std"]
 std = []

Cargo.toml

@@ -1,13 +1,12 @@
 # unicode-normalization
 
-[![Build Status](https://travis-ci.org/unicode-rs/unicode-normalization.svg)](https://travis-ci.org/unicode-rs/unicode-normalization)
 [![Docs](https://docs.rs/unicode-normalization/badge.svg)](https://docs.rs/unicode-normalization/)
 
 Unicode character composition and decomposition utilities
 as described in
 [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/).
 
-This crate requires Rust 1.36+.
+This crate requires Rust 1.84+.
 
 ```rust
 extern crate unicode_normalization;
@@ -31,21 +30,9 @@ to your `Cargo.toml`:
 
 ```toml
 [dependencies]
-unicode-normalization = "0.1.24"
+unicode-normalization = "0.2.0"
 ```
 
 ## `no_std` + `alloc` support
 
 This crate is completely `no_std` + `alloc` compatible. This can be enabled by disabling the `std` feature, i.e. specifying `default-features = false` for this crate on your `Cargo.toml`.
-
-## Note about MSRV
-
-Dependencies' MSRVs evolve independently of this crate's MSRV.
-Old versions of cargo will always try to get the most recent versions of the dependencies.
-Therefore, if you are having troubles compiling on an old Rust version, try to install an older version of the incompatible dependency.
-
-For instance, to compile on Rust 1.36, `tinyvec` must be `<=1.6.0`
-
-```sh
-cargo update -p tinyvec --precise 1.6.0
-```

README.md

@@ -4,74 +4,66 @@
 extern crate libfuzzer_sys;
 
 use unicode_normalization::{
-    is_nfc, is_nfc_quick, is_nfc_stream_safe, is_nfc_stream_safe_quick, is_nfd, is_nfd_quick,
-    is_nfd_stream_safe, is_nfd_stream_safe_quick, is_nfkc, is_nfkc_quick, is_nfkd, is_nfkd_quick,
-    IsNormalized, UnicodeNormalization,
+    check_nfc_quick, check_nfc_stream_safe_quick, check_nfd_quick, check_nfd_stream_safe_quick,
+    check_nfkc_quick, check_nfkd_quick, is_nfc, is_nfc_stream_safe, is_nfd, is_nfd_stream_safe,
+    is_nfkc, is_nfkd, UnicodeNormalization,
 };
 
-fn from_bool(is_normalized: bool) -> IsNormalized {
-    if is_normalized {
-        IsNormalized::Yes
-    } else {
-        IsNormalized::No
-    }
-}
-
 fuzz_target!(|input: String| {
     // The full predicates imply the quick predicates.
-    assert_ne!(is_nfc_quick(input.chars()), from_bool(!is_nfc(&input)));
-    assert_ne!(is_nfd_quick(input.chars()), from_bool(!is_nfd(&input)));
-    assert_ne!(is_nfkc_quick(input.chars()), from_bool(!is_nfkc(&input)));
-    assert_ne!(is_nfkd_quick(input.chars()), from_bool(!is_nfkd(&input)));
-    assert_ne!(
-        is_nfc_stream_safe_quick(input.chars()),
-        from_bool(!is_nfc_stream_safe(&input))
+    assert_eq!(check_nfc_quick(&input).is_ok(), is_nfc(&input));
+    assert_eq!(check_nfd_quick(&input).is_ok(), is_nfd(&input));
+    assert_eq!(check_nfkc_quick(&input).is_ok(), is_nfkc(&input));
+    assert_eq!(check_nfkd_quick(&input).is_ok(), is_nfkd(&input));
+    assert_eq!(
+        check_nfc_stream_safe_quick(&input).is_ok(),
+        is_nfc_stream_safe(&input)
     );
-    assert_ne!(
-        is_nfd_stream_safe_quick(input.chars()),
-        from_bool(!is_nfd_stream_safe(&input))
+    assert_eq!(
+        check_nfd_stream_safe_quick(&input).is_ok(),
+        is_nfd_stream_safe(&input)
     );
 
     // Check NFC, NFD, NFKC, and NFKD normalization.
-    let nfc = input.chars().nfc().collect::<String>();
+    let nfc = input.nfc().collect::<String>();
     assert_eq!(nfc.is_empty(), input.is_empty());
-    assert_ne!(is_nfc_quick(nfc.chars()), IsNormalized::No);
+    assert!(check_nfc_quick(&nfc).is_ok());
     assert!(is_nfc(&nfc));
 
-    let nfd = input.chars().nfd().collect::<String>();
+    let nfd = input.nfd().collect::<String>();
     assert!(nfd.len() >= nfc.len());
-    assert_ne!(is_nfd_quick(nfd.chars()), IsNormalized::No);
+    assert!(check_nfd_quick(&nfd).is_ok());
     assert!(is_nfd(&nfd));
 
-    let nfkc = input.chars().nfkc().collect::<String>();
+    let nfkc = input.nfkc().collect::<String>();
     assert_eq!(nfkc.is_empty(), input.is_empty());
-    assert_ne!(is_nfkc_quick(nfkc.chars()), IsNormalized::No);
+    assert!(check_nfkc_quick(&nfkc).is_ok());
     assert!(is_nfkc(&nfkc));
 
-    let nfkd = input.chars().nfkd().collect::<String>();
+    let nfkd = input.nfkd().collect::<String>();
     assert!(nfkd.len() >= nfkc.len());
-    assert_ne!(is_nfkd_quick(nfkd.chars()), IsNormalized::No);
+    assert!(check_nfkd_quick(&nfkd).is_ok());
     assert!(is_nfkd(&nfkd));
 
     // Check stream-safe.
-    let nfc_ss = nfc.chars().stream_safe().collect::<String>();
+    let nfc_ss = nfc.stream_safe().collect::<String>();
     assert!(nfc_ss.len() >= nfc.len());
-    assert_ne!(is_nfc_stream_safe_quick(nfc_ss.chars()), IsNormalized::No);
+    assert!(check_nfc_stream_safe_quick(&nfc_ss).is_ok());
     assert!(is_nfc_stream_safe(&nfc_ss));
 
-    let nfd_ss = nfd.chars().stream_safe().collect::<String>();
+    let nfd_ss = nfd.stream_safe().collect::<String>();
     assert!(nfd_ss.len() >= nfd.len());
-    assert_ne!(is_nfd_stream_safe_quick(nfd_ss.chars()), IsNormalized::No);
+    assert!(check_nfd_stream_safe_quick(&nfd_ss).is_ok());
     assert!(is_nfd_stream_safe(&nfd_ss));
 
     // Check that NFC and NFD preserve stream-safe.
-    let ss_nfc = input.chars().stream_safe().nfc().collect::<String>();
+    let ss_nfc = input.stream_safe().nfc().collect::<String>();
     assert_eq!(ss_nfc.is_empty(), input.is_empty());
-    assert_ne!(is_nfc_stream_safe_quick(ss_nfc.chars()), IsNormalized::No);
+    assert!(check_nfc_stream_safe_quick(&ss_nfc).is_ok());
     assert!(is_nfc_stream_safe(&ss_nfc));
 
-    let ss_nfd = input.chars().stream_safe().nfd().collect::<String>();
+    let ss_nfd = input.stream_safe().nfd().collect::<String>();
     assert_eq!(ss_nfd.is_empty(), input.is_empty());
-    assert_ne!(is_nfd_stream_safe_quick(ss_nfd.chars()), IsNormalized::No);
+    assert!(check_nfd_stream_safe_quick(&ss_nfd).is_ok());
     assert!(is_nfd_stream_safe(&ss_nfd));
 });

fuzz/fuzz_targets/unicode-normalization.rs

@@ -463,7 +463,8 @@ def gen_public_assigned(general_category_public_assigned, out):
     # This could be done as a hash but the table is somewhat small.
     out.write("#[inline]\n")
     out.write("pub fn is_public_assigned(c: char) -> bool {\n")
-    out.write("    match c {\n")
+    out.write("    matches!(\n")
+    out.write("        c,\n")
 
     start = True
     for first, last in general_category_public_assigned:
@@ -476,10 +477,9 @@ def gen_public_assigned(general_category_public_assigned, out):
             out.write("'\\u{%s}'" % hexify(first))
         else:
             out.write("'\\u{%s}'..='\\u{%s}'" % (hexify(first), hexify(last)))
-    out.write(" => true,\n")
+    out.write(",\n")
 
-    out.write("        _ => false,\n")
-    out.write("    }\n")
+    out.write("    )\n")
     out.write("}\n")
 
 def gen_stream_safe(leading, trailing, out):

scripts/unicode.py

@@ -34,7 +34,7 @@
 //!
 //! ```toml
 //! [dependencies]
-//! unicode-normalization = "0.1.20"
+#![doc = concat!("unicode-normalization = \"", env!("CARGO_PKG_VERSION"), "\"")]
 //! ```
 
 #![deny(missing_docs, unsafe_code)]
@@ -54,9 +54,10 @@ extern crate tinyvec;
 
 pub use crate::decompose::Decompositions;
 pub use crate::quick_check::{
-    is_nfc, is_nfc_quick, is_nfc_stream_safe, is_nfc_stream_safe_quick, is_nfd, is_nfd_quick,
-    is_nfd_stream_safe, is_nfd_stream_safe_quick, is_nfkc, is_nfkc_quick, is_nfkd, is_nfkd_quick,
-    IsNormalized,
+    check_nfc, check_nfc_quick, check_nfc_stream_safe, check_nfc_stream_safe_quick, check_nfd,
+    check_nfd_quick, check_nfd_stream_safe, check_nfd_stream_safe_quick, check_nfkc,
+    check_nfkc_quick, check_nfkd, check_nfkd_quick, is_nfc, is_nfc_stream_safe, is_nfd,
+    is_nfd_stream_safe, is_nfkc, is_nfkd, NormalizationError, QuickCheck,
 };
 pub use crate::recompose::Recompositions;
 pub use crate::replace::Replacements;