0.7.35 - wordlist and replacement improvements.

Author: finnbear

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "rustrict"
 authors = ["Finn Bear"]
-version = "0.7.34"
+version = "0.7.35"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/finnbear/rustrict/"

README.md

@@ -177,7 +177,7 @@ is used as a dataset. Positive accuracy is the percentage of profanity detected
 
 | Crate | Accuracy | Positive Accuracy | Negative Accuracy | Time |
 |-------|----------|-------------------|-------------------|------|
-| [rustrict](https://crates.io/crates/rustrict) | 80.00%   | 93.98%            | 76.52%            | 9s   |
+| [rustrict](https://crates.io/crates/rustrict) | 80.00%   | 94.01%            | 76.50%            | 9s   |
 | [censor](https://crates.io/crates/censor) | 76.16%   | 72.76%            | 77.01%            | 23s  |
 | [stfu](https://crates.io/crates/stfu) | 91.74% | 77.69% | 95.25% | 45s |
 | [profane-rs](https://crates.io/crates/profane-rs) | 80.47% | 73.79% | 82.14% | 52s |

src/censor.rs

@@ -566,7 +566,7 @@ impl<I: Iterator<Item = char>> Iterator for Censor<I> {
                         // space.
                         // ( and ) are for ignoring appositive phrases.
                         // Checking node.last is to collapse multiple spaces into one
-                        let new_space = matches!(c, ' ' | '.' | ',' | ':' | ';' | '…' | '(' | ')')
+                        let new_space = matches!(c, ' ' | '.' | ',' | ':' | ';' | '…' | '(' | ')' | '_' | '-')
                             && m.node.last != Some(' ');
                         let new_repetition: bool = !new_space && c == m.last;
                         let new_skip = !new_space && skippable && !ignore_sep && !new_repetition;

src/character_analyzer.rs

@@ -47,7 +47,7 @@ fn main() {
         if let Some(c) = char::from_u32(u) {
             let max_width = match c {
                 '🐿' => 20,
-                '𒐫' => 40,
+                '𒐫' => 80,
                 '𒈙' => 35,
                 '༺' | '༻' => 25,
                 _ => {

src/character_widths.bin

@@ -87,6 +87,8 @@ faggetaboutit
 farming xp
 fatty acid
 fatty food
+femboys are awesome
+femboys are cool
 few secs
 ffa game
 fire cracker
@@ -99,6 +101,7 @@ freakin
 fuchs dystrophy
 fugia
 gaya
+gg german
 ght, its
 glhf
 graham cracker
@@ -196,6 +199,7 @@ pc master race
 pegging the
 plss
 plsss
+plz stop
 plzz
 plzzz
 pocock

src/dictionary_extra.txt

@@ -1049,6 +1049,8 @@ arco on
 arco vary
 arco ward
 arctocephalus
+are africans
+are asians
 areas hole
 ared skins
 arena holes
@@ -3490,6 +3492,7 @@ buzz ext
 buzz hilt
 buzz hit
 buzz lut
+buzz off
 buzz perm
 bytes cumulative
 bytes ext
@@ -3529,6 +3532,7 @@ caliphate
 cam girl
 camel tox
 campoo
+can't it
 canal
 canberra appeal
 canberra appear
@@ -5270,6 +5274,7 @@ directions lut
 directions perm
 directions seeks
 dirty juan
+dirty muslim
 disco jones
 disco on
 disco vary
@@ -6570,6 +6575,8 @@ felt chuck
 felt church
 felt xhtml
 females squirting
+femboys are awesome
+femboys are cool
 fend yourself
 fennig
 fers cumulative
@@ -7203,6 +7210,7 @@ geyan
 geyerite
 geylies
 geyser
+gg ger
 ghastful
 ghettoized
 ghettoizes
@@ -12187,6 +12195,8 @@ nu destin
 nu destroy
 nu destruct
 nu ger
+nuke iran
+nuke israel
 nurses cumulative
 nurses ext
 nurses hilt
@@ -13567,6 +13577,7 @@ plumbaginaceae
 plumbaginaceous
 plumbum
 plumigerous
+plz stop
 plzz
 pmsg
 pn lips
@@ -13581,6 +13592,7 @@ pockets perm
 pockets seeks
 pocock
 pogeys
+poggers
 pogonips
 points cumulative
 points ext
@@ -15285,6 +15297,20 @@ rico ward
 rid dicke
 rid licking
 rid ongoing
+ride mea
+ride mech
+ride med
+ride mee
+ride meg
+ride mel
+ride mem
+ride men
+ride mer
+ride mes
+ride met
+ride mexica
+ride mexico
+ride meyer
 riders cumulative
 riders ext
 riders hilt
@@ -17056,6 +17082,7 @@ spleening
 spleninii
 splice
 splicing
+splix
 sponsible peer
 spoorn
 sporadic
@@ -18770,6 +18797,7 @@ twattle
 twattling
 tweenies
 tweesht
+tweezer
 twigger
 twilit
 twilt
@@ -19447,6 +19475,7 @@ wan kr
 wan kurt
 wan kuwait
 wan ky
+wang ker
 wantwit
 wap anti
 wap peru

src/false_positives.txt

@@ -88,9 +88,10 @@ pub fn is_whitespace(c: char) -> bool {
     // https://www.compart.com/en/unicode/U+FFA0
     c.is_whitespace()
         || c.is_other()
+        || c.is_format()
         || matches!(
             c,
-            '\u{115F}' | '\u{1160}' | '\u{2800}' | '\u{3164}' | '\u{FFA0}'
+            '\u{115F}' | '\u{1160}' | '\u{2800}' | '\u{3164}' | '\u{FFA0}' | '\u{FFFC}'
         )
 }
 
@@ -113,7 +114,7 @@ mod tests {
         // Special cases.
         assert_eq!(
             crate::trim_whitespace(
-                "\u{0488}\u{1160}\u{0489}\u{1160}\u{0488}\u{1160}\u{0489}abc\u{0488}\u{0489}"
+                "\u{FFF9}\u{FFFA}\u{FFFB}\u{FFFC}\u{0488}\u{1160}\u{0489}\u{1160}\u{0488}\u{1160}\u{0489}abc\u{0488}\u{0489}"
             ),
             "abc\u{0488}\u{0489}"
         )