fix: handle UTF-8 characters in name truncation

digizeph · digizeph · commit bd5d29030978 · 2026-02-28T06:52:29.000-08:00
Fixed panic when truncating organization names containing multi-byte UTF-8 characters (e.g., Arabic, Chinese, emoji). Changed from byte-based to character-based string operations. Changes: - Fixed truncate_name to use chars().count() and chars().take() instead of byte-based slicing that could panic on multi-byte characters - Moved utils module from lens::utils to crate-level utils for better organization (eliminates misleading structure) - Added tests for Arabic, mixed scripts, and emoji character handling Fixes #115
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,22 +4,6 @@ All notable changes to this project will be documented in this file.
 
 ## Unreleased changes
 
-### Breaking Changes
-
-* Moved config, data, and cache paths to follow XDG base directory specification
-  * Config file default is now `$XDG_CONFIG_HOME/monocle/monocle.toml` (fallback: `~/.config/monocle/monocle.toml`)
-  * Data directory default is now `$XDG_DATA_HOME/monocle` (fallback: `~/.local/share/monocle`)
-  * Cache directory default is now `$XDG_CACHE_HOME/monocle` (fallback: `~/.cache/monocle`)
-  * Existing SQLite data under `~/.monocle` is no longer used by default and will be rebuilt in the new data location
-    * Legacy config migration: when the new config directory is empty, monocle copies `~/.monocle/monocle.toml` to the new config path
-    * Old database file will not be copied over. Once the updated monocle has been executed at least once, old `~/.monocle` can be safely deleted
-* Added `--use-cache` flag to `monocle search` to use the default XDG cache path (`$XDG_CACHE_HOME/monocle`)
-  * Value set by `--cache-dir` overrides `--use-cache` when both are provided
-
-### Dependencies
-
-* Switched directory resolution library from `dirs` to `etcetera`
-
 ### New Features
 
 * Added BGP community filtering support to `monocle parse` and `monocle search`
@@ -35,9 +19,34 @@ All notable changes to this project will be documented in this file.
   * `--ts-start` is now accepted as an alias for `--start-ts`
   * `--ts-end` is now accepted as an alias for `--end-ts`
 
+### Bug Fixes
+
+* Fixed panic when truncating names containing non-ASCII UTF-8 characters
+  * Used character-based truncation instead of byte-based slicing
+  * Affects RPKI ASPA display and inspect lens name truncation
+
+### Directory Changes
+
+* Changed config, data, and cache paths to follow XDG base directory specification
+  * Config file default is now `$XDG_CONFIG_HOME/monocle/monocle.toml` (fallback: `~/.config/monocle/monocle.toml`)
+  * Data directory default is now `$XDG_DATA_HOME/monocle` (fallback: `~/.local/share/monocle`)
+  * Cache directory default is now `$XDG_CACHE_HOME/monocle` (fallback: `~/.cache/monocle`)
+  * Existing SQLite data under `~/.monocle` is no longer used by default and will be rebuilt in the new data location
+    * Legacy config migration: when the new config directory is empty, monocle copies `~/.monocle/monocle.toml` to the new config path
+    * Old database file will not be copied over. Once the updated monocle has been executed at least once, old `~/.monocle` can be safely deleted
+* Added `--use-cache` flag to `monocle search` to use the default XDG cache path (`$XDG_CACHE_HOME/monocle`)
+  * Value set by `--cache-dir` overrides `--use-cache` when both are provided
+
 ### Code Improvements
 
 * Updated README command help examples to match current CLI help output from the release binary
+* Moved `utils` module from `lens::utils` to crate-level `utils`
+  * Eliminates misleading module structure since utilities are used throughout the codebase
+  * Updated all imports from `crate::lens::utils` and `monocle::lens::utils` to `crate::utils` and `monocle::utils`
+
+### Dependencies
+
+* Switched directory resolution library from `dirs` to `etcetera`
 
 ## v1.1.0 - 2025-02-10
 
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -158,7 +158,7 @@ dateparser = { version = "0.2", optional = true }
 humantime = { version = "2.1", optional = true }
 bgpkit-broker = { version = "0.10.1", optional = true }
 bgpkit-parser = { version = "0.15.0", features = ["serde"], optional = true }
-bgpkit-commons = { version = "0.10.1", features = ["asinfo", "rpki", "countries"], optional = true }
+bgpkit-commons = { version = "0.10.2", features = ["asinfo", "rpki", "countries"], optional = true }
 itertools = { version = "0.14", optional = true }
 radar-rs = { version = "0.1.0", optional = true }
 rayon = { version = "1.8", optional = true }
diff --git a/src/bin/commands/as2rel.rs b/src/bin/commands/as2rel.rs
@@ -1,7 +1,7 @@
 use clap::Args;
 use monocle::database::MonocleDatabase;
 use monocle::lens::as2rel::{As2relLens, As2relSearchArgs};
-use monocle::lens::utils::{truncate_name, OutputFormat, DEFAULT_NAME_MAX_LEN};
+use monocle::utils::{truncate_name, OutputFormat, DEFAULT_NAME_MAX_LEN};
 use monocle::MonocleConfig;
 use serde::Serialize;
 use serde_json::json;
diff --git a/src/bin/commands/config.rs b/src/bin/commands/config.rs
@@ -6,8 +6,8 @@ use monocle::config::{
 };
 use monocle::database::{MonocleDatabase, Pfx2asDbRecord};
 use monocle::lens::rpki::RpkiLens;
-use monocle::lens::utils::OutputFormat;
 use monocle::server::ServerConfig;
+use monocle::utils::OutputFormat;
 use monocle::MonocleConfig;
 use serde::Serialize;
 use std::path::Path;
diff --git a/src/bin/commands/country.rs b/src/bin/commands/country.rs
@@ -1,6 +1,6 @@
 use clap::Args;
 use monocle::lens::country::{CountryEntry, CountryLens, CountryLookupArgs};
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 use tabled::settings::Style;
 use tabled::Table;
 
diff --git a/src/bin/commands/elem_format.rs b/src/bin/commands/elem_format.rs
@@ -5,7 +5,7 @@
 //! multiple output format support.
 
 use bgpkit_parser::BgpElem;
-use monocle::lens::utils::{OrderByField, OrderDirection, OutputFormat, TimestampFormat};
+use monocle::utils::{OrderByField, OrderDirection, OutputFormat, TimestampFormat};
 use serde_json::json;
 use tabled::builder::Builder;
 use tabled::settings::Style;
diff --git a/src/bin/commands/inspect.rs b/src/bin/commands/inspect.rs
@@ -8,7 +8,7 @@ use monocle::lens::inspect::{
     InspectDataSection, InspectDisplayConfig, InspectLens, InspectQueryOptions, InspectQueryType,
     InspectResult,
 };
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 use monocle::MonocleConfig;
 use std::collections::HashSet;
 
diff --git a/src/bin/commands/ip.rs b/src/bin/commands/ip.rs
@@ -1,6 +1,6 @@
 use clap::Args;
 use monocle::lens::ip::{IpInfo, IpLens, IpLookupArgs};
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 use serde_json::json;
 use std::net::IpAddr;
 use tabled::settings::Style;
diff --git a/src/bin/commands/parse.rs b/src/bin/commands/parse.rs
@@ -6,7 +6,7 @@ use bgpkit_parser::BgpElem;
 use clap::Args;
 
 use monocle::lens::parse::{ParseFilters, ParseLens};
-use monocle::lens::utils::{OrderByField, OrderDirection, OutputFormat, TimestampFormat};
+use monocle::utils::{OrderByField, OrderDirection, OutputFormat, TimestampFormat};
 
 use super::elem_format::{
     available_fields_help, format_elem, format_elems_table, get_header, parse_fields, sort_elems,
diff --git a/src/bin/commands/pfx2as.rs b/src/bin/commands/pfx2as.rs
@@ -7,7 +7,7 @@ use clap::Args;
 use monocle::database::MonocleDatabase;
 use monocle::lens::pfx2as::{Pfx2asLens, Pfx2asSearchArgs};
 use monocle::lens::rpki::RpkiLens;
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 use monocle::MonocleConfig;
 
 /// Arguments for the Pfx2as command
diff --git a/src/bin/commands/rpki.rs b/src/bin/commands/rpki.rs
@@ -5,7 +5,7 @@ use monocle::lens::rpki::{
     RpkiAspaLookupArgs, RpkiAspaTableEntry, RpkiDataSource, RpkiLens, RpkiRoaEntry,
     RpkiRoaLookupArgs, RpkiViewsCollectorOption,
 };
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 use monocle::MonocleConfig;
 use std::collections::HashSet;
 use tabled::settings::object::Columns;
diff --git a/src/bin/commands/search.rs b/src/bin/commands/search.rs
@@ -11,7 +11,7 @@ use bgpkit_parser::BgpElem;
 use clap::Args;
 use monocle::database::MsgStore;
 use monocle::lens::search::SearchFilters;
-use monocle::lens::utils::{OrderByField, OrderDirection, OutputFormat, TimestampFormat};
+use monocle::utils::{OrderByField, OrderDirection, OutputFormat, TimestampFormat};
 use monocle::MonocleConfig;
 use rayon::prelude::*;
 use tracing::{info, warn};
diff --git a/src/bin/commands/time.rs b/src/bin/commands/time.rs
@@ -1,6 +1,6 @@
 use clap::Args;
 use monocle::lens::time::{TimeBgpTime, TimeLens, TimeParseArgs};
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 use tabled::settings::Style;
 use tabled::Table;
 
diff --git a/src/bin/monocle.rs b/src/bin/monocle.rs
@@ -3,7 +3,7 @@
 #![deny(clippy::expect_used)]
 
 use clap::{Args, Parser, Subcommand};
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 use monocle::*;
 use tracing::Level;
 
diff --git a/src/lens/README.md b/src/lens/README.md
@@ -177,7 +177,7 @@ These lenses do not require a persistent database reference:
 
 ```rust,ignore
 use monocle::lens::time::{TimeLens, TimeParseArgs};
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 
 let lens = TimeLens::new();
 let args = TimeParseArgs::new(vec![
@@ -195,7 +195,7 @@ println!("{}", out);
 ```rust,ignore
 use monocle::database::MonocleDatabase;
 use monocle::lens::inspect::{InspectLens, InspectQueryOptions};
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 
 let db = MonocleDatabase::open_in_dir("~/.local/share/monocle")?;
 let lens = InspectLens::new(&db);
@@ -221,7 +221,7 @@ for r in results {
 ```rust,ignore
 use monocle::database::MonocleDatabase;
 use monocle::lens::as2rel::{As2relLens, As2relSearchArgs};
-use monocle::lens::utils::OutputFormat;
+use monocle::utils::OutputFormat;
 
 let db = MonocleDatabase::open_in_dir("~/.local/share/monocle")?;
 let lens = As2relLens::new(&db);
diff --git a/src/lens/as2rel/args.rs b/src/lens/as2rel/args.rs
@@ -7,7 +7,7 @@
 use serde::{Deserialize, Serialize};
 
 use super::types::{As2relOutputFormat, As2relSortOrder};
-use crate::lens::utils::{bool_from_str, u32_or_vec};
+use crate::utils::{bool_from_str, u32_or_vec};
 
 /// Filter for relationship type perspective
 ///
diff --git a/src/lens/as2rel/mod.rs b/src/lens/as2rel/mod.rs
@@ -13,7 +13,7 @@ pub use types::{
 };
 
 // Re-export common utilities for convenience
-pub use crate::lens::utils::{truncate_name, DEFAULT_NAME_MAX_LEN};
+pub use crate::utils::{truncate_name, DEFAULT_NAME_MAX_LEN};
 
 use crate::database::{MonocleDatabase, BGPKIT_AS2REL_URL};
 use anyhow::Result;
@@ -63,8 +63,8 @@ impl<'a> As2relLens<'a> {
     /// Check why the data needs update, if at all
     ///
     /// Returns `Some(RefreshReason)` if update is needed, `None` if data is current.
-    pub fn update_reason(&self) -> Option<crate::lens::utils::RefreshReason> {
-        use crate::lens::utils::RefreshReason;
+    pub fn update_reason(&self) -> Option<crate::utils::RefreshReason> {
+        use crate::utils::RefreshReason;
 
         let as2rel = self.db.as2rel();
 
diff --git a/src/lens/as2rel/types.rs b/src/lens/as2rel/types.rs
@@ -3,7 +3,7 @@
 //! This module defines the types used by the AS2Rel lens for relationship
 //! queries and result formatting.
 
-use crate::lens::utils::{truncate_name, DEFAULT_NAME_MAX_LEN};
+use crate::utils::{truncate_name, DEFAULT_NAME_MAX_LEN};
 use serde::{Deserialize, Serialize};
 
 /// Sort order for search results
diff --git a/src/lens/inspect/mod.rs b/src/lens/inspect/mod.rs
@@ -2294,10 +2294,10 @@ impl<'a> InspectLens<'a> {
 
     /// Truncate a name based on display config
     fn truncate_name(&self, name: &str, config: &InspectDisplayConfig) -> String {
-        if !config.truncate_names || name.len() <= config.name_max_width {
+        if !config.truncate_names {
             name.to_string()
         } else {
-            format!("{}...", &name[..config.name_max_width.saturating_sub(3)])
+            crate::utils::truncate_name(name, config.name_max_width)
         }
     }
 }
diff --git a/src/lens/mod.rs b/src/lens/mod.rs
@@ -54,11 +54,6 @@
 //! use monocle::lens::inspect::{InspectLens, InspectQueryOptions};
 //! ```
 
-// =============================================================================
-// Utility module (always available when lib feature is enabled)
-// =============================================================================
-pub mod utils;
-
 // =============================================================================
 // All lenses (require lib feature)
 // =============================================================================
diff --git a/src/lens/pfx2as/mod.rs b/src/lens/pfx2as/mod.rs
@@ -39,7 +39,7 @@
 
 use crate::database::MonocleDatabase;
 use crate::lens::rpki::RpkiLens;
-use crate::lens::utils::{truncate_name, OutputFormat, DEFAULT_NAME_MAX_LEN};
+use crate::utils::{truncate_name, OutputFormat, DEFAULT_NAME_MAX_LEN};
 use anyhow::Result;
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
@@ -381,8 +381,8 @@ impl<'a> Pfx2asLens<'a> {
     pub fn refresh_reason(
         &self,
         ttl: std::time::Duration,
-    ) -> Result<Option<crate::lens::utils::RefreshReason>> {
-        use crate::lens::utils::RefreshReason;
+    ) -> Result<Option<crate::utils::RefreshReason>> {
+        use crate::utils::RefreshReason;
 
         let pfx2as = self.db.pfx2as();
 
diff --git a/src/lens/rpki/commons.rs b/src/lens/rpki/commons.rs
@@ -4,6 +4,7 @@
 //! from bgpkit-commons, supporting both current (Cloudflare) and historical
 //! (RIPE NCC, RPKIviews) data sources.
 
+use crate::utils::truncate_name;
 use anyhow::{anyhow, Result};
 use bgpkit_commons::rpki::{HistoricalRpkiSource, RpkiTrie, RpkiViewsCollector};
 use chrono::NaiveDate;
@@ -53,17 +54,6 @@ pub struct RpkiAspaTableEntry {
 /// Default max width for customer name in table display
 const DEFAULT_NAME_MAX_WIDTH: usize = 20;
 
-/// Truncate a name to fit within max_width, adding "..." if truncated
-fn truncate_name(name: &str, max_width: usize) -> String {
-    if name.len() <= max_width {
-        name.to_string()
-    } else if max_width <= 3 {
-        "...".to_string()
-    } else {
-        format!("{}...", &name[..max_width - 3])
-    }
-}
-
 impl From<&RpkiAspaEntry> for RpkiAspaTableEntry {
     fn from(entry: &RpkiAspaEntry) -> Self {
         RpkiAspaTableEntry {
diff --git a/src/lens/rpki/mod.rs b/src/lens/rpki/mod.rs
@@ -20,7 +20,7 @@ pub use commons::{RpkiAspaEntry, RpkiAspaProvider, RpkiAspaTableEntry, RpkiRoaEn
 pub use rtr::RtrClient;
 
 use crate::database::MonocleDatabase;
-use crate::lens::utils::option_u32_from_str;
+use crate::utils::option_u32_from_str;
 use anyhow::Result;
 use bgpkit_commons::rpki::RpkiTrie;
 use chrono::NaiveDate;
@@ -396,8 +396,8 @@ impl<'a> RpkiLens<'a> {
     pub fn refresh_reason(
         &self,
         ttl: std::time::Duration,
-    ) -> Result<Option<crate::lens::utils::RefreshReason>> {
-        use crate::lens::utils::RefreshReason;
+    ) -> Result<Option<crate::utils::RefreshReason>> {
+        use crate::utils::RefreshReason;
 
         let rpki = self.db.rpki();
 
diff --git a/src/lib.rs b/src/lib.rs
@@ -193,12 +193,18 @@ pub use config::{
 pub use database::*;
 
 // =============================================================================
-// Lens Module - Feature-gated exports
+// Utility Module
 // =============================================================================
 
+pub mod utils;
+
 // Output format utilities (lib feature)
 #[cfg(feature = "lib")]
-pub use lens::utils::OutputFormat;
+pub use utils::OutputFormat;
+
+// =============================================================================
+// Lens Module - Feature-gated exports
+// =============================================================================
 
 // =============================================================================
 // Server Module (WebSocket API) - requires "server" feature
diff --git a/src/utils.rs b/src/utils.rs
