Merge branch 'main' into dependabot/cargo/cargo-920260e05e

doublegate · web-flow · commit 30d16559f649 · 2026-01-24T21:53:46.000-05:00
diff --git a/Cargo.toml b/Cargo.toml
@@ -43,7 +43,9 @@ rusqlite = { version = "0.31", features = ["bundled"] }
 
 # Memory-mapped I/O (Sprint 6.6)
 memmap2 = "0.9"
-rkyv = "0.8.14"
+
+# Zero-copy serialization (rkyv migration)
+rkyv = { version = "0.8.14", features = ["std", "alloc"] }
 
 # CSV export (promote to workspace)
 csv = "1.3"
diff --git a/crates/prtip-core/src/lib.rs b/crates/prtip-core/src/lib.rs
@@ -136,4 +136,7 @@ pub use resource_monitor::{
 pub use retry::{retry_with_backoff, RetryConfig};
 pub use service_db::{ServiceMatch, ServiceProbe, ServiceProbeDb};
 pub use templates::{ScanTemplate, TemplateManager};
-pub use types::{PortRange, PortState, Protocol, ScanResult, ScanResultRkyv, ScanTarget, ScanType, TimingTemplate};
+pub use types::{
+    PortRange, PortState, Protocol, ScanResult, ScanResultRkyv, ScanTarget, ScanType,
+    TimingTemplate,
+};
diff --git a/crates/prtip-core/src/types.rs b/crates/prtip-core/src/types.rs
@@ -641,6 +641,128 @@ impl fmt::Display for ScanResult {
     }
 }
 
+/// rkyv-compatible serialization format for ScanResult
+///
+/// This type is optimized for zero-copy deserialization using rkyv.
+/// It stores all data in a format that can be directly interpreted from
+/// memory-mapped files without allocation.
+///
+/// # Alignment Requirements
+///
+/// This structure must maintain proper alignment for rkyv's zero-copy
+/// deserialization. The fixed-size entry buffer (512 bytes) provides
+/// adequate alignment for typical rkyv requirements.
+#[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[rkyv(derive(Debug))]
+pub struct ScanResultRkyv {
+    /// Target IP address (16 bytes for IPv6 compatibility)
+    pub target_ip_bytes: [u8; 16],
+    /// Whether the IP is IPv4 (true) or IPv6 (false)
+    pub is_ipv4: bool,
+    /// Port number
+    pub port: u16,
+    /// Port state as u8 (Open=0, Closed=1, Filtered=2, Unknown=3)
+    pub state: u8,
+    /// Response time in nanoseconds (u64 to avoid truncation)
+    pub response_time_nanos: u64,
+    /// Timestamp in nanoseconds since Unix epoch
+    pub timestamp_nanos: i64,
+    /// Optional banner (max 128 bytes)
+    pub banner: Option<String>,
+    /// Optional service name (max 32 bytes)
+    pub service: Option<String>,
+    /// Optional service version (max 64 bytes)
+    pub version: Option<String>,
+    /// Optional raw response (limited to 256 bytes to fit in entry)
+    pub raw_response: Option<Vec<u8>>,
+}
+
+impl From<&ScanResult> for ScanResultRkyv {
+    fn from(result: &ScanResult) -> Self {
+        // Convert IpAddr to bytes
+        let (target_ip_bytes, is_ipv4) = match result.target_ip {
+            IpAddr::V4(ipv4) => {
+                let mut bytes = [0u8; 16];
+                bytes[..4].copy_from_slice(&ipv4.octets());
+                (bytes, true)
+            }
+            IpAddr::V6(ipv6) => (ipv6.octets(), false),
+        };
+
+        // Convert PortState to u8
+        let state = match result.state {
+            PortState::Open => 0,
+            PortState::Closed => 1,
+            PortState::Filtered => 2,
+            PortState::Unknown => 3,
+        };
+
+        // Convert response time to u64 nanoseconds (avoid truncation issues)
+        // Note: u64 can represent up to ~584 years, which is more than sufficient
+        // for network response times. We clamp to u64::MAX to avoid overflow.
+        let response_time_nanos = result.response_time.as_nanos().min(u64::MAX as u128) as u64;
+
+        // Convert timestamp with proper error handling
+        let timestamp_nanos = result
+            .timestamp
+            .timestamp_nanos_opt()
+            .expect("timestamp out of range for nanosecond representation");
+
+        Self {
+            target_ip_bytes,
+            is_ipv4,
+            port: result.port,
+            state,
+            response_time_nanos,
+            timestamp_nanos,
+            banner: result.banner.clone(),
+            service: result.service.clone(),
+            version: result.version.clone(),
+            raw_response: result.raw_response.clone(),
+        }
+    }
+}
+
+impl From<ScanResultRkyv> for ScanResult {
+    fn from(rkyv: ScanResultRkyv) -> Self {
+        // Convert bytes back to IpAddr
+        let target_ip = if rkyv.is_ipv4 {
+            let mut octets = [0u8; 4];
+            octets.copy_from_slice(&rkyv.target_ip_bytes[..4]);
+            IpAddr::V4(std::net::Ipv4Addr::from(octets))
+        } else {
+            IpAddr::V6(std::net::Ipv6Addr::from(rkyv.target_ip_bytes))
+        };
+
+        // Convert u8 back to PortState
+        let state = match rkyv.state {
+            0 => PortState::Open,
+            1 => PortState::Closed,
+            2 => PortState::Filtered,
+            _ => PortState::Unknown,
+        };
+
+        // Convert u64 nanoseconds back to Duration
+        // Safe: u64::MAX nanoseconds fits within Duration's range
+        let response_time = Duration::from_nanos(rkyv.response_time_nanos);
+
+        // Convert i64 nanoseconds back to DateTime
+        let timestamp = DateTime::from_timestamp_nanos(rkyv.timestamp_nanos);
+
+        Self {
+            target_ip,
+            port: rkyv.port,
+            state,
+            response_time,
+            timestamp,
+            banner: rkyv.banner,
+            service: rkyv.service,
+            version: rkyv.version,
+            raw_response: rkyv.raw_response,
+        }
+    }
+}
+
 /// Port filtering for exclusion/inclusion lists
 ///
 /// Provides efficient port filtering using hash sets for O(1) lookups.
diff --git a/crates/prtip-scanner/Cargo.toml b/crates/prtip-scanner/Cargo.toml
@@ -87,7 +87,7 @@ pcap-file = "2.0"
 
 # Memory-mapped I/O
 memmap2 = { workspace = true }
-rkyv = { workspace = true, features = ["alloc"] }
+rkyv = { workspace = true }
 
 [dev-dependencies]
 tokio = { workspace = true }
diff --git a/crates/prtip-scanner/src/output/mmap_reader.rs b/crates/prtip-scanner/src/output/mmap_reader.rs
@@ -8,7 +8,7 @@ use std::path::Path;
 
 const HEADER_SIZE: usize = 64;
 const ENTRY_SIZE: usize = 512;
-const LENGTH_PREFIX_SIZE: usize = 8; // 8 bytes for length to maintain alignment
+const LENGTH_PREFIX_SIZE: usize = 8; // u64 length prefix for each entry
 
 /// Memory-mapped result reader
 pub struct MmapResultReader {
@@ -34,10 +34,21 @@ impl MmapResultReader {
 
         // Parse header
         let version = u64::from_le_bytes(mmap[0..8].try_into().unwrap());
-        if version != 1 {
+        if version == 1 {
             return Err(io::Error::new(
                 io::ErrorKind::InvalidData,
-                format!("Unsupported version: {}", version),
+                "Incompatible file format: version 1 (bincode) is no longer supported. \
+                 This file was created with an older version of the scanner. \
+                 Please regenerate scan results with the current version (rkyv format, version 2).",
+            ));
+        }
+        if version != 2 {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!(
+                    "Unsupported version: {}. Expected version 2 (rkyv format).",
+                    version
+                ),
             ));
         }
 
@@ -85,14 +96,33 @@ impl MmapResultReader {
             return None;
         }
 
-        // Copy data to an aligned buffer (rkyv requires alignment)
-        let entry_bytes = &self.mmap[offset + LENGTH_PREFIX_SIZE..offset + LENGTH_PREFIX_SIZE + len];
-        let aligned_data: Vec<u8> = entry_bytes.to_vec();
+        // Read length prefix (u64 in little-endian)
+        let len = u64::from_le_bytes(
+            entry_bytes[..LENGTH_PREFIX_SIZE]
+                .try_into()
+                .expect("LENGTH_PREFIX_SIZE is 8 bytes"),
+        ) as usize;
+
+        // Validate length
+        if len == 0 || len + LENGTH_PREFIX_SIZE > self.entry_size {
+            eprintln!(
+                "MmapResultReader: invalid entry length {} at index {}",
+                len, index
+            );
+            return None;
+        }
 
-        // Deserialize using rkyv
-        match rkyv::from_bytes::<ScanResultRkyv, rkyv::rancor::Error>(&aligned_data) {
-            Ok(rkyv_result) => Some(rkyv_result.into()),
-            Err(_) => None,
+        // Use zero-copy deserialization without unnecessary allocation
+        let data_bytes = &entry_bytes[LENGTH_PREFIX_SIZE..LENGTH_PREFIX_SIZE + len];
+        match rkyv::from_bytes::<ScanResultRkyv, rkyv::rancor::Error>(data_bytes) {
+            Ok(rkyv_result) => Some(ScanResult::from(rkyv_result)),
+            Err(e) => {
+                eprintln!(
+                    "MmapResultReader: failed to deserialize entry at index {} with length {}: {}",
+                    index, len, e
+                );
+                None
+            }
         }
     }
 
@@ -230,4 +260,43 @@ mod tests {
         assert!(reader.get_entry(1).is_none());
         assert!(reader.get_entry(100).is_none());
     }
+
+    #[test]
+    fn test_mmap_version_1_rejected() {
+        use std::io::Write;
+
+        let temp = NamedTempFile::new().unwrap();
+        let path = temp.path().to_owned();
+
+        // Create a file with version 1 header (old bincode format)
+        {
+            let mut file = std::fs::OpenOptions::new()
+                .write(true)
+                .create(true)
+                .truncate(true)
+                .open(&path)
+                .unwrap();
+
+            // Write a version 1 header
+            file.write_all(&1u64.to_le_bytes()).unwrap(); // version = 1
+            file.write_all(&0u64.to_le_bytes()).unwrap(); // entry_count = 0
+            file.write_all(&(ENTRY_SIZE as u64).to_le_bytes()).unwrap(); // entry_size
+            file.write_all(&0u64.to_le_bytes()).unwrap(); // checksum
+                                                          // Pad to HEADER_SIZE
+            file.write_all(&[0u8; HEADER_SIZE - 32]).unwrap();
+        }
+
+        // Attempt to open should fail with clear error message
+        let result = MmapResultReader::open(&path);
+        assert!(result.is_err());
+        if let Err(err) = result {
+            assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+            let err_msg = err.to_string();
+            assert!(
+                err_msg.contains("version 1") && err_msg.contains("bincode"),
+                "Error message should mention version 1 and bincode format: {}",
+                err_msg
+            );
+        }
+    }
 }
diff --git a/crates/prtip-scanner/src/output/mmap_writer.rs b/crates/prtip-scanner/src/output/mmap_writer.rs
@@ -3,6 +3,13 @@
 //! Uses memory-mapped files to reduce RAM usage by 20-50% compared to
 //! in-memory buffering. Results are written to a binary format with
 //! fixed-size entries for zero-copy random access.
+//!
+//! # Alignment Requirements
+//!
+//! The ENTRY_SIZE (512 bytes) is carefully chosen to provide adequate alignment
+//! for rkyv's zero-copy deserialization. rkyv typically requires 8-byte alignment,
+//! and 512 is a multiple of 16 bytes, ensuring proper alignment for all common
+//! data types.
 
 use memmap2::{MmapMut, MmapOptions};
 use prtip_core::{ScanResult, ScanResultRkyv};
@@ -12,7 +19,19 @@ use std::path::Path;
 
 const HEADER_SIZE: usize = 64; // Version, entry_count, entry_size, checksum
 const ENTRY_SIZE: usize = 512; // Fixed-size entries (padded if needed)
-const LENGTH_PREFIX_SIZE: usize = 8; // 8 bytes for length to maintain alignment
+const LENGTH_PREFIX_SIZE: usize = 8; // u64 length prefix for each entry
+
+// Compile-time assertion to verify ENTRY_SIZE alignment
+const _: () = assert!(
+    ENTRY_SIZE % 16 == 0,
+    "ENTRY_SIZE must be a multiple of 16 bytes for rkyv alignment"
+);
+
+// Compile-time assertion to verify LENGTH_PREFIX_SIZE alignment
+const _: () = assert!(
+    LENGTH_PREFIX_SIZE == 8,
+    "LENGTH_PREFIX_SIZE must be 8 bytes for proper alignment"
+);
 
 /// Memory-mapped result writer
 pub struct MmapResultWriter {
@@ -57,31 +76,36 @@ impl MmapResultWriter {
         }
 
         let offset = HEADER_SIZE + (self.entry_count * ENTRY_SIZE);
-        
-        // Convert to rkyv-compatible type and serialize
-        let rkyv_result: ScanResultRkyv = result.into();
-        let entry_bytes = rkyv::to_bytes::<rkyv::rancor::Error>(&rkyv_result)
-            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?;
 
-        // Check if data + length prefix fits
+        // Convert to rkyv-compatible format
+        let rkyv_result = ScanResultRkyv::from(result);
+
+        // Serialize using rkyv with improved error handling
+        let entry_bytes = rkyv::to_bytes::<rkyv::rancor::Error>(&rkyv_result).map_err(|e| {
+            let msg = format!("rkyv serialization error (rkyv::rancor::Error): {e:?}");
+            io::Error::new(io::ErrorKind::InvalidData, msg)
+        })?;
+
+        // Check if entry fits (accounting for length prefix)
         if entry_bytes.len() + LENGTH_PREFIX_SIZE > ENTRY_SIZE {
             return Err(io::Error::new(
                 io::ErrorKind::InvalidData,
                 format!(
-                    "Entry size {} (+{} length) exceeds maximum {}",
+                    "Entry size {} (+ {} length prefix) exceeds maximum {}",
                     entry_bytes.len(),
                     LENGTH_PREFIX_SIZE,
                     ENTRY_SIZE
                 ),
             ));
         }
 
-        // Write length prefix (8 bytes for alignment)
-        let len = entry_bytes.len() as u64;
-        self.mmap[offset..offset + LENGTH_PREFIX_SIZE].copy_from_slice(&len.to_le_bytes());
+        // Write length prefix (u64 in little-endian)
+        let len_bytes = (entry_bytes.len() as u64).to_le_bytes();
+        self.mmap[offset..offset + LENGTH_PREFIX_SIZE].copy_from_slice(&len_bytes);
 
         // Write serialized data after length prefix
-        self.mmap[offset + LENGTH_PREFIX_SIZE..offset + LENGTH_PREFIX_SIZE + entry_bytes.len()].copy_from_slice(&entry_bytes);
+        let data_offset = offset + LENGTH_PREFIX_SIZE;
+        self.mmap[data_offset..data_offset + entry_bytes.len()].copy_from_slice(&entry_bytes);
 
         // Zero-fill remaining space
         for i in (LENGTH_PREFIX_SIZE + entry_bytes.len())..ENTRY_SIZE {
@@ -113,8 +137,9 @@ impl MmapResultWriter {
     }
 
     fn write_header(&mut self) -> io::Result<()> {
-        // Version: 1
-        self.mmap[0..8].copy_from_slice(&1u64.to_le_bytes());
+        // Version: 2 (rkyv format with length prefix)
+        // Version 1 was bincode format (deprecated)
+        self.mmap[0..8].copy_from_slice(&2u64.to_le_bytes());
         // Entry count: 0
         self.mmap[8..16].copy_from_slice(&0u64.to_le_bytes());
         // Entry size: ENTRY_SIZE
