11use chrono:: Duration ;
2- use std:: collections:: HashMap ;
2+ use std:: collections:: { HashMap , HashSet } ;
33use uuid:: Uuid ;
44
55use crate :: episode:: { DecayConfig , EpisodeStore } ;
66use crate :: procedural:: { Pattern , ProceduralStore } ;
7- use crate :: storage:: memory_index:: MemoryIndex ;
7+ use crate :: storage:: memory_index:: { cosine_similarity , MemoryIndex } ;
88use crate :: storage:: traits:: StorageBackend ;
99use crate :: types:: * ;
1010use crate :: CortexError ;
1111
12+ /// Minimum cosine similarity for embedding-based promotion grouping.
13+ const SEMANTIC_SIMILARITY_THRESHOLD : f32 = 0.85 ;
14+
1215/// Report from a consolidation cycle.
1316#[ derive( Debug , Default ) ]
1417pub struct ConsolidationReport {
1518 pub episodes_scanned : usize ,
1619 pub promoted_to_semantic : usize ,
1720 pub decayed_updated : usize ,
21+ /// Count of memories archived (moved to cold storage, not deleted).
1822 pub decayed_swept : usize ,
1923 pub patterns_detected : usize ,
2024 pub contradictions_found : usize ,
@@ -107,14 +111,20 @@ impl<'a> ConsolidationEngine<'a> {
107111 Ok ( report)
108112 }
109113
110- /// Find episodic memories that repeat (same content observed 3+ times).
111- /// Groups by content text similarity (exact match for now).
114+ /// Find episodic memories that repeat, using two strategies:
115+ /// 1. Exact content hash matching (fast, strict)
116+ /// 2. Embedding-based semantic similarity for ungrouped memories (cosine >= 0.85)
117+ ///
112118 /// Processes in pages to avoid loading all episodic memories at once.
113119 fn find_promotion_candidates (
114120 & self ,
115121 min_occurrences : usize ,
116122 ) -> Result < Vec < Vec < Uuid > > , CortexError > {
117123 let mut fact_groups: HashMap < String , Vec < Uuid > > = HashMap :: new ( ) ;
124+ // Collect ungrouped memories with embeddings for the semantic pass.
125+ // Store (id, embedding) — clone the Arc's inner vec only for memories
126+ // that didn't match by hash, keeping the hot path allocation-free.
127+ let mut ungrouped_with_embeddings: Vec < ( Uuid , Vec < f32 > ) > = Vec :: new ( ) ;
118128 let page_size = 1000 ;
119129 let mut offset = 0 ;
120130
@@ -140,6 +150,9 @@ impl<'a> ConsolidationEngine<'a> {
140150 } ;
141151 if let Some ( k) = key {
142152 fact_groups. entry ( k) . or_default ( ) . push ( mem. id ) ;
153+ } else if let Some ( ref emb) = mem. embedding {
154+ // No hash key (e.g. Relationship, Pattern, Event) but has embedding
155+ ungrouped_with_embeddings. push ( ( mem. id , emb. as_ref ( ) . clone ( ) ) ) ;
143156 }
144157 }
145158
@@ -149,10 +162,82 @@ impl<'a> ConsolidationEngine<'a> {
149162 offset += page_size;
150163 }
151164
152- Ok ( fact_groups
165+ // Collect IDs that were already grouped by hash so we can skip them
166+ // in the semantic pass.
167+ let hash_grouped_ids: HashSet < Uuid > = fact_groups
168+ . values ( )
169+ . flat_map ( |ids| ids. iter ( ) . copied ( ) )
170+ . collect ( ) ;
171+
172+ // Also add memories that had a hash key but whose group is too small
173+ // (they won't be promoted by hash, so give them a chance via embedding).
174+ // We need to re-scan fact_groups to find singleton/small groups with embeddings.
175+ // To avoid a second storage round-trip, collect their IDs now and fetch
176+ // embeddings only for those that need it.
177+ let mut extra_candidates: Vec < Uuid > = Vec :: new ( ) ;
178+ for ids in fact_groups. values ( ) {
179+ if ids. len ( ) < min_occurrences {
180+ extra_candidates. extend ( ids) ;
181+ }
182+ }
183+ for id in & extra_candidates {
184+ // Skip if already in the ungrouped list
185+ if ungrouped_with_embeddings. iter ( ) . any ( |( uid, _) | uid == id) {
186+ continue ;
187+ }
188+ if let Some ( mem) = self . storage . get_memory ( * id) ? {
189+ if let Some ( ref emb) = mem. embedding {
190+ ungrouped_with_embeddings. push ( ( mem. id , emb. as_ref ( ) . clone ( ) ) ) ;
191+ }
192+ }
193+ }
194+
195+ // --- Pass 1: collect hash-based groups that meet the threshold ---
196+ let mut results: Vec < Vec < Uuid > > = fact_groups
153197 . into_values ( )
154198 . filter ( |ids| ids. len ( ) >= min_occurrences)
155- . collect ( ) )
199+ . collect ( ) ;
200+
201+ // --- Pass 2: greedy single-pass embedding clustering for ungrouped memories ---
202+ // Remove any memory that already belongs to a promoted hash group.
203+ ungrouped_with_embeddings. retain ( |( id, _) | !hash_grouped_ids. contains ( id) ) ;
204+
205+ if ungrouped_with_embeddings. len ( ) >= min_occurrences {
206+ let semantic_groups = self . cluster_by_embedding ( & ungrouped_with_embeddings) ;
207+ for group in semantic_groups {
208+ if group. len ( ) >= min_occurrences {
209+ results. push ( group) ;
210+ }
211+ }
212+ }
213+
214+ Ok ( results)
215+ }
216+
217+ /// Greedy single-pass clustering of memories by embedding similarity.
218+ /// Each unassigned memory is compared to existing cluster centroids (the
219+ /// first member's embedding). If similar enough, it joins; otherwise it
220+ /// starts a new cluster.
221+ fn cluster_by_embedding ( & self , items : & [ ( Uuid , Vec < f32 > ) ] ) -> Vec < Vec < Uuid > > {
222+ // Each cluster is represented by the embedding of its first member
223+ // (used as the centroid) and the list of member IDs.
224+ let mut clusters: Vec < ( Vec < Uuid > , & [ f32 ] ) > = Vec :: new ( ) ;
225+
226+ for ( id, embedding) in items {
227+ let mut assigned = false ;
228+ for ( members, centroid) in & mut clusters {
229+ if cosine_similarity ( embedding, centroid) >= SEMANTIC_SIMILARITY_THRESHOLD {
230+ members. push ( * id) ;
231+ assigned = true ;
232+ break ;
233+ }
234+ }
235+ if !assigned {
236+ clusters. push ( ( vec ! [ * id] , embedding. as_slice ( ) ) ) ;
237+ }
238+ }
239+
240+ clusters. into_iter ( ) . map ( |( ids, _) | ids) . collect ( )
156241 }
157242
158243 /// Promote a group of episodic memories to a single semantic memory.
@@ -200,16 +285,23 @@ impl<'a> ConsolidationEngine<'a> {
200285 procedural_store. detect_patterns ( 3 )
201286 }
202287
203- /// Soft-delete episodic memories below salience threshold.
204- /// Returns count of deleted memories.
288+ /// Archive episodic memories below salience threshold.
289+ /// Returns count of archived memories.
205290 pub fn sweep_decayed ( & self , threshold : f32 ) -> Result < usize , CortexError > {
206291 let decayed = self
207292 . storage
208293 . list_by_salience_below ( MemoryTier :: Episodic , threshold) ?;
209294 if decayed. is_empty ( ) {
210295 return Ok ( 0 ) ;
211296 }
212- let ids: Vec < uuid:: Uuid > = decayed. iter ( ) . map ( |m| m. id ) . collect ( ) ;
213- self . storage . delete_memories_batch ( & ids)
297+ let ids_to_sweep: Vec < uuid:: Uuid > = decayed. iter ( ) . map ( |m| m. id ) . collect ( ) ;
298+ // Archive instead of delete - preserve memories in cold storage
299+ let mut archived_count = 0 ;
300+ for id in & ids_to_sweep {
301+ if self . storage . archive_memory ( * id) . is_ok ( ) {
302+ archived_count += 1 ;
303+ }
304+ }
305+ Ok ( archived_count)
214306 }
215307}
0 commit comments