[amazon_rose_forest] add hierarchical clustering (#71)

Author: kalisam

src/core/hierarchical.rs

@@ -0,0 +1,104 @@
+use crate::core::vector::Vector;
+
+#[derive(Debug, Clone)]
+pub struct Cluster {
+    pub centroid: Vector,
+    pub members: Vec<Vector>,
+}
+
+impl Cluster {
+    fn new(vector: Vector) -> Self {
+        Self {
+            centroid: vector.clone(),
+            members: vec![vector],
+        }
+    }
+
+    fn recompute_centroid(&mut self) {
+        if self.members.is_empty() {
+            return;
+        }
+        let dimensions = self.members[0].dimensions;
+        let mut sums = vec![0.0f32; dimensions];
+        for v in &self.members {
+            for (i, val) in v.values.iter().enumerate() {
+                sums[i] += val;
+            }
+        }
+        let len_inv = 1.0 / self.members.len() as f32;
+        for sum in &mut sums {
+            *sum *= len_inv;
+        }
+        self.centroid = Vector::new(sums);
+    }
+}
+
+/// Perform a simple agglomerative clustering using Euclidean distance.
+/// Clusters are merged until the closest pair has distance greater than
+/// `threshold`.
+pub fn cluster_vectors(vectors: &[Vector], threshold: f32) -> Vec<Cluster> {
+    let mut clusters: Vec<Cluster> = vectors.iter().cloned().map(Cluster::new).collect();
+    if clusters.is_empty() {
+        return clusters;
+    }
+    loop {
+        let mut best_dist = f32::MAX;
+        let mut best_pair: Option<(usize, usize)> = None;
+        for i in 0..clusters.len() {
+            for j in (i + 1)..clusters.len() {
+                let dist = clusters[i]
+                    .centroid
+                    .euclidean_distance(&clusters[j].centroid);
+                if dist < best_dist {
+                    best_dist = dist;
+                    best_pair = Some((i, j));
+                }
+            }
+        }
+        match best_pair {
+            Some((i, j)) if best_dist <= threshold => {
+                let mut members = clusters[i].members.clone();
+                members.extend(clusters[j].members.clone());
+                clusters[i].members = members;
+                clusters[i].recompute_centroid();
+                clusters.remove(j);
+            }
+            _ => break,
+        }
+    }
+    clusters
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basic_clustering() {
+        let v1 = Vector::new(vec![0.0, 0.0]);
+        let v2 = Vector::new(vec![0.1, -0.1]);
+        let v3 = Vector::new(vec![5.0, 5.0]);
+        let v4 = Vector::new(vec![5.2, 4.9]);
+        let clusters = cluster_vectors(&[v1, v2, v3, v4], 0.5);
+        assert_eq!(clusters.len(), 2);
+    }
+
+    #[test]
+    fn test_single_cluster_when_threshold_large() {
+        let vectors = vec![
+            Vector::new(vec![0.0, 0.0]),
+            Vector::new(vec![1.0, 0.0]),
+            Vector::new(vec![0.0, 1.0]),
+        ];
+        let clusters = cluster_vectors(&vectors, 10.0);
+        assert_eq!(clusters.len(), 1);
+        assert_eq!(clusters[0].members.len(), 3);
+    }
+
+    #[test]
+    fn test_no_merge_when_threshold_small() {
+        let vectors = vec![Vector::new(vec![0.0, 0.0]), Vector::new(vec![1.0, 1.0])];
+        let clusters = cluster_vectors(&vectors, 0.1);
+        assert_eq!(clusters.len(), 2);
+    }
+}

src/core/mod.rs

@@ -1,4 +1,5 @@
 pub mod centroid;
 pub mod centroid_crdt;
+pub mod hierarchical;
 pub mod metrics;
 pub mod vector;

src/lib.rs

@@ -1,26 +1,26 @@
+pub mod ad4m;
+pub mod code_analysis;
 pub mod consciousness;
 pub mod core;
 pub mod darwin;
+pub mod evaluation;
 pub mod governance;
+pub mod hypothesis;
 pub mod intelligence;
+pub mod llm;
 pub mod nerv;
 pub mod network;
 pub mod server;
 pub mod sharding;
 pub mod utils;
-pub mod llm;
-pub mod code_analysis;
-pub mod hypothesis;
-pub mod evaluation;
-pub mod ad4m;
-
 
 // Export common types for easier access
 pub use crate::consciousness::ad4m_bridge::Ad4mBridge;
 pub use crate::consciousness::introspection::Introspection;
 pub use crate::consciousness::swarm::Swarm;
 pub use crate::core::centroid::Centroid;
 pub use crate::core::centroid_crdt::CentroidCRDT;
+pub use crate::core::hierarchical::{cluster_vectors, Cluster};
 pub use crate::core::vector::Vector;
 pub use crate::darwin::self_improvement::SelfImprovementEngine;
 pub use crate::governance::dao::Dao;

src/nerv/replication.rs

@@ -51,15 +51,11 @@ impl ReplicationManager {
         info!("Removed peer {} from replication manager", peer_id);
     }
 
-<<<<<<< Updated upstream
     pub async fn start_replication(
         self: Arc<Self>,
         shard_id: Uuid,
         target_node: &str,
     ) -> Result<Uuid> {
-=======
-    pub async fn start_replication(&self, shard_id: Uuid, target_node: &str) -> Result<Uuid> {
->>>>>>> Stashed changes
         // Verify target node is in peers
         if !self.peers.read().await.contains(target_node) {
             return Err(anyhow!("Target node {} is not a known peer", target_node));
@@ -83,11 +79,7 @@ impl ReplicationManager {
 
         // Spawn task to handle replication
         let task_id_clone = task_id;
-<<<<<<< Updated upstream
         let self_clone = Arc::clone(&self);
-=======
-        let self_clone = Arc::new(self.clone());
->>>>>>> Stashed changes
 
         tokio::spawn(async move {
             if let Err(e) = self_clone.execute_replication(task_id_clone).await {
@@ -167,9 +159,6 @@ impl ReplicationManager {
     }
 }
 
-<<<<<<< Updated upstream
-// Support cloning for the manager to allow sharing between threads
-=======
 // Support cloning for the manager to allow sharing between threads
 impl Clone for ReplicationManager {
     fn clone(&self) -> Self {
@@ -183,4 +172,3 @@ impl Clone for ReplicationManager {
         }
     }
 }
->>>>>>> Stashed changes

src/sharding/manager.rs

@@ -467,3 +467,22 @@ impl ShardManager {
         Ok(distribution)
     }
 }
+
+// Support cloning for the manager to allow sharing between threads
+impl Clone for ShardManager {
+    fn clone(&self) -> Self {
+        // Note: This creates a new instance with the same node_id and metrics
+        // but empty collections. The collections are meant to be
+        // accessed through the original instance's RwLocks.
+        Self {
+            metrics: self.metrics.clone(),
+            node_id: self.node_id.clone(),
+            shards: RwLock::new(HashMap::new()),
+            shard_assignments: RwLock::new(HashMap::new()),
+            migrations: RwLock::new(HashMap::new()),
+            indices: RwLock::new(HashMap::new()),
+            shard_loads: RwLock::new(HashMap::new()),
+        }
+    }
+}
+