Added calculating mean versus nearest neighbor search speed test.

Forgot to add TSVQ header eariler.

Author: cmdevries

src/EMTree.cpp

@@ -87,9 +87,9 @@ void sigEMTreeCluster(vector<SVector<bool>*> &vectors) {
     typedef KMeans<vecType, seederType, distanceType, protoType> clustererType;
 
     // EMTree
-    int depth = 5;
+    int depth = 3;
     int iters = 2;
-    vector<int> nodeSizes = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20};
+    vector<int> nodeSizes = {100};//{10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20};
     for (int m : nodeSizes) {
         std::cout << "-------------------" << std::endl;
         EMTree<vecType, clustererType, distanceType, protoType> emt(m, depth);
@@ -343,28 +343,63 @@ void reduceDims(const set<int>& topbits, vector<SVector<bool>*>& vectors,
     }
 }
 
-int main(int argc, char** argv) {
-    vector<SVector<bool>*> vectors;
-    int veccount = -1;
-    loadWikiSignatures(vectors, veccount);
-    vector<SVector<bool>*> subset;
-
+void testHistogram(vector<SVector<bool>*>& vectors) {
     if (!vectors.empty()) {
+        vector<SVector<bool>*> subset;
         int dims = vectors[0]->size();
-        set<int> topbits = dimensionHistogram(vectors, dims);
+        set<int> topbits;
+        {
+            boost::timer::auto_cpu_timer seed("calculating histogram: %w seconds\n");
+            topbits = dimensionHistogram(vectors, dims);
+        }       
         loadSubset(vectors, subset, "/Users/chris/LMW-tree/data/inex_xml_mining_subset_2010.txt");
         cout << "filtered " << subset.size() << " vectors to create a subet" << endl;
         vector<SVector<bool>*> reducedSubset;
         cout << "reducing dimensionality to " << topbits.size() << endl;
         reduceDims(topbits, subset, reducedSubset);
-        //sigKTreeCluster(vectors);
-        //sigTSVQCluster(subset);
-        //sigEMTreeCluster(vectors);
         sigKmeansCluster(subset, "/Users/chris/LMW-tree/data/fulldim_clusters.txt");
         sigKmeansCluster(reducedSubset, "/Users/chris/LMW-tree/data/reduceddim_clusters.txt");
-        //testReadVectors();
-        //TestSigEMTree();
     }
-    return 0;
+}
+
+void testMeanVersusNNSpeed(vector<SVector<bool>*>& vectors) {
+    if (!vectors.empty()) {
+        const int dims = vectors[0]->size();
+        SVector<bool> mean(dims);
+        vector<int> weights;
+        {
+            boost::timer::auto_cpu_timer time("calculating mean: %w seconds\n");
+            meanBitPrototype2 proto;
+            proto(&mean, vectors, weights);
+        }
+        {
+            boost::timer::auto_cpu_timer hammingTime("hamming distance: %w seconds\n");
+            hammingDistance distance;
+            uint64_t sum = 0;
+            for (auto vector : vectors) {
+                sum += distance(&mean, vector);
+            }
+            cout << sum << endl;
+        }        
+    }
+}
+
+int main(int argc, char** argv) {
+    vector < SVector<bool>*> vectors;
+    //int veccount = -1;
+    int veccount = 100000;
+    {
+        boost::timer::auto_cpu_timer seed("loading signatures: %w seconds\n");
+        loadWikiSignatures(vectors, veccount);
+    }
+
+    //sigKTreeCluster(vectors);
+    //sigTSVQCluster(vectors);
+    sigEMTreeCluster(vectors);
+    //testHistogram(vectors);
+    //testMeanVersusNNSpeed(vectors);
+    //testReadVectors();
+    //TestSigEMTree();
+    return EXIT_SUCCESS;
 }
 

src/TSVQ.h

@@ -0,0 +1,199 @@
+#ifndef TSVQ_H
+#define	TSVQ_H
+
+#include "StdIncludes.h"
+
+#include "Node.h"
+#include "threadpool.hpp"
+#include "threadpool/size_policies.hpp"
+
+using namespace boost::threadpool;
+
+template <typename T, typename ClustererType, typename DistanceType, typename ProtoType>
+class TSVQ {
+private:
+    // The order of this tree
+    int _m;
+
+    // The order of this tree
+    int _depth;
+
+    // The root of the tree.
+    Node<T> *_root;
+
+    ClustererType _clusterer;
+    DistanceType _distF;
+    ProtoType _protoF;
+
+public:
+    TSVQ(int order, int depth, int maxiters) : _m(order), _depth(depth),
+            _root(new Node<T>()) {
+        _clusterer.setNumClusters(_m);
+        _clusterer.setMaxIters(maxiters);
+    }
+
+    int getClusterCount() {
+        return clusterCount(_root);
+    }
+
+    int getObjCount() {
+        return objCount(_root);
+    }
+
+    int getLevelCount() {
+        return levelCount(_root);
+    }
+
+    int getMaxLevelCount() {
+        return maxLevelCount(_root);
+    }
+
+    int getMinLevelCount() {
+        return minLevelCount(_root);
+    }
+    
+    void printStats() {
+        std::cout << "\nNumber of objects: " << getObjCount();
+        std::cout << "\nCluster count: " << getClusterCount();
+        std::cout << "\nLevel count (node 0): " << getLevelCount();
+        std::cout << "\nMax depth: " << getMaxLevelCount();
+        std::cout << "\nRMSE: " << getRMSE();
+    }
+
+    void cluster(vector<T*> &data) {
+        // make the root a leaf containing all data
+        _root->addAll(data);
+        cluster(_root, _depth);
+    }
+
+    void cluster(Node<T>* current, int depth) {
+        if (depth == 1) {
+            return;
+        } else {
+            vector<Cluster<T>*> clusters = _clusterer.cluster(current->getKeys());
+            current->clearKeysAndChildren();
+            for (Cluster<T>* c : clusters) {
+                Node<T>* child = new Node<T>();
+                child->addAll(c->getNearestList());
+                current->add(c->getCentroid(), child);
+            }
+            for (Node<T>* n : current->getChildren()) {
+                cluster(n, depth - 1);
+            }
+        }
+    }
+
+    double getRMSE() {
+        return RMSE();
+    }
+
+
+private:
+
+    double RMSE() {
+        double RMSE = sumSquaredError(NULL, _root);
+        int size = getObjCount();
+        RMSE /= size;
+        RMSE = sqrt(RMSE);
+        return RMSE;
+    }
+
+    double sumSquaredError(T* parentKey, Node<T> *child) {
+
+        double distance = 0.0;
+        double dis;
+
+        if (child->isLeaf()) {
+            vector<T*> &keys = child->getKeys();
+            for (T* key : keys) {
+                dis = _distF(key, parentKey);
+                distance += dis * dis;
+            }
+        } else {
+            int numEntries = child->size();
+
+            vector<T*> &keys = child->getKeys();
+            vector<Node<T>*> &children = child->getChildren();
+
+            for (int i = 0; i < numEntries; i++) {
+                distance += sumSquaredError(keys[i], children[i]);
+            }
+        }
+
+        return distance;
+    }
+
+    int objCount(Node<T>* current) {
+
+        if (current->isLeaf()) {
+            return current->size();
+        } else {
+            int localCount = 0;
+            vector<Node<T>*>& children = current->getChildren();
+            for (Node<T> *child : children) {
+                localCount += objCount(child);
+
+            }
+            return localCount;
+        }
+    }
+
+    int clusterCount(Node<T>* current) {
+        if (current->isLeaf()) {
+            return 1;
+        } else {
+            int localCount = 0;
+            vector<Node<T>*>& children = current->getChildren();
+            for (Node<T> *child : children) {
+                localCount += clusterCount(child);
+            }
+            return localCount;
+        }
+    }
+
+    int levelCount(Node<T>* current) {
+        if (current->isLeaf()) {
+            return 1;
+        } else {
+            return levelCount(current->getChild(0)) + 1;
+        }
+    }
+
+    int maxLevelCount(Node<T>* current) {
+        if (current->isLeaf()) {
+            return 1;
+        }
+        else {
+            int count = 0;
+            int maxCount = 0;
+            vector<Node<T>*>& children = current->getChildren();
+            for (Node<T> *child : children) {
+                count = maxLevelCount(child);
+                if (count > maxCount) maxCount = count;
+            }
+            return maxCount + 1;
+        }
+    }
+    
+    int minLevelCount(Node<T>* current) {
+        if (current->isLeaf()) {
+            return 1;
+        }
+        else {
+            int count = 0;
+            int minCount = 0;
+            vector<Node<T>*>& children = current->getChildren();
+            for (Node<T> *child : children) {
+                count = maxLevelCount(child);
+                if (count < minCount) minCount = count;
+            }
+            return minCount + 1;
+        }
+    }
+
+
+};
+
+
+#endif	/* TSVQ_H */
+