fix: clean-up

Author: pminev

code/ac/llama/LogitComparer.cpp

@@ -46,29 +46,7 @@ bool LogitComparer::compare(const TokenDataVector& data1, const TokenDataVector&
     return jsd(prob_map, prob_map2) < 0.01f; // 1% divergence allowed
 }
 
-float LogitComparer::JSD(const TokenDataVector& data1, const TokenDataVector& data2) {
-    std::unordered_map<int32_t, float> prob_map, prob_map2;
-
-    for (const auto& p : data1) prob_map[p.token] = p.prob;
-    for (const auto& p : data2) prob_map2[p.token] = p.prob;
-
-    return jsd(prob_map, prob_map2);
-}
-
-float LogitComparer::cosineDistance(const TokenDataVector& data1, const TokenDataVector& data2) {
-    assert(data1.size() == data2.size());
-    float dot = 0.0f, normA = 0.0f, normB = 0.0f;
-    for (size_t i = 0; i < data1.size(); ++i) {
-        dot += data1[i].logit * data2[i].logit;
-        normA += data1[i].logit * data1[i].logit;
-        normB += data2[i].logit * data2[i].logit;
-    }
-    return 1.0f - (dot / (std::sqrt(normA) * std::sqrt(normB)));
-}
-
 float LogitComparer::logitSimilarity(const TokenDataVector& data1, const TokenDataVector& data2) {
-    float res = 0.0f;
-
     assert(data1.size() == data2.size());
     std::unordered_map<int32_t, float> l_map, l2_map;
 

code/ac/llama/LogitComparer.hpp

@@ -12,10 +12,6 @@ class LogitComparer {
 public:
     static bool compare(const TokenDataVector& data1, const TokenDataVector& data2);
 
-    static float JSD(const TokenDataVector& data1, const TokenDataVector& data2);
-
-    static float cosineDistance(const TokenDataVector& data1, const TokenDataVector& data2);
-
     static float logitSimilarity(const TokenDataVector& data1, const TokenDataVector& data2);
 
 private:

example/e-verify.cpp

@@ -128,32 +128,6 @@ class Model {
     ac::llama::Session* m_session;
 };
 
-// -- Helper function to compute normalized entropy --
-float normalizedEntropy(const ac::llama::TokenDataVector& data) {
-    std::vector<float> probs(data.size());
-    float sum = 0.0f;
-
-    // Calculate softmax probabilities
-    for (auto& val : data) {
-        sum += std::exp(val.logit);
-    }
-    for (size_t i = 0; i < data.size(); ++i) {
-        probs[i] = std::exp(data[i].logit) / sum;
-    }
-
-    // Calculate entropy
-    float entropy = 0.0f;
-    for (float p : probs) {
-        if (p > 0.0f) {
-            entropy -= p * std::log(p);
-        }
-    }
-
-    // Normalize entropy by maximum possible entropy (log(number of classes))
-    float maxEntropy = std::log(float(probs.size()));
-    return entropy / maxEntropy;
-}
-
 
 std::vector<Model::GenerationResult> modelGeneration(Model& m1, Model& m2, std::string_view prompt, uint32_t maxTokens) {
     auto res = m1.generate(prompt, maxTokens);
@@ -280,66 +254,20 @@ Model::GenerationResult deserialize(std::string_view filename) {
 void runCompare(Model::GenerationResult& r1, Model::GenerationResult& r2) {
     std::vector<float> jsdResults;
     std::vector<float> similarityResults;
-    float totalWeightedDist = 0.0f;
-    float totalWeight = 0.0f;
 
     for (size_t i = 0; i < r1.steps.size(); i++) {
         auto& step1 = r1.steps[i];
         auto& step2 = r2.steps[i];
 
-        // Calculate distance
-        float dist = ac::llama::LogitComparer::cosineDistance(step1.data, step2.data);
-
-        // Calculate weight based on normalized entropy
-        float weight = normalizedEntropy(step1.data);
-        totalWeightedDist += weight * dist;
-        totalWeight += weight;
-
-        // Calculate JSD
-        float jsd = ac::llama::LogitComparer::JSD(step1.data, step2.data);
-        jsdResults.push_back(jsd);
-
         // Calculate similarity
         float similarity = ac::llama::LogitComparer::logitSimilarity(step1.data, step2.data);
         similarityResults.push_back(similarity);
 
         std::cout << "Token: " << step1.tokenStr
-                << ", Weight: " << weight
-                << ", JSD: " << jsd
                 << ", Similarity: " << similarity
-                << ", Distance: " << dist
                 << "\n";
     }
 
-
-    {
-        // Final step: Normalize
-
-        // Score range | Interpretation
-        // 0.0 | Perfect match (identical predictions)
-        // 0.0001 - 0.001 | Practically indistinguishable
-        // 0.001 - 0.01 | Very close, slight variation
-        // 0.01 - 0.1 | Moderate variation, likely different versions/settings
-        // 0.1 - 1.0 | Large differences, likely different models
-        float finalScore = (totalWeight > 0.0f) ? (totalWeightedDist / totalWeight) : 0.0f;
-        std::cout << "Final weighted distance score: " << finalScore << "\n";
-    }
-
-    {
-        // Final score interpretation
-        // average JSD score
-        // 0.0 | Perfect match (identical predictions)
-        // 0.0001 - 0.001 | Practically indistinguishable
-        // 0.001 - 0.01 | Moderate variation, likely different versions/settings
-        // 0.01 - 0.1 | Large differences, likely different models
-        float jsdSum = 0.0f;
-        for (const auto& jsd : jsdResults) {
-            jsdSum += jsd;
-        }
-        float jsdAvg = jsdSum / jsdResults.size();
-        std::cout << "Average JSD score: " << jsdAvg << "\n";
-    }
-
     {
         float similaritySum = 0.0f;
         for (const auto& similarity : similarityResults) {
@@ -362,8 +290,8 @@ int main() try {
 
     // load model
     std::string tmpFolder = AC_TEST_DATA_LLAMA_DIR "/../../../tmp/";
-    std::string modelGguf = "Meta-Llama-3.1-70B-Instruct-Q5_K_S.gguf";
-    // std::string modelGguf = "Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf";
+    // std::string modelGguf = "Meta-Llama-3.1-70B-Instruct-Q5_K_S.gguf";
+    std::string modelGguf = "Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf";
     // std::string modelGguf = "BgGPT-Gemma-2-2B-IT-v1.0.Q8_0.gguf";
     // std::string modelGguf = "Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf";
     // std::string modelGguf2 = "Meta-Llama-3.1-70B-Instruct-Q5_K_S.gguf";