Fix policies minfloat (#22)

* Add policy minfloat

* Set alpha min = 0

* Add minfloat handling for RPO

* Fix pi difference

* Fix logarithmic policy

Author: sorgfresser

src/graph/htps.cpp

@@ -891,6 +891,13 @@ Simulation HTPS::find_leaves_to_expand(std::vector<std::shared_ptr<theorem>> &te
             tactic_id = std::distance(node_policy.begin(), std::max_element(node_policy.begin(), node_policy.end()));
         } else {
             // Normal softmax with temperature, i.e. exp(p / temperature)
+            // But take logarithm of policy first, as done in evariste
+            for (size_t i = 0; i < node_policy.size(); i++) {
+                if (node_policy[i] > MIN_FLOAT)
+                    node_policy[i] = std::log(node_policy[i]);
+                else
+                    node_policy[i] = MIN_FLOAT;
+            }
             double p_sum = 0;
             for (auto &p: node_policy) {
                 p = std::exp(p / params.policy_temperature);

src/model/policy.cpp

@@ -36,6 +36,7 @@ void Policy::get_policy(const std::vector<double> &q_values, const std::vector<d
         throw std::invalid_argument("No valid q-values");
     }
     if (valid_count == 1) {
+        std::fill(result.begin(), result.end(), MIN_FLOAT);
         result[valid_indices[0]] = 1;
         return;
     }
@@ -52,7 +53,11 @@ void Policy::get_policy(const std::vector<double> &q_values, const std::vector<d
     bool is_nan = std::any_of(result.begin(), result.end(), [](double d) { return std::isnan(d); });
     assert (!is_nan);
     assert (q_values.size() == result.size());
-    double sum = std::accumulate(result.begin(), result.end(), 0.0);
+    double sum = 0.0;
+    for (size_t i = 0; i < result.size(); i++) {
+        if (result[i] > MIN_FLOAT)
+            sum += result[i];
+    }
     assert (sum > 0.99 && sum < 1.01);
 }
 
@@ -68,10 +73,10 @@ void Policy::alpha_zero(const std::vector<double> &q_values, const std::vector<d
         if (pi_values[i] > MIN_FLOAT && q_values[i] > MIN_FLOAT) {
             scores[i] = q_values[i] + exploration * pi_values[i] * std::sqrt(count_sum) / (1 + counts_d[i]);
             valid_count++;
+            score_sum += scores[i];
         } else {
-            scores[i] = 0;
+            scores[i] = MIN_FLOAT;
         }
-        score_sum += scores[i];
     }
     assert (valid_count > 0);
     // If score sum is 0, simply return the uniform distribution over valid actions
@@ -80,14 +85,17 @@ void Policy::alpha_zero(const std::vector<double> &q_values, const std::vector<d
             if (q_values[i] > MIN_FLOAT && pi_values[i] > MIN_FLOAT) {
                 result[i] = 1.0 / static_cast<double>(valid_count);
             } else {
-                result[i] = 0;
+                result[i] = MIN_FLOAT;
             }
         }
         return;
     }
     // Normalize the scores
     for (size_t i = 0; i < q_values.size(); i++) {
-        result[i] = scores[i] / score_sum;
+        if (scores[i] > MIN_FLOAT)
+            result[i] = scores[i] / score_sum;
+        else
+            result[i] = MIN_FLOAT;
     }
 }
 
@@ -108,7 +116,7 @@ double Policy::find_rpo_alpha(double alpha_min, double alpha_max, const std::vec
             }
             pi_difference_sum += scaled_pi_values[i] / diff;
         }
-        if ((pi_difference_sum - 1) < TOLERANCE) {
+        if (std::abs(pi_difference_sum - 1) < TOLERANCE) {
             return alpha_mid;
         }
         if (pi_difference_sum > 1)
@@ -133,7 +141,7 @@ void Policy::mcts_rpo(const std::vector<double> &q_values, const std::vector<dou
                 q_sum += q_values[i];
                 valid_count++;
             } else {
-                result[i] = 0;
+                result[i] = MIN_FLOAT;
             }
         }
         // If q sum is 0, simply return the uniform distribution over valid actions
@@ -143,21 +151,25 @@ void Policy::mcts_rpo(const std::vector<double> &q_values, const std::vector<dou
                 if (q_values[i] > MIN_FLOAT && pi_values[i] > MIN_FLOAT) {
                     result[i] = 1.0 / static_cast<double>(valid_count);
                 } else {
-                    result[i] = 0;
+                    result[i] = MIN_FLOAT;
                 }
             }
             return;
         }
 
         for (size_t i = 0; i < q_values.size(); i++) {
-            result[i] /= q_sum;
+            if (q_values[i] > MIN_FLOAT) {
+                result[i] = result[i] / q_sum;
+            } else {
+                result[i] = MIN_FLOAT;
+            }
         }
         return;
     }
 
     std::vector<double> scaled_pi_values(q_values.size());
 
-    double alpha_min, alpha_max = 0;
+    double alpha_min = 0, alpha_max = 0;
 
     for (size_t i = 0; i < q_values.size(); i++) {
         scaled_pi_values[i] = pi_values[i] * multiplier;
@@ -167,11 +179,19 @@ void Policy::mcts_rpo(const std::vector<double> &q_values, const std::vector<dou
     double alpha = find_rpo_alpha(alpha_min, alpha_max, q_values, scaled_pi_values);
     double result_sum = 0;
     for (size_t i = 0; i < q_values.size(); i++) {
-        result[i] = scaled_pi_values[i] / std::max((alpha - q_values[i]), EPSILON);
-        result_sum += result[i];
+        if (q_values[i] > MIN_FLOAT) {
+            result[i] = scaled_pi_values[i] / std::max((alpha - q_values[i]), EPSILON);
+            result_sum += result[i];
+        }
+        else {
+            result[i] = MIN_FLOAT;
+        }
     }
     for (size_t i = 0; i < q_values.size(); i++) {
-        result[i] /= result_sum;
+        if (result[i] > MIN_FLOAT)
+            result[i] = result[i] / result_sum;
+        else
+            result[i] = MIN_FLOAT;
     }
 }