Battle env fixes

Author: Joseph Suarez

pufferlib/config/ocean/battle.ini

@@ -12,22 +12,39 @@ input_size = 512
 hidden_size = 512
 
 [vec]
-num_envs = 8
+num_envs = 16
 
 [env]
-num_envs = 8
+num_envs = 4
 num_agents = 128
 num_armies = 2
 size_x = 2
-size_y = 2
+size_y = 1.0
 size_z = 2
 
 [train]
-total_timesteps = 100_000_000
+total_timesteps = 50_000_000
 
-learning_rate = 0.0015534438005054883
-gamma = 0.9923382806478448
-minibatch_size = 32768
+#adam_beta1 = 0.9672322418397323
+#adam_beta2 = 0.9877607751795193
+#adam_eps = 3.1721115738865995e-12
+#clip_coef = 0.43568934504743784
+#ent_coef = 0.0009836417478975427
+#gae_lambda = 0.9668222538234107
+#gamma = 0.990709789440733
+#learning_rate = 0.006246420318636455
+#max_grad_norm = 1.7919049246329588
+#minibatch_size = 65536
+#prio_alpha = 0.09999999999999998
+#prio_beta0 = 0.7406397128300295
+#vf_clip_coef = 1.6190073090306314
+#vf_coef = 3.4918587292978454
+#vtrace_c_clip = 0.5344573247342275
+#vtrace_rho_clip = 1.2893540729776307
+
+#learning_rate = 0.0015534438005054883
+#gamma = 0.9923382806478448
+#minibatch_size = 32768
 
 #adam_beta1 = 0.5797997352318079
 #adam_beta2 = 0.9001752474216785

pufferlib/ocean/battle/battle.h

@@ -477,6 +477,12 @@ void scripted_move(Battle* env, Entity* agent, bool is_air) {
     float dx = target->x - agent->x;
     float dy = target->y - agent->y;
     float dz = target->z - agent->z;
+
+    // Add some noise
+    dx += randf(-0.1f, 0.1f);
+    dy += randf(-0.1f, 0.1f);
+    dz += randf(-0.1f, 0.1f);
+
     float dd = dx*dx + dz*dz;
     if (is_air) {
         dd += dy*dy;
@@ -669,11 +675,12 @@ void compute_observations(Battle* env) {
             o->dx = dx;
             o->dy = dy;
             o->dz = dz;
-            o->distance = distance;
             if (other->army == agent->army) {
                 o->same_team = 1.0f;
+                o->distance = 99999.0f;
             } else {
                 o->same_team = 0.0f;
+                o->distance = distance;
             }
             o->idx = i;
         }
@@ -817,13 +824,14 @@ void c_step(Battle* env) {
             if (i < env->num_agents/2) {
                 env->rewards[i] = reward;
                 env->terminals[i] = 1;
+                env->log.score = env->log.episode_return;
+                env->log.episode_length += agent->episode_length;
+                env->log.episode_return += agent->episode_return;
+                env->log.collision_rate += collision;
+                env->log.oob_rate += oob;
+                env->log.n++;
+
             }
-            env->log.score = (1.0f - collision) * env->log.episode_return;
-            env->log.episode_length += agent->episode_length;
-            env->log.episode_return += agent->episode_return;
-            env->log.collision_rate += collision;
-            env->log.oob_rate += oob;
-            env->log.n++;
             agent->episode_length = 0;
             agent->episode_return = 0;
         }
@@ -870,8 +878,8 @@ void c_step(Battle* env) {
             agent->target = j;
             if (i < env->num_agents/2) {
                 env->rewards[i] += 0.25f;
+                agent->episode_return += 0.25f;
             }
-            agent->episode_return += 0.25f;
             target->health -= agent->attack_damage;
             break;
         }

pufferlib/ocean/battle/battle.py

@@ -12,7 +12,6 @@ def __init__(self, num_envs=1, width=1920, height=1080, size_x=1.0,
             num_armies=4, render_mode=None, log_interval=128, buf=None, seed=0):
         self.single_observation_space = gymnasium.spaces.Box(low=0, high=1,
             shape=(num_armies*3 + 4*16 + 22 + 8,), dtype=np.float32)
-        #self.single_action_space = gymnasium.spaces.MultiDiscrete([9, 9, 9])
         self.single_action_space = gymnasium.spaces.Box(
                 low=-1, high=1, shape=(3,), dtype=np.float32)
         self.render_mode = render_mode