Merge branch '3.0' of https://github.com/pufferai/pufferlib into 3.0

Author: Joseph Suarez

pufferlib/config/ocean/asteroids.ini

@@ -12,5 +12,26 @@ num_envs = 1024
 size = 500
 
 [train]
-total_timesteps = 150_000_000
-minibatch_size = 32768
+adam_beta1 = 0.975493290069733
+adam_beta2 = 0.9999436458974764
+adam_eps = 6.915036275112011e-08
+anneal_lr = true
+batch_size = auto
+bptt_horizon = 64
+checkpoint_interval = 200
+clip_coef = 0.18588778503512546
+ent_coef = 0.0016620361911332262
+gae_lambda = 0.8400278040617952
+gamma = 0.9998708818940873
+learning_rate = 0.00502237062536979
+max_grad_norm = 0.7306435358436453
+max_minibatch_size = 32768
+minibatch_size = 8192
+prio_alpha = 0.9165093859993415
+prio_beta0 = 0.8869674411376214
+total_timesteps = 100_000_000
+update_epochs = 1
+vf_clip_coef = 0.1
+vf_coef = 2.960148388519086
+vtrace_c_clip = 1.0767718761515104
+vtrace_rho_clip = 4.132507367126342

pufferlib/config/ocean/battle.ini

@@ -12,22 +12,39 @@ input_size = 512
 hidden_size = 512
 
 [vec]
-num_envs = 8
+num_envs = 16
 
 [env]
-num_envs = 8
+num_envs = 4
 num_agents = 128
 num_armies = 2
 size_x = 2
-size_y = 2
+size_y = 1.0
 size_z = 2
 
 [train]
-total_timesteps = 100_000_000
+total_timesteps = 50_000_000
 
-learning_rate = 0.0015534438005054883
-gamma = 0.9923382806478448
-minibatch_size = 32768
+#adam_beta1 = 0.9672322418397323
+#adam_beta2 = 0.9877607751795193
+#adam_eps = 3.1721115738865995e-12
+#clip_coef = 0.43568934504743784
+#ent_coef = 0.0009836417478975427
+#gae_lambda = 0.9668222538234107
+#gamma = 0.990709789440733
+#learning_rate = 0.006246420318636455
+#max_grad_norm = 1.7919049246329588
+#minibatch_size = 65536
+#prio_alpha = 0.09999999999999998
+#prio_beta0 = 0.7406397128300295
+#vf_clip_coef = 1.6190073090306314
+#vf_coef = 3.4918587292978454
+#vtrace_c_clip = 0.5344573247342275
+#vtrace_rho_clip = 1.2893540729776307
+
+#learning_rate = 0.0015534438005054883
+#gamma = 0.9923382806478448
+#minibatch_size = 32768
 
 #adam_beta1 = 0.5797997352318079
 #adam_beta2 = 0.9001752474216785

pufferlib/config/ocean/g2048.ini

@@ -4,9 +4,36 @@ env_name = puffer_g2048
 policy_name = G2048
 rnn_name = Recurrent
 
+[policy]
+hidden_size = 256
+
+[rnn]
+input_size = 256
+hidden_size = 256
+
+[vec]
+num_envs = 4
+
 [env]
-num_envs = 4096
+num_envs = 4024
 
 [train]
-total_timesteps = 1_000_000_000
-minibatch_size = 32768
+total_timesteps = 600_000_000
+adam_beta1 = 0.9529488439604378
+adam_beta2 = 0.9993901829477296
+adam_eps = 2.745365927413118e-7
+bptt_horizon = 64
+clip_coef = 0.596573170393339
+ent_coef = 0.02107417730003862
+gae_lambda = 0.9940613415815854
+gamma = 0.9889857974154952
+learning_rate = 0.0032402460796988127
+max_grad_norm = 1.0752406726589745
+minibatch_size = 16384
+prio_alpha = 0.25297099593586336
+prio_beta0 = 0.940606268942572
+vf_clip_coef = 0.1
+vf_coef = 1.6362878279900643
+vtrace_c_clip = 0
+vtrace_rho_clip = 1.2917509971869054
+anneal_lr = False

pufferlib/ocean/battle/battle.h

@@ -477,6 +477,12 @@ void scripted_move(Battle* env, Entity* agent, bool is_air) {
     float dx = target->x - agent->x;
     float dy = target->y - agent->y;
     float dz = target->z - agent->z;
+
+    // Add some noise
+    dx += randf(-0.1f, 0.1f);
+    dy += randf(-0.1f, 0.1f);
+    dz += randf(-0.1f, 0.1f);
+
     float dd = dx*dx + dz*dz;
     if (is_air) {
         dd += dy*dy;
@@ -669,11 +675,12 @@ void compute_observations(Battle* env) {
             o->dx = dx;
             o->dy = dy;
             o->dz = dz;
-            o->distance = distance;
             if (other->army == agent->army) {
                 o->same_team = 1.0f;
+                o->distance = 99999.0f;
             } else {
                 o->same_team = 0.0f;
+                o->distance = distance;
             }
             o->idx = i;
         }
@@ -817,13 +824,14 @@ void c_step(Battle* env) {
             if (i < env->num_agents/2) {
                 env->rewards[i] = reward;
                 env->terminals[i] = 1;
+                env->log.score = env->log.episode_return;
+                env->log.episode_length += agent->episode_length;
+                env->log.episode_return += agent->episode_return;
+                env->log.collision_rate += collision;
+                env->log.oob_rate += oob;
+                env->log.n++;
+
             }
-            env->log.score = (1.0f - collision) * env->log.episode_return;
-            env->log.episode_length += agent->episode_length;
-            env->log.episode_return += agent->episode_return;
-            env->log.collision_rate += collision;
-            env->log.oob_rate += oob;
-            env->log.n++;
             agent->episode_length = 0;
             agent->episode_return = 0;
         }
@@ -870,8 +878,8 @@ void c_step(Battle* env) {
             agent->target = j;
             if (i < env->num_agents/2) {
                 env->rewards[i] += 0.25f;
+                agent->episode_return += 0.25f;
             }
-            agent->episode_return += 0.25f;
             target->health -= agent->attack_damage;
             break;
         }

pufferlib/ocean/battle/battle.py

@@ -12,7 +12,6 @@ def __init__(self, num_envs=1, width=1920, height=1080, size_x=1.0,
             num_armies=4, render_mode=None, log_interval=128, buf=None, seed=0):
         self.single_observation_space = gymnasium.spaces.Box(low=0, high=1,
             shape=(num_armies*3 + 4*16 + 22 + 8,), dtype=np.float32)
-        #self.single_action_space = gymnasium.spaces.MultiDiscrete([9, 9, 9])
         self.single_action_space = gymnasium.spaces.Box(
                 low=-1, high=1, shape=(3,), dtype=np.float32)
         self.render_mode = render_mode