Merge branch 'PufferAI:3.0' into 3.0

Author: y-hesse

pufferlib/config/metta.ini

@@ -5,7 +5,8 @@ policy_name = Policy
 rnn_name = Recurrent
 
 [vec]
-num_envs = 8
+num_envs = 64
+num_workers = 16
 
 [env]
 render_mode = auto

pufferlib/config/ocean/drone.ini pufferlib/config/ocean/drone_race.inipufferlib/config/ocean/drone.ini renamed to pufferlib/config/ocean/drone_race.ini

@@ -1,6 +1,6 @@
 [base]
 package = ocean
-env_name = puffer_drone
+env_name = puffer_drone_race
 policy_name = Policy
 rnn_name = Recurrent
 

pufferlib/config/ocean/drone_swarm.ini

@@ -0,0 +1,57 @@
+[base]
+package = ocean
+env_name = puffer_drone_swarm
+policy_name = Policy
+rnn_name = Recurrent
+
+[policy]
+hidden_size = 128
+
+[rnn]
+input_size = 128
+hidden_size = 128
+
+[vec]
+num_envs = 8
+
+[env]
+num_envs = 16
+num_drones = 64
+max_rings = 10
+
+[train]
+adam_beta1 = 0.9610890980775877
+adam_beta2 = 0.9999260775286266
+adam_eps = 7.782906079040132e-10
+anneal_lr = true
+batch_size = auto
+bptt_horizon = 64
+checkpoint_interval = 200
+clip_coef = 0.05982655642208556
+ent_coef = 0.002465076521024325
+gae_lambda = 0.9641173414828333
+gamma = 0.997472126425902
+learning_rate = 0.010933756713881205
+#learning_rate = 0.005
+max_grad_norm = 1.6317688647793107
+max_minibatch_size = 32768
+minibatch_size = 32768
+prio_alpha = 0.8968873016577552
+prio_beta0 = 0.8672928227817938
+total_timesteps = 500_000_000
+update_epochs = 1
+#use_rnn = false
+vf_clip_coef = 0.5869845581530236
+vf_coef = 2.1319065538539963
+vtrace_c_clip = 2.714930379733876
+vtrace_rho_clip = 3.8183814893708057
+
+[sweep]
+downsample = 0
+
+[sweep.train.total_timesteps]
+distribution = log_normal
+min = 2e8
+max = 4e8
+mean = 2e8
+scale = time

pufferlib/config/ocean/matsci.ini

@@ -0,0 +1,17 @@
+[base]
+package = ocean
+env_name = puffer_matsci
+policy_name = Policy
+
+[vec]
+num_envs = 8
+
+[env]
+num_envs = 8
+num_atoms = 128
+
+[train]
+total_timesteps = 50_000_000
+minibatch_size = 32768
+
+

pufferlib/environments/metta/environment.py

@@ -9,40 +9,24 @@
 from metta.mettagrid.curriculum.core import SingleTaskCurriculum
 from metta.mettagrid.replay_writer import ReplayWriter
 
-#from mettagrid.mettagrid_env import MettaGridEnv
-#from mettagrid.curriculum import SingleTaskCurriculum
-
 def env_creator(name='metta'):
     return functools.partial(make, name)
 
 def make(name, config='pufferlib/environments/metta/metta.yaml', render_mode='auto', buf=None, seed=0,
-         ore_reward=0.25, heart_reward=0.5, battery_reward=0.25):
-    '''Crafter creation function'''
-    #return MettaPuff(config, render_mode, buf)
-    #import mettagrid.mettagrid_env
-    #from omegaconf import OmegaConf
+         ore_reward=0.17088483842567775, battery_reward=0.9882859711234822, heart_reward=1.0):
+    '''Metta creation function'''
+    
     OmegaConf.register_new_resolver("div", oc_divide, replace=True)
     cfg = OmegaConf.load(config)
-    reward_cfg = cfg['game']['agent']['rewards']
-    '''
-    env_overrides = {
-        'game': {
-            'agent': {
-                'rewards': {
-                    'ore.red': 0.25,
-                    'ore.blue': 0.25,
-                    'ore.green': 0.25,
-                    'heart': 0.5,
-                    'battery': 0.25,
-                }
-            }
-        }
-    '''
-    reward_cfg['ore.red'] = float(ore_reward)
-    reward_cfg['heart'] = float(heart_reward)
-    reward_cfg['battery.red'] = float(battery_reward)
-    cfg = SingleTaskCurriculum('puffer', cfg)
-    return MettaPuff(cfg, render_mode=render_mode, buf=buf)
+    
+    # Update rewards under the new structure: agent.rewards.inventory
+    inventory_rewards = cfg['game']['agent']['rewards']['inventory']
+    inventory_rewards['ore_red'] = float(ore_reward)
+    inventory_rewards['heart'] = float(heart_reward)
+    inventory_rewards['battery_red'] = float(battery_reward)
+    
+    curriculum = SingleTaskCurriculum('puffer', cfg)
+    return MettaPuff(curriculum, render_mode=render_mode, buf=buf, seed=seed)
 
 def oc_divide(a, b):
     """
@@ -56,12 +40,17 @@ def oc_divide(a, b):
     return result
 
 class MettaPuff(MettaGridEnv):
-    def __init__(self, config, render_mode='human', buf=None, seed=0):
+    def __init__(self, curriculum, render_mode='human', buf=None, seed=0):
         self.replay_writer = None
         #if render_mode == 'auto':
         #    self.replay_writer = ReplayWriter("metta/")
 
-        super().__init__(config, render_mode=render_mode, buf=buf, replay_writer=self.replay_writer)
+        super().__init__(
+            curriculum=curriculum,
+            render_mode=render_mode,
+            buf=buf,
+            replay_writer=self.replay_writer
+        )
         self.action_space = pufferlib.spaces.joint_space(self.single_action_space, self.num_agents)
         self.actions = self.actions.astype(np.int32)