minor changes to fix a bug in the env and add sweeped params and a cnn

Author: y-hesse

pufferlib/config/ocean/g2048.ini

@@ -2,11 +2,26 @@
 package = ocean
 env_name = puffer_g2048
 policy_name = G2048
-rnn_name = Recurrent
 
 [env]
-num_envs = 4096
+num_envs = 4024
 
 [train]
-total_timesteps = 1_000_000_000
+total_timesteps = 10_000_000_000 #4.6B
+adam_beta1 = 0.8081024539479613
+adam_beta2 = 0.9978536811174212
+adam_eps = 1.4542006937471102e-9
+bptt_horizon = 64
+clip_coef = 0.095627870395359
+ent_coef = 0.08439222625935798
+gae_lambda = 0.9155041484587568
+gamma = 0.9661669903070148
+learning_rate = 0.0014756768275156805
+max_grad_norm = 0.8813109722891985
 minibatch_size = 32768
+prio_alpha = 0.565686548517019
+prio_beta0 = 0.811234742153397
+vf_clip_coef = 0.1
+vf_coef = 2.2900867366155664
+vtrace_c_clip = 0
+vtrace_rho_clip = 0.8647738667214979

pufferlib/ocean/g2048/2048.h

@@ -21,16 +21,15 @@ typedef struct {
     float n;
 } Log;
 
-// Required struct for env_binding.h compatibility
 typedef struct {
     Log log;                        // Required
-    unsigned char* observations;            // Required (flattened 256 floats: 16 tiles * 16 one-hot)
+    unsigned char* observations;    // Cheaper in memory if encoded in uint_8
     int* actions;                   // Required
     float* rewards;                 // Required
     unsigned char* terminals;       // Required
     int score;
     int tick;
-    unsigned char grid[SIZE][SIZE];         // Store tile values directly as floats
+    unsigned char grid[SIZE][SIZE];
     float episode_reward;           // Accumulate episode reward
 } Game;
 
@@ -43,27 +42,15 @@ void c_step(Game* env);
 void c_render(Game* env);
 void c_close(Game* env);
 
-// Update the observation vector to be one-hot encoded for all tiles
 static void update_observations(Game* game) {
-    for (int i = 0; i < SIZE; i++) {
-        for (int j = 0; j < SIZE; j++) {
-            game->observations[i * SIZE + j] = game->grid[i][j];
-        }
-    }
+    memcpy(game->observations, game->grid, sizeof(unsigned char) * SIZE * SIZE);
 }
 
 // --- Implementation ---
 
 void add_log(Game* game) {
-    int max_tile = 0;
-    for (int i = 0; i < SIZE; i++) {
-        for (int j = 0; j < SIZE; j++) {
-            int tile = (int)(game->grid[i][j]);
-            if (tile > max_tile) max_tile = tile;
-        }
-    }
-    game->log.score = (float)pow(2,max_tile);
-    game->log.perf += (game->rewards[0] > 0) ? 1 : 0;
+    game->log.score = pow(2,(float)game->score);
+    game->log.perf += (float)game->score/11.;
     game->log.episode_length += game->tick;
     game->log.episode_return += game->episode_reward;
     game->log.n += 1;
@@ -103,22 +90,12 @@ void add_random_tile(Game* game) {
     }
     if (count > 0) {
         int random_index = rand() % count;
-        int value = (rand() % 10 == 0) ? 4 : 2;
-        game->grid[empty_cells[random_index][0]][empty_cells[random_index][1]] = (float)value;
+        int value = (rand() % 10 == 0) ? 2 : 1;
+        game->grid[empty_cells[random_index][0]][empty_cells[random_index][1]] = value;
     }
     update_observations(game);
 }
 
-void print_grid(Game* game) {
-    for (int i = 0; i < SIZE; i++) {
-        for (int j = 0; j < SIZE; j++) {
-            printf("%4.0f ", game->grid[i][j]);
-        }
-        printf("\n");
-    }
-    printf("Score: %d\n", game->score);
-}
-
 bool slide_and_merge_row(float* row, float* reward) {
     bool moved = false;
     // Slide left
@@ -231,11 +208,11 @@ bool is_game_over(Game* game) {
     return true;
 }
 
-int calc_score(Game* game) {
-    int max_tile = 0;
+unsigned char calc_score(Game* game) {
+    unsigned char max_tile = 0;
     for (int i = 0; i < SIZE; i++) {
         for (int j = 0; j < SIZE; j++) {
-            int tile = (int)(game->grid[i][j]);
+            int tile = (game->grid[i][j]);
             if (tile > max_tile) max_tile = tile;
         }
     }
@@ -248,7 +225,7 @@ void c_step(Game* game) {
     game->tick += 1;
     if (did_move) {
         add_random_tile(game);
-        game->score = calc_score(game);
+        game->score = (float)calc_score(game);
     }
 
     game->terminals[0] = is_game_over(game) ? 1 : 0;

pufferlib/ocean/torch.py

@@ -190,15 +190,20 @@ def decode_actions(self, hidden):
         return action, value
 
 class G2048(nn.Module):
-    def __init__(self, env, hidden_size=128):
+    def __init__(self, env, cnn_channels=32, hidden_size=128):
         super().__init__()
         self.hidden_size = hidden_size
         self.is_continuous = False
 
-        self.encoder= torch.nn.Sequential(
-            nn.Linear(12*np.prod(env.single_observation_space.shape), hidden_size),
+        self.cnn = nn.Sequential(
+            pufferlib.pytorch.layer_init(
+                nn.Conv2d(16, cnn_channels, 2, stride=1)),
             nn.GELU(),
+            pufferlib.pytorch.layer_init(
+                nn.Conv2d(cnn_channels, cnn_channels, 2, stride=1)),
+            nn.Flatten(),
         )
+
         self.decoder = pufferlib.pytorch.layer_init(
             nn.Linear(hidden_size, env.single_action_space.n), std=0.01)
         self.value = pufferlib.pytorch.layer_init(
@@ -213,9 +218,8 @@ def forward(self, x, state=None):
         return self.forward_eval(x, state)
 
     def encode_observations(self, observations, state=None):
-        observations = F.one_hot(observations.long(), 12).view(-1, 12*16).float()
-        #observations = observations.float().view(-1, 16) / 11.0
-        return self.encoder(observations)
+        observations = F.one_hot(observations.long(), 16).view(-1, 16, 4, 4).float()
+        return self.cnn(observations)
 
     def decode_actions(self, hidden):
         action = self.decoder(hidden)