Merge pull request #3 from y-hesse/3.0

g2048 fix 1

Author: kywch

pufferlib/config/ocean/g2048.ini

@@ -5,11 +5,11 @@ policy_name = Policy
 rnn_name = Recurrent
 
 [policy]
-hidden_size = 128
+hidden_size = 1024
 
 [rnn]
-input_size = 128
-hidden_size = 128
+input_size = 1024
+hidden_size = 1024
 
 [vec]
 num_envs = 4
@@ -19,22 +19,21 @@ num_envs = 4096
 
 [train]
 total_timesteps = 5_000_000_000
-adam_beta1 = 0.9529488439604378
-adam_beta2 = 0.9993901829477296
-adam_eps = 2.745365927413118e-7
+adam_beta1 = 0.982603624444803
+adam_beta2 = 0.982603624444803
+adam_eps = 3.2888696338626164e-11
 bptt_horizon = 64
-clip_coef = 0.596573170393339
-ent_coef = 0.02107417730003862
-gae_lambda = 0.9940613415815854
-gamma = 0.9889857974154952
+clip_coef = 0.2709219986085283
+ent_coef = 0.09221187601118314
+gae_lambda = 0.5999999999999999
+gamma = 0.9913033082924563
 #learning_rate = 0.0032402460796988127
-learning_rate = 0.001
-max_grad_norm = 1.0752406726589745
-minibatch_size = 16384
-prio_alpha = 0.25297099593586336
-prio_beta0 = 0.940606268942572
-vf_clip_coef = 0.1
-vf_coef = 1.6362878279900643
-vtrace_c_clip = 0
-vtrace_rho_clip = 1.2917509971869054
-anneal_lr = False
+learning_rate = 0.001370087925623787
+max_grad_norm = 3.382578348055827
+minibatch_size = 32768
+prio_alpha = 0.09999999999999998
+prio_beta0 = 0.941336023531629
+vf_clip_coef = 0.3229933703598912
+vf_coef = 3.591594736259073
+vtrace_c_clip = 1.405090934486193
+vtrace_rho_clip = 0.836535302835556

pufferlib/ocean/g2048/2048.h

@@ -14,7 +14,7 @@
 #define RIGHT 4
 
 // Precomputed constants
-#define REWARD_MULTIPLIER 0.09090909f
+#define REWARD_MULTIPLIER 0.0625f
 #define INVALID_MOVE_PENALTY -0.05f
 #define GAME_OVER_PENALTY -1.0f
 
@@ -93,9 +93,24 @@ static inline void update_empty_count(Game* game) {
     game->empty_count = count;
 }
 
+// Optimized score calculation
+static inline unsigned char calc_score(Game* game) {
+    unsigned char max_tile = 0;
+    // Unroll loop for better performance
+    for (int i = 0; i < SIZE; i++) {
+        for (int j = 0; j < SIZE; j++) {
+            if (game->grid[i][j] > max_tile) {
+                max_tile = game->grid[i][j];
+            }
+        }
+    }
+    return max_tile;
+}
+
 void add_log(Game* game) {
-    game->log.score = (float)(1 << game->score);
-    game->log.perf += ((float)game->score) * REWARD_MULTIPLIER;
+    unsigned char s = calc_score(game);
+    game->log.score = (float)(1 << s);
+    game->log.perf += ((float)s) * 0.0909f;
     game->log.episode_length += game->tick;
     game->log.episode_return += game->episode_reward;
     game->log.n += 1;
@@ -162,7 +177,7 @@ void add_random_tile(Game* game) {
 }
 
 // Optimized slide and merge with fewer memory operations
-static inline bool slide_and_merge(unsigned char* row, float* reward) {
+static inline bool slide_and_merge(unsigned char* row, float* reward, float* score_increase) {
     bool moved = false;
     int write_pos = 0;
 
@@ -183,6 +198,7 @@ static inline bool slide_and_merge(unsigned char* row, float* reward) {
         if (row[i] != EMPTY && row[i] == row[i + 1]) {
             row[i]++;
             *reward += ((float)row[i]) * REWARD_MULTIPLIER;
+            *score_increase += (float)(1 << (int)row[i]);
             // Shift remaining elements left
             for (int j = i + 1; j < SIZE - 1; j++) {
                 row[j] = row[j + 1];
@@ -195,7 +211,7 @@ static inline bool slide_and_merge(unsigned char* row, float* reward) {
     return moved;
 }
 
-bool move(Game* game, int direction, float* reward) {
+bool move(Game* game, int direction, float* reward, float* score_increase) {
     bool moved = false;
     unsigned char temp[SIZE];
 
@@ -207,7 +223,7 @@ bool move(Game* game, int direction, float* reward) {
                 temp[i] = game->grid[idx][col];
             }
 
-            if (slide_and_merge(temp, reward)) {
+            if (slide_and_merge(temp, reward, score_increase)) {
                 moved = true;
                 // Write back column
                 for (int i = 0; i < SIZE; i++) {
@@ -224,7 +240,7 @@ bool move(Game* game, int direction, float* reward) {
                 temp[i] = game->grid[row][idx];
             }
 
-            if (slide_and_merge(temp, reward)) {
+            if (slide_and_merge(temp, reward, score_increase)) {
                 moved = true;
                 // Write back row
                 for (int i = 0; i < SIZE; i++) {
@@ -280,28 +296,15 @@ bool is_game_over(Game* game) {
     return true;
 }
 
-// Optimized score calculation
-static inline unsigned char calc_score(Game* game) {
-    unsigned char max_tile = 0;
-    // Unroll loop for better performance
-    for (int i = 0; i < SIZE; i++) {
-        for (int j = 0; j < SIZE; j++) {
-            if (game->grid[i][j] > max_tile) {
-                max_tile = game->grid[i][j];
-            }
-        }
-    }
-    return max_tile;
-}
-
 void c_step(Game* game) {
     float reward = 0.0f;
-    bool did_move = move(game, game->actions[0] + 1, &reward);
+    float score_add = 0.0f;
+    bool did_move = move(game, game->actions[0] + 1, &reward, &score_add);
     game->tick++;
 
     if (did_move) {
         add_random_tile(game);
-        game->score = calc_score(game);
+        game->score += score_add;
         update_empty_count(game); // Update after adding tile
     }
 
@@ -369,7 +372,7 @@ void c_render(Game* game) {
     }
 
     // Draw score (format once per frame)
-    snprintf(score_text, sizeof(score_text), "Score: %d", 1 << game->score);
+    snprintf(score_text, sizeof(score_text), "Score: %d", game->score);
     DrawText(score_text, 10, px * SIZE + 10, 24, PUFF_WHITE);
 
     EndDrawing();

pufferlib/ocean/g2048/g2048.c

@@ -22,18 +22,19 @@ int main() {
 
     // Main game loop
     int frame = 0;
+    int action = -1;
     while (!WindowShouldClose()) {
         c_render(&env);
         frame++;
-
-        int action = 0;
+        
         if (IsKeyDown(KEY_LEFT_SHIFT)) {
-            if (IsKeyPressed(KEY_W) || IsKeyPressed(KEY_UP)) action = UP;
-            else if (IsKeyPressed(KEY_S) || IsKeyPressed(KEY_DOWN)) action = DOWN;
-            else if (IsKeyPressed(KEY_A) || IsKeyPressed(KEY_LEFT)) action = LEFT;
-            else if (IsKeyPressed(KEY_D) || IsKeyPressed(KEY_RIGHT)) action = RIGHT;
+            action = -1;
+            if (IsKeyDown(KEY_W) || IsKeyDown(KEY_UP)) action = UP;
+            else if (IsKeyDown(KEY_S) || IsKeyDown(KEY_DOWN)) action = DOWN;
+            else if (IsKeyDown(KEY_A) || IsKeyDown(KEY_LEFT)) action = LEFT;
+            else if (IsKeyDown(KEY_D) || IsKeyDown(KEY_RIGHT)) action = RIGHT;
             env.actions[0] = action - 1;
-        } else if (frame % 10 != 0) {
+        } else if (frame % 1 != 0) {
             continue;
         } else {
             action = 1;
@@ -43,9 +44,15 @@ int main() {
             forward_linearlstm(net, net->obs, env.actions);
         }
 
-        if (action != 0) {
+        if (action > 0) {
             c_step(&env);
         }
+        
+        if (IsKeyDown(KEY_LEFT_SHIFT) && action > 0) {
+            WaitTime(0.05);
+        }
+
+
     }
 
     free_linearlstm(net);