revert score to max tile based, lengthen tick limit

Author: kywch

pufferlib/config/ocean/g2048.ini

@@ -5,11 +5,11 @@ policy_name = Policy
 rnn_name = Recurrent
 
 [policy]
-hidden_size = 128
+hidden_size = 256
 
 [rnn]
-input_size = 128
-hidden_size = 128
+input_size = 256
+hidden_size = 256
 
 [vec]
 num_envs = 4
@@ -18,6 +18,7 @@ num_envs = 4
 num_envs = 4096
 
 [train]
+# https://wandb.ai/kywch/pufferlib/runs/n8xml0u9?nw=nwuserkywch
 total_timesteps = 3_000_000_000
 anneal_lr = True
 batch_size = auto
@@ -31,7 +32,7 @@ clip_coef = 0.1
 ent_coef = 0.02
 gae_lambda = 0.6
 gamma = 0.985
-learning_rate = 0.003
+learning_rate = 0.001
 max_grad_norm = 1.0
 prio_alpha = 0.99
 prio_beta0 = 0.40

pufferlib/ocean/g2048/binding.c

@@ -12,7 +12,7 @@ static int my_init(Env* env, PyObject* args, PyObject* kwargs) {
 static int my_log(PyObject* dict, Log* log) {
     assign_to_dict(dict, "perf", log->perf);
     assign_to_dict(dict, "score", log->score);
-    assign_to_dict(dict, "max_tile", log->max_tile);
+    assign_to_dict(dict, "merge_score", log->merge_score);
     assign_to_dict(dict, "episode_return", log->episode_return);
     assign_to_dict(dict, "episode_length", log->episode_length);
     return 0;

pufferlib/ocean/g2048/g2048.c

@@ -29,10 +29,10 @@ int main() {
 
         if (IsKeyDown(KEY_LEFT_SHIFT)) {
             action = -1;
-            if (IsKeyDown(KEY_W) || IsKeyDown(KEY_UP)) action = UP;
-            else if (IsKeyDown(KEY_S) || IsKeyDown(KEY_DOWN)) action = DOWN;
-            else if (IsKeyDown(KEY_A) || IsKeyDown(KEY_LEFT)) action = LEFT;
-            else if (IsKeyDown(KEY_D) || IsKeyDown(KEY_RIGHT)) action = RIGHT;
+            if (IsKeyPressed(KEY_W) || IsKeyPressed(KEY_UP)) action = UP;
+            else if (IsKeyPressed(KEY_S) || IsKeyPressed(KEY_DOWN)) action = DOWN;
+            else if (IsKeyPressed(KEY_A) || IsKeyPressed(KEY_LEFT)) action = LEFT;
+            else if (IsKeyPressed(KEY_D) || IsKeyPressed(KEY_RIGHT)) action = RIGHT;
             env.actions[0] = action - 1;
         } else if (frame % 1 != 0) {
             continue;
@@ -48,11 +48,6 @@ int main() {
             c_step(&env);
         }
 
-        if (IsKeyDown(KEY_LEFT_SHIFT) && action > 0) {
-            WaitTime(0.05);
-        }
-
-
     }
 
     free_linearlstm(net);

pufferlib/ocean/g2048/g2048.h

@@ -20,13 +20,13 @@
 #define INVALID_MOVE_PENALTY -0.05f
 #define GAME_OVER_PENALTY -1.0f
 
-// To normalize perf from 0 to 1. Update when beaten.
-#define OBSERVED_MAX_SCORE 100000.0f
+// To normalize perf from 0 to 1. Reachable with hidden size 256.
+#define OBSERVED_MAX_TILE 4096.0f
 
 typedef struct {
     float perf;
     float score;
-    float max_tile;
+    float merge_score;
     float episode_return;
     float episode_length;
     float n;
@@ -42,6 +42,7 @@ typedef struct {
     int tick;
     unsigned char grid[SIZE][SIZE];
     float episode_reward;           // Accumulate episode reward
+    int moves_made;
     int max_episode_ticks;          // Dynamic max_ticks based on score
 
     // Cached values to avoid recomputation
@@ -115,9 +116,9 @@ static inline unsigned char get_max_tile(Game* game) {
 
 void add_log(Game* game) {
     unsigned char s = get_max_tile(game);
-    game->log.max_tile += (float)(1 << s);
-    game->log.score += (float)game->score;
-    game->log.perf += (float)game->score / OBSERVED_MAX_SCORE;
+    game->log.score += (float)(1 << s);
+    game->log.perf += (float)(1 << s) / OBSERVED_MAX_TILE;
+    game->log.merge_score += (float)game->score;
     game->log.episode_length += game->tick;
     game->log.episode_return += game->episode_reward;
     game->log.n += 1;
@@ -136,6 +137,7 @@ void c_reset(Game* game) {
     game->empty_count = SIZE * SIZE;
     game->game_over_cached = false;
     game->grid_changed = true;
+    game->moves_made = 0;
     game->max_episode_ticks = BASE_MAX_TICKS;
 
     if (game->terminals) game->terminals[0] = 0;
@@ -176,6 +178,7 @@ void add_random_tile(Game* game) {
     if (chosen_pos >= 0) {
         int i = chosen_pos / SIZE;
         int j = chosen_pos % SIZE;
+        // Implement the 90% 2, 10% 4 rule
         game->grid[i][j] = (rand() % 10 == 0) ? 2 : 1;
         game->empty_count--;
         game->grid_changed = true;
@@ -309,11 +312,13 @@ void c_step(Game* game) {
     game->tick++;
 
     if (did_move) {
+        game->moves_made++;
         add_random_tile(game);
         game->score += score_add;
         update_empty_count(game); // Update after adding tile
         // This is to limit infinite invalid moves during eval
-        game->max_episode_ticks = max(BASE_MAX_TICKS, game->score / 20);
+        // Don't need to be tight. Don't need to show to user?
+        game->max_episode_ticks = max(BASE_MAX_TICKS, game->score / 10);
     } else {
         reward = INVALID_MOVE_PENALTY;
     }
@@ -385,6 +390,9 @@ void c_render(Game* game) {
     // Draw score (format once per frame)
     snprintf(score_text, sizeof(score_text), "Score: %d", game->score);
     DrawText(score_text, 10, px * SIZE + 10, 24, PUFF_WHITE);
+
+    snprintf(score_text, sizeof(score_text), "Moves: %d", game->moves_made);
+    DrawText(score_text, 210, px * SIZE + 10, 24, PUFF_WHITE);
 
     EndDrawing();
 }