polish(zjow): rename 'eval reward' -> 'episode return' (#536)

* fix typo 'final eval reward' -> 'eval episode return'; 'episode reward' -> 'episode return'

* formatting

* fix typo

* Polish code.

Author: zjowowen

ding/design/serial_evaluator-activity.puml

@@ -26,6 +26,6 @@ repeat
   endif
 repeat while (evaluate episodes are not enough?)
 |#FFCCCC|evaluator|
-:return eval_episode_reward;
+:return eval_episode_return;
 stop
 @enduml

ding/design/serial_main.puml

@@ -31,7 +31,7 @@ loop
     evaluator -> evaluator: eval_performance
     alt reach eval stop_value
       learner -> learner: save checkpoint and exit
-    else eval_reward is new highest
+    else episode_return is new highest
       learner -> learner: save checkpoint
     end
   end

ding/entry/application_entry.py

@@ -72,10 +72,10 @@ def eval(
 
     # Evaluate
     _, episode_info = evaluator.eval()
-    reward = [e['final_eval_reward'] for e in episode_info]
-    eval_reward = np.mean(to_ndarray(reward))
-    print('Eval is over! The performance of your RL policy is {}'.format(eval_reward))
-    return eval_reward
+    reward = [e['eval_episode_return'] for e in episode_info]
+    episode_return = np.mean(to_ndarray(reward))
+    print('Eval is over! The performance of your RL policy is {}'.format(episode_return))
+    return episode_return
 
 
 def collect_demo_data(
@@ -271,8 +271,8 @@ def episode_to_transitions_filter(data_path: str, expert_data_path: str, nstep:
         _dict = pickle.load(f)  # class is list; length is cfg.reward_model.collect_count
     post_process_data = []
     for i in range(len(_dict)):
-        episode_rewards = torch.stack([_dict[i][j]['reward'] for j in range(_dict[i].__len__())], axis=0)
-        if episode_rewards.sum() < min_episode_return:
+        episode_returns = torch.stack([_dict[i][j]['reward'] for j in range(_dict[i].__len__())], axis=0)
+        if episode_returns.sum() < min_episode_return:
             continue
         data = get_nstep_return_data(_dict[i], nstep)
         post_process_data.extend(data)

ding/entry/serial_entry.py

@@ -121,7 +121,7 @@ def serial_pipeline(
     import pickle
     import numpy as np
     with open(os.path.join(cfg.exp_name, 'result.pkl'), 'wb') as f:
-        eval_value_raw = [d['final_eval_reward'] for d in eval_info]
+        eval_value_raw = [d['eval_episode_return'] for d in eval_info]
         final_data = {
             'stop': stop,
             'env_step': collector.envstep,

ding/entry/serial_entry_gail.py

@@ -127,7 +127,7 @@ def serial_pipeline_gail(
         # Evaluate policy performance
         if evaluator.should_eval(learner.train_iter):
             stop, reward = evaluator.eval(learner.save_checkpoint, learner.train_iter, collector.envstep)
-            reward_mean = np.array([r['final_eval_reward'] for r in reward]).mean()
+            reward_mean = np.array([r['eval_episode_return'] for r in reward]).mean()
             if reward_mean >= best_reward:
                 save_reward_model(cfg.exp_name, reward_model, 'best')
                 best_reward = reward_mean

ding/entry/serial_entry_onpolicy.py

@@ -102,7 +102,7 @@ def serial_pipeline_onpolicy(
     import pickle
     import numpy as np
     with open(os.path.join(cfg.exp_name, 'result.pkl'), 'wb') as f:
-        eval_value_raw = [d['final_eval_reward'] for d in eval_info]
+        eval_value_raw = [d['eval_episode_return'] for d in eval_info]
         final_data = {
             'stop': stop,
             'env_step': collector.envstep,

ding/entry/tests/test_application_entry.py

@@ -34,16 +34,16 @@ def test_eval(self, setup_state_dict):
         cfg_for_stop_value = compile_config(cartpole_offppo_config, auto=True, create_cfg=cartpole_offppo_create_config)
         stop_value = cfg_for_stop_value.env.stop_value
         config = deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)
-        eval_reward = eval(config, seed=0, state_dict=setup_state_dict['eval'])
-        assert eval_reward >= stop_value
+        episode_return = eval(config, seed=0, state_dict=setup_state_dict['eval'])
+        assert episode_return >= stop_value
         config = deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)
-        eval_reward = eval(
+        episode_return = eval(
             config,
             seed=0,
             env_setting=[CartPoleEnv, None, [{} for _ in range(5)]],
             state_dict=setup_state_dict['eval']
         )
-        assert eval_reward >= stop_value
+        assert episode_return >= stop_value
 
     def test_collect_demo_data(self, setup_state_dict):
         config = deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)

ding/envs/env/default_wrapper.py

@@ -2,14 +2,14 @@
 from typing import Optional, List
 import copy
 
-final_eval_reward_wrapper = EasyDict(type='final_eval_reward')
+eval_episode_return_wrapper = EasyDict(type='eval_episode_return')
 
 
 def get_default_wrappers(env_wrapper_name: str, env_id: Optional[str] = None) -> List[dict]:
     if env_wrapper_name == 'mujoco_default':
         return [
             EasyDict(type='delay_reward', kwargs=dict(delay_reward_step=3)),
-            copy.deepcopy(final_eval_reward_wrapper),
+            copy.deepcopy(eval_episode_return_wrapper),
         ]
     elif env_wrapper_name == 'atari_default':
         wrapper_list = []
@@ -23,14 +23,14 @@ def get_default_wrappers(env_wrapper_name: str, env_id: Optional[str] = None) ->
         wrapper_list.append(EasyDict(type='scaled_float_frame'))
         wrapper_list.append(EasyDict(type='clip_reward'))
         wrapper_list.append(EasyDict(type='frame_stack', kwargs=dict(n_frames=4)))
-        wrapper_list.append(copy.deepcopy(final_eval_reward_wrapper))
+        wrapper_list.append(copy.deepcopy(eval_episode_return_wrapper))
         return wrapper_list
     elif env_wrapper_name == 'gym_hybrid_default':
         return [
             EasyDict(type='gym_hybrid_dict_action'),
-            copy.deepcopy(final_eval_reward_wrapper),
+            copy.deepcopy(eval_episode_return_wrapper),
         ]
     elif env_wrapper_name == 'default':
-        return [copy.deepcopy(final_eval_reward_wrapper)]
+        return [copy.deepcopy(eval_episode_return_wrapper)]
     else:
         raise NotImplementedError()

ding/envs/env/env_implementation_check.py

@@ -74,7 +74,7 @@ def check_step(env: BaseEnv) -> None:
         for ndarray, space, name in zip([obs, rew], [env.observation_space, env.reward_space], ['obs', 'rew']):
             check_array_space(ndarray, space, name)
         if done:
-            assert 'final_eval_reward' in info, "info dict should have 'final_eval_reward' key."
+            assert 'eval_episode_return' in info, "info dict should have 'eval_episode_return' key."
             done_times += 1
             _ = env.reset()
         if done_times == 3:
@@ -163,7 +163,7 @@ def demonstrate_correct_procedure(env_fn: Callable) -> None:
         action = env.random_action()
         obs, rew, done, info = env.step(action)
         if done:
-            assert 'final_eval_reward' in info
+            assert 'eval_episode_return' in info
             done_times += 1
             obs = env.reset()
             # Seed will not change unless `seed` method is called again.

ding/envs/env/tests/demo_env.py

@@ -59,7 +59,7 @@ def step(self, action: Any) -> 'BaseEnv.timestep':
             done = False
         info = {}
         if done:
-            info['final_eval_reward'] = self.reward_space.sample() * 30
+            info['eval_episode_return'] = self.reward_space.sample() * 30
         return BaseEnvTimestep(obs, rew, done, info)
 
     def seed(self, seed: int) -> None: