polish(nyz): polish example demos (#568)

* polish(nyz): polish example demos

* fix(nyz): fix unittest bugs

* fix(nyz): fix trex unittest bugs

Author: PaParaZz1

ding/config/config.py

@@ -341,6 +341,7 @@ def compile_config(
         create_cfg: dict = None,
         save_cfg: bool = True,
         save_path: str = 'total_config.py',
+        renew_dir: bool = True,
 ) -> EasyDict:
     """
     Overview:
@@ -361,6 +362,7 @@ def compile_config(
         - create_cfg (:obj:`dict`): Input create config dict
         - save_cfg (:obj:`bool`): Save config or not
         - save_path (:obj:`str`): Path of saving file
+        - renew_dir (:obj:`bool`): Whether to new a directory for saving config.
     Returns:
         - cfg (:obj:`EasyDict`): Config after compiling
     """
@@ -460,7 +462,7 @@ def compile_config(
     if 'exp_name' not in cfg:
         cfg.exp_name = 'default_experiment'
     if save_cfg:
-        if os.path.exists(cfg.exp_name):
+        if os.path.exists(cfg.exp_name) and renew_dir:
             cfg.exp_name += datetime.datetime.now().strftime("_%y%m%d_%H%M%S")
         try:
             os.makedirs(cfg.exp_name)

ding/entry/serial_entry_preference_based_irl.py

@@ -47,7 +47,7 @@ def serial_pipeline_preference_based_irl(
     create_cfg.policy.type = create_cfg.policy.type + '_command'
     create_cfg.reward_model = dict(type=cfg.reward_model.type)
     env_fn = None if env_setting is None else env_setting[0]
-    cfg = compile_config(cfg, seed=seed, env=env_fn, auto=True, create_cfg=create_cfg, save_cfg=True)
+    cfg = compile_config(cfg, seed=seed, env=env_fn, auto=True, create_cfg=create_cfg, save_cfg=True, renew_dir=False)
     cfg_bak = copy.deepcopy(cfg)
     # Create main components: env, policy
     if env_setting is None:

ding/entry/serial_entry_preference_based_irl_onpolicy.py

@@ -46,7 +46,7 @@ def serial_pipeline_preference_based_irl_onpolicy(
     create_cfg.policy.type = create_cfg.policy.type + '_command'
     create_cfg.reward_model = dict(type=cfg.reward_model.type)
     env_fn = None if env_setting is None else env_setting[0]
-    cfg = compile_config(cfg, seed=seed, env=env_fn, auto=True, create_cfg=create_cfg, save_cfg=True)
+    cfg = compile_config(cfg, seed=seed, env=env_fn, auto=True, create_cfg=create_cfg, save_cfg=True, renew_dir=False)
     # Create main components: env, policy
     if env_setting is None:
         env_fn, collector_env_cfg, evaluator_env_cfg = get_vec_env_setting(cfg.env)

ding/entry/tests/test_application_entry.py

@@ -3,8 +3,8 @@
 import os
 import pickle
 
-from dizoo.classic_control.cartpole.config.cartpole_offppo_config import cartpole_offppo_config, \
-    cartpole_offppo_create_config  # noqa
+from dizoo.classic_control.cartpole.config.cartpole_ppo_offpolicy_config import cartpole_ppo_offpolicy_config, \
+    cartpole_ppo_offpolicy_create_config  # noqa
 from dizoo.classic_control.cartpole.config.cartpole_trex_offppo_config import cartpole_trex_offppo_config,\
      cartpole_trex_offppo_create_config
 from dizoo.classic_control.cartpole.envs import CartPoleEnv
@@ -15,7 +15,7 @@
 
 @pytest.fixture(scope='module')
 def setup_state_dict():
-    config = deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)
+    config = deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)
     try:
         policy = serial_pipeline(config, seed=0)
     except Exception:
@@ -31,12 +31,14 @@ def setup_state_dict():
 class TestApplication:
 
     def test_eval(self, setup_state_dict):
-        cfg_for_stop_value = compile_config(cartpole_offppo_config, auto=True, create_cfg=cartpole_offppo_create_config)
+        cfg_for_stop_value = compile_config(
+            cartpole_ppo_offpolicy_config, auto=True, create_cfg=cartpole_ppo_offpolicy_create_config
+        )
         stop_value = cfg_for_stop_value.env.stop_value
-        config = deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)
+        config = deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)
         episode_return = eval(config, seed=0, state_dict=setup_state_dict['eval'])
         assert episode_return >= stop_value
-        config = deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)
+        config = deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)
         episode_return = eval(
             config,
             seed=0,
@@ -46,7 +48,7 @@ def test_eval(self, setup_state_dict):
         assert episode_return >= stop_value
 
     def test_collect_demo_data(self, setup_state_dict):
-        config = deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)
+        config = deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)
         collect_count = 16
         expert_data_path = './expert.data'
         collect_demo_data(

ding/entry/tests/test_application_entry_trex_collect_data.py

@@ -8,8 +8,8 @@
 
 from dizoo.classic_control.cartpole.config.cartpole_trex_offppo_config import cartpole_trex_offppo_config,\
      cartpole_trex_offppo_create_config
-from dizoo.classic_control.cartpole.config.cartpole_offppo_config import cartpole_offppo_config,\
-     cartpole_offppo_create_config
+from dizoo.classic_control.cartpole.config.cartpole_ppo_offpolicy_config import cartpole_ppo_offpolicy_config,\
+     cartpole_ppo_offpolicy_create_config
 from ding.entry.application_entry_trex_collect_data import collect_episodic_demo_data_for_trex, trex_collecting_data
 from ding.entry import serial_pipeline
 
@@ -18,7 +18,7 @@
 def test_collect_episodic_demo_data_for_trex():
     exp_name = "test_collect_episodic_demo_data_for_trex_expert"
     expert_policy_state_dict_path = os.path.join(exp_name, 'expert_policy.pth.tar')
-    config = [deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)]
+    config = [deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)]
     config[0].exp_name = exp_name
     expert_policy = serial_pipeline(config, seed=0)
     torch.save(expert_policy.collect_mode.state_dict(), expert_policy_state_dict_path)
@@ -41,7 +41,7 @@ def test_collect_episodic_demo_data_for_trex():
 @pytest.mark.unittest
 def test_trex_collecting_data():
     expert_policy_dir = 'test_trex_collecting_data_expert'
-    config = [deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)]
+    config = [deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)]
     config[0].exp_name = expert_policy_dir
     config[0].policy.learn.learner.hook.save_ckpt_after_iter = 100
     serial_pipeline(config, seed=0)

ding/entry/tests/test_serial_entry.py

@@ -9,8 +9,8 @@
 from dizoo.classic_control.cartpole.config.cartpole_dqn_stdim_config import cartpole_dqn_stdim_config, \
     cartpole_dqn_stdim_create_config
 from dizoo.classic_control.cartpole.config.cartpole_ppo_config import cartpole_ppo_config, cartpole_ppo_create_config
-from dizoo.classic_control.cartpole.config.cartpole_offppo_config import cartpole_offppo_config, \
-    cartpole_offppo_create_config
+from dizoo.classic_control.cartpole.config.cartpole_ppo_offpolicy_config import cartpole_ppo_offpolicy_config, \
+    cartpole_ppo_offpolicy_create_config
 from dizoo.classic_control.cartpole.config.cartpole_impala_config import cartpole_impala_config, cartpole_impala_create_config  # noqa
 from dizoo.classic_control.cartpole.config.cartpole_rainbow_config import cartpole_rainbow_config, cartpole_rainbow_create_config  # noqa
 from dizoo.classic_control.cartpole.config.cartpole_iqn_config import cartpole_iqn_config, cartpole_iqn_create_config  # noqa
@@ -209,7 +209,7 @@ def test_qrdqn():
 @pytest.mark.platformtest
 @pytest.mark.unittest
 def test_ppo():
-    config = [deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)]
+    config = [deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)]
     config[0].policy.learn.update_per_collect = 1
     config[0].exp_name = 'ppo_offpolicy_unittest'
     try:
@@ -221,7 +221,7 @@ def test_ppo():
 @pytest.mark.platformtest
 @pytest.mark.unittest
 def test_ppo_nstep_return():
-    config = [deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)]
+    config = [deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)]
     config[0].policy.learn.update_per_collect = 1
     config[0].policy.nstep_return = True
     try:

ding/entry/tests/test_serial_entry_bc.py

@@ -14,7 +14,7 @@
 from ding.utils import POLICY_REGISTRY
 from ding.utils.data import default_collate, default_decollate
 from dizoo.classic_control.cartpole.config import cartpole_dqn_config, cartpole_dqn_create_config, \
-    cartpole_offppo_config, cartpole_offppo_create_config
+    cartpole_ppo_offpolicy_config, cartpole_ppo_offpolicy_create_config
 from dizoo.classic_control.pendulum.config import pendulum_sac_config, pendulum_sac_create_config
 
 
@@ -53,22 +53,22 @@ def _monitor_vars_learn(self) -> list:
 @pytest.mark.unittest
 def test_serial_pipeline_bc_ppo():
     # train expert policy
-    train_config = [deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)]
+    train_config = [deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)]
     train_config[0].exp_name = 'test_serial_pipeline_bc_ppo'
     expert_policy = serial_pipeline(train_config, seed=0)
 
     # collect expert demo data
     collect_count = 10000
     expert_data_path = 'expert_data_ppo_bc.pkl'
     state_dict = expert_policy.collect_mode.state_dict()
-    collect_config = [deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)]
+    collect_config = [deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)]
     collect_config[0].exp_name = 'test_serial_pipeline_bc_ppo_collect'
     collect_demo_data(
         collect_config, seed=0, state_dict=state_dict, expert_data_path=expert_data_path, collect_count=collect_count
     )
 
     # il training 1
-    il_config = [deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)]
+    il_config = [deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)]
     il_config[0].policy.eval.evaluator.multi_gpu = False
     il_config[0].policy.learn.train_epoch = 20
     il_config[1].policy.type = 'ppo_bc'

ding/entry/tests/test_serial_entry_preference_based_irl.py

@@ -9,8 +9,8 @@
 from ding.entry import serial_pipeline_preference_based_irl
 from dizoo.classic_control.cartpole.config.cartpole_trex_offppo_config import cartpole_trex_offppo_config,\
      cartpole_trex_offppo_create_config
-from dizoo.classic_control.cartpole.config.cartpole_offppo_config import cartpole_offppo_config,\
-     cartpole_offppo_create_config
+from dizoo.classic_control.cartpole.config.cartpole_ppo_offpolicy_config import cartpole_ppo_offpolicy_config,\
+     cartpole_ppo_offpolicy_create_config
 from ding.entry.application_entry_trex_collect_data import trex_collecting_data
 from ding.reward_model.trex_reward_model import TrexConvEncoder
 from ding.torch_utils import is_differentiable
@@ -19,16 +19,14 @@
 @pytest.mark.unittest
 def test_serial_pipeline_trex():
     exp_name = 'test_serial_pipeline_trex_expert'
-    config = [deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)]
+    config = [deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)]
     config[0].policy.learn.learner.hook.save_ckpt_after_iter = 100
     config[0].exp_name = exp_name
     expert_policy = serial_pipeline(config, seed=0)
 
     exp_name = 'test_serial_pipeline_trex_collect'
     config = [deepcopy(cartpole_trex_offppo_config), deepcopy(cartpole_trex_offppo_create_config)]
     config[0].exp_name = exp_name
-    config[0].reward_model.data_path = exp_name
-    config[0].reward_model.reward_model_path = exp_name + '/cartpole.params'
     config[0].reward_model.expert_model_path = 'test_serial_pipeline_trex_expert'
     config[0].reward_model.checkpoint_max = 100
     config[0].reward_model.checkpoint_step = 100

ding/entry/tests/test_serial_entry_preference_based_irl_onpolicy.py

@@ -24,8 +24,6 @@ def test_serial_pipeline_trex_onpolicy():
     exp_name = 'test_serial_pipeline_trex_onpolicy_collect'
     config = [deepcopy(cartpole_trex_ppo_onpolicy_config), deepcopy(cartpole_trex_ppo_onpolicy_create_config)]
     config[0].exp_name = exp_name
-    config[0].reward_model.data_path = exp_name
-    config[0].reward_model.reward_model_path = exp_name + '/cartpole.params'
     config[0].reward_model.expert_model_path = 'test_serial_pipeline_trex_onpolicy_expert'
     config[0].reward_model.checkpoint_max = 100
     config[0].reward_model.checkpoint_step = 100

ding/entry/tests/test_serial_entry_reward_model.py

@@ -5,7 +5,7 @@
 from copy import deepcopy
 
 from dizoo.classic_control.cartpole.config.cartpole_dqn_config import cartpole_dqn_config, cartpole_dqn_create_config
-from dizoo.classic_control.cartpole.config.cartpole_offppo_config import cartpole_offppo_config, cartpole_offppo_create_config  # noqa
+from dizoo.classic_control.cartpole.config.cartpole_ppo_offpolicy_config import cartpole_ppo_offpolicy_config, cartpole_ppo_offpolicy_create_config  # noqa
 from dizoo.classic_control.cartpole.config.cartpole_rnd_onppo_config import cartpole_ppo_rnd_config, cartpole_ppo_rnd_create_config  # noqa
 from dizoo.classic_control.cartpole.config.cartpole_ppo_icm_config import cartpole_ppo_icm_config, cartpole_ppo_icm_create_config  # noqa
 from ding.entry import serial_pipeline, collect_demo_data, serial_pipeline_reward_model_offpolicy, \
@@ -44,13 +44,13 @@
 @pytest.mark.parametrize('reward_model_config', cfg)
 def test_irl(reward_model_config):
     reward_model_config = EasyDict(reward_model_config)
-    config = deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)
+    config = deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)
     expert_policy = serial_pipeline(config, seed=0, max_train_iter=2)
     # collect expert demo data
     collect_count = 10000
     expert_data_path = 'expert_data.pkl'
     state_dict = expert_policy.collect_mode.state_dict()
-    config = deepcopy(cartpole_offppo_config), deepcopy(cartpole_offppo_create_config)
+    config = deepcopy(cartpole_ppo_offpolicy_config), deepcopy(cartpole_ppo_offpolicy_create_config)
     collect_demo_data(
         config, seed=0, state_dict=state_dict, expert_data_path=expert_data_path, collect_count=collect_count
     )