Support state with history length = 1

Author: garlicdevs

fruit/buffers/replay.py

@@ -53,7 +53,10 @@ def append(self, state, action, reward, next_state, terminal):
         else:
             self.start_index = (self.start_index + 1) % self.max_size
 
-        self.states[insert_index] = state[-1]
+        if self.state_history > 1:
+            self.states[insert_index] = state[-1]
+        else:
+            self.states[insert_index] = state
 
     def get_state(self, index):
         if self.current_size < self.max_size:

fruit/buffers/tree.py

@@ -173,7 +173,10 @@ def append(self, state=None, action=0, reward=0, next_state=None, terminal=False
             self.start_index = (self.start_index + 1) % self.max_size
             self.__modify(self.num_of_levels-1, insert_index, pre_p**self.alpha, priority**self.alpha)
 
-        self.states[insert_index] = state[-1]
+        if self.state_history > 1:
+            self.states[insert_index] = state[-1]
+        else:
+            self.states[insert_index] = state
 
     def __update(self, new_level, old_index, new_value):
         new_index = int(old_index/2)

fruit/learners/dqn.py

@@ -19,7 +19,8 @@ def __init__(self, agent, name, environment, network, global_dict, report_freque
         global experience_replay
         with global_dict[AgentMonitor.Q_LOCK]:
             if experience_replay is None:
-                experience_replay = SyncExperienceReplay(experience_replay_size)
+                experience_replay = SyncExperienceReplay(experience_replay_size,
+                                                         state_history=network.network_config.get_history_length())
         self.replay = experience_replay
         self.batch_size = batch_size
         self.warmup_steps = warmup_steps

fruit/monitor/monitor.py

@@ -208,5 +208,10 @@ def run_epochs(self, learners):
         self.shared_dict[AgentMonitor.Q_FINISH] = True
         for t in threads:
             t.join()
+
+        current_epoch = self.shared_dict[AgentMonitor.Q_GLOBAL_STEPS] / self.epoch_steps
+        et = time.time()
+        self.__print_log(et - st, current_epoch)
+
         print('All threads stopped')
         return self.shared_dict[AgentMonitor.Q_REWARD_LIST]

fruit/samples/multi_objectives_test.py

@@ -40,12 +40,19 @@ def train_multi_objective_agent_mountain_car():
     agent.train()
 
 
-def train_multi_objective_dqn_agent(is_linear=True, extended_config=True):
-    # Create a Deep Sea Treasure game
-    game = DeepSeaTreasure(graphical_state=True, width=5, seed=100, render=False, max_treasure=100, speed=1000)
+def train_multi_objective_dqn_agent(is_linear=False, extended_config=False):
+    if extended_config:
+        # Create a Deep Sea Treasure game
+        game = DeepSeaTreasure(graphical_state=True, width=5, seed=100, render=False, max_treasure=100, speed=1000)
+
+        # Put game into fruit wrapper
+        environment = FruitEnvironment(game, max_episode_steps=60, state_processor=AtariProcessor())
+    else:
+        # Create a Deep Sea Treasure game
+        game = DeepSeaTreasure(graphical_state=False, width=5, seed=100, render=False, max_treasure=100, speed=1000)
 
-    # Put game into fruit wrapper
-    environment = FruitEnvironment(game, max_episode_steps=60, state_processor=AtariProcessor())
+        # Put game into fruit wrapper
+        environment = FruitEnvironment(game, max_episode_steps=60)
 
     # Get treasures
     treasures = game.get_treasure()

requirements.txt

@@ -5,4 +5,5 @@ Pillow==4.3.0
 psutil==5.6.2
 statsmodels==0.10.1
 tensorflow-gpu==1.12.0
-matplotlib==3.1.1
+matplotlib==3.1.1
+pygame==1.9.6