Fixes on most agents and added the option for log demand link in hyper training

Author: miTTimmiTTim

ddopai/_modidx.py

@@ -86,11 +86,10 @@ def parameter_update(self):
         self.alpha = results.params[:self.environment_info.observation_space['features'].shape[0]]
         self.beta = results.params[self.environment_info.observation_space['features'].shape[0]:]
 
-    def update_env(self, env):
+    def update_task(self, env):
         self.environment_info = env.mdp_info
         self.X = np.empty((0, self.environment_info.observation_space['features'].shape[0] * 2))
         self.Y = np.empty((0, 1))
-        self.actionprocessors[-1] = ClipAction(self.environment_info.action_space.low, self.environment_info.action_space.high)
         self.M = [[np.power(x,2)+i for x in range(0, int(np.sqrt(self.environment_info.horizon)))] for i in range(0, 2)]
         self.t = 0 
 
@@ -127,8 +126,8 @@ def fit(self, dataset, **kwargs):
         action = dataset[0][1]
         self.policy.fit(X, Y, action)
 
-    def update_env(self, env):
-        self.policy.update_env(env)
+    def update_task(self, env):
+        self.policy.update_task(env)
 
 
 # %% ../../../nbs/30_agents/42_DP_agents/11_ILQX_agent.ipynb 6
@@ -157,6 +156,6 @@ def __init__(self,
                                    price_function = price_function, 
                                    g = g)
         super().__init__(environment_info = environment_info, obsprocessors = obsprocessors, agent_name = agent_name)
-    def update_env(self, env: object):
+    def update_task(self, env: object):
         """ Update the environment specific parameters of the agent """
-        self.agent.update_env(env)
+        self.agent.update_task(env)

ddopai/agents/dynamic_pricing/ILQX.py

@@ -147,7 +147,7 @@ def draw_action(self, observation: np.ndarray) -> np.ndarray:
             price = self.price_function(x_feat, alpha, beta)
         for proc in self.actionprocessors:
             price = proc(price)
-        return np.asarray([price], dtype=float)   # keep shape (1,)
+        return price   # keep shape (1,)
 
     # --------------------------------------------------
     # Online update after receiving (x, price, demand)
@@ -157,7 +157,7 @@ def fit(self, X: np.ndarray, Y: np.ndarray, action: float):
         self.t += 1
         m = np.concatenate([X, X * action]).astype(float)       # (2d,)
         self.X_buf = np.vstack([self.X_buf, m])
-        self.Y_buf = np.vstack([self.Y_buf, [[Y]]])
+        self.Y_buf = np.vstack([self.Y_buf, [Y]])
 
         # update posterior *after* burn‑in
         if self.t >= self.t_e:
@@ -166,7 +166,7 @@ def fit(self, X: np.ndarray, Y: np.ndarray, action: float):
     # --------------------------------------------------
     # End of epoch – build OLS & possibly refresh meta‑prior
     # --------------------------------------------------
-    def update_env(self, env):
+    def update_task(self, env):
         """Call this after each product/epoch ends."""
         # ---------- compute OLS (full‑rank guaranteed by burn‑in) ----------
         V = self.X_buf.T @ self.X_buf               # (2d,2d)
@@ -254,8 +254,8 @@ def fit(self, dataset, **kwargs):
         Y = kwargs["demand"][0]
         action = dataset[0][1]
         self.policy.fit(X, Y, action)
-    def update_env(self, env):
-        self.policy.update_env(env)
+    def update_task(self, env):
+        self.policy.update_task(env)
 
 
 # %% ../../../nbs/30_agents/42_DP_agents/12_MTS_agent.ipynb 7
@@ -284,7 +284,7 @@ def __init__(self,
                                   agent_name=agent_name, ex_prices=ex_prices, price_function=price_function, g=g)
         super().__init__(environment_info=environment_info, obsprocessors=obsprocessors, agent_name=agent_name)
 
-    def update_env(self, env: object):
+    def update_task(self, env: object):
         """ Update the environment specific parameters of the agent """
-        self.agent.update_env(env)
+        self.agent.update_task(env)
 

ddopai/agents/dynamic_pricing/MTS.py

@@ -119,7 +119,7 @@ def parameter_update(self):
         self.alpha = results.params[:self.environment_info.observation_space['features'].shape[0]]
         self.beta = results.params[self.environment_info.observation_space['features'].shape[0]:]
 
-    def update_env(self, env):
+    def update_task(self, env):
         self.environment_info = env.mdp_info
         self.X = np.empty((0, self.environment_info.observation_space['features'].shape[0] * 2))
         self.Y = np.empty((0, 1))
@@ -159,8 +159,8 @@ def fit(self, dataset, **kwargs):
         Y = kwargs["demand"][0]
         action = dataset[0][1]
         self.policy.fit(X, Y, action)
-    def update_env(self, env):
-        self.policy.update_env(env)
+    def update_task(self, env):
+        self.policy.update_task(env)
 
 # %% ../../../nbs/30_agents/42_DP_agents/12_TS_agent.ipynb 6
 class TSAgent(PricingMushroomBaseAgent):
@@ -190,6 +190,6 @@ def __init__(self,
                                  price_function=price_function, 
                                  g=g)
         super().__init__(environment_info=environment_info, obsprocessors=obsprocessors, agent_name=agent_name)
-    def update_env(self, env: object):
+    def update_task(self, env: object):
         """ Update the environment specific parameters of the agent """
-        self.agent.update_env(env)
+        self.agent.update_task(env)

ddopai/agents/dynamic_pricing/TS.py

@@ -119,7 +119,7 @@ def parameter_update(self):
         self.alpha = results.params[:self.environment_info.observation_space['features'].shape[0]]
         self.beta = results.params[self.environment_info.observation_space['features'].shape[0]:]
 
-    def update_env(self, env):
+    def update_task(self, env):
         self.environment_info = env.mdp_info
         self.X = np.empty((0, self.environment_info.observation_space['features'].shape[0] * 2))
         self.Y = np.empty((0, 1))
@@ -162,8 +162,8 @@ def fit(self, dataset, **kwargs):
         action = dataset[0][1]
         self.policy.fit(X, Y, action)
 
-    def update_env(self, env):
-        self.policy.update_env(env)
+    def update_task(self, env):
+        self.policy.update_task(env)
 
 # %% ../../../nbs/30_agents/42_DP_agents/13_UCB_agent.ipynb 6
 class UCBAgent(PricingMushroomBaseAgent):
@@ -193,6 +193,6 @@ def __init__(self,
                                   price_function=price_function, 
                                   g=g)
         super().__init__(environment_info=environment_info, obsprocessors=obsprocessors, agent_name=agent_name)
-    def update_env(self, env: object):
+    def update_task(self, env: object):
         """ Update the environment specific parameters of the agent """
-        self.agent.update_env(env)
+        self.agent.update_task(env)

ddopai/agents/dynamic_pricing/UCB.py

@@ -118,7 +118,7 @@ def draw_action_(self, obs: np.ndarray) -> np.ndarray:   # DDOP naming
                                         latent_logvar=self.latent_logvar)
 
         with torch.no_grad():
-            _, action, _ = self.policy.act(state=state_t.view(-1),
+            _, action, _ = self.policy.act(state=state_t,
                                            latent=latent,
                                            belief=None, task=None,
                                            deterministic=self.deterministic)

ddopai/agents/rl/hyper.py

@@ -129,7 +129,27 @@ def __init__(self,
                                 dropout=self.dropout,)
 
         critic_params = merge_dictionaries(critic_params, network_critic_params)
-
+        self.agent_params = {
+            "mdp_info": environment_info,
+            "actor_mu_params": actor_mu_params,
+            "actor_sigma_params": actor_sigma_params,
+            "actor_optimizer": actor_optimizer,
+            "critic_params": critic_params,
+            "batch_size": batch_size,
+            "initial_replay_size": initial_replay_size,
+            "max_replay_size": max_replay_size,
+            "warmup_transitions": warmup_transitions,
+            "tau": tau,
+            "lr_alpha": lr_alpha,
+            "use_log_alpha_loss": use_log_alpha_loss,
+            "log_std_min": log_std_min,
+            "log_std_max": log_std_max,
+            "target_entropy": target_entropy,
+            "critic_fit_params": None
+        }
+        self._obsprocessors = obsprocessors 
+        self.device = device
+        self.agent_name = agent_name
         self.agent = SAC(
             mdp_info=environment_info,
             actor_mu_params=actor_mu_params,
@@ -228,6 +248,33 @@ def predict_(self, observation: np.ndarray) -> np.ndarray: #
         action = action.cpu().detach().numpy()
 
         return action
+    
+    def update_task(self, env):
+        self.agent = SAC(
+            mdp_info=env.mdp_info,
+            actor_mu_params=self.agent_params["actor_mu_params"],
+            actor_sigma_params=self.agent_params["actor_sigma_params"],
+            actor_optimizer=self.agent_params["actor_optimizer"],
+            critic_params=self.agent_params["critic_params"],
+            batch_size=self.agent_params["batch_size"],
+            initial_replay_size=self.agent_params["initial_replay_size"],
+            max_replay_size=self.agent_params["max_replay_size"],
+            warmup_transitions=self.agent_params["warmup_transitions"],
+            tau=self.agent_params["tau"],
+            lr_alpha=self.agent_params["lr_alpha"],
+            use_log_alpha_loss=self.agent_params["use_log_alpha_loss"],
+            log_std_min=self.agent_params["log_std_min"],
+            log_std_max=self.agent_params["log_std_max"],
+            target_entropy=self.agent_params["target_entropy"],
+            critic_fit_params=self.agent_params["critic_fit_params"]
+        )
+        self.obsprocessors = self._obsprocessors
+        super().__init__(
+            environment_info=env.mdp_info,
+            obsprocessors=self._obsprocessors,
+            device=self.device,
+            agent_name=self.agent_name
+        )
 
 # %% ../../../nbs/30_agents/51_RL_agents/10_SAC_agents.ipynb 6
 class SACAgent(SACBaseAgent):

ddopai/agents/rl/sac.py

@@ -223,7 +223,7 @@ def create_online_data(
             size = parameter["horizon"]
             noise_std = parameter["noise_std"]
             if nb_features > 1:
-                scale = 1 / np.sqrt(nb_features-1)
+                scale = 1 / np.sqrt(nb_features)
                 X = np.random.uniform(0, scale, size=(size, nb_features))
             else:
                 X = np.ones((size, 1))

ddopai/experiments/meta_experiment_functions.py

@@ -123,11 +123,10 @@
     "        self.alpha = results.params[:self.environment_info.observation_space['features'].shape[0]]\n",
     "        self.beta = results.params[self.environment_info.observation_space['features'].shape[0]:]\n",
     "    \n",
-    "    def update_env(self, env):\n",
+    "    def update_task(self, env):\n",
     "        self.environment_info = env.mdp_info\n",
     "        self.X = np.empty((0, self.environment_info.observation_space['features'].shape[0] * 2))\n",
     "        self.Y = np.empty((0, 1))\n",
-    "        self.actionprocessors[-1] = ClipAction(self.environment_info.action_space.low, self.environment_info.action_space.high)\n",
     "        self.M = [[np.power(x,2)+i for x in range(0, int(np.sqrt(self.environment_info.horizon)))] for i in range(0, 2)]\n",
     "        self.t = 0 \n",
     "        \n",
@@ -171,8 +170,8 @@
     "        action = dataset[0][1]\n",
     "        self.policy.fit(X, Y, action)\n",
     "\n",
-    "    def update_env(self, env):\n",
-    "        self.policy.update_env(env)\n"
+    "    def update_task(self, env):\n",
+    "        self.policy.update_task(env)\n"
    ]
   },
   {
@@ -207,9 +206,9 @@
     "                                   price_function = price_function, \n",
     "                                   g = g)\n",
     "        super().__init__(environment_info = environment_info, obsprocessors = obsprocessors, agent_name = agent_name)\n",
-    "    def update_env(self, env: object):\n",
+    "    def update_task(self, env: object):\n",
     "        \"\"\" Update the environment specific parameters of the agent \"\"\"\n",
-    "        self.agent.update_env(env)"
+    "        self.agent.update_task(env)"
    ]
   }
  ],

nbs/30_agents/42_DP_agents/11_ILQX_agent.ipynb

@@ -189,7 +189,7 @@
     "            price = self.price_function(x_feat, alpha, beta)\n",
     "        for proc in self.actionprocessors:\n",
     "            price = proc(price)\n",
-    "        return np.asarray([price], dtype=float)   # keep shape (1,)\n",
+    "        return price   # keep shape (1,)\n",
     "    \n",
     "    # --------------------------------------------------\n",
     "    # Online update after receiving (x, price, demand)\n",
@@ -199,7 +199,7 @@
     "        self.t += 1\n",
     "        m = np.concatenate([X, X * action]).astype(float)       # (2d,)\n",
     "        self.X_buf = np.vstack([self.X_buf, m])\n",
-    "        self.Y_buf = np.vstack([self.Y_buf, [[Y]]])\n",
+    "        self.Y_buf = np.vstack([self.Y_buf, [Y]])\n",
     "\n",
     "        # update posterior *after* burn‑in\n",
     "        if self.t >= self.t_e:\n",
@@ -208,7 +208,7 @@
     "    # --------------------------------------------------\n",
     "    # End of epoch – build OLS & possibly refresh meta‑prior\n",
     "    # --------------------------------------------------\n",
-    "    def update_env(self, env):\n",
+    "    def update_task(self, env):\n",
     "        \"\"\"Call this after each product/epoch ends.\"\"\"\n",
     "        # ---------- compute OLS (full‑rank guaranteed by burn‑in) ----------\n",
     "        V = self.X_buf.T @ self.X_buf               # (2d,2d)\n",
@@ -302,8 +302,8 @@
     "        Y = kwargs[\"demand\"][0]\n",
     "        action = dataset[0][1]\n",
     "        self.policy.fit(X, Y, action)\n",
-    "    def update_env(self, env):\n",
-    "        self.policy.update_env(env)\n"
+    "    def update_task(self, env):\n",
+    "        self.policy.update_task(env)\n"
    ]
   },
   {
@@ -338,9 +338,9 @@
     "                                  agent_name=agent_name, ex_prices=ex_prices, price_function=price_function, g=g)\n",
     "        super().__init__(environment_info=environment_info, obsprocessors=obsprocessors, agent_name=agent_name)\n",
     "        \n",
-    "    def update_env(self, env: object):\n",
+    "    def update_task(self, env: object):\n",
     "        \"\"\" Update the environment specific parameters of the agent \"\"\"\n",
-    "        self.agent.update_env(env)\n"
+    "        self.agent.update_task(env)\n"
    ]
   }
  ],