Changes to quad training scripts.

Correction of an error in the PPO_perso model
Added two-way flip: clockwise and counterclockwise

Author: ArzhelaR

environment/gymnasium_envs/quadmesh_env/envs/quadmesh.py

@@ -46,8 +46,8 @@ def __init__(self, mesh=None, max_episode_steps=30, n_darts_selected=20, deep=6,
         self.darts_selected = [] # darts id observed
         self.deep = deep*2 if self.degree_observation else deep
         self.observation_space = spaces.Box(
-            low=-6,  # nodes min degree : -15
-            high=2,  # nodes max degree : 15
+            low=-6,  # nodes min degree : -6
+            high=2,  # nodes max degree : 2
             shape=(self.n_darts_selected, deep),
             dtype=np.int64
         )
@@ -102,8 +102,7 @@ def _get_info(self, terminated, valid_act, action, mesh_reward):
             "split": 1.0 if action[0]==Actions.SPLIT.value else 0.0,
             "collapse": 1.0 if action[0]==Actions.COLLAPSE.value else 0.0,
             "cleanup": 1.0 if action[0]==Actions.CLEANUP.value else 0.0,
-            "invalid_flip_cw": 1.0 if action[0]==Actions.FLIP_CW.value and not valid_action else 0.0,
-            "invalid_flip_cntcw": 1.0 if action[0]==Actions.FLIP_CNTCW.value and not valid_action else 0.0,
+            "invalid_flip": 1.0 if (action[0]==Actions.FLIP_CW.value or action[0]==Actions.FLIP_CNTCW.value) and not valid_action else 0.0,
             "invalid_split": 1.0 if action[0]==Actions.SPLIT.value and not valid_action else 0.0,
             "invalid_collapse": 1.0 if action[0]==Actions.COLLAPSE.value and not valid_action else 0.0,
             "invalid_cleanup": 1.0 if action[0]==Actions.CLEANUP.value and not valid_action else 0.0,

main.py

@@ -1,7 +1,7 @@
 import sys
 
 from user_game import user_game
-from training.train_quadmesh import train
+from training.train import train
 from training.exploit import exploit
 #from mesh_model.reader import read_gmsh
 #from mesh_display import MeshDisplay

mesh_model/mesh_analysis/global_mesh_analysis.py

@@ -233,7 +233,7 @@ def test_degree(n: Node) -> bool:
     :param n: a Node
     :return: True if the degree is lower than 10, False otherwise
     """
-    if degree(n) > 10:
+    if degree(n) >= 10:
         return False
     else:
         return True

mesh_model/mesh_analysis/quadmesh_analysis.py

@@ -2,29 +2,43 @@
 
 from mesh_model.mesh_struct.mesh_elements import Dart, Node, Face
 from mesh_model.mesh_struct.mesh import Mesh
-from mesh_model.mesh_analysis.global_mesh_analysis import test_degree, on_boundary, adjacent_faces_id
+from mesh_model.mesh_analysis.global_mesh_analysis import test_degree, on_boundary, adjacent_faces_id, degree
+
+FLIP_CW = 0 # flip clockwise
+FLIP_CCW = 1 # flip counterclockwise
+SPLIT = 2
+COLLAPSE = 3
+CLEANUP = 4
+TEST_ALL = 5 # test if all actions are valid
+ONE_VALID = 6 # test if at least one action is valid
 
 
 def isValidAction(mesh: Mesh, dart_id: int, action: int) -> (bool, bool):
-    flip_ccw = 0
-    split = 1
-    collapse = 2
-    cleanup =3
-    test_all = 4
-    one_valid = 5
+    """
+    Test if an action is valid. You can select the ype of action between {flip clockwise, flip counterclockwise, split, collapse, cleanup, all action, one action no matter wich one}.    :param mesh:
+    :param mesh: a mesh
+    :param dart_id: a dart on which to test the action
+    :param action: an action type
+    :return:
+    """
     d = Dart(mesh, dart_id)
     if d.get_beta(2) is None:
         return False, True
-    elif action == flip_ccw:
+    elif action == FLIP_CW:
+        return isFlipCWOk(d)
+    elif action == FLIP_CCW:
         return isFlipCCWOk(d)
-    elif action == split:
+    elif action == SPLIT:
         return isSplitOk(d)
-    elif action == collapse:
+    elif action == COLLAPSE:
         return isCollapseOk(d)
-    elif action == cleanup:
+    elif action == CLEANUP:
         return isCleanupOk(d)
-    elif action == test_all:
+    elif action == TEST_ALL:
         topo, geo = isFlipCCWOk(d)
+        if not (topo and geo):
+            return False, False
+        topo, geo = isFlipCWOk(d)
         if not (topo and geo):
             return False, False
         topo, geo = isSplitOk(d)
@@ -35,8 +49,11 @@ def isValidAction(mesh: Mesh, dart_id: int, action: int) -> (bool, bool):
             return False, False
         elif topo and geo:
             return True, True
-    elif action == one_valid:
+    elif action == ONE_VALID:
         topo_flip, geo_flip = isFlipCCWOk(d)
+        if (topo_flip and geo_flip):
+            return True, True
+        topo_flip, geo_flip = isFlipCWOk(d)
         if (topo_flip and geo_flip):
             return True, True
         topo_split, geo_split = isSplitOk(d)
@@ -54,30 +71,26 @@ def isFlipCCWOk(d: Dart) -> (bool, bool):
     mesh = d.mesh
     topo = True
     geo = True
+
     # if d is on boundary, flip is not possible
     if d.get_beta(2) is None:
         topo = False
         return topo, geo
     else:
         d2, d1, d11, d111, d21, d211, d2111, n1, n2, n3, n4, n5, n6 = mesh.active_quadrangles(d)
-    # if degree are
+
+    # if degree will not too high
     if not test_degree(n5) or not test_degree(n3):
         topo = False
         return topo, geo
 
+    # if two faces share two edges
     if d211.get_node() == d111.get_node() or d11.get_node() == d2111.get_node():
         topo = False
         return topo, geo
-    topo = isValidQuad(n5, n6, n2, n3) and isValidQuad(n1, n5, n3, n4)
-
-    """
-    # Check angle at d limits to avoid edge reversal
-    angle_A = get_angle_by_coord(n5.x(), n5.y(), n1.x(), n1.y(), n3.x(), n3.y())
 
-    if angle_A <= 90 or angle_A >= 180:
-        topo = False
-        return topo, geo
-    """
+    # check validity of the two modified quads
+    geo = isValidQuad(n5, n6, n2, n3) and isValidQuad(n1, n5, n3, n4)
 
     return topo, geo
 
@@ -99,7 +112,7 @@ def isFlipCWOk(d: Dart) -> (bool, bool):
     if d211.get_node() == d111.get_node() or d11.get_node() == d2111.get_node():
         topo = False
         return topo, geo
-    topo = isValidQuad(n4, n6, n2, n3) and isValidQuad(n1, n5, n6, n4)
+    geo = isValidQuad(n4, n6, n2, n3) and isValidQuad(n1, n5, n6, n4)
 
     return topo, geo
 
@@ -123,7 +136,7 @@ def isSplitOk(d: Dart) -> (bool, bool):
         return topo, geo
 
     n10 = mesh.add_node((n1.x() + n2.x()) / 2, (n1.y() + n2.y()) / 2)
-    topo = isValidQuad(n4, n1, n5, n10) and isValidQuad(n4, n10, n2, n3) and isValidQuad(n10, n5, n6, n2)
+    geo = isValidQuad(n4, n1, n5, n10) and isValidQuad(n4, n10, n2, n3) and isValidQuad(n10, n5, n6, n2)
     mesh.del_node(n10)
     return topo, geo
 
@@ -142,7 +155,7 @@ def isCollapseOk(d: Dart) -> (bool, bool):
         topo = False
         return topo, geo
 
-    if not test_degree(n3) and not test_degree(n1):
+    if (degree(n3)+degree(n1)-2) > 10:
         topo = False
         return topo, geo
 
@@ -198,7 +211,7 @@ def isCollapseOk(d: Dart) -> (bool, bool):
                 D=n10
 
             if not isValidQuad(A, B, C, D):
-                topo = False
+                geo = False
                 mesh.del_node(n10)
                 return topo, geo
 

mesh_model/random_quadmesh.py

@@ -15,7 +15,7 @@ def random_mesh() -> Mesh:
     :param num_nodes_max: number of nodes of the final mesh
     :return: a random mesh
     """
-    filename = os.path.join(os.path.dirname(__file__), '../mesh_files/t1_quad.msh')
+    filename = os.path.join(os.path.dirname(__file__), '../mesh_files/simple_quad.msh')
     #filename = os.path.join('../mesh_files/', 't1_quad.msh')
     #mesh = read_gmsh("/home/ropercha/PycharmProjects/tune/mesh_files/t1_quad.msh")
     mesh = read_gmsh(filename)

model_RL/PPO_model_pers.py

@@ -1,6 +1,7 @@
 from mesh_model.mesh_analysis.global_mesh_analysis import global_score
 import copy
 import random
+import json
 from tqdm import tqdm
 import numpy as np
 import torch
@@ -58,8 +59,8 @@ def select_action(self, observation, info):
             action = dist.sample()
             action = action.tolist()
             prob = pmf[action]
-            action_dart = int(action/3)
-            action_type = action % 3
+            action_dart = int(action/4)
+            action_type = action % 4
             dart_id = info["darts_selected"][action_dart]
             i = 0
             while not isValidAction(info["mesh"], dart_id, action_type):
@@ -70,8 +71,8 @@ def select_action(self, observation, info):
                 action = dist.sample()
                 action = action.tolist()
                 prob = pmf[action]
-                action_dart = int(action/3)
-                action_type = action % 3
+                action_dart = int(action/4)
+                action_type = action % 4
                 dart_id = info["darts_selected"][action_dart]
                 i += 1
         action_list = [action, dart_id, action_type]
@@ -139,7 +140,7 @@ def learn(self, critic_loss):
 class PPO:
     def __init__(self, env, lr, gamma, nb_iterations, nb_episodes_per_iteration, nb_epochs, batch_size):
         self.env = env
-        self.actor = Actor(env, 10*8, 3*10, lr=0.0001)
+        self.actor = Actor(env, 10*8, 4*10, lr=0.0001)
         self.critic = Critic(8*10, lr=0.0001)
         self.lr = lr
         self.gamma = gamma
@@ -165,16 +166,14 @@ def train(self, dataset):
                 critic_loss = []
                 actor_loss = []
                 self.critic.optimizer.zero_grad()
-                G = 0
-                for _, (s, o, a, r, old_prob, next_o, done) in enumerate(batch, 1):
+                for _, (s, o, a, r, G, old_prob, next_o, done) in enumerate(batch, 1):
                     o = torch.tensor(o.flatten(), dtype=torch.float32)
                     next_o = torch.tensor(next_o.flatten(), dtype=torch.float32)
                     value = self.critic(o)
                     pmf = self.actor.forward(o)
                     log_prob = torch.log(pmf[a[0]])
                     next_value = torch.tensor(0.0, dtype=torch.float32) if done else self.critic(next_o)
                     delta = r + 0.9 * next_value - value
-                    G = (r + 0.9 * G) / 10
                     _, st, ideal_s, _ = global_score(s) # Comparaison à l'état s et pas s+1 ?
                     if st == ideal_s:
                         continue
@@ -221,6 +220,7 @@ def learn(self, writer):
                     ep_reward = 0
                     ep_mesh_reward = 0
                     ideal_reward = info["mesh_ideal_rewards"]
+                    G = 0
                     done = False
                     step = 0
                     while step < 40:
@@ -230,20 +230,21 @@ def learn(self, writer):
                         if action is None:
                             wins.append(0)
                             break
-                        gym_action = [action[2],int(action[0]/3)]
+                        gym_action = [action[2],int(action[0]/4)]
                         next_obs, reward, terminated, truncated, info = self.env.step(gym_action)
                         ep_reward += reward
                         ep_mesh_reward += info["mesh_reward"]
+                        G = info["mesh_reward"] + 0.9 * G
                         if terminated:
                             if truncated:
                                 wins.append(0)
-                                trajectory.append((state, obs, action, reward, prob, next_obs, done))
+                                trajectory.append((state, obs, action, reward, G, prob, next_obs, done))
                             else:
                                 wins.append(1)
                                 done = True
-                                trajectory.append((state, obs, action, reward, prob, next_obs, done))
+                                trajectory.append((state, obs, action, reward, G, prob, next_obs, done))
                             break
-                        trajectory.append((state, obs, action, reward, prob, next_obs, done))
+                        trajectory.append((state, obs, action, reward, G, prob, next_obs, done))
                         step += 1
                     if len(trajectory) != 0:
                         rewards.append(ep_reward)
@@ -252,7 +253,8 @@ def learn(self, writer):
                         len_ep.append(len(trajectory))
                     nb_episodes += 1
                     writer.add_scalar("episode_reward", ep_reward, nb_episodes)
-                    writer.add_scalar("normalized return", (ep_reward/ideal_reward), nb_episodes)
+                    writer.add_scalar("episode_mesh_reward", ep_mesh_reward, nb_episodes)
+                    writer.add_scalar("normalized return", (ep_mesh_reward/ideal_reward), nb_episodes)
                     writer.add_scalar("len_episodes", len(trajectory), nb_episodes)
 
                 self.train(dataset)
@@ -263,4 +265,5 @@ def learn(self, writer):
         except NaNExceptionCritic:
             print("NaN Exception on Critic Network")
             return None, None, None, None
-        return self.actor, rewards, wins, len_ep
+
+        return self.actor, rewards, wins, len_ep, info["observation_count"]

model_RL/parameters/ppo_config.json

@@ -6,6 +6,6 @@
     "learning_rate": 0.0001,
     "gamma": 0.9,
     "verbose": 1,
-    "tensorboard_log": "./results/quad/",
+    "tensorboard_log": "training/results/quad/",
     "total_timesteps": 80000
 }