Merge pull request #115 from web-genie-ai/feat/avg-score

Feat/avg score

Author: sangar-1028

neurons/validators/genie_validator.py

@@ -19,6 +19,7 @@
     SESSION_WINDOW_BLOCKS,
     BLOCK_IN_SECONDS,
     __VERSION__,
+    MAX_NUMBER_OF_TASKS_PER_SESSION,
 )
 from webgenie.challenges import (
     AccuracyChallenge,
@@ -289,7 +290,6 @@ async def forward(self):
                 else:
                     task_index = self.neuron.score_manager.number_of_tasks
 
-            MAX_NUMBER_OF_TASKS_PER_SESSION = 18
             if task_index >= MAX_NUMBER_OF_TASKS_PER_SESSION:
                 return
 

neurons/validators/score_manager.py

@@ -50,6 +50,11 @@ def load_scores(self):
                 -1
             )
 
+            self.solved_tasks = data.get(
+                f"solved_tasks", 
+                np.zeros(self.neuron.metagraph.n, dtype=np.float32),
+            )
+
             self.total_scores = data.get(
                 f"total_scores_{__STATE_VERSION__}", 
                 np.zeros(self.neuron.metagraph.n, dtype=np.float32),
@@ -62,6 +67,7 @@ def load_scores(self):
             bt.logging.error(f"Error loading state: {e}")
             self.hotkeys = copy.deepcopy(self.neuron.metagraph.hotkeys)
             self.current_session = -1
+            self.solved_tasks = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
             self.total_scores = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
             self.last_set_weights_session = -1
             self.number_of_tasks = 0
@@ -76,6 +82,7 @@ def save_scores(self):
                 **{f"current_session": self.current_session},
                 last_set_weights_session=self.last_set_weights_session,
                 number_of_tasks=self.number_of_tasks,
+                solved_tasks=self.solved_tasks,
                 **{f"total_scores_{__STATE_VERSION__}": self.total_scores},
                 session_results= self.session_results,
                 allow_pickle=True,
@@ -91,6 +98,7 @@ def set_new_hotkeys(self, new_hotkeys: List[str]):
         for uid, hotkey in enumerate(self.hotkeys):
             if hotkey != new_hotkeys[uid]:
                 self.total_scores[uid] = 0
+                self.solved_tasks[uid] = 0
 
         # Check to see if the metagraph has changed size.
         # If so, we need to add new hotkeys and moving averages.
@@ -100,6 +108,11 @@ def set_new_hotkeys(self, new_hotkeys: List[str]):
             new_total_scores[:min_len] = self.total_scores[:min_len]
             self.total_scores = new_total_scores
 
+            new_solved_tasks = np.zeros((len(new_hotkeys)))
+            min_len = min(len(self.hotkeys), len(self.solved_tasks))
+            new_solved_tasks[:min_len] = self.solved_tasks[:min_len]
+            self.solved_tasks = new_solved_tasks
+
         # Update the hotkeys.
         self.hotkeys = copy.deepcopy(new_hotkeys)
         with self.lock:
@@ -113,16 +126,27 @@ def update_scores(self, rewards: np.ndarray, uids: List[int], challenge: Challen
             # This is a new session, reset the scores and winners.
             self.current_session = session
             self.number_of_tasks = 0
+            self.solved_tasks = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
             self.total_scores = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
         # Update accumulated scores and track best performer
         self.number_of_tasks += 1
         self.total_scores[uids] += rewards
+        self.solved_tasks[uids] += 1
 
+        avg_scores = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
+        for uid in range(self.neuron.metagraph.n):
+            if self.solved_tasks[uid] >= max(1, self.number_of_tasks / 2):
+                avg_scores[uid] = self.total_scores[uid] / self.solved_tasks[uid]
+            else:
+                avg_scores[uid] = 0
+        winner = np.argmax(avg_scores) if max(avg_scores) > 0 else -1
+        
         current_session_results = {
             "session": session,
             "competition_type": competition_type,
             "number_of_tasks": self.number_of_tasks,
-            "winner": np.argmax(self.total_scores),
+            "winner": winner,
+            "solved_tasks": self.solved_tasks,
             "scores": self.total_scores,
         }
 
@@ -162,42 +186,51 @@ def get_scores(self, session_upto: int):
         # return np.power(scores, 9)
 
     def print_session_result(self, session_upto: int, console: Console):
-        session_result = self.session_results[session_upto]
-
-        number_of_tasks = session_result["number_of_tasks"]
-        session = session_result["session"]
-        competition_type = session_result["competition_type"]
-        winner = session_result["winner"]
-        scores = session_result["scores"]
-
-        total_scores_table = Table(
-            title=(
-                f"📊 Total Scores Summary\n"
-                f"🔄 Session: #{session}\n"
-                f"📝 Number of Tasks: #{number_of_tasks}\n" 
-                f"🏆 Competition: {competition_type}\n"
-                f"👑 Winner: #{winner}\n"
-            ),
-            show_header=True,
-            header_style="bold magenta", 
-            title_style="bold blue",
-            border_style="blue"
-        )
-
-        total_scores_table.add_column("Rank", justify="right", style="red", header_style="bold red")
-        total_scores_table.add_column("UID", justify="right", style="cyan", header_style="bold cyan")
-        total_scores_table.add_column("Total Score", justify="right", style="green")
-        total_scores_table.add_column("Average Score", justify="right", style="yellow")
-        scored_uids = [(uid, score) for uid, score in enumerate(scores) if score > 0]
-        scored_uids.sort(key=lambda x: x[1], reverse=True)
-        for rank, (uid, score) in enumerate(scored_uids):
-            total_scores_table.add_row(
-                str(rank + 1),
-                str(uid),
-                f"{score:.4f}",
-                f"{score / number_of_tasks:.4f}",
+        try:
+            session_result = self.session_results[session_upto]
+
+            number_of_tasks = session_result["number_of_tasks"]
+            session = session_result["session"]
+            competition_type = session_result["competition_type"]
+            winner = session_result["winner"]
+            scores = session_result["scores"]
+            solved_tasks = session_result["solved_tasks"]
+            
+            avg_scores = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
+            for uid in range(self.neuron.metagraph.n):
+                if solved_tasks[uid] >= max(1, number_of_tasks / 2):
+                    avg_scores[uid] = scores[uid] / solved_tasks[uid]
+                else:
+                    avg_scores[uid] = 0
+            
+            total_scores_table = Table(
+                title=(
+                    f"📊 Total Scores Summary\n"
+                    f"🔄 Session: #{session}\n"
+                    f"📝 Number of Tasks: #{number_of_tasks}\n" 
+                    f"🏆 Competition: {competition_type}\n"
+                    f"👑 Winner: #{winner}\n"
+                ),
+                show_header=True,
+                header_style="bold magenta", 
+                title_style="bold blue",
+                border_style="blue"
             )
-        console.print(total_scores_table)
+
+            total_scores_table.add_column("Rank", justify="right", style="red", header_style="bold red")
+            total_scores_table.add_column("UID", justify="right", style="cyan", header_style="bold cyan")
+            total_scores_table.add_column("Average Score", justify="right", style="yellow")
+            scored_uids = [(uid, avg_scores[uid]) for uid in range(self.neuron.metagraph.n) if avg_scores[uid] > 0]
+            scored_uids.sort(key=lambda x: x[1], reverse=True)
+            for rank, (uid, score) in enumerate(scored_uids):
+                total_scores_table.add_row(
+                    str(rank + 1),
+                    str(uid),
+                    f"{score:.4f}",
+                )
+            console.print(total_scores_table)
+        except Exception as e:
+            bt.logging.warning(f"Error printing session result: {e}")
 
     def save_session_result_to_file(self, session_upto: int):
         try:
@@ -211,3 +244,4 @@ def save_session_result_to_file(self, session_upto: int):
         except Exception as e:
             bt.logging.error(f"Error saving session result to file: {e}")
             raise e
+

webgenie/constants.py

@@ -3,7 +3,7 @@
 
 # Change this value when updating your code base.
 # Define the version of the webgenie.
-__VERSION__ = "1.1.16" # version
+__VERSION__ = "1.1.17" # version
 
 SPEC_VERSION = (
     (1000 * int(__VERSION__.split(".")[0]))
@@ -87,3 +87,4 @@
 
 NEURON_EPOCH_LENGTH = int(os.getenv("NEURON_EPOCH_LENGTH", 25)) # neuron epoch length
 
+MAX_NUMBER_OF_TASKS_PER_SESSION = 18 # max number of tasks per session