Merge pull request #124 from web-genie-ai/feat/reserved-weights

Feat/reserved weights

Author: sangar-1028

neurons/validators/genie_validator.py

@@ -82,6 +82,8 @@ async def query_miners(self):
             available_challenges_classes = [
                 AccuracyChallenge, 
                 SeoChallenge,
+                #QualityChallenge, 
+                SeoChallenge,
                 SeoChallenge,
                 SeoChallenge,
             ]  

neurons/validators/score_manager.py

@@ -8,13 +8,9 @@
 from typing import List
 
 from webgenie.base.neuron import BaseNeuron
-from webgenie.challenges.challenge import Challenge
-from webgenie.constants import (
-    CONSIDERING_SESSION_COUNTS,
-    __STATE_VERSION__,
-    WORK_DIR,
-    MAX_UNANSWERED_TASKS,
-)
+
+from webgenie.challenges.challenge import Challenge, RESERVED_WEIGHTS
+from webgenie.constants import CONSIDERING_SESSION_COUNTS, __STATE_VERSION__, WORK_DIR
 from webgenie.helpers.weights import save_file_to_wandb
 
 class ScoreManager:
@@ -183,30 +179,37 @@ def get_winner(self, total_scores: np.ndarray, solved_tasks: np.ndarray, number_
 
     def get_scores(self, session_upto: int):
         scores = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
-        tiny_weight = 1 / 128
-        big_weight = 1.0
         for session_number in self.session_results:
             if (session_number <= session_upto - CONSIDERING_SESSION_COUNTS or 
                 session_number > session_upto):
                 continue
-                
-            winner = self.get_winner(
-                self.session_results[session_number]["scores"],
-                self.session_results[session_number]["solved_tasks"],
-                self.session_results[session_number]["number_of_tasks"],
-            )
-            if winner == -1:
-                continue
-            if session_number == session_upto:
-                scores[winner] += big_weight
-            else:
-                scores[winner] += tiny_weight
-                
-        for uid in range(self.neuron.metagraph.n):
-            if self.is_blacklisted(uid):
-                scores[uid] = 0
-        
+
+            try:
+                winner = self.session_results[session_number]["winner"]
+                competition_type = self.session_results[session_number]["competition_type"]
+                if winner == -1:
+                    continue
+                scores[winner] += RESERVED_WEIGHTS[competition_type]
+            except Exception as e:
+                bt.logging.warning(f"Error getting scores: {e}")
+
         return scores
+        # scores = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
+        # tiny_weight = 1 / 128
+        # big_weight = 1.0
+        # for session_number in self.session_results:
+        #     if (session_number <= session_upto - CONSIDERING_SESSION_COUNTS or 
+        #         session_number > session_upto):
+        #         continue
+                
+        #     winner = self.session_results[session_number]["winner"]
+        #     if winner == -1:
+        #         continue
+        #     if session_number == session_upto:
+        #         scores[winner] += big_weight
+        #     else:
+        #         scores[winner] += tiny_weight
+        # return scores
 
         # if session_upto in self.session_results:
         #     scores = self.session_results[session_upto]["scores"]

neurons/validators/validator.py

@@ -69,7 +69,7 @@ def __init__(self, config=None):
         self.query_miners_thread: Union[threading.Thread, None] = None
         self.score_thread: Union[threading.Thread, None] = None
         self.sync_thread: Union[threading.Thread, None] = None
-        self.lock = threading.RLock()
+        self.lock = threading.Lock()
 
         self.genie_validator = GenieValidator(neuron=self)
         self.score_manager = ScoreManager(neuron=self)
@@ -345,7 +345,7 @@ def sync_loop(self):
             try:
                 with self.lock:
                     self.sync()
-                self.set_weights()
+                    self.set_weights()
             except Exception as e:
                 bt.logging.error(f"Error during sync: {str(e)}")
             if self.should_exit:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "web-genie-ai"
-version = "1.1.26"
+version = "1.1.20"
 description = "The first bittensor subnet for web generation"
 readme = "README.md"
 requires-python = ">=3.12.4"

webgenie/challenges/challenge.py

@@ -43,7 +43,7 @@ async def calculate_scores(self) -> dict[str, np.ndarray]:
         scores = await self.task.generator.calculate_scores(self.task, self.solutions)
         accuracy_scores = scores[ACCURACY_METRIC_NAME]
         seo_scores = scores[SEO_METRIC_NAME]
-        aggregated_scores = np.where(accuracy_scores > 0.9, seo_scores, 0)
+        aggregated_scores = np.where(accuracy_scores > 0.7, seo_scores, 0)
         return aggregated_scores, scores
 
 
@@ -54,9 +54,10 @@ async def calculate_scores(self) -> dict[str, np.ndarray]:
         scores = await self.task.generator.calculate_scores(self.task, self.solutions)
         accuracy_scores = scores[ACCURACY_METRIC_NAME]
         quality_scores = scores[QUALITY_METRIC_NAME]
-        aggregated_scores = np.where(accuracy_scores > 0.9, quality_scores, 0)
+        aggregated_scores = np.where(accuracy_scores > 0.7, quality_scores, 0)
         return aggregated_scores, scores
 
+
 class BalancedChallenge(Challenge):
     competition_type: str = Field(default=BALANCED_COMPETITION_TYPE, description="The type of competition")
 
@@ -65,13 +66,13 @@ async def calculate_scores(self) -> dict[str, np.ndarray]:
         accuracy_scores = scores[ACCURACY_METRIC_NAME]
         quality_scores = scores[QUALITY_METRIC_NAME]
         seo_scores = scores[SEO_METRIC_NAME]
-        aggregated_scores = accuracy_scores * 0.4 + quality_scores * 0.3 + seo_scores * 0.3
+        aggregated_scores = accuracy_scores * 0.6 + quality_scores * 0.2 + seo_scores * 0.2
         return aggregated_scores, scores
 
 
 RESERVED_WEIGHTS = {
-    ACCURACY_COMPETITION_TYPE: 50,
-    BALANCED_COMPETITION_TYPE: 30,
+    ACCURACY_COMPETITION_TYPE: 70,
+    BALANCED_COMPETITION_TYPE: 10,
     SEO_COMPETITION_TYPE: 10,
     QUALITY_COMPETITION_TYPE: 10,
 }

webgenie/constants.py

@@ -3,7 +3,8 @@
 import psutil
 # Change this value when updating your code base.
 # Define the version of the webgenie.
-__VERSION__ = "1.1.29" # version
+
+__VERSION__ = "1.1.20" # version
 
 SPEC_VERSION = (
     (1000 * int(__VERSION__.split(".")[0]))