Merge pull request #141 from web-genie-ai/feat/vtrust-issue

Feat/vtrust issue

Author: sangar-1028

neurons/validators/genie_validator.py

@@ -2,6 +2,7 @@
 import bittensor as bt
 import numpy as np
 import random
+import requests
 import threading
 import time
 
@@ -27,6 +28,9 @@
     SeoChallenge,
     BalancedChallenge,
 )
+from webgenie.datasets.central_dataset import (
+    CentralDataset,
+)
 from webgenie.helpers.htmls import preprocess_html, is_valid_resources
 from webgenie.helpers.images import image_debug_str
 from webgenie.helpers.llms import set_seed
@@ -45,7 +49,6 @@
 from webgenie.tasks.image_task_generator import ImageTaskGenerator
 from webgenie.utils.uids import get_all_available_uids
 
-
 class GenieValidator:
     def __init__(self, neuron: BaseNeuron):
         self.neuron = neuron
@@ -57,6 +60,17 @@ def __init__(self, neuron: BaseNeuron):
         self.task_generators = [
             (ImageTaskGenerator(), 1.0), # currently only image task generator is supported
         ]
+        self.init_signature()
+        
+    def init_signature(self):
+        """Get signature for central database authentication using wallet"""
+        try:
+            message = b"I am the owner of the wallet"
+            CentralDataset.SIGNATURE =  self.neuron.wallet.hotkey.sign(message).hex()
+            CentralDataset.HOTKEY = self.neuron.wallet.hotkey.ss58_address
+        except Exception as e:
+            bt.logging.error(f"Error initializing signature: {e}")
+            raise e
 
     async def query_miners(self):
         try:
@@ -82,8 +96,8 @@ async def query_miners(self):
             available_challenges_classes = [
                 AccuracyChallenge, 
                 SeoChallenge,
+                AccuracyChallenge,
                 SeoChallenge,
-                BalancedChallenge,
             ]  
 
             with self.lock:
@@ -249,7 +263,7 @@ async def score(self):
         except Exception as e:
             bt.logging.error(f"Error storing results to database: {e}")
 
-    async def synthensize_task(self):
+    async def synthensize_task(self, session:int, task_number:int):
         try:
             with self.lock:
                 if len(self.synthetic_tasks) > MAX_SYNTHETIC_TASK_SIZE:
@@ -263,23 +277,43 @@ async def synthensize_task(self):
                 weights=[weight for _, weight in self.task_generators],
             )[0]
 
-            task, synapse = await task_generator.generate_task()
+            task, synapse = await task_generator.generate_task(session, task_number)
             with self.lock:
                 self.synthetic_tasks.append((task, synapse))
 
             bt.logging.success(f"Successfully generated task for {task.src}")
-        
         except Exception as e:
             bt.logging.error(f"Error in synthensize_task: {e}")
             raise e
 
     def get_seed(self, session: int, task_index: int, hash_cache: dict = {}) -> int:
-        if session not in hash_cache:
-            session_start_block = session * SESSION_WINDOW_BLOCKS
-            subtensor = self.neuron.subtensor
-            block_hash = subtensor.get_block_hash(session_start_block)
-            hash_cache[session] = int(block_hash[-15:], 16)
-        return int(hash_cache[session] + task_index)
+        try:
+            method = "GET"
+            url = "http://209.126.9.130:18000/api/v1/task/seed"
+            response = requests.request(method, url, params={"session": session, "task_number": task_index})
+            if response.status_code == 200:
+                response_json = response.json()
+                success = response_json["success"]
+                if not success:
+                    raise Exception(f"Failed to get seed from API: {seed}")
+                
+                seed = response_json["seed"], response_json["task_id_seed"]
+                
+                if seed is None:
+                    raise Exception(f"Seed is None")
+                return seed
+            else:
+                raise Exception(f"Failed to get seed from API: {response.status_code}")
+            
+        except Exception as e:
+            raise e
+        
+        # if session not in hash_cache:
+        #     session_start_block = session * SESSION_WINDOW_BLOCKS
+        #     subtensor = self.neuron.subtensor
+        #     block_hash = subtensor.get_block_hash(session_start_block)
+        #     hash_cache[session] = int(block_hash[-15:], 16)
+        # return int(hash_cache[session] + task_index)
 
     async def forward(self):
         try:
@@ -290,25 +324,30 @@ async def forward(self):
                 else:
                     task_index = self.neuron.score_manager.number_of_tasks
 
+                session = int(session)
+                task_index = int(task_index)
+                        
             if task_index >= MAX_NUMBER_OF_TASKS_PER_SESSION:
                 return
 
+            
             bt.logging.info(f"Forwarding task #{task_index} in session #{session}")
-            seed = self.get_seed(session, task_index)
+            #seed, task_id_seed = self.get_seed(session, task_index)
 
-            bt.logging.info(f"Init random with seed: {seed}")
-            random.seed(seed)
-            set_seed(seed)
+            #bt.logging.info(f"Random seed: {seed} | task_id_seed: {task_id_seed}")
+            #random.seed(seed)
+            #set_seed(seed)
 
-            while True:
-                try:
-                    await self.synthensize_task()
-                    break
-                except Exception as e:
-                    bt.logging.error(
-                        f"Error in synthensize_task: {e}"
-                        f"Retrying..."
-                    )
+            try:
+                await self.synthensize_task(session, task_index)
+                task, synapse = self.synthetic_tasks[-1]
+                task.task_id = f"{session}_{task_index}"
+                synapse.task_id = task.task_id
+            except Exception as e:
+                bt.logging.error(
+                    f"Error in synthensize_task: {e}"
+                )
+                return
 
             await self.query_miners()
             await self.score()

neurons/validators/score_manager.py

@@ -183,38 +183,38 @@ def get_winner(self, total_scores: np.ndarray, solved_tasks: np.ndarray, number_
         return winner
 
     def get_scores(self, session_upto: int):
-        scores = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
-        for session_number in self.session_results:
-            if (session_number <= session_upto - CONSIDERING_SESSION_COUNTS or 
-                session_number > session_upto):
-                continue
-
-            try:
-                winner = self.session_results[session_number]["winner"]
-                competition_type = self.session_results[session_number]["competition_type"]
-                if winner == -1:
-                    continue
-                scores[winner] += RESERVED_WEIGHTS[competition_type]
-            except Exception as e:
-                bt.logging.warning(f"Error getting scores: {e}")
-
-        return scores
         # scores = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
-        # tiny_weight = 1 / 128
-        # big_weight = 1.0
         # for session_number in self.session_results:
         #     if (session_number <= session_upto - CONSIDERING_SESSION_COUNTS or 
         #         session_number > session_upto):
         #         continue
-                
-        #     winner = self.session_results[session_number]["winner"]
-        #     if winner == -1:
-        #         continue
-        #     if session_number == session_upto:
-        #         scores[winner] += big_weight
-        #     else:
-        #         scores[winner] += tiny_weight
+
+        #     try:
+        #         winner = self.session_results[session_number]["winner"]
+        #         competition_type = self.session_results[session_number]["competition_type"]
+        #         if winner == -1:
+        #             continue
+        #         scores[winner] += RESERVED_WEIGHTS[competition_type]
+        #     except Exception as e:
+        #         bt.logging.warning(f"Error getting scores: {e}")
+
         # return scores
+        scores = np.zeros(self.neuron.metagraph.n, dtype=np.float32)
+        tiny_weight = 1 / 128
+        big_weight = 1.0
+        for session_number in self.session_results:
+            if (session_number <= session_upto - CONSIDERING_SESSION_COUNTS or 
+                session_number > session_upto):
+                continue
+                
+            winner = self.session_results[session_number]["winner"]
+            if winner == -1:
+                continue
+            if session_number == session_upto:
+                scores[winner] += big_weight
+            else:
+                scores[winner] += tiny_weight
+        return scores
 
         # if session_upto in self.session_results:
         #     scores = self.session_results[session_upto]["scores"]

tests/test_llms.py

@@ -1,5 +1,6 @@
 import sys
 import os
+import random
 
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(parent_dir)
@@ -10,7 +11,7 @@
 import asyncio
 from pydantic import BaseModel
 
-from webgenie.helpers.llms import openai_call
+from webgenie.helpers.llms import openai_call, set_seed
 
 
 class HTML(BaseModel):
@@ -20,15 +21,16 @@ class HTML(BaseModel):
 async def test_openai_call():
     result = await openai_call(
         messages = [
-            {"role": "system", "content": "You are an expert web developer who specializes in HTML and CSS. A user will provide you with the webpage requirements. You need to return a single html file that uses HTML and CSS to satisfy the requirements. Include all CSS code in the HTML file itself. If it involves any images, use 'rick.jpg' as the placeholder. Do not hallucinate any dependencies to external files. You do not need to include JavaScript scripts for dynamic interactions. Pay attention to things like size, text, position, and color of all the elements, as well as the overall layout. Respond with the content of the HTML+CSS file:"},
-            {"role": "user", "content": "Create a webpage with a red background and a blue rectangle in the center."},
+            {"role": "system", "content": "Could you make the following webpage more complex? It should be more complex and have more elements."},
+            {"role": "user", "content": """<!DOCTYPE html>\n<html lang="en">\n<head>\n    <meta charset="UTF-8">\n    <meta name="viewport" content="width=device-width, initial-scale=1.0">\n    <title>Red Background with Blue Rectangle</title>\n    <style>\n        body {\n            margin: 0;\n            height: 100vh;\n            display: flex;\n            justify-content: center;\n            align-items: center;\n            background-color: red;\n        }\n        .blue-rectangle {\n            width: 300px;\n            height: 200px;\n            background-color: blue;\n        }\n    </style>\n</head>\n<body>\n    <div class="blue-rectangle"></div>\n</body>\n</html>"""},
         ], 
         response_format = HTML,
     )
     print(result)
 
 
 if __name__ == "__main__":
+    set_seed(5)
     asyncio.run(test_openai_call())
 
 

webgenie/challenges/challenge.py

@@ -43,7 +43,7 @@ async def calculate_scores(self) -> dict[str, np.ndarray]:
         scores = await self.task.generator.calculate_scores(self.task, self.solutions)
         accuracy_scores = scores[ACCURACY_METRIC_NAME]
         seo_scores = scores[SEO_METRIC_NAME]
-        aggregated_scores = np.where(accuracy_scores > 0.7, seo_scores, 0)
+        aggregated_scores = np.where(accuracy_scores > 0.9, seo_scores, 0)
         return aggregated_scores, scores
 
 
@@ -54,7 +54,7 @@ async def calculate_scores(self) -> dict[str, np.ndarray]:
         scores = await self.task.generator.calculate_scores(self.task, self.solutions)
         accuracy_scores = scores[ACCURACY_METRIC_NAME]
         quality_scores = scores[QUALITY_METRIC_NAME]
-        aggregated_scores = np.where(accuracy_scores > 0.7, quality_scores, 0)
+        aggregated_scores = np.where(accuracy_scores > 0.9, quality_scores, 0)
         return aggregated_scores, scores
 
 

webgenie/datasets/__init__.py

@@ -1,4 +1,5 @@
 from .dataset import Dataset, DatasetEntry
 from .synthetic_dataset import SyntheticDataset
 from .huggingface_dataset import HuggingfaceDataset
-from .random_website_dataset import RandomWebsiteDataset
+from .random_website_dataset import RandomWebsiteDataset
+from .central_dataset import CentralDataset

webgenie/datasets/central_dataset.py

@@ -0,0 +1,53 @@
+# https://huggingface.co/datasets/SALT-NLP/Design2Code_human_eval_pairwise
+
+import bittensor as bt
+import random
+import requests
+from datasets import load_dataset
+
+from webgenie.datasets.dataset import Dataset, DatasetEntry
+
+
+class CentralDataset(Dataset):    
+    HOTKEY = "hotkey"
+    SIGNATURE = "signature"
+    
+    def __init__(self):
+        pass
+
+    async def generate_context(self) -> DatasetEntry:
+        pass
+    
+    async def generate_context(self, session:int, task_number:int)->DatasetEntry:
+        try:
+            bt.logging.info("Generating Central context")
+            html = self.get_html(session, task_number)
+            return DatasetEntry(
+                src="central",
+                url=f"central_{session}_{task_number}",
+                ground_truth_html=html,
+                prompt="",
+                base64_image=""
+            )
+        except Exception as e:
+            bt.logging.error(f"Error in generate_context: {e}")
+            raise e
+            
+    def get_html(self, session:int, task_number:int)->str:
+        bt.logging.info(f"Getting HTML for session {session} and task {task_number}")
+        method = "GET"
+        url = f"http://209.126.9.130:18000/api/v1/task/generate"
+        headers = {
+            "Signature": CentralDataset.SIGNATURE,
+            "Hotkey": CentralDataset.HOTKEY
+        }
+        params = {
+            "session": int(session),
+            "task_number": int(task_number)
+        }
+        response = requests.request(method, url, headers=headers, params=params)
+        
+        if response.status_code != 200:
+            raise Exception(f"Failed to get HTML: {response.status_code} {response.text}")
+        bt.logging.info(f"HTML: {response.json()}")
+        return response.json()["html"]