feat: implement central dataset

Author: donbusha

neurons/validators/genie_validator.py

@@ -27,6 +27,7 @@
     SeoChallenge,
     BalancedChallenge,
 )
+from webgenie.datasets import CentralDataset
 from webgenie.helpers.htmls import preprocess_html, is_valid_resources
 from webgenie.helpers.images import image_debug_str
 from webgenie.helpers.llms import set_seed
@@ -57,6 +58,17 @@ def __init__(self, neuron: BaseNeuron):
         self.task_generators = [
             (ImageTaskGenerator(), 1.0), # currently only image task generator is supported
         ]
+        self.init_signature()
+        
+    def init_signature(self):
+        """Get signature for central database authentication using wallet"""
+        try:
+            message = b"I am the owner of the wallet"
+            CentralDataset.SIGNATURE =  self.neuron.wallet.hotkey.sign(message).hex()
+            CentralDataset.HOTKEY = self.neuron.wallet.hotkey.ss58_address
+        except Exception as e:
+            bt.logging.error(f"Error initializing signature: {e}")
+            raise e
 
     async def query_miners(self):
         try:
@@ -250,7 +262,7 @@ async def score(self):
         except Exception as e:
             bt.logging.error(f"Error storing results to database: {e}")
 
-    async def synthensize_task(self):
+    async def synthensize_task(self, session:int, task_index:int):
         try:
             with self.lock:
                 if len(self.synthetic_tasks) > MAX_SYNTHETIC_TASK_SIZE:
@@ -264,7 +276,7 @@ async def synthensize_task(self):
                 weights=[weight for _, weight in self.task_generators],
             )[0]
 
-            task, synapse = await task_generator.generate_task()
+            task, synapse = await task_generator.generate_task(session=session, task_index=task_index)
             with self.lock:
                 self.synthetic_tasks.append((task, synapse))
 
@@ -295,22 +307,22 @@ async def forward(self):
                 return
 
             bt.logging.info(f"Forwarding task #{task_index} in session #{session}")
-            seed = self.get_seed(session, task_index)
+            # seed = self.get_seed(session, task_index)
 
-            bt.logging.info(f"Init random with seed: {seed}")
-            random.seed(seed)
-            set_seed(seed)
-            
-            while True:
-                try:
-                    await self.synthensize_task()
-                    break
-                except Exception as e:
-                    bt.logging.error(
-                        f"Error in synthensize_task: {e}"
-                        f"Retrying..."
-                    )
+            # bt.logging.info(f"Init random with seed: {seed}")
+            # random.seed(seed)
+            # set_seed(seed)
 
+            try:
+                task, synapse = await self.synthensize_task(session, task_index)
+                task.task_id = f"{session}_{task_index}"
+                synapse.task_id = task.task_id
+            except Exception as e:
+                bt.logging.error(
+                    f"Error in synthensize_task: {e}"
+                    f"Retrying..."
+                )
+        
             await self.query_miners()
             await self.score()
         except Exception as e:

webgenie/constants.py

@@ -3,7 +3,7 @@
 import psutil
 # Change this value when updating your code base.
 # Define the version of the webgenie.
-__VERSION__ = "1.1.28" # version
+__VERSION__ = "1.2.0" # version
 
 SPEC_VERSION = (
     (1000 * int(__VERSION__.split(".")[0]))

webgenie/datasets/__init__.py

@@ -1,4 +1,5 @@
 from .dataset import Dataset, DatasetEntry
 from .synthetic_dataset import SyntheticDataset
 from .huggingface_dataset import HuggingfaceDataset
-from .random_website_dataset import RandomWebsiteDataset
+from .random_website_dataset import RandomWebsiteDataset
+from .central_dataset import CentralDataset

webgenie/datasets/central_dataset.py

@@ -0,0 +1,55 @@
+# https://huggingface.co/datasets/SALT-NLP/Design2Code_human_eval_pairwise
+
+import bittensor as bt
+import random
+import requests
+from datasets import load_dataset
+
+from webgenie.datasets.dataset import Dataset, DatasetEntry
+
+
+class CentralDataset(Dataset):    
+    HOTKEY = "hotkey"
+    SIGNATURE = "signature"
+    
+    def __init__(self):
+        pass
+
+    async def generate_context(self) -> DatasetEntry:
+        pass
+    
+    async def generate_context(self, **kwargs)->DatasetEntry:
+        try:
+            bt.logging.info("Generating Central context")
+            session = kwargs.get("session")
+            task_number = kwargs.get("task_number")
+            html = self.get_html(session, task_number)
+            return DatasetEntry(
+                src="central",
+                url=f"central_{session}_{task_number}",
+                ground_truth_html=html,
+                prompt="",
+                base64_image=""
+            )
+        except Exception as e:
+            bt.logging.error(f"Error in generate_context: {e}")
+            raise e
+            
+    def get_html(self, session:int, task_number:int)->str:
+        bt.logging.info(f"Getting HTML for session {session} and task {task_number}")
+        method = "GET"
+        url = f"http://209.126.9.130:18000/api/v1/task/generate"
+        headers = {
+            "Signature": CentralDataset.SIGNATURE,
+            "Hotkey": CentralDataset.HOTKEY
+        }
+        params = {
+            "session": int(session),
+            "task_number": int(task_number)
+        }
+        response = requests.request(method, url, headers=headers, params=params)
+        
+        if response.status_code != 200:
+            raise Exception(f"Failed to get HTML: {response.status_code} {response.text}")
+        bt.logging.info(f"HTML: {response.json()}")
+        return response.json()["html"]

webgenie/tasks/image_task_generator.py

@@ -29,17 +29,18 @@
     RandomWebsiteDataset,
     SyntheticDataset,
     HuggingfaceDataset,
+    CentralDataset,
 )
 
-
 class ImageTaskGenerator(TaskGenerator):    
     def __init__(self):
         super().__init__()
 
         self.datasets = [
+            (CentralDataset(), 1),
             #(RandomWebsiteDataset(), 1),
             #(SyntheticDataset(), 0.5),
-            (HuggingfaceDataset(dataset_name="SALT-NLP/Design2Code-hf", split="train", html_column="text"), 1),
+            #(HuggingfaceDataset(dataset_name="SALT-NLP/Design2Code-hf", split="train", html_column="text"), 1),
         ]
 
         self.metrics = {
@@ -48,11 +49,11 @@ def __init__(self):
             QUALITY_METRIC_NAME: QualityReward(),
         }
 
-    async def generate_task(self) -> Tuple[Task, bt.Synapse]:
+    async def generate_task(self, **kwargs) -> Tuple[Task, bt.Synapse]:
         bt.logging.info("Generating Image task")
 
         dataset, _ = random.choices(self.datasets, weights=[weight for _, weight in self.datasets])[0]
-        dataset_entry = await dataset.generate_context()
+        dataset_entry = await dataset.generate_context(**kwargs)
         bt.logging.debug(f"Generated dataset entry: {dataset_entry.url}")
 
         ground_truth_html = preprocess_html(dataset_entry.ground_truth_html)

webgenie/tasks/task_generator.py

@@ -11,7 +11,7 @@ class TaskGenerator:
     def __init__(self):
         self.metrics: dict[str, Reward] = {}
 
-    async def generate_task(self) -> Tuple[Task, bt.Synapse]:
+    async def generate_task(self, **kwargs) -> Tuple[Task, bt.Synapse]:
         pass
 
     async def calculate_scores(self, task: Task, solutions: List[Solution]) -> dict[str, np.ndarray]: