sync

Author: jcapp4

chebai/models/base.py

@@ -261,17 +261,16 @@ def _execute(
         model_output = self(data, **data.get("model_kwargs", dict()))
         pr, tar = self._get_prediction_and_labels(data, labels, model_output)
         d = dict(data=data, labels=labels, output=model_output, preds=pr)
+        torch.save(d,"d.pt")
         if log:
             if self.criterion is not None:
-                f.init_weights()
                 loss_data, loss_labels, loss_kwargs_candidates = self._process_for_loss(
                     model_output, labels, data.get("loss_kwargs", dict())
                 )
                 loss_kwargs = dict()
                 if self.pass_loss_kwargs:
                     loss_kwargs = loss_kwargs_candidates
-                dict_weights = f.get_weights(data['idents'])
-                loss_kwargs['weights'] = f.create_data_weights(len(data['idents']),data['labels'].size(dim=1),dict_weights,data['idents'])
+                torch.save(loss_data,"loss_data.pt")
                 loss_kwargs["current_epoch"] = self.trainer.current_epoch
                 loss = self.criterion(loss_data, loss_labels, **loss_kwargs)
                 if isinstance(loss, tuple):

chebai/preprocessing/collate.py

@@ -77,7 +77,6 @@ def __call__(self, data: List[Union[Dict, Tuple]]) -> XYData:
         model_kwargs: Dict = dict()
         # Indices of non-null labels are stored in key `non_null_labels` of loss_kwargs.
         loss_kwargs: Dict = dict()
-
         if isinstance(data[0], tuple):
             # For legacy data
             x, y, idents = zip(*data)
@@ -106,6 +105,11 @@ def __call__(self, data: List[Union[Dict, Tuple]]) -> XYData:
         lens = torch.tensor(list(map(len, x)))
         model_kwargs["mask"] = torch.arange(max(lens))[None, :] < lens[:, None]
         model_kwargs["lens"] = lens
+        for d in data:
+            id = d["ident"]
+            weight = d["weight"]
+            loss_kwargs["ident"] = weight
+
 
         return XYData(
             pad_sequence([torch.tensor(a) for a in x], batch_first=True),

chebai/preprocessing/datasets/base.py

@@ -18,6 +18,10 @@
 from torch.utils.data import DataLoader
 
 from chebai.preprocessing import reader as dr
+import sys
+sys.path.insert(1,'/home/programmer/Bachelorarbeit/python-chebai')
+
+import extras.weight_loader as f
 
 
 class XYBaseDataModule(LightningDataModule):
@@ -1123,6 +1127,7 @@ def _retrieve_splits_from_csv(self) -> None:
             os.path.join(self.processed_dir, filename)
         )
         df_data = pd.DataFrame(data)
+        
 
         train_ids = splits_df[splits_df["split"] == "train"]["id"]
         validation_ids = splits_df[splits_df["split"] == "validation"]["id"]
@@ -1165,6 +1170,9 @@ def load_processed_data(
             raise ValueError(
                 "Either kind or filename is required to load the correct dataset, both are None"
             )
+        if kind == "train":
+            print("loading train data")
+
 
         # If both kind and filename are given, use filename
         if kind is not None and filename is None:
@@ -1174,10 +1182,19 @@ def load_processed_data(
                 ]
             else:
                 data_df = self.dynamic_split_dfs[kind]
-                return data_df.to_dict(orient="records")
+                data = data_df.to_dict(orient="records")
+                if kind == "train":
+                    data = f.add_train_weights(data)
+                if kind == "validation":
+                    print(kind)
+                    data = f.add_val_weights(data)
+                torch.save(data,"gewicht.pt")
+
+                return data
 
         # If filename is provided
-        return self.load_processed_data_from_file(filename)
+        data = self.load_processed_data_from_file(filename)
+        return data
 
     def load_processed_data_from_file(self, filename):
         return torch.load(os.path.join(filename), weights_only=False)

extras/weight_loader.py

@@ -15,7 +15,7 @@ def init_weights(path="../weights/first_it.csv",path_to_split="../split/splits.c
                 reader = csv.reader(csvfile)
                 weight = 1 / get_size(path_to_split)
                 for row in reader:
-                    if row[1] == "train" or row[1] == "validation":
+                    if row[1] == "train":
                         #print(type(row[0]))
                         writer.writerow([int(row[0]),row[1],weight])
 
@@ -28,7 +28,7 @@ def mock_init_weights(path="../weights/first_it.csv",path_to_split="../split/spl
             reader = csv.reader(csvfile)
             weight = 1
             for row in reader:
-                if row[1] == "train" or row[1] == "validation":
+                if row[1] == "train":
                     writer.writerow([int(row[0]),row[1],weight])
                     weight = weight + 1
 
@@ -71,16 +71,16 @@ def find_label(id:int,path="../split/splits.csv")-> str:
 
 #to do 
 # return should be a tuple of weigths matching the sequenece of the target and label tensor
-def create_data_weights(batchsize:int,dim:int,weights:dict[str,float],idents:tuple[int,...])-> torch.tensor:
-    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-    weight = torch.empty(batchsize,dim,device=device)
-    index = 0
-    for i in idents:
-        w = weights[str(i)]
-        for j in range(0,dim):
-            weight[index][j] = float(w)
-        index = index + 1
-    return weight
+# def create_data_weights(batchsize:int,dim:int,weights:dict[str,float],idents:tuple[int,...])-> torch.tensor:
+#     device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+#     weight = torch.empty(batchsize,dim,device=device)
+#     index = 0
+#     for i in idents:
+#         w = weights[str(i)]
+#         for j in range(0,dim):
+#             weight[index][j] = float(w)
+#         index = index + 1
+#     return weight
 
 def testing():
     print("hello world")
@@ -89,15 +89,25 @@ def testing():
 def create_weight_tensor(weight:float)-> torch.tensor:
     pass
 
+def add_val_weights(ids):
+    for i in ids:
+        weight = 1
+        i["weight"] = weight
+    return ids
 
 
+def add_train_weights(ids):
+    for i in ids:
+        ident = i["ident"]
+        weight = find_weight("/home/programmer/Bachelorarbeit/weights/first_it.csv",ident=ident)
+        i["weight"] = weight
+    return ids
 
+def check_weights(data):
+    for i in data:
+        print(f"({i["ident"]} , {i["weight"]}")
 
 
-
-def create_class_weights()-> torch.tensor:
-    pass
-
 #mock_init_weights()
 # print(get_weights((233713,51990)))