G

Author: Programmer-RD-AI

ML/__init__.py

@@ -18,13 +18,15 @@
 from torchvision import transforms
 from torchvision.models import *
 from tqdm import tqdm
-from wandb import AlertLevel
+from wandb import *
 from torch.nn import *
 from torchvision.models import *
 import torchtext
 from torchtext.transforms import *
 from torchtext.models import *
-from sklearn.metrics import classification_report
+from sklearn.metrics import *
+from torch.hub import *
+import torchtext.functional as F
 
 print(torch.__version__, torchvision.__version__, torchtext.__version__)
 os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
@@ -40,5 +42,4 @@
 
 from ML.dataset import *
 from ML.helper_functions import *
-from ML.metrics import *
 from ML.modelling import *

ML/dataset/loader.py

@@ -2,10 +2,10 @@
 
 
 class Loader(Dataset):
-    def __init__(self, path: str, transform=None) -> None:
+    def __init__(self, path: str, transform: bool = None) -> None:
         self.path = path
         self.transform = transform
-        self.data: pd.DataFrame = pd.read_csv(self.path)
+        self.data: pd.DataFrame = pd.read_csv(self.path).iloc[:5000]
 
     def __len__(self) -> int:
         return len(self.data)

ML/dataset/main_loaders.py

@@ -1,9 +1,17 @@
 from ML import *
+from ML.dataset.loader import *
 
 
 class Main_DL(Loader):
-    def __init__(self, train: bool = True, test_split: float = 0.125, seed: int = 42) -> None:
-        super().__init__()
+    def __init__(
+        self,
+        train: bool = True,
+        test_split: float = 0.125,
+        seed: int = 42,
+        batch_size: int = 32,
+        **kwargs
+    ) -> None:
+        super().__init__(**kwargs)
         self.X = self.data["text"].to_numpy()
         self.y = self.data["target"].to_numpy()
         self.train = train
@@ -12,16 +20,44 @@ def __init__(self, train: bool = True, test_split: float = 0.125, seed: int = 42
         self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
             self.X, self.y, test_size=test_split, random_state=seed
         )
+        self.X_train, self.X_test, self.y_train, self.y_test = (
+            np.array(self.X_train),
+            np.array(self.X_test),
+            np.array(self.y_train),
+            np.array(self.y_test),
+        )
+        self.batch_size = batch_size
+        # self.get_batches()
+        
+
+    def get_batches(self):
+        X = self.X_train if self.train else self.X_test
+        y = self.y_train if self.train else self.y_test
+        X_batches = []
+        y_batches = []
+        for i in range(0, len(X), self.batch_size):
+            X_iter = X[i : i + self.batch_size]
+            y_iter = y[i : i + self.batch_size]
+            X_batches.append(X_iter)
+            y_batches.append(y_iter)
+        if self.train:
+            self.X_train = F.to_tensor(X_batches, padding_value=1)
+            self.y_train = np.array(y_batches)
+        else:
+            self.X_test = F.to_tensor(X_batches, padding_value=1)
+            self.y_test = np.array(y_batches)
+
+        print(X_batches[0], y_batches[0])
 
     def __getitem__(self, index) -> Tuple[torch.tensor, torch.tensor]:
         if self.train:
             return (
-                self.transform(self.X_train[index]) if self.transform else self.X_train[index],
-                self.y_train[index],
+                self.transform(self.X_train[index]),
+                [self.y_train[index]],
             )
         return (
-            self.transform(self.X_test[index]) if self.transform else self.X_test[index],
-            self.y_test[index],
+            self.transform(self.X_test[index]),
+            [self.y_test[index]],
         )
 
     def __len__(self) -> int:

ML/dataset/valid_loaders.py

@@ -1,10 +1,11 @@
 from ML import *
+from ML.dataset.loader import *
 
 
 class Valid_Loader(Loader):
-    def __init__(self) -> None:
-        super().__init__()
+    def __init__(self, *args) -> None:
+        super().__init__(*args)
         self.X = self.data["text"].to_numpy()
 
     def __getitem__(self, index) -> np.array:
-        return self.X[index]
+        return self.transform(self.X[index])

ML/helper_functions/load_data.py

@@ -26,33 +26,31 @@ def __init__(
     def ld(self) -> Tuple[DataLoader, DataLoader, DataLoader]:
         self.train_data_loader = DataLoader(
             self.dataset_main(
-                self.main_path,
-                self.main_transform,
+                path=self.main_path,
+                transform=self.main_transform,
                 train=True,
                 test_split=self.test_split,
                 seed=self.seed,
             ),
-            batch_size=self.main_batch_size,
+            batch_size=None,
             shuffle=True,
             num_workers=round(os.cpu_count() / 2),
         )
         self.test_data_loader = DataLoader(
             self.dataset_main(
-                self.main_path,
-                self.main_transform,
+                path=self.main_path,
+                transform=self.main_transform,
                 train=False,
                 test_split=self.test_split,
                 seed=self.seed,
             ),
-            batch_size=self.main_batch_size,
+            batch_size=None,
             shuffle=True,
             num_workers=round(os.cpu_count() / 2),
         )
         self.valid_data_loader = DataLoader(
-            self.dataset_valid(
-                self.valid_path,
-            ),
-            batch_size=self.valid_batch_size,
+            self.dataset_valid(self.valid_path, None),
+            batch_size=None,
             shuffle=False,
             num_workers=round(os.cpu_count() / 2),
         )

ML/helper_functions/test.py

@@ -24,6 +24,8 @@ def test(self):
         n = 0
         with torch.inference_mode():
             for X, y in self.test_dataloader:
+                X = torch.tensor(X).to("cuda").view(1, -1)
+                y = torch.tensor(y).to("cuda")
                 preds = torch.argmax(torch.softmax(self.model(X), dim=1), dim=1)
                 results = classification_report(preds, y, class_names=["0", "1"])
                 precision = results["weighted avg"]["precision"]
@@ -38,6 +40,6 @@ def test(self):
         return {
             f"{self.name} precision": p_tot / n,
             f"{self.name} recall": r_tot / n,
-            f"{self.name} f1-score": f_tot / n,
+            f"{self.name} f1-score": f1_tot / n,
             f"{self.name} accuracy": a_tot / n,
         }

ML/helper_functions/train.py

@@ -1,4 +1,5 @@
 from ML import *
+import torchtext.functional as F
 
 
 class Train:
@@ -12,6 +13,7 @@ def __init__(
         valid_dataloader: DataLoader,
         criterion: torch.nn,
         optimizer: torch.optim,
+        lr_schedular: bool = None,
     ) -> None:
         self.model = model
         self.epochs = epochs
@@ -21,21 +23,29 @@ def __init__(
         self.valid_dataloader = valid_dataloader
         self.criterion = criterion
         self.optimizer = optimizer
+        self.lr_schedular = lr_schedular
 
     def train(self, run_name):
         print(torchinfo.summary(self.model))
-        wandb.init(project=PROJECT_NAME, entity=run_name)
+        wandb.init(project=PROJECT_NAME, name=run_name, config=self.config)
         wandb.watch(self.model, log="all")
         iterator = tqdm(range(self.epochs))
         for _ in iterator:
+            torch.cuda.empty_cache()
             for i, (X, y) in enumerate(self.train_dataloader):
+                torch.cuda.empty_cache()
+                X = torch.tensor(X).to("cuda").view(1, -1)
+                y = torch.tensor(y).to("cuda")
+                print(X.shape, y.shape)
                 self.optimizer.zero_grad()
                 loss = self.criterion(self.model(X), y)
                 loss.backward()
-                self.optimizer.step(f"{i}/{len(self.train_dataloader)}")
-                iterator.set_description()
+                self.optimizer.step()
+                iterator.set_description(f"{i}/{len(self.train_dataloader)}")
             if self.lr_schedular:
                 self.lr_schedular.step()
+            iterator.set_description(f"Testing...")
+            self.model.eval()
             wandb.log(
                 Test(
                     self.test_dataloader, self.valid_dataloader, self.criterion, self.model, "Test"
@@ -50,5 +60,7 @@ def train(self, run_name):
                     "Train",
                 ).test()
             )
+            iterator.set_description(f"Testing Done")
+            self.model.train()
         wandb.save()
         wandb.finish()

ML/helper_functions/transformer.py

@@ -7,9 +7,9 @@ def __init__(
         padding_idx: int = 1,
         beg_idx: int = 0,
         end_idx: int = 2,
-        max_seq_len: int = 256,
-        vocab_path: str = r"https://download.pytorch.org/models/text/xlmr.vocab.pt",
-        spm_model_path: str = r"https://download.pytorch.org/models/text/xlmr.sentencepiece.bpe.model",
+        max_seq_len: int = 256 - 2,
+        vocab_path=r"https://download.pytorch.org/models/text/xlmr.vocab.pt",
+        spm_model_path=r"https://download.pytorch.org/models/text/xlmr.sentencepiece.bpe.model",
         tokenizer: torchtext.transforms = SentencePieceTokenizer,
         vocab_transform: torchtext.transforms = VocabTransform,
         truncate: torchtext.transforms = Truncate,
@@ -24,13 +24,13 @@ def __init__(
         self.vocab_transform = vocab_transform
         self.truncate = truncate
 
-    def transform(self):
-        t = torchtext.transforms.Compose(
-            self.tokenizer(self.vocab_path),
-            self.vocab_transform(self.spm_model_path),
+    def transform(self) -> torchtext.transforms.Sequential:
+        t = torchtext.transforms.Sequential(
+            self.tokenizer(self.spm_model_path),
+            self.vocab_transform(load_state_dict_from_url(self.vocab_path)),
             self.truncate(self.max_seq_len),
             AddToken(self.beg_idx, begin=True),
-            AddToken(self.end_idx, end=True),
+            AddToken(self.end_idx, begin=False),
         )
         return t
 

ML/modelling/tt.py

@@ -9,10 +9,11 @@ def __init__(
         classifier_head: torchtext.models = RobertaClassificationHead,
         model: torchtext.models = XLMR_BASE_ENCODER,
     ) -> None:
+        super().__init__()
         self.num_classes = num_classes
         self.input_dim = input_dim
         self.classifier_head = classifier_head(num_classes, input_dim)
-        self.model = model(self.classifier_head).to(device)
+        self.model = model.get_model(head=self.classifier_head).to(device)
 
     def forward(self, X):
         return self.model(X)