gg

Author: Programmer-RD-AI

.gitignore

@@ -158,3 +158,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+.history/

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
+    "python.formatting.provider": "none"
+}

ML/__init__.py

@@ -0,0 +1,44 @@
+import math
+import os
+import random
+import threading
+from typing import *
+import time
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import torch
+import torchinfo
+import torchvision
+import wandb
+from PIL import Image
+from sklearn.model_selection import train_test_split
+from torch import nn, optim
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+from torchvision.models import *
+from tqdm import tqdm
+from wandb import AlertLevel
+from torch.nn import *
+from torchvision.models import *
+import torchtext
+from torchtext.transforms import *
+from torchtext.models import *
+from sklearn.metrics import classification_report
+
+print(torch.__version__, torchvision.__version__, torchtext.__version__)
+os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
+# os.environ["WANDB_SILENT"] = "true"
+PROJECT_NAME = "NLP-Disaster Tweets"
+device = torch.device("cuda")
+BATCH_SIZE = 32
+torch.backends.cudnn.benchmark = True
+torch.cuda.empty_cache()
+torch.manual_seed(42)
+np.random.seed(42)
+torch.cuda.manual_seed(42)
+
+from ML.dataset import *
+from ML.helper_functions import *
+from ML.metrics import *
+from ML.modelling import *

ML/dataset/__init__.py

@@ -0,0 +1,3 @@
+from ML.dataset.loader import *
+from ML.dataset.main_loaders import *
+from ML.dataset.valid_loaders import *

ML/dataset/loader.py

@@ -0,0 +1,11 @@
+from ML import *
+
+
+class Loader(Dataset):
+    def __init__(self, path: str, transform=None) -> None:
+        self.path = path
+        self.transform = transform
+        self.data: pd.DataFrame = pd.read_csv(self.path)
+
+    def __len__(self) -> int:
+        return len(self.data)

ML/dataset/main_loaders.py

@@ -0,0 +1,28 @@
+from ML import *
+
+
+class Main_DL(Loader):
+    def __init__(self, train: bool = True, test_split: float = 0.125, seed: int = 42) -> None:
+        super().__init__()
+        self.X = self.data["text"].to_numpy()
+        self.y = self.data["target"].to_numpy()
+        self.train = train
+        self.test_split = test_split
+        self.seed = seed
+        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
+            self.X, self.y, test_size=test_split, random_state=seed
+        )
+
+    def __getitem__(self, index) -> Tuple[torch.tensor, torch.tensor]:
+        if self.train:
+            return (
+                self.transform(self.X_train[index]) if self.transform else self.X_train[index],
+                self.y_train[index],
+            )
+        return (
+            self.transform(self.X_test[index]) if self.transform else self.X_test[index],
+            self.y_test[index],
+        )
+
+    def __len__(self) -> int:
+        return len(self.y_train) if self.train else len(self.y_test)

ML/dataset/test_loaders.py

@@ -0,0 +1,10 @@
+from ML import *
+
+
+class Valid_Loader(Loader):
+    def __init__(self) -> None:
+        super().__init__()
+        self.X = self.data["text"].to_numpy()
+
+    def __getitem__(self, index) -> np.array:
+        return self.X[index]

ML/dataset/train_loaders.py

@@ -0,0 +1,4 @@
+from ML.helper_functions.load_data import *
+from ML.helper_functions.test import *
+from ML.helper_functions.train import *
+from ML.helper_functions.transformer import *