Merge branch 'pythonlessons:main' into main

Author: pythonlessons

Tutorials/05_sound_to_text/configs.py

@@ -3,6 +3,7 @@
 
 from mltu.configs import BaseModelConfigs
 
+
 class ModelConfigs(BaseModelConfigs):
     def __init__(self):
         super().__init__()

mltu/configs.py

@@ -1,15 +1,25 @@
 import os
 import yaml
 
+
 class BaseModelConfigs:
     def __init__(self):
         self.model_path = None
 
     def serialize(self):
-        # get object attributes
-        return self.__dict__
+        class_attributes = {key: value
+                            for (key, value)
+                            in type(self).__dict__.items()
+                            if key not in ['__module__', '__init__', '__doc__', '__annotations__']}
+        instance_attributes = self.__dict__
+
+        # first init with class attributes then apply instance attributes overwriting any existing duplicate attributes
+        all_attributes = class_attributes.copy()
+        all_attributes.update(instance_attributes)
+
+        return all_attributes
 
-    def save(self, name: str="configs.yaml"):
+    def save(self, name: str = "configs.yaml"):
         if self.model_path is None:
             raise Exception("Model path is not specified")
 

mltu/dataProvider.py

@@ -9,24 +9,23 @@
 from .transformers import Transformer
 
 import logging
-logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s: %(message)s")
 
 
 class DataProvider:
     def __init__(
-        self, 
-        dataset: typing.Union[str, list, pd.DataFrame],
-        data_preprocessors: typing.List[typing.Callable] = None,
-        batch_size: int = 4,
-        shuffle: bool = True,
-        initial_epoch: int = 1,
-        augmentors: typing.List[Augmentor] = None,
-        transformers: typing.List[Transformer] = None,
-        skip_validation: bool = True,
-        limit: int = None,
-        use_cache: bool = False,
-        log_level: int = logging.INFO,
-        ) -> None:
+            self,
+            dataset: typing.Union[str, list, pd.DataFrame],
+            data_preprocessors: typing.List[typing.Callable] = None,
+            batch_size: int = 4,
+            shuffle: bool = True,
+            initial_epoch: int = 1,
+            augmentors: typing.List[Augmentor] = None,
+            transformers: typing.List[Transformer] = None,
+            skip_validation: bool = True,
+            limit: int = None,
+            use_cache: bool = False,
+            log_level: int = logging.INFO,
+    ) -> None:
         """ Standardised object for providing data to a model while training.
 
         Attributes:
@@ -61,7 +60,7 @@ def __init__(
 
         # Validate dataset
         if not skip_validation:
-            self._dataset = self.validate(dataset, skip_validation, limit)
+            self._dataset = self.validate(dataset)
         else:
             self.logger.info("Skipping Dataset validation...")
 
@@ -91,8 +90,6 @@ def augmentors(self, augmentors: typing.List[Augmentor]):
             else:
                 self.logger.warning(f"Augmentor {augmentor} is not an instance of Augmentor.")
 
-        return self._augmentors
-
     @property
     def transformers(self) -> typing.List[Transformer]:
         """ Return transformers """
@@ -111,8 +108,6 @@ def transformers(self, transformers: typing.List[Transformer]):
             else:
                 self.logger.warning(f"Transformer {transformer} is not an instance of Transformer.")
 
-        return self._transformers
-
     @property
     def epoch(self) -> int:
         """ Return Current Epoch"""
@@ -131,28 +126,28 @@ def on_epoch_end(self):
 
         # Remove any samples that were marked for removal
         for remove in self._on_epoch_end_remove:
-            self.logger.warn(f"Removing {remove} from dataset.")
+            self.logger.warning(f"Removing {remove} from dataset.")
             self._dataset.remove(remove)
         self._on_epoch_end_remove = []
 
-    def validate_list_dataset(self, dataset: list, skip_validation: bool = False) -> list:
+    def validate_list_dataset(self, dataset: list) -> list:
         """ Validate a list dataset """
         validated_data = [data for data in tqdm(dataset, desc="Validating Dataset") if os.path.exists(data[0])]
         if not validated_data:
             raise FileNotFoundError("No valid data found in dataset.")
 
         return validated_data
 
-    def validate(self, dataset: typing.Union[str, list, pd.DataFrame], skip_validation: bool) -> list:
+    def validate(self, dataset: typing.Union[str, list, pd.DataFrame]) -> typing.Union[list, str]:
         """ Validate the dataset and return the dataset """
 
         if isinstance(dataset, str):
             if os.path.exists(dataset):
                 return dataset
         elif isinstance(dataset, list):
-            return self.validate_list_dataset(dataset, skip_validation)
+            return self.validate_list_dataset(dataset)
         elif isinstance(dataset, pd.DataFrame):
-            return self.validate_list_dataset(dataset.values.tolist(), skip_validation)
+            return self.validate_list_dataset(dataset.values.tolist())
         else:
             raise TypeError("Dataset must be a path, list or pandas dataframe.")
 
@@ -176,7 +171,7 @@ def split(self, split: float = 0.9, shuffle: bool = True) -> typing.Tuple[typing
 
         return train_data_provider, val_data_provider
 
-    def to_csv(self, path: str, index: bool=False) -> None:
+    def to_csv(self, path: str, index: bool = False) -> None:
         """ Save the dataset to a csv file 
 
         Args:
@@ -230,8 +225,8 @@ def process_data(self, batch_data):
 
         # Then augment, transform and postprocess the batch data
         for objects in [self._augmentors, self._transformers]:
-            for object in objects:
-                data, annotation = object(data, annotation)
+            for _object in objects:
+                data, annotation = _object(data, annotation)
 
         # Convert to numpy array if not already
         if not isinstance(data, np.ndarray):
@@ -261,4 +256,4 @@ def __getitem__(self, index: int):
             batch_data.append(data)
             batch_annotations.append(annotation)
 
-        return np.array(batch_data), np.array(batch_annotations)
+        return np.array(batch_data), np.array(batch_annotations)

mltu/preprocessors.py

@@ -9,8 +9,6 @@
 
 from . import Image
 
-logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s: %(message)s")
-matplotlib.interactive(False)
 
 
 class ImageReader:
@@ -56,17 +54,20 @@ class WavReader:
         frame_step (int): Step size between frames in samples.
         fft_length (int): Number of FFT components.
     """
+
     def __init__(
-        self, 
-        frame_length: int = 256,
-        frame_step: int = 160,
-        fft_length: int = 384,
-        *args, **kwargs
-        ) -> None:
+            self,
+            frame_length: int = 256,
+            frame_step: int = 160,
+            fft_length: int = 384,
+            *args, **kwargs
+    ) -> None:
         self.frame_length = frame_length
         self.frame_step = frame_step
         self.fft_length = fft_length
 
+        matplotlib.interactive(False)
+
     @staticmethod
     def get_spectrogram(wav_path: str, frame_length: int, frame_step: int, fft_length: int) -> np.ndarray:
         """Compute the spectrogram of a WAV file

mltu/tensorflow/__init__.py

@@ -5,9 +5,6 @@
 from . import Image
 
 import logging
-logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s: %(message)s")
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class Transformer:

mltu/transformers.py

@@ -3,9 +3,9 @@ tqdm
 pandas
 numpy
 opencv-python
-Pillow==9.4.0
-onnxruntime # onnxruntime-gpu for GPU support
-librosa==0.9.2
+Pillow>=9.4.0
+onnxruntime>=1.15.0  # onnxruntime-gpu for GPU support
+librosa>=0.9.2
 matplotlib
-onnx==1.14.0
-tf2onnx==1.14.0
+onnx>=1.14.0
+tf2onnx>=1.14.0