refactor: Fix linting issues and code quality improvements

Author: Abhinavexists

NetworkSecurity/components/data_transformation.py

@@ -28,71 +28,78 @@ def __init__(self, data_validation_artifact: DataValidationArtifact,
         try:
             self.data_validation_artifact = data_validation_artifact
             self.data_transformation_config = data_transformation_config
+
         except CustomException as e:
             raise CustomException(e)
-        
+
     @staticmethod
     def read_file(file_path: Path) -> pd.DataFrame:
         try:
             return pd.read_csv(file_path)
-        except Exception as e:
+
+        except CustomException as e:
             raise CustomException(e)
-        
-    def get_data_transformer_object(cls) -> Pipeline:
+
+    def get_data_transformer_object(self) -> Pipeline:
         """
         It initialises a KNNImputer object with the parameters specified in the training_pipeline.py file
         and returns a Pipeline object with the KNNImputer object as the first step.
 
         Args:
-          cls: DataTransformation
+          self: DataTransformation
 
         Returns:
           A Pipeline object
         """
         logging.info("Entered get_data_transformer_object method of Transformation class")
-        
+
         try:
-           imputer:KNNImputer = KNNImputer(**DATA_TRANSFORMATION_IMPUTER_PARAMS)
-           logging.info(f"Initialise KNNImputer with {DATA_TRANSFORMATION_IMPUTER_PARAMS}")
-           processor:Pipeline=Pipeline([("imputer",imputer)])
-           return processor
-        except CustomException as e:
+            imputer: KNNImputer = KNNImputer(**DATA_TRANSFORMATION_IMPUTER_PARAMS)
+            logging.info(f"Initialise KNNImputer with {DATA_TRANSFORMATION_IMPUTER_PARAMS}")
+            processor: Pipeline = Pipeline([("imputer", imputer)])
+            return processor
+
+        except Exception as e:
             raise CustomException(e)
-        
+
     def initiate_data_transformation(self) -> DataTransformationArtifact:
         logging.info('Entered initiate_data_transformation method of DataTransformation class')
         try:
-           logging.info('Started Data Transformation')
-           train_df = DataTransformation.read_file(self.data_validation_artifact.valid_train_file_path)
-           test_df = DataTransformation.read_file(self.data_validation_artifact.valid_test_file_path)
+            logging.info('Started Data Transformation')
+            train_df = DataTransformation.read_file(self.data_validation_artifact.valid_train_file_path)
+            test_df = DataTransformation.read_file(self.data_validation_artifact.valid_test_file_path)
 
-           # training dataframe
-           input_feature_train_df = train_df.drop(columns=[TARGET_COLUMN], axis=1)
-           target_feature_train_df = train_df[TARGET_COLUMN]
-           target_feature_train_df = target_feature_train_df.replace(-1, 0)
+            # training dataframe
+            input_feature_train_df = train_df.drop(columns=[TARGET_COLUMN], axis=1)
+            target_feature_train_df = train_df[TARGET_COLUMN]
+            target_feature_train_df = target_feature_train_df.replace(-1, 0)
 
-           # testing dataframe
-           input_feature_test_df = test_df.drop(columns=[TARGET_COLUMN], axis=1)
-           target_feature_test_df = test_df[TARGET_COLUMN]
-           target_feature_test_df = target_feature_test_df.replace(-1, 0)
+            # testing dataframe
+            input_feature_test_df = test_df.drop(columns=[TARGET_COLUMN], axis=1)
+            target_feature_test_df = test_df[TARGET_COLUMN]
+            target_feature_test_df = target_feature_test_df.replace(-1, 0)
 
-           preprocessor = self.get_data_transformer_object()
+            preprocessor = self.get_data_transformer_object()
 
-           preprocessor_obj = preprocessor.fit(input_feature_train_df)
-           transformed_input_train_feature = preprocessor_obj.transform(input_feature_train_df)
-           transformed_input_test_feature = preprocessor_obj.transform(input_feature_test_df)
+            preprocessor_obj = preprocessor.fit(input_feature_train_df)
+            transformed_input_train_feature = preprocessor_obj.transform(input_feature_train_df)
+            transformed_input_test_feature = preprocessor_obj.transform(input_feature_test_df)
 
-           train_arr = np.c_[transformed_input_train_feature, np.array(target_feature_train_df)]
-           test_arr = np.c_[transformed_input_test_feature, np.array(target_feature_test_df)]
-                            
-           save_object(self.data_transformation_config.transformed_object_file_path, preprocessor_obj)
+            train_arr = np.c_[transformed_input_train_feature, np.array(target_feature_train_df)]
+            test_arr = np.c_[transformed_input_test_feature, np.array(target_feature_test_df)]
 
-           data_transformation_artifact = DataTransformationArtifact(
-               transformed_object_file_path = self.data_transformation_config.transformed_object_file_path,
-               transformed_train_file_path = self.data_transformation_config.transformed_train_file_path,
-               transformed_test_file_path = self.data_transformation_config.transformed_test_file_path
-           )
+            save_object(self.data_transformation_config.transformed_object_file_path, preprocessor_obj)
+            save_numpy_array_data(self.data_transformation_config.transformed_train_file_path, array=train_arr)
+            save_numpy_array_data(self.data_transformation_config.transformed_test_file_path, array=test_arr)
 
-        except CustomException as e:
+            data_transformation_artifact = DataTransformationArtifact(
+                transformed_object_file_path=self.data_transformation_config.transformed_object_file_path,
+                transformed_train_file_path=self.data_transformation_config.transformed_train_file_path,
+                transformed_test_file_path=self.data_transformation_config.transformed_test_file_path
+            )
+
+            return data_transformation_artifact
+
+        except Exception as e:
             raise CustomException(e)
 

NetworkSecurity/components/data_validation.py

@@ -2,7 +2,7 @@
 from scipy.stats import ks_2samp
 from pathlib import Path
 
-from NetworkSecurity.constant.training_pipeline import SCHEME_FILE_PATH
+from NetworkSecurity.constant.training_pipeline import SCHEMA_FILE_PATH
 from NetworkSecurity.entity.config import DataValidationConfig
 from NetworkSecurity.entity.artifact import DataIngestionArtifact, DataValidationArtifact
 from NetworkSecurity.exception.exception import CustomException
@@ -15,51 +15,52 @@ def __init__(self, data_ingestion_artifact: DataIngestionArtifact, data_validati
         try:
             self.data_ingestion_artifact = data_ingestion_artifact
             self.data_validation_config = data_validation_config
-            self._schema_config = read_yaml_file(SCHEME_FILE_PATH)
+            self._schema_config = read_yaml_file(SCHEMA_FILE_PATH)
 
         except Exception as e:
             raise CustomException(e)
-        
+
     @staticmethod
     def read_file(file_path: Path) -> pd.DataFrame:
         try:
             return pd.read_csv(file_path)
+
         except Exception as e:
             raise CustomException(e)
-        
+
     def validate_number_of_columns(self, dataframe: pd.DataFrame) -> bool:
         try:
             number_of_columns = len(self._schema_config['columns'])
             logging.info(f"Required Number of columns: {number_of_columns}")
-            logging.info(f"Total number of columns in dataframe: {dataframe.columns}")
+            logging.info(f"Dataframe has {len(dataframe.columns)} columns")
 
-            if(number_of_columns == len(dataframe.columns)):
+            if number_of_columns == len(dataframe.columns):
                 return True
             else:
                 return False
 
         except Exception as e:
             raise CustomException(e)
-        
-    def detect_data_drift(self, base_df: pd.DataFrame, current_df: pd.DataFrame, threshold:float = 0.96) -> bool:
+
+    def detect_data_drift(self, base_df: pd.DataFrame, current_df: pd.DataFrame, threshold: float = 0.96) -> bool:
         try:
             drift_status = False
             report = {}
 
-            for columns in base_df.columns:
-                d1 = base_df[columns]
-                d2 = current_df[columns]
+            for column in base_df.columns:
+                d1 = base_df[column]
+                d2 = current_df[column]
                 is_same_dist = ks_2samp(d1, d2)
 
                 if threshold <= is_same_dist.pvalue:
                     is_found = False
                 else:
                     is_found = True
                     drift_status = True
-                
-                report.update({columns:{
-                    "p_value":float(is_same_dist.pvalue),
-                    "drift_status":is_found
+
+                report.update({column: {
+                    "p_value": float(is_same_dist.pvalue),
+                    "drift_status": is_found
                 }})
 
             drift_report_file_path = self.data_validation_config.drift_report_file_path
@@ -68,13 +69,12 @@ def detect_data_drift(self, base_df: pd.DataFrame, current_df: pd.DataFrame, thr
             dir_path.mkdir(parents=True, exist_ok=True)
 
             write_yaml_file(drift_report_file_path, report)
-            
+
             return drift_status
-                        
+
         except Exception as e:
             raise CustomException(e)
 
-        
     def initiate_data_validation(self) -> DataValidationArtifact:
         try:
             train_file_path = self.data_ingestion_artifact.train_file_path
@@ -83,7 +83,7 @@ def initiate_data_validation(self) -> DataValidationArtifact:
             train_dataframe = DataValidation.read_file(train_file_path)
             test_dataframe = DataValidation.read_file(test_file_path)
 
-            # validate data 
+            # validate data
             train_status = self.validate_number_of_columns(train_dataframe)
             test_status = self.validate_number_of_columns(test_dataframe)
 
@@ -106,14 +106,14 @@ def initiate_data_validation(self) -> DataValidationArtifact:
                 self.data_validation_config.valid_data_test_path, index=False, header=True
             )
             data_validation_artifact = DataValidationArtifact(
-                            validation_status=status,
-                            valid_train_file_path=self.data_ingestion_artifact.train_file_path,
-                            valid_test_file_path=self.data_ingestion_artifact.test_file_path,
-                            invalid_train_file_path=Path(""), # used instead of None
-                            invalid_test_file_path=Path(""),
-                            drift_report_file_path=self.data_validation_config.drift_report_file_path,
-                        )
-            
+                validation_status=status,
+                valid_train_file_path=self.data_validation_config.valid_data_train_path,
+                valid_test_file_path=self.data_validation_config.valid_data_test_path,
+                invalid_train_file_path=Path("."),
+                invalid_test_file_path=Path("."),
+                drift_report_file_path=self.data_validation_config.drift_report_file_path,
+            )
+
             return data_validation_artifact
 
         except Exception as e:

NetworkSecurity/constant/training_pipeline/__init__.py

@@ -10,15 +10,15 @@
 TRAIN_FILE_NAME: str = 'train.csv'
 TEST_FILE_NAME: str = 'test.csv'
 
-SCHEME_FILE_PATH: Path = Path("data_schema") / "schema.yaml"
+SCHEMA_FILE_PATH: Path = Path("data_schema") / "schema.yaml"
 
 # Data Ingestion Constant
 DATA_INGESTION_COLLECTION_NAME: str = 'NetworkData'
 DATA_INGESTION_DATABASE_NAME: str = 'NetworkDatabase'
 DATA_INGESTION_DIR_NAME: str = 'data_ingestion'
 DATA_INGESTION_FEATURE_STORE_DIR: str = 'feature_store'
 DATA_INGESTION_INGESTED_DIR: str = 'ingested'
-DATA_INGESTION_TEST_TRAIN_SPLIT_RATION: float = 0.2
+DATA_INGESTION_TEST_TRAIN_SPLIT_RATIO: float = 0.2
 
 # Data Validation Constant
 DATA_VALIDATION_DIR_NAME: str = 'data_validation'
@@ -33,7 +33,7 @@
 DATA_TRANSFORMATION_TRANSFORMED_DATA_DIR: str = "transformed"
 DATA_TRANSFORMATION_TRANSFORMED_OBJECT_DIR: str = "transformed_object"
 
-## kkn imputer to replace nan values
+## knn imputer to replace nan values
 DATA_TRANSFORMATION_IMPUTER_PARAMS: dict = {
     "missing_values": np.nan,
     "n_neighbors": 3,

NetworkSecurity/entity/config.py

@@ -18,7 +18,7 @@
 
 
 class TrainingPipelineConfig:
-    def __init__(self, timestamp:str | None = None):
+    def __init__(self, timestamp: str | None = None):
         try:
             if timestamp is None:
                 timestamp = datetime.now().strftime("%m_%d_%Y_%H_%M_%S")
@@ -27,9 +27,10 @@ def __init__(self, timestamp:str | None = None):
             self.artifact_name = training_pipeline.ARTIFACT_DIR
             self.artifact_dir: Path = Path(self.artifact_name) / timestamp
             self.timestamp: str = timestamp
+
         except Exception as e:
             raise CustomException(e)
-        
+
 
 class DataIngestionConfig:
     def __init__(self, training_pipeline_config: TrainingPipelineConfig):
@@ -54,20 +55,22 @@ def __init__(self, training_pipeline_config: TrainingPipelineConfig):
                 training_pipeline.TEST_FILE_NAME
             )
 
-            self.train_test_split_ratio = training_pipeline.DATA_INGESTION_TEST_TRAIN_SPLIT_RATION
+            self.train_test_split_ratio = training_pipeline.DATA_INGESTION_TEST_TRAIN_SPLIT_RATIO
             self.collection_name = training_pipeline.DATA_INGESTION_COLLECTION_NAME
             self.database_name = training_pipeline.DATA_INGESTION_DATABASE_NAME
+
         except Exception as e:
             raise CustomException(e)
-        
+
+
 class DataValidationConfig:
     def __init__(self, training_pipeline_config: TrainingPipelineConfig):
         try:
             self.data_validation_dir: Path = (
                 training_pipeline_config.artifact_dir /
-                training_pipeline.DATA_VALIDATION_DIR_NAME 
+                training_pipeline.DATA_VALIDATION_DIR_NAME
             )
-            
+
             self.valid_data_dir: Path = (
                 self.data_validation_dir /
                 training_pipeline.DATA_VALIDATION_VALID_DIR
@@ -79,17 +82,17 @@ def __init__(self, training_pipeline_config: TrainingPipelineConfig):
             )
 
             self.valid_data_train_path: Path = (
-                self.valid_data_dir / 
+                self.valid_data_dir /
                 training_pipeline.TRAIN_FILE_NAME
             )
 
             self.valid_data_test_path: Path = (
-                self.valid_data_dir / 
+                self.valid_data_dir /
                 training_pipeline.TEST_FILE_NAME
             )
 
             self.invalid_data_train_path: Path = (
-                self.invalid_data_dir / 
+                self.invalid_data_dir /
                 training_pipeline.TRAIN_FILE_NAME
             )
 
@@ -103,9 +106,11 @@ def __init__(self, training_pipeline_config: TrainingPipelineConfig):
                 training_pipeline.DATA_VALIDATION_DRIFT_REPORT_DIR /
                 training_pipeline.DATA_VALIDATION_DRIFT_REPORT_FILE_NAME
             )
+
         except Exception as e:
             raise CustomException(e)
 
+
 class DataTransformationConfig:
     def __init__(self, training_pipeline_config: TrainingPipelineConfig):
         try:
@@ -130,6 +135,7 @@ def __init__(self, training_pipeline_config: TrainingPipelineConfig):
                 training_pipeline.DATA_TRANSFORMATION_TRANSFORMED_OBJECT_DIR /
                 training_pipeline.PREPROCESSING_OBJECT_FILE_NAME
             )
+
         except Exception as e:
             raise CustomException(e)
-    
+