Changed for generalisation and better accuracy score

CPPavithra · CPPavithra · commit d25454897d2a · 2025-03-23T18:43:35.000+05:30
diff --git a/scripts/fetch_metrics.py b/scripts/fetch_metrics.py
@@ -1,11 +1,10 @@
 import requests
 import pandas as pd
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 
 PROMETHEUS_URL = "http://localhost:9090/api/v1/query"
 
-# Define metrics to fetch
 METRICS = {
     "cpu_usage": "container_cpu_usage_seconds_total",
     "memory_usage": "container_memory_usage_bytes",
@@ -24,16 +23,45 @@ def fetch_metric(metric_name):
 
     results = []
     for item in data.get("data", {}).get("result", []):
-        timestamp = datetime.utcfromtimestamp(float(item["value"][0])).strftime("%Y-%m-%d %H:%M:%S")
-        value = float(item["value"][1])
-        results.append({"timestamp": timestamp, "value": value})
+        try:
+            timestamp = datetime.fromtimestamp(float(item["value"][0]), tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
+            value = float(item["value"][1])
+            results.append({"timestamp": timestamp, metric_name: value})
+        except Exception as e:
+            print(f"❌ Error processing {metric_name}: {e}")
 
-    return pd.DataFrame(results)
+    df = pd.DataFrame(results)
+
+    if not df.empty:
+        df["timestamp"] = pd.to_datetime(df["timestamp"], errors="coerce")
+
+    return df
 
 # Fetch all metrics
+all_data = None
+
 for metric_key, query in METRICS.items():
     df = fetch_metric(query)
-    save_path = os.path.join(SAVE_DIR, f"{metric_key}.csv")
-    df.to_csv(save_path, index=False)
-    print(f"✅ {metric_key} data saved to {save_path}")
+
+    if df.empty:
+        print(f"⚠️ Warning: No data for {metric_key}, skipping merge.")
+        continue
+
+    if all_data is None:
+        all_data = df
+    else:
+        print(f"Merging {metric_key}...")
+        print("Before merge, all_data columns:", list(all_data.columns))
+        print("Before merge, df columns:", list(df.columns))
+
+        all_data = pd.merge(all_data, df, on="timestamp", how="outer")
+
+# Save collected data
+if all_data is not None and not all_data.empty:
+    save_path = os.path.join(SAVE_DIR, "merged_data.csv")
+    all_data.to_csv(save_path, index=False)
+    print(f"✅ Merged data saved to {save_path}")
+    print(all_data.head())  # Preview first few rows
+else:
+    print("⚠️ No data was fetched, skipping save.")
 
diff --git a/scripts/train_model.py b/scripts/train_model.py
@@ -1,96 +1,109 @@
+import xgboost as xgb
 import pandas as pd
 import numpy as np
-import joblib
 import matplotlib.pyplot as plt
 import seaborn as sns
-from sklearn.model_selection import train_test_split, cross_val_score
-from sklearn.preprocessing import StandardScaler
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.metrics import classification_report, confusion_matrix
+import joblib
+from sklearn.model_selection import train_test_split, RandomizedSearchCV
+from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from sklearn.impute import SimpleImputer
 from imblearn.over_sampling import SMOTE
-from xgboost import XGBClassifier
 
 # Load dataset
-data = pd.read_csv("data/merged_data.csv")
+data = pd.read_csv("/home/pavithra/k8s-failure-prediction/data/merged_data.csv")
+
+# Convert datetime columns to numeric timestamps
+for col in data.select_dtypes(include=['object', 'datetime']):
+    try:
+        data[col] = pd.to_datetime(data[col]).astype(int) / 10**9
+    except:
+        pass
+
+# Handle categorical features
+categorical_cols = data.select_dtypes(include=['object']).columns
+data[categorical_cols] = data[categorical_cols].apply(LabelEncoder().fit_transform)
 
-# Check if target column exists
-if "target" not in data.columns:
-    raise KeyError("❌ 'target' column not found in the dataset!")
+# Handle missing values
+imputer = SimpleImputer(strategy='mean')
+data.iloc[:, :] = imputer.fit_transform(data)
 
-# Remove non-numeric columns and separate features/target
-X = data.drop(columns=["timestamp", "target"], errors="ignore")
+# Split into features and target
+X = data.drop(columns=["target"])
 y = data["target"]
 
-# Standardize features
+# Handle Class Imbalance with SMOTE
+smote = SMOTE(sampling_strategy=0.6, random_state=42)
+X_resampled, y_resampled = smote.fit_resample(X, y)
+
+# Feature Scaling
 scaler = StandardScaler()
-X_scaled = scaler.fit_transform(X)
+X_scaled = scaler.fit_transform(X_resampled)
+
+# Train-Test Split
+X_train, X_test, y_train, y_test = train_test_split(X_scaled, y_resampled, test_size=0.2, random_state=42, stratify=y_resampled)
 
-# Handle class imbalance using SMOTE
-smote = SMOTE(random_state=42)
-X_resampled, y_resampled = smote.fit_resample(X_scaled, y)
+# Hyperparameter Tuning
 
-# Split into train & test sets
-X_train, X_test, y_train, y_test = train_test_split(X_resampled, y_resampled, test_size=0.2, random_state=42)
+param_grid = {
+    'n_estimators': [400, 500, 600],  # More trees to learn better
+    'max_depth': [10, 12, 15],        # Allow deeper trees
+    'learning_rate': [0.1, 0.2, 0.3], # Increase learning rate
+    'min_child_weight': [1, 2],       # Reduce constraints
+    'subsample': [0.9, 1.0],          # Use more data per tree
+    'colsample_bytree': [0.9, 1.0],   # Use more features per tree
+    'gamma': [0, 0.1],                # Reduce penalty on splits
+    'reg_lambda': [0, 1],             # Reduce L2 regularization
+    'reg_alpha': [0, 1],              # Reduce L1 regularization
+    'scale_pos_weight': [1]           # Balance class weights normally
+}
 
-# Train model with class weighting
 
-model = XGBClassifier(
-    n_estimators=500,
-    max_depth=10,
-    learning_rate=0.01,
-    subsample=0.8,
-    colsample_bytree=0.8,
-    scale_pos_weight=1,
-    use_label_encoder=False,
-    eval_metric="logloss"
-)
+xgb_model = xgb.XGBClassifier(objective='binary:logistic', eval_metric='logloss')
 
-model.fit(X_train, y_train)
+search = RandomizedSearchCV(xgb_model, param_distributions=param_grid, n_iter=30, scoring='accuracy', cv=5, verbose=1, n_jobs=-1, random_state=42)
+search.fit(X_train, y_train)
+
+best_model = search.best_estimator_
 
 # Predictions
-y_pred = model.predict(X_test)
-
-# Evaluate model
-train_acc = model.score(X_train, y_train)
-test_acc = model.score(X_test, y_test)
-cv_acc = np.mean(cross_val_score(model, X_resampled, y_resampled, cv=5))
-
-# Confusion matrix
-cm = confusion_matrix(y_test, y_pred)
-
-# Feature importance
-feature_importances = model.feature_importances_
-sorted_indices = np.argsort(feature_importances)[::-1]
-top_features = X.columns[sorted_indices]
-
-# Print results
-print("\n📊 MODEL PERFORMANCE METRICS")
-print("────────────────────────────────")
-print(f"🏋️ Training Accuracy: {train_acc:.4f}")
-print(f"🛠️ Test Accuracy: {test_acc:.4f}")
-print(f"🎯 Cross-Validation Accuracy: {cv_acc:.4f}")
-
-# Print classification report
-print("\n📜 Classification Report:\n", classification_report(y_test, y_pred))
-
-# Print confusion matrix
-print("\n🖼️ Confusion Matrix:")
-print(cm)
-
-# Show top features
-print("\n🔍 Top 5 Most Important Features:")
-for i in range(min(5, len(top_features))):
-    print(f"   {i+1}. {top_features[i]} ({feature_importances[sorted_indices[i]]:.4f})")
-
-# Save trained model
-joblib.dump(model, "models/failure_predictor.pkl")
-print("\n✅ Model saved successfully!")
-
-# Plot confusion matrix
-plt.figure(figsize=(6, 5))
-sns.heatmap(cm, annot=True, fmt="d", cmap="Blues", xticklabels=["No Failure", "Failure"], yticklabels=["No Failure", "Failure"])
+y_train_pred = best_model.predict(X_train)
+y_test_pred = best_model.predict(X_test)
+
+# Accuracy Scores
+train_accuracy = accuracy_score(y_train, y_train_pred) * 100
+test_accuracy = accuracy_score(y_test, y_test_pred) * 100
+
+print(f"\n🔥 Train Accuracy: {train_accuracy:.2f}%")
+print(f"🔥 Test Accuracy: {test_accuracy:.2f}%")
+
+# Classification Report
+print("\n📊 Classification Report:")
+print(classification_report(y_test, y_test_pred))
+
+joblib.dump(best_model, "k8s_failure_model.pkl")
+print("\nMODEL SAVED\n")
+
+# Confusion Matrix
+conf_matrix = confusion_matrix(y_test, y_test_pred)
+plt.figure(figsize=(6, 4))
+sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues', xticklabels=['Class 0', 'Class 1'], yticklabels=['Class 0', 'Class 1'])
 plt.xlabel("Predicted")
 plt.ylabel("Actual")
 plt.title("Confusion Matrix")
 plt.show()
 
+# Feature Importance Graph
+feature_importances = best_model.feature_importances_
+features = data.drop(columns=["target"]).columns
+
+# Sort feature importances
+sorted_idx = np.argsort(feature_importances)[::-1]
+
+plt.figure(figsize=(10, 5))
+sns.barplot(x=feature_importances[sorted_idx][:10], y=[features[i] for i in sorted_idx[:10]], palette="coolwarm")
+plt.xlabel("Feature Importance Score")
+plt.ylabel("Top 10 Features")
+plt.title("Feature Importance (Top 10)")
+plt.show()
+
diff --git a/scripts/train_model1.py b/scripts/train_model1.py
@@ -0,0 +1,99 @@
+import pandas as pd
+import numpy as np
+import os
+import joblib
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
+from imblearn.over_sampling import BorderlineSMOTE
+from xgboost import XGBClassifier
+
+# ✅ Load Dataset
+CSV_PATH = "/home/pavithra/k8s-failure-prediction/data/merged_data.csv"
+df = pd.read_csv(CSV_PATH)
+
+# ✅ Preprocessing
+df.columns = df.columns.str.strip().str.replace(r'\s+', '_', regex=True).str.lower()
+df["timestamp"] = pd.to_datetime(df["timestamp"])
+df.set_index("timestamp", inplace=True)
+
+# ✅ Feature Engineering
+for col in df.columns:
+    df[f"{col}_avg"] = df[col].rolling(window=5, min_periods=1).mean()
+
+# ✅ Target Variable
+df["target"] = (df["container_restart_count"].diff().fillna(0) > 1).astype(int)
+df.drop(columns=["container_restart_count"], inplace=True)
+
+# ✅ Prepare Data
+X = df.drop(columns=["target"])
+y = df["target"]
+
+# ✅ Handle Class Imbalance
+if y.value_counts().min() >= 5:
+    smote = BorderlineSMOTE(sampling_strategy='auto', random_state=42)
+    X_resampled, y_resampled = smote.fit_resample(X, y)
+else:
+    X_resampled, y_resampled = X, y
+
+# ✅ Train-Test Split
+X_train, X_test, y_train, y_test = train_test_split(X_resampled, y_resampled, test_size=0.2, random_state=42)
+
+# ✅ Reduce Overfitting (Final Fix)
+rf = RandomForestClassifier(
+    n_estimators=300,  # More trees
+    max_depth=10,  # Reduce tree depth
+    min_samples_split=20,  # More samples needed per split
+    min_samples_leaf=10,  # Prevent small branches
+    bootstrap=True,
+    random_state=42
+)
+
+# ✅ Ensemble Model (Random Forest + XGBoost)
+xgb = XGBClassifier(n_estimators=200, learning_rate=0.05, max_depth=7, subsample=0.8, colsample_bytree=0.8, random_state=42)
+rf.fit(X_train, y_train)
+xgb.fit(X_train, y_train)
+
+# ✅ Predictions
+y_pred_rf = rf.predict(X_test)
+y_pred_xgb = xgb.predict(X_test)
+
+# ✅ Combine Predictions (Soft Voting)
+y_pred_ensemble = (y_pred_rf + y_pred_xgb) // 2
+
+# ✅ Evaluate Model
+train_acc = rf.score(X_train, y_train) * 100
+test_acc = accuracy_score(y_test, y_pred_ensemble) * 100
+print(f"\n🎯 Train Accuracy: {train_acc:.2f} %")
+print(f"🎯 Test Accuracy: {test_acc:.2f} %")
+print("\n🔹 Classification Report:\n", classification_report(y_test, y_pred_ensemble))
+
+# ✅ Save Model
+MODEL_PATH = "../models/k8s_failure_model.pkl"
+joblib.dump(rf, MODEL_PATH)
+model = joblib.load("models/k8s_failure_model.pkl")
+print("The features in model are\n")
+print(model.feature_names_in_)
+print(f"\n✅ Model saved at {MODEL_PATH}")
+
+# 🔥 Confusion Matrix Plot
+cm = confusion_matrix(y_test, y_pred_ensemble)
+plt.figure(figsize=(6, 4))
+sns.heatmap(cm, annot=True, fmt='d', cmap="Blues", xticklabels=["No Failure", "Failure"], yticklabels=["No Failure", "Failure"])
+plt.title("Confusion Matrix")
+plt.xlabel("Predicted")
+plt.ylabel("Actual")
+plt.show()
+
+# 🔥 Feature Importance Plot
+feature_importances = pd.DataFrame({'Feature': X_train.columns, 'Importance': rf.feature_importances_})
+feature_importances = feature_importances.sort_values(by='Importance', ascending=False).head(15)
+
+plt.figure(figsize=(10, 6))
+sns.barplot(x='Importance', y='Feature', data=feature_importances, palette="viridis")
+plt.title("Top 15 Important Features")
+plt.show()
+
