Merge pull request #126 from dataiku/release/2.0.1

Release/2.0.1

Author: david-behar

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [Version 2.0.1] - Patch Release - 2025-11
+
+* Small UI improvements
+* Bugfix on explicit Train/Test
+* Performance improvement
+
 ## [Version 2.0.0] - New Feature Release - 2025-10
 
 * Visual Webapp to train GLMs and assess their fit

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "generalized-linear-models",
-    "version": "2.0.0",
+    "version": "2.0.1",
     "meta": {
         "label": "Generalized Linear Models",
         "description": "Train and deploy Generalized Linear Models",

python-lib/backend/api_utils.py

@@ -24,8 +24,12 @@ def format_models(global_dku_mltask):
         if is_conform:
             model_name = model_details.get_user_meta()['name']
             matches = re.findall(model_id_pattern, model_name)
-            date = [v['value'] for v in model_details.get_user_meta()['labels'] if v['key'] == 'model:date'][0]
-            models.append({"id": ml_id, "name": matches[0], "date": date, "project_key": project_key, "ml_task_id": ml_task_id, "analysis_id": analysis_id})
+            found_date = [v['value'] for v in model_details.get_user_meta()['labels'] if v['key'] == 'model:date']
+            if (len(found_date) > 0) and (len(matches) > 0):
+                date = found_date[0]
+                models.append({"id": ml_id, "name": matches[0], "date": date, "project_key": project_key, "ml_task_id": ml_task_id, "analysis_id": analysis_id})
+            else:
+                current_app.logger.info(f"model {ml_id} missing date or name info")
         else:
             current_app.logger.info(f"model {ml_id} is not conform")
     return models

python-lib/backend/fetch_api.py

@@ -3,6 +3,13 @@
 
 fetch_api = Blueprint("fetch_api", __name__, url_prefix="/api")
 
+@fetch_api.errorhandler(Exception)
+def handle_fetch_api_exception(e):
+    current_app.logger.error(f"Error in fetch_api: {str(e)}")
+    response = jsonify({"error": str(e)})
+    response.status_code = 400
+    return response
+
 @fetch_api.route("/send_webapp_id", methods=["POST"])
 def update_config():
     data_service = current_app.data_service

python-lib/backend/services.py

@@ -23,6 +23,7 @@ class MockDataService:
     def train_model(self, request_json: dict):
         current_app.logger.info("Local set up: No model training completed")
         time.sleep(2)
+        raise ValueError("Model training error: Simulated training error for testing purposes.")
         return {'message': 'Model training initiated successfully.'}
 
     def deploy_model(self, request_json: dict):
@@ -102,16 +103,6 @@ def export_one_way(self, request_json: dict):
         csv_data = variable_level_stats_df.to_csv(index=False).encode('utf-8')
         return csv_data
 
-    # def get_excluded_columns(self):
-    #     exposure_column = "Exposure"
-    #     target_column = "ClaimAmount"
-        
-    #     cols_json = {
-    #         "target_column": target_column,
-    #         "exposure_column": exposure_column
-    #     }
-    #     return cols_json
-    
     def get_dataset_columns(self, request_json: dict):
         dataset_name = "claim_train"
         exposure_column = "exposure"

python-lib/dku_visual_ml/dku_base.py

@@ -1,4 +1,5 @@
 import dataiku
+import pandas as pd
 from logging_assist.logging import logger
 
 class DataikuClientProject:
@@ -20,8 +21,8 @@ def format_ml_task(self, ml_task_config):
         if split_params['ttPolicy'] == 'SPLIT_SINGLE_DATASET':
             test_set = ""
             split_policy = "random"
-        elif split_params['ttPolicy'] == 'EXPLICIT_TEST_SET':
-            test_set = "REPLACE_ME"
+        elif split_params['ttPolicy'] == 'EXPLICIT_FILTERING_TWO_DATASETS':
+            test_set = split_params['eftdTest']['datasetSmartName']
             split_policy = "explicit"
         else:
             test_set = ""
@@ -77,5 +78,11 @@ def get_datasets(self):
     def get_variables_for_dataset(self, dataset_name):
         dataset = dataiku.Dataset(dataset_name)
         columns = dataset.get_config()['schema']['columns']
-        column_names = [{'name': column['name']} for column in columns]
-        return column_names
+        df = dataset.get_dataframe(limit=100)
+        numeric_columns = []
+        for column in columns:
+            col_name = column['name']
+            if col_name in df.columns:
+                if pd.api.types.is_numeric_dtype(df[col_name]):
+                    numeric_columns.append({'name': col_name})
+        return numeric_columns

python-lib/dku_visual_ml/dku_model_trainer.py

@@ -55,20 +55,19 @@ def setup_using_existing_ml_task(self, mltask_id, analysis_id):
         logger.debug(f"Updating the ml task with analysis id {analysis_id} and mltask_id {mltask_id}")
 
         self.mltask = self.project.get_ml_task(mltask_id=mltask_id, analysis_id=analysis_id)
-        self.remove_failed_trainings()
 
         logger.info(f"Successfully update the existing ML task")
 
 
     def assign_train_test_policy(self):
-        logger.info(f"Assigning train test policy")   
-
+        logger.info(f"Assigning train test policy")
         if hasattr(self.visual_ml_config, "policy"):
-            if self.visual_ml_config.policy == "explicit_test_set":
+            if self.visual_ml_config.policy == "Explicit":
                 logger.info(f"Configuration specifies test set, assigning")   
                 settings = self.mltask.get_settings()
                 settings.split_params.set_split_explicit(
                     dku_dataset_selection_params, 
+                    dku_dataset_selection_params,
                     dataset_name=self.visual_ml_config.input_dataset,
                     test_dataset_name=self.visual_ml_config.test_dataset_string)
                 settings.save()
@@ -236,14 +235,6 @@ def set_code_env_settings(self,code_env_string):
         settings.mltask_settings['envSelection']['envName'] = code_env_string
         settings.save()
         logger.info(f"set code env settings to {self.mltask.get_settings().mltask_settings.get('envSelection')} ")
-    
-    def remove_failed_trainings(self):
-        
-        ids = self.mltask.get_trained_models_ids()
-        for model_id in ids:
-            state = self.mltask.get_trained_model_details(model_id).details.get('trainInfo').get('state')
-            if state == "FAILED":
-                self.mltask.delete_trained_model(model_id)
 
 
     def get_latest_model(self):
@@ -316,7 +307,6 @@ def train_model(self, code_env_string, session_name=None):
         if status == "FAILED":
             if error_message == "Failed to train : <class 'numpy.linalg.LinAlgError'> : Matrix is singular.":
                 error_message = error_message + "Check colinearity of variables added to the model"
-            self.remove_failed_trainings()
             return None, error_message
         else:
             return None, error_message

python-lib/glm_handler/dku_relativites_calculator.py

@@ -48,7 +48,12 @@ def __init__(self, data_handler, model_retriever, prepared_train_set=None, prepa
             logger.error(f"Error initializing RelativitiesCalculator: {e}")
             self.train_set = None
             self.test_set = None
-        
+    
+    def _predict_from_df(self, df):
+        preprocessed_data = self.model_retriever.predictor.preprocess(df)
+        predictions_array = self.model_retriever.predictor._clf.predict(preprocessed_data[0])
+        return predictions_array
+
     def compute_base_values(self):
         logger.info("Computing base values on initiation.")
         params = self.model_retriever.predictor.params
@@ -102,7 +107,7 @@ def initialize_baseline(self):
 
     def calculate_baseline_prediction(self, sample_train_row):
         logger.info("Calculating baseline prediction")
-        return self.model_retriever.predictor.predict(sample_train_row).iloc[0][0]
+        return self._predict_from_df(sample_train_row)[0]
 
     def construct_relativities_df(self):
         logger.info("constructing relativites DF")
@@ -129,6 +134,7 @@ def construct_relativities_interaction_df(self):
     def get_relativities_df(self):
         """
         Computes and returns the relativities DataFrame for the model.
+        (Optimized with batch prediction)
         Returns:
             pd.DataFrame: The relativities DataFrame.
         """
@@ -139,11 +145,13 @@ def get_relativities_df(self):
         self.relativities = {'base': {'base': baseline_prediction}}
         used_features = self.model_retriever.get_used_features()
 
+        dfs_to_predict = []
+        features_and_values = [] # To map results back
+
         for feature in used_features:
             feature_type = self.model_retriever.features[feature]['type']
             base_value = self.base_values[feature]
-            self.relativities[feature] = {base_value: 1.0}
-            train_row_copy = sample_train_row.copy()
+            self.relativities[feature] = {}
 
             exposure_col = self.model_retriever.exposure_columns
             exposure_per_modality = self.train_set.groupby(feature)[exposure_col].sum()
@@ -153,8 +161,22 @@ def get_relativities_df(self):
                 values_to_process.append(base_value)
 
             for value in values_to_process:
+                if value == base_value:
+                    self.relativities[feature][value] = 1.0
+                    continue
+                
+                train_row_copy = sample_train_row.copy()
                 train_row_copy[feature] = value
-                prediction = self.model_retriever.predictor.predict(train_row_copy).iloc[0][0]
+                dfs_to_predict.append(train_row_copy)
+                features_and_values.append((feature, value))
+
+        if dfs_to_predict:
+            logger.info(f"Predicting batch of {len(dfs_to_predict)} rows for relativities...")
+            batch_df = pd.concat(dfs_to_predict, ignore_index=True)
+            batch_predictions = self._predict_from_df(batch_df)
+            
+            for i, (feature, value) in enumerate(features_and_values):
+                prediction = batch_predictions[i]
                 relativity = prediction / baseline_prediction
                 self.relativities[feature][value] = relativity
 
@@ -165,6 +187,7 @@ def get_relativities_df(self):
     def get_relativities_interactions_df(self):
         """
         Computes and returns the relativities DataFrame for the model.
+        (Optimized with batch prediction)
         Returns:
             pd.DataFrame: The relativities DataFrame.
         """
@@ -174,43 +197,58 @@ def get_relativities_interactions_df(self):
 
         self.relativities_interaction = {}
         interactions = self.model_retriever.get_interactions()
+        
+        dfs_to_predict = []
+        features_and_values_list = [] # To map results back
 
         for interaction in interactions:
             interaction_first = interaction[0]
             interaction_second = interaction[1]
 
             base_value_first = self.base_values[interaction_first]
             base_value_second = self.base_values[interaction_second]
-            try:
-                self.relativities_interaction[interaction_first][interaction_second] = {base_value_first: {base_value_second: 1.0}}
-            except KeyError:
-                self.relativities_interaction[interaction_first] = {interaction_second:  {base_value_first: {base_value_second: 1.0}}}
-            train_row_copy = sample_train_row.copy()
+            
+            # Initialize the nested dictionary structure
+            if interaction_first not in self.relativities_interaction:
+                self.relativities_interaction[interaction_first] = {}
+            if interaction_second not in self.relativities_interaction[interaction_first]:
+                self.relativities_interaction[interaction_first][interaction_second] = {}
+            if base_value_first not in self.relativities_interaction[interaction_first][interaction_second]:
+                 self.relativities_interaction[interaction_first][interaction_second][base_value_first] = {}
+            
+            # Set base relativity
+            self.relativities_interaction[interaction_first][interaction_second][base_value_first][base_value_second] = 1.0
 
             type_first = self.variable_types.get(interaction_first)
             type_second = self.variable_types.get(interaction_second)
 
-            if type_first == 'CATEGORICAL':
-                values_to_process_first = self.modalities[interaction_first]
-            else:
-                values_to_process_first = [base_value_first]
-
-            if type_second == 'CATEGORICAL':
-                values_to_process_second = self.modalities[interaction_second]
-            else: 
-                values_to_process_second = [base_value_second]
-                
+            values_to_process_first = self.modalities[interaction_first] if type_first == 'CATEGORICAL' else [base_value_first]
+            values_to_process_second = self.modalities[interaction_second] if type_second == 'CATEGORICAL' else [base_value_second]
 
             for value_first in values_to_process_first:
                 for value_second in values_to_process_second:
+                    if value_first == base_value_first and value_second == base_value_second:
+                        continue # Skip base case, already set to 1.0
+
+                    train_row_copy = sample_train_row.copy()
                     train_row_copy[interaction_first] = value_first
                     train_row_copy[interaction_second] = value_second
-                    prediction = self.model_retriever.predictor.predict(train_row_copy).iloc[0][0]
-                    relativity = prediction / baseline_prediction
-                    try:
-                        self.relativities_interaction[interaction_first][interaction_second][value_first][value_second] = relativity
-                    except KeyError:
-                        self.relativities_interaction[interaction_first][interaction_second][value_first] = {value_second: relativity}
+                    dfs_to_predict.append(train_row_copy)
+                    features_and_values_list.append((interaction_first, interaction_second, value_first, value_second))
+
+        # Predict on the entire batch at once
+        if dfs_to_predict:
+            logger.info(f"Predicting batch of {len(dfs_to_predict)} rows for interactions...")
+            batch_df = pd.concat(dfs_to_predict, ignore_index=True)
+            batch_predictions = self._predict_from_df(batch_df)
+            
+            # Map results back
+            for i, (f1, f2, v1, v2) in enumerate(features_and_values_list):
+                prediction = batch_predictions[i]
+                relativity = prediction / baseline_prediction
+                if v1 not in self.relativities_interaction[f1][f2]:
+                    self.relativities_interaction[f1][f2][v1] = {}
+                self.relativities_interaction[f1][f2][v1][v2] = relativity
 
         relativities_interaction_df = self.construct_relativities_interaction_df()
         logger.info("Relativities DataFrame computed")
@@ -245,7 +283,7 @@ def prepare_dataset(self, dataset_type='train'):
         else:
             raise ValueError("dataset_type must be either 'train' or 'test'")
 
-        predicted = self.model_retriever.predictor.predict(dataset)
+        predicted = self._predict_from_df(dataset)
         dataset['predicted'] = predicted
         dataset['weight'] = 1 if self.model_retriever.exposure_columns is None else dataset[self.model_retriever.exposure_columns]
 
@@ -311,12 +349,9 @@ def weighted_mean(x):
             if other_feature != feature:
                 feature_df[other_feature] = self.base_values[other_feature]
 
-        logger.debug("predictions")
-        logger.debug(feature_df)
-        predictions = self.model_retriever.predictor.predict(feature_df)
-        logger.debug(predictions)
+        predictions = self._predict_from_df(feature_df)
         base_data[feature] = pd.DataFrame({
-            f'base_{feature}': predictions['prediction'],
+            f'base_{feature}': predictions,
             feature: feature_df[feature]
         })