update max_model when trial fails (#6596)

* update max_model when trial fails

* add nullable mark to cfo tuner

* update exception message

Author: LittleLittleCloud

src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs

@@ -282,9 +282,16 @@ void handler(object o, EventArgs e)
                             }
                         }
                     }
-                    catch (OperationCanceledException ex) when (aggregateTrainingStopManager.IsStopTrainingRequested() == false)
+                    catch (Exception ex) when (aggregateTrainingStopManager.IsStopTrainingRequested() == false)
                     {
-                        logger.Trace($"trial cancelled - {JsonSerializer.Serialize(trialSettings)}, continue training");
+                        var exceptionMessage = $@"
+Exception thrown during Trial {trialSettings.TrialId} with configuration {JsonSerializer.Serialize(trialSettings)}
+
+Exception Details: ex.Message
+
+Abandoning Trial {trialSettings.TrialId} and continue training.
+";
+                        logger.Trace(exceptionMessage);
                         trialSettings.EndedAtUtc = DateTime.UtcNow;
                         monitor?.ReportFailTrial(trialSettings, ex);
                         var trialResult = new TrialResult
@@ -296,22 +303,8 @@ void handler(object o, EventArgs e)
                         tuner.Update(trialResult);
                         trialResultManager?.AddOrUpdateTrialResult(trialResult);
                         aggregateTrainingStopManager.Update(trialResult);
-                        continue;
-                    }
-                    catch (OperationCanceledException) when (aggregateTrainingStopManager.IsStopTrainingRequested())
-                    {
-                        logger.Trace($"trial cancelled - {JsonSerializer.Serialize(trialSettings)}, stop training");
-
-                        break;
-                    }
-                    catch (Exception ex)
-                    {
-                        logger.Trace($"trial failed - {JsonSerializer.Serialize(trialSettings)}, stop training");
 
-                        trialSettings.EndedAtUtc = DateTime.UtcNow;
-                        monitor?.ReportFailTrial(trialSettings, ex);
-
-                        if (!aggregateTrainingStopManager.IsStopTrainingRequested() && _bestTrialResult == null)
+                        if (ex is not OperationCanceledException && _bestTrialResult == null)
                         {
                             logger.Trace($"trial fatal error - {JsonSerializer.Serialize(trialSettings)}, stop training");
 
@@ -321,6 +314,13 @@ void handler(object o, EventArgs e)
                             // when error is fatal (like schema mismatch).
                             throw;
                         }
+                        continue;
+                    }
+                    catch (Exception) when (aggregateTrainingStopManager.IsStopTrainingRequested())
+                    {
+                        logger.Trace($"trial cancelled - {JsonSerializer.Serialize(trialSettings)}, stop training");
+
+                        break;
                     }
                     finally
                     {

src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs

@@ -24,7 +24,7 @@ internal class CostFrugalTuner : ITuner
         private double _bestLoss;
 
         public CostFrugalTuner(AutoMLExperiment.AutoMLExperimentSettings settings, ITrialResultManager trialResultManager = null)
-            : this(settings.SearchSpace, settings.SearchSpace.SampleFromFeatureSpace(settings.SearchSpace.Default), trialResultManager.GetAllTrialResults(), settings.Seed)
+            : this(settings.SearchSpace, settings.SearchSpace.SampleFromFeatureSpace(settings.SearchSpace.Default), trialResultManager?.GetAllTrialResults(), settings.Seed)
         {
         }