Skip to content

Commit b0b3266

Browse files
committed
fix: normalize logits to probabilities before computing adaptive temperature
In AdaptiveTeacherModel.ComputeAdaptiveTemperature, difficulty was computed from raw logits which broke bounds. Now: - Convert logits to probabilities using softmax before difficulty computation - Clamp resulting difficulty to [0,1] for all strategies - Ensures temperature stays within [_minTemperature, _maxTemperature] bounds
1 parent 8beeed2 commit b0b3266

File tree

1 file changed

+8
-2
lines changed

1 file changed

+8
-2
lines changed

src/KnowledgeDistillation/Teachers/AdaptiveTeacherModel.cs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,18 +178,21 @@ public void UpdateStudentPerformance(int sampleIndex, Vector<T> studentPredictio
178178
/// </summary>
179179
private double ComputeAdaptiveTemperature(Vector<T> logits, double baseTemperature)
180180
{
181+
// Convert logits to normalized probabilities
182+
var probs = ApplyTemperatureSoftmax(logits, 1.0);
183+
181184
double difficulty = 0;
182185

183186
switch (_strategy)
184187
{
185188
case AdaptiveStrategy.ConfidenceBased:
186189
// Lower confidence = harder sample = lower temperature
187-
difficulty = 1.0 - GetMaxConfidence(logits);
190+
difficulty = 1.0 - GetMaxConfidence(probs);
188191
break;
189192

190193
case AdaptiveStrategy.EntropyBased:
191194
// Higher entropy = harder sample = lower temperature
192-
difficulty = ComputeEntropy(logits);
195+
difficulty = ComputeEntropy(probs);
193196
break;
194197

195198
case AdaptiveStrategy.AccuracyBased:
@@ -198,6 +201,9 @@ private double ComputeAdaptiveTemperature(Vector<T> logits, double baseTemperatu
198201
break;
199202
}
200203

204+
// Clamp difficulty to [0,1] to ensure temperature stays within bounds
205+
difficulty = Math.Max(0.0, Math.Min(1.0, difficulty));
206+
201207
// Map difficulty [0,1] to temperature [min, max]
202208
// High difficulty (0) -> min temp (sharper)
203209
// Low difficulty (1) -> max temp (softer)

0 commit comments

Comments
 (0)