Skip to content

Commit dc2c80a

Browse files
gyrdymgyrdym
committed
Linear Regression: Newton method added (#237)
1 parent 20515b8 commit dc2c80a

18 files changed

+310
-6
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# Changelog
22

3+
## 16.16.0
4+
- LinearRegressor:
5+
- Newton method added
6+
37
## 16.15.2
48
- LinearRegressor, LogisticRegressor, SoftmaxRegressor:
59
- Set `fitIntercept` param to `true` by default

‎README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,12 @@ it in web applications.
7373

7474
- [LinearRegressor.SGD](https://pub.dev/documentation/ml_algo/latest/ml_algo/LinearRegressor/LinearRegressor.SGD.html)
7575
Implementation of the linear regression algorithm based on stochastic gradient descent with L2 regularisation
76+
77+
- [LinearRegressor.BGD](https://pub.dev/documentation/ml_algo/latest/ml_algo/LinearRegressor/LinearRegressor.BGD.html)
78+
Implementation of the linear regression algorithm based on batch gradient descent with L2 regularisation
79+
80+
- [LinearRegressor.newton](https://pub.dev/documentation/ml_algo/latest/ml_algo/LinearRegressor/LinearRegressor.newton.html)
81+
Implementation of the linear regression algorithm based on Newton method with L2 regularisation
7682

7783
- [KnnRegressor](https://pub.dev/documentation/ml_algo/latest/ml_algo/KnnRegressor-class.html)
7884
A class that makes predictions for each new observation based on the first `k` closest observations from

‎e2e/linear_regressor/linear_regressor_newton_test.dart‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
import 'package:ml_algo/ml_algo.dart';
2+
import 'package:ml_dataframe/ml_dataframe.dart';
3+
import 'package:test/test.dart';
4+
5+
num trainHousingModel(MetricType metricType, DType dtype) {
6+
final data = getHousingDataFrame().shuffle();
7+
final samples = splitData(data, [0.8]);
8+
final trainSamples = samples.first;
9+
final testSamples = samples.last;
10+
final targetName = 'MEDV';
11+
12+
final model = LinearRegressor.newton(
13+
trainSamples,
14+
targetName,
15+
dtype: dtype,
16+
);
17+
18+
return model.assess(testSamples, metricType);
19+
}
20+
21+
num trainWineModel(MetricType metricType, DType dtype) {
22+
final data = getWineQualityDataFrame().shuffle();
23+
final samples = splitData(data, [0.8]);
24+
final trainSamples = samples.first;
25+
final testSamples = samples.last;
26+
final targetName = 'quality';
27+
28+
final model = LinearRegressor.newton(
29+
trainSamples,
30+
targetName,
31+
dtype: dtype,
32+
);
33+
34+
return model.assess(testSamples, metricType);
35+
}
36+
37+
void main() {
38+
group('LinearRegressor, Newton method, housing dataset', () {
39+
test(
40+
'should return adequate error on mape metric, '
41+
'dtype=DType.float32', () {
42+
final error = trainHousingModel(MetricType.mape, DType.float32);
43+
44+
print('MAPE is $error');
45+
46+
expect(error, lessThan(0.5));
47+
});
48+
49+
test(
50+
'should return adequate error on mape metric, '
51+
'dtype=DType.float64', () {
52+
final error = trainHousingModel(MetricType.mape, DType.float64);
53+
54+
print('MAPE is $error');
55+
56+
expect(error, lessThan(0.2));
57+
});
58+
});
59+
60+
group('LinearRegressor, Newton method, wine dataset', () {
61+
test(
62+
'should return adequate error on mape metric, '
63+
'dtype=DType.float32', () {
64+
final error = trainWineModel(MetricType.mape, DType.float32);
65+
66+
print('MAPE is $error');
67+
68+
expect(error, lessThan(0.2));
69+
});
70+
71+
test(
72+
'should return adequate error on mape metric, '
73+
'dtype=DType.float64', () {
74+
final error = trainWineModel(MetricType.mape, DType.float64);
75+
76+
print('MAPE is $error');
77+
78+
expect(error, lessThan(0.5));
79+
});
80+
});
81+
}

‎example/logistic_regression.dart‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import 'package:ml_algo/ml_algo.dart';
2+
import 'package:ml_dataframe/ml_dataframe.dart';
3+
4+
void main() {
5+
final samples = getPimaIndiansDiabetesDataFrame().shuffle();
6+
final splits = splitData(samples, [0.8]);
7+
final model = LogisticRegressor(
8+
splits.first,
9+
'Outcome',
10+
batchSize: splits.first.rows.length,
11+
learningRateType: LearningRateType.exponential,
12+
decay: 0.7,
13+
collectLearningData: true,
14+
);
15+
16+
print('ACURACY:');
17+
print(model.assess(splits.last, MetricType.accuracy));
18+
19+
print('RECALL:');
20+
print(model.assess(splits.last, MetricType.recall));
21+
22+
print('PRECISION:');
23+
print(model.assess(splits.last, MetricType.precision));
24+
25+
print('LD: ');
26+
print(splits.last['Outcome'].data.take(10));
27+
print(model
28+
.predict(splits.last.dropSeries(names: ['Outcome']))
29+
.series
30+
.first
31+
.data
32+
.take(10));
33+
}

‎lib/src/cost_function/cost_function.dart‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ import 'package:ml_linalg/matrix.dart';
33
abstract class CostFunction {
44
double getCost(Matrix x, Matrix w, Matrix y);
55
Matrix getGradient(Matrix x, Matrix w, Matrix y);
6+
Matrix getHessian(Matrix x, Matrix w, Matrix y);
67
}

‎lib/src/cost_function/least_square_cost_function.dart‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,7 @@ class LeastSquareCostFunction implements CostFunction {
1010
@override
1111
Matrix getGradient(Matrix x, Matrix w, Matrix y) =>
1212
x.transpose() * -2 * (y - x * w);
13+
14+
@override
15+
Matrix getHessian(Matrix x, Matrix w, Matrix y) => x.transpose() * x * 2;
1316
}

‎lib/src/cost_function/log_likelihood_cost_function.dart‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,10 @@ class LogLikelihoodCostFunction implements CostFunction {
3535

3636
return x.transpose() * (yNormalized - _linkFunction.link(x * w));
3737
}
38+
39+
@override
40+
Matrix getHessian(Matrix x, Matrix w, Matrix y) {
41+
// TODO: implement getHessian
42+
throw UnimplementedError();
43+
}
3844
}

‎lib/src/linear_optimizer/closed_form_optimizer/closed_form_optimizer.dart‎ renamed to ‎lib/src/linear_optimizer/closed_form_optimizer.dart‎

File renamed without changes.

‎lib/src/linear_optimizer/coordinate_optimizer/least_squares_coordinate_descent_optimizer.dart‎ renamed to ‎lib/src/linear_optimizer/least_squares_coordinate_descent_optimizer.dart‎

File renamed without changes.

‎lib/src/linear_optimizer/least_squares_newton_optimizer.dart‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import 'package:ml_algo/src/cost_function/cost_function.dart';
2+
import 'package:ml_algo/src/linear_optimizer/linear_optimizer.dart';
3+
import 'package:ml_linalg/matrix.dart';
4+
import 'package:xrange/xrange.dart';
5+
6+
class LeastSquaresNewtonOptimizer implements LinearOptimizer {
7+
LeastSquaresNewtonOptimizer(
8+
{required Matrix features,
9+
required Matrix labels,
10+
required CostFunction costFunction,
11+
required int iterationLimit,
12+
required num minCoefficientsUpdate,
13+
num lambda = 0})
14+
: _features = features,
15+
_labels = labels,
16+
_costFunction = costFunction,
17+
_iterations = integers(0, iterationLimit),
18+
_minCoefficientsUpdate = minCoefficientsUpdate,
19+
_lambda = lambda;
20+
21+
final Matrix _features;
22+
final Matrix _labels;
23+
final CostFunction _costFunction;
24+
final Iterable<int> _iterations;
25+
final List<num> _costPerIteration = [];
26+
final num _lambda;
27+
final num _minCoefficientsUpdate;
28+
29+
@override
30+
List<num> get costPerIteration => _costPerIteration;
31+
32+
@override
33+
Matrix findExtrema(
34+
{Matrix? initialCoefficients,
35+
bool isMinimizingObjective = true,
36+
bool collectLearningData = false}) {
37+
var dtype = _features.dtype;
38+
var coefficients = initialCoefficients ??
39+
Matrix.column(List.filled(_features.first.length, 0), dtype: dtype);
40+
var prevCoefficients = coefficients;
41+
var coefficientsUpdate = double.maxFinite;
42+
43+
final regularizingTerm =
44+
Matrix.scalar(_lambda.toDouble(), _features.columnsNum, dtype: dtype);
45+
// Since we perfectly know that Hessian matrix calculation of least squares
46+
// function doesn't depend on coefficient vector, Hessian matrix will be
47+
// constant throughout the entire optimization procedure, let's calculate it
48+
// only once in the beginning of the procedure:
49+
final hessian = _costFunction.getHessian(_features, coefficients, _labels);
50+
final regularizedInverseHessian = _lambda == 0
51+
? hessian.inverse()
52+
: (hessian + regularizingTerm).inverse();
53+
54+
for (final _ in _iterations) {
55+
if (coefficientsUpdate.isNaN ||
56+
coefficientsUpdate <= _minCoefficientsUpdate) {
57+
break;
58+
}
59+
60+
final gradient =
61+
_costFunction.getGradient(_features, coefficients, _labels);
62+
63+
coefficients = coefficients - regularizedInverseHessian * gradient;
64+
coefficientsUpdate = (coefficients - prevCoefficients).norm();
65+
prevCoefficients = coefficients;
66+
67+
if (collectLearningData) {
68+
final cost = _costFunction.getCost(_features, coefficients, _labels);
69+
70+
_costPerIteration.add(cost);
71+
}
72+
}
73+
74+
return coefficients;
75+
}
76+
}

0 commit comments

Comments
 (0)