Greedy splitter: the case of a split column consisting of the same values (#213)

Author: gyrdym

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 16.7.1
+- DecisionTreeClassifier:
+    - Fixed greedy splitter in case of a split column consisting of the same values
+
 ## 16.7.0
 - DecisionTreeClassifier:
     - Added `saveAsSvg` method which returns '.svg' file with a graphical representation of a tree

benchmark/decision_tree_classifier.dart

@@ -0,0 +1,53 @@
+// MacBook Air 13.3 mid 2017: ~ 6.5 sec
+
+import 'dart:math';
+
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:ml_algo/ml_algo.dart';
+import 'package:ml_dataframe/ml_dataframe.dart';
+import 'package:ml_linalg/linalg.dart';
+
+const observationsNum = 300;
+const columnsNum = 11;
+
+class DecisionTreeClassifierBenchmark extends BenchmarkBase {
+  DecisionTreeClassifierBenchmark()
+      : super('Decision tree classifier benchmark');
+
+  late DataFrame _data;
+
+  static void main() {
+    DecisionTreeClassifierBenchmark().report();
+  }
+
+  @override
+  void run() {
+    DecisionTreeClassifier(
+      _data,
+      'col_10',
+      maxDepth: 4,
+      minError: 0.4,
+      minSamplesCount: 10,
+    );
+  }
+
+  @override
+  void setup() {
+    final random = Random(1);
+    final observations =
+        Matrix.random(observationsNum, columnsNum - 1, seed: 1);
+    final outcomes = Vector.fromList([
+      ...List.filled(observationsNum ~/ 2, 1),
+      ...List.filled(observationsNum ~/ 2, 0)
+    ]..shuffle(random));
+
+    _data = DataFrame(observations.insertColumns(columnsNum - 1, [outcomes]),
+        headerExists: false);
+  }
+
+  void tearDown() {}
+}
+
+Future main() async {
+  DecisionTreeClassifierBenchmark.main();
+}

benchmark/logistic_regressor.dart

@@ -1,11 +1,12 @@
+// MacBook Air 13.3 mid 2017: ~ 4 sec
+
 import 'package:benchmark_harness/benchmark_harness.dart';
 import 'package:ml_algo/ml_algo.dart';
 import 'package:ml_dataframe/ml_dataframe.dart';
 import 'package:ml_linalg/matrix.dart';
-import 'package:ml_linalg/vector.dart';
 
-const observationsNum = 200;
-const columnsNum = 21;
+const observationsNum = 20000;
+const columnsNum = 101;
 
 class LogisticRegressorBenchmark extends BenchmarkBase {
   LogisticRegressorBenchmark() : super('Logistic regressor');
@@ -28,8 +29,7 @@ class LogisticRegressorBenchmark extends BenchmarkBase {
 
   @override
   void setup() {
-    final observations = Matrix.fromRows(
-        List.generate(observationsNum, (i) => Vector.randomFilled(columnsNum)));
+    final observations = Matrix.random(observationsNum, columnsNum, seed: 1);
 
     _data = DataFrame.fromMatrix(observations);
   }

e2e/decision_tree_classifier/decision_tree_classifier_save_as_svg_test.dart

@@ -36,9 +36,9 @@ void main() async {
       final classifier = DecisionTreeClassifier(
         samples,
         'class variable (0 or 1)',
-        minError: 0.1,
-        minSamplesCount: 2,
-        maxDepth: 4,
+        minError: 0.15,
+        minSamplesCount: 1,
+        maxDepth: 5,
       );
 
       await classifier

e2e/decision_tree_classifier/iris_tree.svg

@@ -12,10 +12,9 @@ class MajorityTreeSplitAssessor implements TreeSplitAssessor {
   double getAggregatedError(Iterable<Matrix> splitObservations, int targetId) {
     var errorCount = 0;
     var totalCount = 0;
-    for (final nodeObservations in splitObservations) {
-      if (nodeObservations.columnsNum == 0) {
-        throw Exception('Observations on the node are empty');
-      }
+
+    for (final nodeObservations in splitObservations
+        .where((observations) => observations.columnsNum > 0)) {
       if (targetId >= nodeObservations.columnsNum) {
         throw ArgumentError.value(
             targetId,
@@ -26,6 +25,7 @@ class MajorityTreeSplitAssessor implements TreeSplitAssessor {
       errorCount += _getErrorCount(nodeObservations.getColumn(targetId));
       totalCount += nodeObservations.rowsNum;
     }
+
     return errorCount / totalCount;
   }
 

e2e/decision_tree_classifier/pima_indians_tree.svg

@@ -35,13 +35,16 @@ class GreedyTreeSplitter implements TreeSplitter {
   Map<TreeNode, Matrix> _createByNumericalValue(
       Matrix samples, int splittingIdx, int targetId) {
     final errors = <double, List<Map<TreeNode, Matrix>>>{};
-    final sortedRows = samples.sort((row) => row[splittingIdx], Axis.rows).rows;
+    final sortedRows =
+        samples.sort((row) => row[splittingIdx], Axis.rows).rows.toList();
+    final rowsWithoutFirst = sortedRows.skip(1).toList();
     var prevValue = sortedRows.first[splittingIdx];
 
-    for (final row in sortedRows.skip(1)) {
+    for (var i = 0; i < rowsWithoutFirst.length; i++) {
+      final row = rowsWithoutFirst[i];
       final nextValue = row[splittingIdx];
 
-      if (prevValue == nextValue) {
+      if (prevValue == nextValue && i < sortedRows.length - 2) {
         continue;
       }
 

lib/src/tree_trainer/split_assessor/majority_split_assessor.dart

@@ -1,6 +1,6 @@
 name: ml_algo
 description: Machine learning algorithms, Machine learning models performance evaluation functionality
-version: 16.7.0
+version: 16.7.1
 homepage: https://github.com/gyrdym/ml_algo
 
 environment:

lib/src/tree_trainer/splitter/greedy_splitter.dart

@@ -120,7 +120,7 @@ void main() {
     });
 
     test(
-        'should return majority-based error, that is equal to 0, if all '
+        'should return majority-based error that is equal to 0, if all '
         'nodes in the stump have only one observation', () {
       final node1 = Matrix.fromList([
         [50, 70, 0],
@@ -141,9 +141,7 @@ void main() {
       expect(error, 0);
     });
 
-    test(
-        'should throw an error if at least one node in the stump does not '
-        'have observations at all', () {
+    test('should ignore empty split matrices', () {
       final node1 = Matrix.fromList([]);
 
       final node2 = Matrix.fromList([
@@ -156,10 +154,7 @@ void main() {
 
       final stump = [node1, node2, node3];
 
-      expect(
-        () => const MajorityTreeSplitAssessor().getAggregatedError(stump, 2),
-        throwsException,
-      );
+      expect(const MajorityTreeSplitAssessor().getAggregatedError(stump, 2), 0);
     });
 
     test(