Added KDTree algorithm (#208)

Author: gyrdym

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 16.6.0
+- Added `KDTree` algorithm
+
 ## 16.5.2
 - Add ecosystem notes to `README.md`
 

README.md

@@ -68,6 +68,9 @@ it in the web applications.
     A class that makes prediction for each new observation basing on first `k` closest observations from 
     training data. It may catch non-linear pattern of the data.
 
+- #### Clustering and retrieval algorithms
+    - [KDTree](https://github.com/gyrdym/ml_algo/blob/master/lib/src/retrieval/kd_tree/kd_tree.dart)
+    
 For more information on the library's API, please visit [API reference](https://pub.dev/documentation/ml_algo/latest/ml_algo/ml_algo-library.html) 
 
 ## Examples

benchmark/data/sample_regression_data.json renamed to benchmark/data/sample_data.json

@@ -0,0 +1,37 @@
+// 0.5 sec (MacBook Air mid 2017)
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:ml_algo/src/retrieval/kd_tree/kd_tree.dart';
+import 'package:ml_dataframe/ml_dataframe.dart';
+
+late DataFrame trainData;
+
+class KDTreeBuildingBenchmark extends BenchmarkBase {
+  KDTreeBuildingBenchmark() : super('KDTree building benchmark');
+
+  static void main() {
+    KDTreeBuildingBenchmark().report();
+  }
+
+  @override
+  void run() {
+    KDTree(trainData);
+  }
+
+  void tearDown() {}
+}
+
+Future main() async {
+  final file = File('benchmark/data/sample_data.json');
+  final dataAsString = await file.readAsString();
+  final decoded = jsonDecode(dataAsString) as Map<String, dynamic>;
+
+  trainData = DataFrame.fromJson(decoded);
+
+  print(
+      'Data dimension: ${trainData.rows.length}x${trainData.rows.first.length}');
+
+  KDTreeBuildingBenchmark.main();
+}

benchmark/kd_tree_building.dart

@@ -0,0 +1,47 @@
+// 0.04 sec (MacBook Air mid 2017)
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:ml_algo/src/retrieval/kd_tree/kd_tree.dart';
+import 'package:ml_dataframe/ml_dataframe.dart';
+import 'package:ml_linalg/linalg.dart';
+import 'package:ml_linalg/vector.dart';
+
+final k = 10;
+
+late DataFrame trainData;
+late KDTree tree;
+late Vector point;
+
+class KDTreeQueryingBenchmark extends BenchmarkBase {
+  KDTreeQueryingBenchmark() : super('KDTree querying benchmark');
+
+  static void main() {
+    KDTreeQueryingBenchmark().report();
+  }
+
+  @override
+  void run() {
+    tree.query(point, k);
+  }
+
+  void tearDown() {}
+}
+
+Future main() async {
+  final file = File('benchmark/data/sample_data.json');
+  final dataAsString = await file.readAsString();
+  final decodedPoints = jsonDecode(dataAsString) as Map<String, dynamic>;
+
+  trainData = DataFrame.fromJson(decodedPoints);
+  tree = KDTree(trainData);
+  point = Vector.randomFilled(trainData.rows.first.length,
+      seed: 10, min: -5000, max: 5000);
+
+  print(
+      'Data dimension: ${trainData.rows.length}x${trainData.rows.first.length}');
+  print('Number of neighbours: $k');
+
+  KDTreeQueryingBenchmark.main();
+}

benchmark/kd_tree_querying.dart

@@ -5,8 +5,6 @@ import 'package:benchmark_harness/benchmark_harness.dart';
 import 'package:ml_algo/ml_algo.dart';
 import 'package:ml_dataframe/ml_dataframe.dart';
 
-const observationsNum = 1000;
-const featuresNum = 100;
 late DataFrame trainData;
 
 class LassoRegressorBenchmark extends BenchmarkBase {
@@ -23,7 +21,7 @@ class LassoRegressorBenchmark extends BenchmarkBase {
 }
 
 Future main() async {
-  final file = File('benchmark/data/sample_regression_data.json');
+  final file = File('benchmark/data/sample_data.json');
   final dataAsString = await file.readAsString();
   final decoded = jsonDecode(dataAsString) as Map<String, dynamic>;
 

benchmark/lasso_regressor.dart

@@ -0,0 +1,10 @@
+class InvalidQueryPointLength implements Exception {
+  InvalidQueryPointLength(int pointLength, int expectedLength)
+      : message =
+            'Invalid query point length: expected length is $expectedLength, but given point\'s length is $pointLength';
+
+  final String message;
+
+  @override
+  String toString() => message;
+}

lib/src/retrieval/kd_tree/exceptions/invalid_query_point_length.dart

@@ -0,0 +1,13 @@
+import 'package:ml_algo/src/retrieval/kd_tree/kd_tree_builder.dart';
+import 'package:ml_algo/src/retrieval/kd_tree/kd_tree_constants.dart';
+import 'package:ml_algo/src/retrieval/kd_tree/kd_tree_impl.dart';
+import 'package:ml_dataframe/ml_dataframe.dart';
+import 'package:ml_linalg/dtype.dart';
+
+KDTreeImpl createKDTree(DataFrame pointsSrc, int leafSize, DType dtype) {
+  final points = pointsSrc.toMatrix(dtype);
+  final builder = KDTreeBuilder(leafSize, points);
+  final root = builder.train();
+
+  return KDTreeImpl(points, leafSize, root, dtype, kdTreeJsonSchemaVersion);
+}

lib/src/retrieval/kd_tree/helpers/create_kd_tree.dart

@@ -0,0 +1,54 @@
+import 'package:ml_algo/src/common/serializable/serializable.dart';
+import 'package:ml_algo/src/retrieval/kd_tree/helpers/create_kd_tree.dart';
+import 'package:ml_algo/src/retrieval/kd_tree/kd_tree_impl.dart';
+import 'package:ml_algo/src/retrieval/kd_tree/kd_tree_neighbour.dart';
+import 'package:ml_dataframe/ml_dataframe.dart';
+import 'package:ml_linalg/dtype.dart';
+import 'package:ml_linalg/matrix.dart';
+import 'package:ml_linalg/vector.dart';
+
+/// KD-tree - an algorithm that provides efficient data retrieval. It splits
+/// the whole searching space into partitions in binary tree form which means
+/// that data querying on average will take O(log(n)) time
+abstract class KDTree implements Serializable {
+  factory KDTree(DataFrame points,
+          {int leafSie = 10, DType dtype = DType.float32}) =>
+      createKDTree(points, leafSie, dtype);
+
+  factory KDTree.fromJson(Map<String, dynamic> json) =>
+      KDTreeImpl.fromJson(json);
+
+  /// Points which were used to build the kd-tree
+  Matrix get points;
+
+  /// A number of points on a leaf node.
+  ///
+  /// The bigger the number, the less effective search is. If [leafSize] is
+  /// equal to the number of [points], a regular KNN-search will take place.
+  ///
+  /// Extremely small [leafSize] leads to ineffective memory usage since in
+  /// this case a lot of kd-tree nodes will be allocated
+  int get leafSize;
+
+  /// Data type for [points] matrix
+  DType get dtype;
+
+  /// Returns [k] nearest neighbours for [point]
+  ///
+  /// The neighbour is represented by an index and the distance between [point]
+  /// and the neighbour itself. The index is a zero-based index of a point in
+  /// the source [points] matrix. Example:
+  ///
+  /// ```dart
+  /// final data = DataFrame([
+  ///   [21, 34, 22, 11],
+  ///   [11, 33, 44, 55],
+  ///   ...,
+  /// ], headerExists: false);
+  /// final kdTree = KDTree(data);
+  /// final neighbours = kdTree.query([1, 2, 3, 4], 2);
+  ///
+  /// print(neighbours.index); // let's say, it outputs `3` which means that the nearest neighbour is kdTree.points[3]
+  /// ```
+  Iterable<KDTreeNeighbour> query(Vector point, int k);
+}

lib/src/retrieval/kd_tree/kd_tree.dart

@@ -0,0 +1,81 @@
+import 'package:ml_algo/src/retrieval/kd_tree/kd_tree_node.dart';
+import 'package:ml_linalg/matrix.dart';
+
+class _Split {
+  _Split(this.left, this.right, this.midPoint);
+
+  final List<int> left;
+  final List<int> right;
+  final int midPoint;
+}
+
+class KDTreeBuilder {
+  KDTreeBuilder(this._leafSize, this._points);
+
+  final int _leafSize;
+  final Matrix _points;
+
+  KDTreeNode train() => _train(_points.rowIndices.toList());
+
+  KDTreeNode _train(List<int> pointIndices) {
+    final isLeaf = pointIndices.length <= _leafSize;
+
+    if (isLeaf) {
+      return KDTreeNode(pointIndices: pointIndices);
+    }
+
+    final points = _points.sample(rowIndices: pointIndices);
+    final splitIdx = _getSplitIdx(points);
+    final splitValue = points.getColumn(splitIdx).median();
+    final split = _splitPoints(pointIndices, splitIdx, splitValue);
+
+    return KDTreeNode(
+      pointIndices: [split.midPoint],
+      splitIndex: splitIdx,
+      left: _train(split.left),
+      right: _train(split.right),
+    );
+  }
+
+  int _getSplitIdx(Matrix points) {
+    final variances = points.variance();
+
+    var colIdx = 0;
+    var maxIdx = colIdx;
+    var max = variances[maxIdx];
+
+    variances.forEach((variance) {
+      if (variance > max) {
+        max = variance;
+        maxIdx = colIdx;
+      }
+      colIdx++;
+    });
+
+    return maxIdx;
+  }
+
+  _Split _splitPoints(List<int> pointIndices, int splitIdx, num splitValue) {
+    final left = <int>[];
+    final right = <int>[];
+    int? midPoint;
+
+    for (var i = 0; i < pointIndices.length; i++) {
+      final pointIndex = pointIndices[i];
+      final point = _points[pointIndex];
+
+      if (point[splitIdx] < splitValue) {
+        left.add(pointIndex);
+        continue;
+      }
+
+      if (midPoint == null || point[splitIdx] < _points[midPoint][splitIdx]) {
+        midPoint = pointIndex;
+      }
+
+      right.add(pointIndex);
+    }
+
+    return _Split(left, right, midPoint!);
+  }
+}