RCF4.0 and PredictiveRCF (#401)

* RCF4.0 and PredictiveRCF

* fixes and spotless

* fixes and improvements

* fixes for normalization equalling forest outputReady

* fixes and internal shingling

* small corrections

* changes and fixes

* test changes

* optimization and stability

* comments etc.

* fix

* shortening noisy test

* changes

* comments

* changes

Author: sudiptoguha

Java/benchmark/pom.xml

@@ -6,7 +6,7 @@
   <parent>
     <groupId>software.amazon.randomcutforest</groupId>
     <artifactId>randomcutforest-parent</artifactId>
-    <version>3.8.0</version>
+    <version>4.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>randomcutforest-benchmark</artifactId>
@@ -50,12 +50,12 @@
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>
-      <version>2.14.2</version>
+      <version>2.16.0</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
-      <version>2.14.2</version>
+      <version>2.16.0</version>
     </dependency>
     <dependency>
       <groupId>io.protostuff</groupId>

Java/core/pom.xml

@@ -6,7 +6,7 @@
   <parent>
     <groupId>software.amazon.randomcutforest</groupId>
     <artifactId>randomcutforest-parent</artifactId>
-    <version>3.8.0</version>
+    <version>4.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>randomcutforest-core</artifactId>
@@ -22,7 +22,7 @@
     <dependency>
       <groupId>org.projectlombok</groupId>
       <artifactId>lombok</artifactId>
-      <version>1.18.24</version>
+      <version>1.18.30</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
@@ -55,5 +55,17 @@
       <artifactId>powermock-api-easymock</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+      <version>2.16.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>2.16.0</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>

Java/core/src/main/java/com/amazon/randomcutforest/CommonUtils.java

@@ -217,6 +217,10 @@ public static double[] toDoubleArray(float[] array) {
         return result;
     }
 
+    public static double[] toDoubleArrayNullable(float[] array) {
+        return (array == null) ? null : toDoubleArray(array);
+    }
+
     public static float[] toFloatArray(double[] array) {
         checkNotNull(array, "array must not be null");
         float[] result = new float[array.length];
@@ -227,6 +231,10 @@ public static float[] toFloatArray(double[] array) {
         return result;
     }
 
+    public static float[] toFloatArrayNullable(double[] array) {
+        return (array == null) ? null : toFloatArray(array);
+    }
+
     public static int[] toIntArray(byte[] values) {
         checkNotNull(values, "array must not be null");
         int[] result = new int[values.length];

Java/core/src/main/java/com/amazon/randomcutforest/MultiVisitor.java

@@ -38,12 +38,13 @@ public interface MultiVisitor<R> extends Visitor<R> {
     boolean trigger(final INodeView node);
 
     /**
-     * Return a copy of this visitor. The original visitor plus the copy will each
-     * traverse one branch of the tree.
+     * Return a partial copy of this visitor. The original visitor plus the copy
+     * will each traverse one branch of the tree. The fields not copied will be
+     * filled in by the branches of the tree
      *
      * @return a copy of this visitor
      */
-    MultiVisitor<R> newCopy();
+    MultiVisitor<R> newPartialCopy();
 
     /**
      * Combine two visitors. The state of the argument visitor should be combined

Java/core/src/main/java/com/amazon/randomcutforest/PredictiveRandomCutForest.java

@@ -36,6 +36,7 @@ public abstract class AbstractForestUpdateExecutor<PointReference, Point> {
 
     protected final IStateCoordinator<PointReference, Point> updateCoordinator;
     protected final ComponentList<PointReference, Point> components;
+    protected boolean currentlySampling = true;
 
     /**
      * Create a new AbstractForestUpdateExecutor.
@@ -59,17 +60,26 @@ protected AbstractForestUpdateExecutor(IStateCoordinator<PointReference, Point>
      * @param point The point used to update the forest.
      */
     public void update(Point point) {
+        update(point, false);
+    }
+
+    public void update(Point point, boolean updateShingleOnly) {
         long internalSequenceNumber = updateCoordinator.getTotalUpdates();
         IPointStore<?, ?> store = updateCoordinator.getStore();
         if (store != null && store.isInternalShinglingEnabled()) {
             internalSequenceNumber -= store.getShingleSize() - 1;
         }
-        update(point, internalSequenceNumber);
+        update(point, internalSequenceNumber, updateShingleOnly);
     }
 
     public void update(Point point, long sequenceNumber) {
-        PointReference updateInput = updateCoordinator.initUpdate(point, sequenceNumber);
-        List<UpdateResult<PointReference>> results = (updateInput == null) ? Collections.emptyList()
+        update(point, sequenceNumber, false);
+    }
+
+    public void update(Point point, long sequenceNumber, boolean updateShingleOnly) {
+        PointReference updateInput = updateCoordinator.initUpdate(point, sequenceNumber, updateShingleOnly);
+        boolean propagate = (updateInput != null) && currentlySampling;
+        List<UpdateResult<PointReference>> results = (!propagate) ? Collections.emptyList()
                 : updateInternal(updateInput, sequenceNumber);
         updateCoordinator.completeUpdate(results, updateInput);
     }
@@ -86,4 +96,8 @@ public void update(Point point, long sequenceNumber) {
      */
     protected abstract List<UpdateResult<PointReference>> updateInternal(PointReference updateInput, long currentIndex);
 
+    public void setCurrentlySampling(boolean value) {
+        currentlySampling = value;
+    }
+
 }

Java/core/src/main/java/com/amazon/randomcutforest/RandomCutForest.java

@@ -33,12 +33,16 @@ public interface IStateCoordinator<PointReference, Point> {
      * Transform the input point into a value that can be submitted to IUpdatable
      * instances.
      *
-     * @param point          The input point.
-     * @param sequenceNumber the sequence number associated with the point
+     * @param point             The input point.
+     * @param sequenceNumber    the sequence number associated with the point
+     * @param updateShingleOnly Only update the shingles (Provide a null reference)
+     *                          or, also update the point store (provide a usable
+     *                          reference)
+     *
      * @return The point transformed into the representation expected by an
      *         IUpdatable instance.
      */
-    PointReference initUpdate(Point point, long sequenceNumber);
+    PointReference initUpdate(Point point, long sequenceNumber, boolean updateShingleOnly);
 
     /**
      * Complete the update. This method is called by IStateCoordinator after all
@@ -56,7 +60,5 @@ public interface IStateCoordinator<PointReference, Point> {
 
     void setTotalUpdates(long totalUpdates);
 
-    default IPointStore<PointReference, Point> getStore() {
-        return null;
-    }
+    IPointStore<PointReference, Point> getStore();
 }

Java/core/src/main/java/com/amazon/randomcutforest/executor/AbstractForestUpdateExecutor.java

@@ -34,7 +34,7 @@
  */
 public class ParallelForestTraversalExecutor extends AbstractForestTraversalExecutor {
 
-    private ForkJoinPool forkJoinPool;
+    ForkJoinPool forkJoinPool;
     private final int threadPoolSize;
 
     public ParallelForestTraversalExecutor(ComponentList<?, ?> treeExecutors, int threadPoolSize) {
@@ -96,7 +96,7 @@ public <R, S> S traverseForestMulti(float[] point, IMultiVisitorFactory<R> visit
                 () -> components.parallelStream().map(c -> c.traverseMulti(point, visitorFactory)).collect(collector));
     }
 
-    private <T> T submitAndJoin(Callable<T> callable) {
+    <T> T submitAndJoin(Callable<T> callable) {
         if (forkJoinPool == null) {
             forkJoinPool = new ForkJoinPool(threadPoolSize);
         }

Java/core/src/main/java/com/amazon/randomcutforest/executor/IStateCoordinator.java

@@ -32,7 +32,7 @@
 public class ParallelForestUpdateExecutor<PointReference, Point>
         extends AbstractForestUpdateExecutor<PointReference, Point> {
 
-    private ForkJoinPool forkJoinPool;
+    ForkJoinPool forkJoinPool;
     private final int threadPoolSize;
 
     public ParallelForestUpdateExecutor(IStateCoordinator<PointReference, Point> updateCoordinator,
@@ -48,7 +48,7 @@ protected List<UpdateResult<PointReference>> updateInternal(PointReference point
                 .filter(UpdateResult::isStateChange).collect(Collectors.toList()));
     }
 
-    private <T> T submitAndJoin(Callable<T> callable) {
+    <T> T submitAndJoin(Callable<T> callable) {
         if (forkJoinPool == null) {
             forkJoinPool = new ForkJoinPool(threadPoolSize);
         }