Implement the removeTestExemplar feature

Add a method to `Label` that indicates whether it consists of all 0's
(matches). Allow configuring `SubcontextList` so that it skips the
adding of such labels. Then pass this configuration from
`AnalogicalModeling`. Change the default to `true` to match the behavior
of Algorithm::AM.

Fixes #26.

Author: garfieldnate

src/main/java/weka/classifiers/lazy/AM/data/SubcontextList.java

@@ -40,6 +40,7 @@ public class SubcontextList implements Iterable<Subcontext> {
     private final HashMap<Label, Subcontext> labelToSubcontext = new HashMap<>();
 
     private final Labeler labeler;
+    private final boolean ignoreFullMatches;
 
     /**
      * @return the number of attributes used to predict an outcome
@@ -54,9 +55,11 @@ public int getCardinality() {
      *
      * @param labeler Labeler for assigning labels to items in data
      * @param data    Training data (exemplars)
+     * @param ignoreFullMatches if true, will not add entirely matching contexts
      */
-    public SubcontextList(Labeler labeler, List<Instance> data) {
+    public SubcontextList(Labeler labeler, List<Instance> data, boolean ignoreFullMatches) {
         this.labeler = labeler;
+        this.ignoreFullMatches = ignoreFullMatches;
         for (Instance se : data)
             add(se);
     }
@@ -66,6 +69,9 @@ public SubcontextList(Labeler labeler, List<Instance> data) {
      */
     void add(Instance exemplar) {
         Label label = labeler.label(exemplar);
+        if(ignoreFullMatches && label.allMatching()) {
+            return;
+        }
         if (!labelToSubcontext.containsKey(label)) {
 			labelToSubcontext.put(label, new Subcontext(label, labeler.getContextString(label)));
 		}

src/main/java/weka/classifiers/lazy/AM/label/BitSetLabel.java

@@ -66,6 +66,11 @@ public Label union(Label other) {
         return new BitSetLabel(bitSet, getCardinality());
     }
 
+    @Override
+    public boolean allMatching() {
+        return labelBits.isEmpty();
+    }
+
     @Override
     public String toString() {
         if (getCardinality() == 0) {

src/main/java/weka/classifiers/lazy/AM/label/IntLabel.java

@@ -104,6 +104,11 @@ public Label union(Label other) {
         return new IntLabel(labelBits & otherLabel.labelBits, getCardinality());
     }
 
+    @Override
+    public boolean allMatching() {
+        return labelBits == 0;
+    }
+
 	@Override
     public String toString() {
     	if(getCardinality() == 0) {

src/main/java/weka/classifiers/lazy/AM/label/Label.java

@@ -77,4 +77,10 @@ public abstract class Label {
      * words, keep all matches from both labels.
      */
     public abstract Label union(Label other);
+
+    /**
+     * @return true if every feature of this label is a match (i.e. this is the
+     * {@link Labeler#getLatticeTop() top of the lattice}; false otherwise
+     */
+    public abstract boolean allMatching();
 }

src/main/java/weka/classifiers/lazy/AM/label/LongLabel.java

@@ -97,6 +97,11 @@ public Label union(Label other) {
         return new LongLabel(labelBits & otherLabel.labelBits, getCardinality());
     }
 
+    @Override
+    public boolean allMatching() {
+        return labelBits == 0L;
+    }
+
 	@Override
     public String toString() {
 		if(getCardinality() == 0) {

src/main/java/weka/classifiers/lazy/AnalogicalModeling.java

@@ -167,7 +167,7 @@ private AMResults classify(Instance testItem) throws InterruptedException, Execu
 		Labeler labeler = new LabelerFactory.CardinalityBasedLabelerFactory().createLabeler(testItem, m_ignoreUnknowns, mdc);
 		// 3 steps to assigning outcome probabilities:
 		// 1. Place each data item in a subcontext
-		SubcontextList subList = new SubcontextList(labeler, trainingExemplars);
+		SubcontextList subList = new SubcontextList(labeler, trainingExemplars, getRemoveTestExemplar());
         // 2. Create a supracontextual lattice and fill it with subcontexts
 		LatticeFactory latticeFactory;
 		if (randomProvider == null) {
@@ -234,7 +234,7 @@ public String ignoreUnknownsTipText() {
     /**
      * By default, we remove any exemplar with the same features as the test exemplar
      */
-    private boolean m_removeTestExemplar = false;
+    private boolean m_removeTestExemplar = true;
 
     /**
      * @return true if we remove a test instance from training before predicting its outcome
@@ -255,8 +255,8 @@ public void setRemoveTestExemplar(boolean removeTestExemplar) {
      */
 	@SuppressWarnings("unused") // used by Weka UI
     public String removeTestExemplarTipText() {
-        return "Set to true if you wish to remove a test instance from the training set before "
-               + "attempting to predict its outcome.";
+        return "Set to true if you wish to remove the test instance from the training set before "
+               + "attempting to classify it.";
     }
 
     /**

src/test/java/weka/classifiers/lazy/AM/data/SubcontextListTest.java

@@ -13,8 +13,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 public class SubcontextListTest {
 
@@ -26,7 +25,7 @@ public void testChapter3Data() throws Exception {
 
         Labeler labeler = new IntLabeler(test, false, MissingDataCompare.MATCH);
 
-        SubcontextList subs = new SubcontextList(labeler, train);
+        SubcontextList subs = new SubcontextList(labeler, train, false);
         assertEquals(subs.getCardinality(), 3);
 
         List<Subcontext> subList = getSubList(subs);
@@ -50,6 +49,22 @@ public void testChapter3Data() throws Exception {
         assertTrue(subList.contains(expected));
     }
 
+    @Test
+    public void testIgnoreFullMatches() throws Exception {
+        Instances train = TestUtils.getDataSet(TestUtils.CHAPTER_3_DATA);
+        Instance test = train.get(0);
+
+        Labeler labeler = new IntLabeler(test, false, MissingDataCompare.MATCH);
+        Subcontext allMatchingSub = new Subcontext(new IntLabel(0b000, 3), "foo");
+        allMatchingSub.add(train.get(0));// 310e
+
+        SubcontextList subs = new SubcontextList(labeler, train, false);
+        assertTrue("Should contain 000 sub when not ignoring full matches", getSubList(subs).contains(allMatchingSub));
+
+        subs = new SubcontextList(labeler, train, true);
+        assertFalse("Should not contain 000 sub when ignoring full matches", getSubList(subs).contains(allMatchingSub));
+    }
+
     private List<Subcontext> getSubList(final SubcontextList subcontextList) {
 		return new ArrayList<>() {
 			{
@@ -67,7 +82,7 @@ public void testAccessors() throws Exception {
 
         Labeler labeler = new IntLabeler(test, false, MissingDataCompare.MATCH);
 
-        SubcontextList subs = new SubcontextList(labeler, train);
+        SubcontextList subs = new SubcontextList(labeler, train, false);
         assertEquals("getLabeler returns the labeler used in the constructor", subs.getLabeler(), labeler);
         assertEquals("getCardinality returns the cardinality of the test item", subs.getCardinality(), 3);
     }

src/test/java/weka/classifiers/lazy/AM/label/LabelTest.java

@@ -127,6 +127,15 @@ public void testUnion() {
 			assertTrue(intersected.matches(i));
     }
 
+    @Test
+    public void testAllMatching() {
+        Labeler labeler = labelerFactory.createLabeler(mockInstance(3), false, MATCH);
+        assertTrue("Label composed of all 0's", labeler.fromBits(0b000).allMatching());
+        for (int bits : List.of(0b100, 0b001, 0b010, 0b111)) {
+            assertFalse("Label with a 1 in it", labeler.fromBits(bits).allMatching());
+        }
+    }
+
     @Test
     public void testMatchesThrowsExceptionForIndexTooLow() {
 		Labeler labeler = labelerFactory.createLabeler(mockInstance(3), false, MATCH);

src/test/java/weka/classifiers/lazy/AM/lattice/HeterogeneousLatticeTest.java

@@ -35,7 +35,7 @@ public void testChapter3Data() throws Exception {
         // cardinality of one
 		Labeler noPartitionLabeler = Mockito.spy(new LabelerFactory.IntLabelerFactory().createLabeler(test, false, MissingDataCompare.MATCH));
 		when(noPartitionLabeler.numPartitions()).thenReturn(1);
-		SubcontextList subList = new SubcontextList(noPartitionLabeler, train);
+		SubcontextList subList = new SubcontextList(noPartitionLabeler, train, false);
 		HeterogeneousLattice heteroLattice = new HeterogeneousLattice(0);
 		heteroLattice.fill(subList);
 

src/test/java/weka/classifiers/lazy/AM/lattice/LatticeTest.java

@@ -72,16 +72,16 @@ private void skipForLatticeClass(String reason, Class<? extends Lattice> clazz)
 	@Test
 	public void testFillingWithEmptySubcontextList() throws Exception {
 		Lattice lattice = latticeSupplier.get();
-		lattice.fill(new SubcontextList(mock(IntLabeler.class), Collections.emptyList()));
+		lattice.fill(new SubcontextList(mock(IntLabeler.class), Collections.emptyList(), false));
 	}
 
 	@Test
 	public void testLatticeCannotBeFilledTwice() throws Exception {
 		Lattice lattice = latticeSupplier.get();
-		lattice.fill(new SubcontextList(mock(IntLabeler.class), Collections.emptyList()));
+		lattice.fill(new SubcontextList(mock(IntLabeler.class), Collections.emptyList(), false));
 		exception.expect(IllegalStateException.class);
 		exception.expectMessage(new StringContains("already filled"));
-		lattice.fill(new SubcontextList(mock(IntLabeler.class), Collections.emptyList()));
+		lattice.fill(new SubcontextList(mock(IntLabeler.class), Collections.emptyList(), false));
 	}
 
 	@Test
@@ -150,7 +150,7 @@ public void testCleanSupraTiming() throws Exception {
     private void testSupras(Instances train, int testIndex, String[] expectedSupras) throws ExecutionException, InterruptedException {
         final Instance test = train.get(testIndex);
         train.remove(testIndex);
-        SubcontextList subList = new SubcontextList(getFullSplitLabeler(test), train);
+        SubcontextList subList = new SubcontextList(getFullSplitLabeler(test), train, false);
         Lattice testLattice = latticeSupplier.get();
         testLattice.fill(subList);
         Set<Supracontext> actualSupras = testLattice.getSupracontexts();