Skip to content

Fix inconsistent numeric filtering across types with indexes #1175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
anidotnet merged 8 commits into from
Oct 25, 2025
Merged

Fix inconsistent numeric filtering across types with indexes #1175

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1b9dae8
Initial plan
Copilot Oct 25, 2025
7f06063
Add test case for numeric filter consistency issue
Copilot Oct 25, 2025
4d90e92
Fix inconsistent filtering of numeric values across types
Copilot Oct 25, 2025
3a586ae
Update tests to reflect cross-type numeric equality behavior
Copilot Oct 25, 2025
69212c0
Optimize EqualsFilter to use fast path for non-numeric values
Copilot Oct 25, 2025
b394fce
Fix EqualsFilter to handle compound indexes correctly
Copilot Oct 25, 2025
5a26121
Fix comparable issues
anidotnet Oct 25, 2025
2df9813
Normalize numeric values to Double for consistent filtering and seria…
anidotnet Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions nitrite/src/main/java/org/dizitart/no2/common/util/ObjectUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,6 @@ public static boolean deepEquals(Object o1, Object o2) {
}

if (o1 instanceof Number && o2 instanceof Number) {
if (o1.getClass() != o2.getClass()) {
return false;
}
// cast to Number and take care of boxing and compare
return Numbers.compare((Number) o1, (Number) o2) == 0;
} else if (o1 instanceof Iterable && o2 instanceof Iterable) {
Expand Down
23 changes: 23 additions & 0 deletions nitrite/src/main/java/org/dizitart/no2/filters/EqualsFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,29 @@ public boolean apply(Pair<NitriteId, Document> element) {

@Override
public List<?> applyOnIndex(IndexMap indexMap) {
// If value is a Number, we need to check for numeric equivalents across types
// Otherwise, use fast direct lookup
if (getValue() instanceof Number) {
// Scan for all numerically equivalent values
List<java.util.NavigableMap<Comparable<?>, Object>> subMaps = new ArrayList<>();
List<NitriteId> nitriteIds = new ArrayList<>();

for (Pair<Comparable<?>, ?> entry : indexMap.entries()) {
if (deepEquals(getValue(), entry.getFirst())) {
Object entryValue = entry.getSecond();
// Handle both single-field indexes (List) and compound indexes (NavigableMap)
processIndexValue(entryValue, subMaps, nitriteIds);
}
}

// Return sub-maps for compound indexes, or nitrite IDs for single-field indexes
if (!subMaps.isEmpty()) {
return subMaps;
}
return nitriteIds;
}

// Fast path for non-numeric values: direct lookup
Object value = indexMap.get((Comparable<?>) getValue());
if (value == null) {
return new ArrayList<>();
Expand Down
55 changes: 42 additions & 13 deletions nitrite/src/main/java/org/dizitart/no2/filters/GreaterEqualFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,24 +66,53 @@ public List<?> applyOnIndex(IndexMap indexMap) {
// maintain the find sorting order
List<NitriteId> nitriteIds = new ArrayList<>();

// Check if this is a compound index by looking at the first value
Comparable firstKey = indexMap.firstKey();
boolean isCompoundIndex = firstKey != null && indexMap.get(firstKey) instanceof NavigableMap;

// For compound indexes or non-numeric comparisons, use the efficient range approach
// For single-field numeric indexes, scan all entries to handle cross-type comparisons
boolean useFullScan = !isCompoundIndex && comparable instanceof Number && firstKey instanceof Number;

if (isReverseScan()) {
// if reverse scan is required, then start from the last key
Comparable lastKey = indexMap.lastKey();
while(lastKey != null && Comparables.compare(lastKey, comparable) >= 0) {
// get the starting value, it can be a navigable-map (compound index)
// or list (single field index)
Object value = indexMap.get(lastKey);
processIndexValue(value, subMaps, nitriteIds);
lastKey = indexMap.lowerKey(lastKey);
if (useFullScan) {
// Full scan with numeric comparison for single-field numeric indexes
while(lastKey != null) {
if (compare((Number) lastKey, (Number) comparable) >= 0) {
Object value = indexMap.get(lastKey);
processIndexValue(value, subMaps, nitriteIds);
}
lastKey = indexMap.lowerKey(lastKey);
}
} else {
// Efficient range scan for compound indexes or non-numeric comparisons
while(lastKey != null && Comparables.compare(lastKey, comparable) >= 0) {
Object value = indexMap.get(lastKey);
processIndexValue(value, subMaps, nitriteIds);
lastKey = indexMap.lowerKey(lastKey);
}
}
} else {
Comparable ceilingKey = indexMap.ceilingKey(comparable);
while (ceilingKey != null) {
// get the starting value, it can be a navigable-map (compound index)
// or list (single field index)
Object value = indexMap.get(ceilingKey);
processIndexValue(value, subMaps, nitriteIds);
ceilingKey = indexMap.higherKey(ceilingKey);
if (useFullScan) {
// Full scan with numeric comparison for single-field numeric indexes
Comparable key = indexMap.firstKey();
while (key != null) {
if (compare((Number) key, (Number) comparable) >= 0) {
Object value = indexMap.get(key);
processIndexValue(value, subMaps, nitriteIds);
}
key = indexMap.higherKey(key);
}
} else {
// Efficient range scan for compound indexes or non-numeric comparisons
Comparable ceilingKey = indexMap.ceilingKey(comparable);
while (ceilingKey != null) {
Object value = indexMap.get(ceilingKey);
processIndexValue(value, subMaps, nitriteIds);
ceilingKey = indexMap.higherKey(ceilingKey);
}
}
}

Expand Down
55 changes: 42 additions & 13 deletions nitrite/src/main/java/org/dizitart/no2/filters/GreaterThanFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,23 +65,52 @@ public List<?> applyOnIndex(IndexMap indexMap) {
List<NavigableMap<Comparable<?>, Object>> subMaps = new ArrayList<>();
List<NitriteId> nitriteIds = new ArrayList<>();

// Check if this is a compound index by looking at the first value
Comparable firstKey = indexMap.firstKey();
boolean isCompoundIndex = firstKey != null && indexMap.get(firstKey) instanceof NavigableMap;

// For compound indexes or non-numeric comparisons, use the efficient range approach
// For single-field numeric indexes, scan all entries to handle cross-type comparisons
boolean useFullScan = !isCompoundIndex && comparable instanceof Number && firstKey instanceof Number;

if (isReverseScan()) {
Comparable lastKey = indexMap.lastKey();
while (lastKey != null && Comparables.compare(lastKey, comparable) > 0) {
// get the starting value, it can be a navigable-map (compound index)
// or list (single field index)
Object value = indexMap.get(lastKey);
processIndexValue(value, subMaps, nitriteIds);
lastKey = indexMap.lowerKey(lastKey);
if (useFullScan) {
// Full scan with numeric comparison for single-field numeric indexes
while (lastKey != null) {
if (compare((Number) lastKey, (Number) comparable) > 0) {
Object value = indexMap.get(lastKey);
processIndexValue(value, subMaps, nitriteIds);
}
lastKey = indexMap.lowerKey(lastKey);
}
} else {
// Efficient range scan for compound indexes or non-numeric comparisons
while (lastKey != null && Comparables.compare(lastKey, comparable) > 0) {
Object value = indexMap.get(lastKey);
processIndexValue(value, subMaps, nitriteIds);
lastKey = indexMap.lowerKey(lastKey);
}
}
} else {
Comparable higherKey = indexMap.higherKey(comparable);
while (higherKey != null) {
// get the starting value, it can be a navigable-map (compound index)
// or list (single field index)
Object value = indexMap.get(higherKey);
processIndexValue(value, subMaps, nitriteIds);
higherKey = indexMap.higherKey(higherKey);
Comparable key = indexMap.firstKey();
if (useFullScan) {
// Full scan with numeric comparison for single-field numeric indexes
while (key != null) {
if (compare((Number) key, (Number) comparable) > 0) {
Object value = indexMap.get(key);
processIndexValue(value, subMaps, nitriteIds);
}
key = indexMap.higherKey(key);
}
} else {
// Efficient range scan for compound indexes or non-numeric comparisons
Comparable higherKey = indexMap.higherKey(comparable);
while (higherKey != null) {
Object value = indexMap.get(higherKey);
processIndexValue(value, subMaps, nitriteIds);
higherKey = indexMap.higherKey(higherKey);
}
}
}

Expand Down
57 changes: 43 additions & 14 deletions nitrite/src/main/java/org/dizitart/no2/filters/LesserEqualFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,23 +64,52 @@ public List<?> applyOnIndex(IndexMap indexMap) {
List<NavigableMap<Comparable<?>, Object>> subMap = new ArrayList<>();
List<NitriteId> nitriteIds = new ArrayList<>();

// Check if this is a compound index by looking at the first value
Comparable firstKey = indexMap.firstKey();
boolean isCompoundIndex = firstKey != null && indexMap.get(firstKey) instanceof NavigableMap;

// For compound indexes or non-numeric comparisons, use the efficient range approach
// For single-field numeric indexes, scan all entries to handle cross-type comparisons
boolean useFullScan = !isCompoundIndex && comparable instanceof Number && firstKey instanceof Number;

if (isReverseScan()) {
Comparable floorKey = indexMap.floorKey(comparable);
while (floorKey != null) {
// get the starting value, it can be a navigable-map (compound index)
// or list (single field index)
Object value = indexMap.get(floorKey);
processIndexValue(value, subMap, nitriteIds);
floorKey = indexMap.lowerKey(floorKey);
Comparable lastKey = indexMap.lastKey();
if (useFullScan) {
// Full scan with numeric comparison for single-field numeric indexes
while (lastKey != null) {
if (compare((Number) lastKey, (Number) comparable) <= 0) {
Object value = indexMap.get(lastKey);
processIndexValue(value, subMap, nitriteIds);
}
lastKey = indexMap.lowerKey(lastKey);
}
} else {
// Efficient range scan for compound indexes or non-numeric comparisons
Comparable floorKey = indexMap.floorKey(comparable);
while (floorKey != null) {
Object value = indexMap.get(floorKey);
processIndexValue(value, subMap, nitriteIds);
floorKey = indexMap.lowerKey(floorKey);
}
}
} else {
Comparable firstKey = indexMap.firstKey();
while (firstKey != null && Comparables.compare(firstKey, comparable) <= 0) {
// get the starting value, it can be a navigable-map (compound index)
// or list (single field index)
Object value = indexMap.get(firstKey);
processIndexValue(value, subMap, nitriteIds);
firstKey = indexMap.higherKey(firstKey);
Comparable key = indexMap.firstKey();
if (useFullScan) {
// Full scan with numeric comparison for single-field numeric indexes
while (key != null) {
if (compare((Number) key, (Number) comparable) <= 0) {
Object value = indexMap.get(key);
processIndexValue(value, subMap, nitriteIds);
}
key = indexMap.higherKey(key);
}
} else {
// Efficient range scan for compound indexes or non-numeric comparisons
while (key != null && Comparables.compare(key, comparable) <= 0) {
Object value = indexMap.get(key);
processIndexValue(value, subMap, nitriteIds);
key = indexMap.higherKey(key);
}
}
}

Expand Down
57 changes: 43 additions & 14 deletions nitrite/src/main/java/org/dizitart/no2/filters/LesserThanFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,23 +64,52 @@ public List<?> applyOnIndex(IndexMap indexMap) {
List<NavigableMap<Comparable<?>, Object>> subMap = new ArrayList<>();
List<NitriteId> nitriteIds = new ArrayList<>();

// Check if this is a compound index by looking at the first value
Comparable firstKey = indexMap.firstKey();
boolean isCompoundIndex = firstKey != null && indexMap.get(firstKey) instanceof NavigableMap;

// For compound indexes or non-numeric comparisons, use the efficient range approach
// For single-field numeric indexes, scan all entries to handle cross-type comparisons
boolean useFullScan = !isCompoundIndex && comparable instanceof Number && firstKey instanceof Number;

if (isReverseScan()) {
Comparable lowerKey = indexMap.lowerKey(comparable);
while (lowerKey != null) {
// get the starting value, it can be a navigable-map (compound index)
// or list (single field index)
Object value = indexMap.get(lowerKey);
processIndexValue(value, subMap, nitriteIds);
lowerKey = indexMap.lowerKey(lowerKey);
Comparable lastKey = indexMap.lastKey();
if (useFullScan) {
// Full scan with numeric comparison for single-field numeric indexes
while (lastKey != null) {
if (compare((Number) lastKey, (Number) comparable) < 0) {
Object value = indexMap.get(lastKey);
processIndexValue(value, subMap, nitriteIds);
}
lastKey = indexMap.lowerKey(lastKey);
}
} else {
// Efficient range scan for compound indexes or non-numeric comparisons
Comparable lowerKey = indexMap.lowerKey(comparable);
while (lowerKey != null) {
Object value = indexMap.get(lowerKey);
processIndexValue(value, subMap, nitriteIds);
lowerKey = indexMap.lowerKey(lowerKey);
}
}
} else {
Comparable firstKey = indexMap.firstKey();
while (firstKey != null && Comparables.compare(firstKey, comparable) < 0) {
// get the starting value, it can be a navigable-map (compound index)
// or list (single field index)
Object value = indexMap.get(firstKey);
processIndexValue(value, subMap, nitriteIds);
firstKey = indexMap.higherKey(firstKey);
Comparable key = indexMap.firstKey();
if (useFullScan) {
// Full scan with numeric comparison for single-field numeric indexes
while (key != null) {
if (compare((Number) key, (Number) comparable) < 0) {
Object value = indexMap.get(key);
processIndexValue(value, subMap, nitriteIds);
}
key = indexMap.higherKey(key);
}
} else {
// Efficient range scan for compound indexes or non-numeric comparisons
while (key != null && Comparables.compare(key, comparable) < 0) {
Object value = indexMap.get(key);
processIndexValue(value, subMap, nitriteIds);
key = indexMap.higherKey(key);
}
}
}

Expand Down
3 changes: 2 additions & 1 deletion nitrite/src/test/java/org/dizitart/no2/common/util/ObjectUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ public void testDeepEquals() {
@Test
public void testDeepEquals3() {
MutableByte o1 = new MutableByte();
assertFalse(ObjectUtils.deepEquals(o1, new MutableDouble()));
// Numbers with equal values but different types should be considered equal
assertTrue(ObjectUtils.deepEquals(o1, new MutableDouble()));
}

@Test
Expand Down
12 changes: 8 additions & 4 deletions ...te/src/test/java/org/dizitart/no2/integration/collection/CollectionCompoundIndexTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,10 +251,12 @@ public void testIssue178() {
assertEquals(cursor.size(), 1);

cursor = collection.find(where("field1").eq(5));
assertEquals(cursor.size(), 1);
// With cross-type numeric equality, both Integer(5) and Double(5.0) match
assertEquals(cursor.size(), 2);

cursor = collection.find(where("field1").eq(5.0));
assertEquals(cursor.size(), 1);
// With cross-type numeric equality, both Integer(5) and Double(5.0) match
assertEquals(cursor.size(), 2);

collection.createIndex("field1", "field2");
cursor = collection.find(and(where("field1").eq(0.03),
Expand All @@ -266,10 +268,12 @@ public void testIssue178() {
assertEquals(cursor.size(), 1);

cursor = collection.find(where("field1").eq(5));
assertEquals(cursor.size(), 1);
// With cross-type numeric equality, both Integer(5) and Double(5.0) match
assertEquals(cursor.size(), 2);

cursor = collection.find(where("field1").eq(5.0));
assertEquals(cursor.size(), 1);
// With cross-type numeric equality, both Integer(5) and Double(5.0) match
assertEquals(cursor.size(), 2);
}

@Test
Expand Down
6 changes: 4 additions & 2 deletions nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,13 +181,15 @@ public void testIssue178() {

collection.insert(doc1, doc2, doc3, doc4, doc5);

// With cross-type numeric equality, both 5 (Integer) and 5.0 (Double) match
DocumentCursor cursor = collection.find(where("field").eq(5));
assertEquals(cursor.size(), 1);
assertEquals(cursor.size(), 2);

collection.createIndex(indexOptions(IndexType.NON_UNIQUE), "field");

// Same behavior with index
cursor = collection.find(where("field").eq(5));
assertEquals(cursor.size(), 1);
assertEquals(cursor.size(), 2);
}

@Test
Expand Down
6 changes: 4 additions & 2 deletions ...src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,13 +206,15 @@ public void testIssue178() {

collection.insert(doc1, doc2, doc3, doc4, doc5);

// With cross-type numeric equality, both 5 (Integer) and 5.0 (Double) match
DocumentCursor cursor = collection.find(where("field").eq(5));
assertEquals(cursor.size(), 1);
assertEquals(cursor.size(), 2);

collection.createIndex(indexOptions(IndexType.NON_UNIQUE), "field");

// Same behavior with index
cursor = collection.find(where("field").eq(5));
assertEquals(cursor.size(), 1);
assertEquals(cursor.size(), 2);
}

@Test
Expand Down
Loading
Loading