Skip to content

Commit 6913b52

Browse files
utk-spartanutk-spartan
authored
feat: Postgres Collection - Implement NEQ filter (#35)
* fix: Mongo DocStore - Fix integration tests for NEQ filter. * feat: Postgres Collection - Implement NEQ filter
1 parent e936b35 commit 6913b52

File tree

4 files changed

+224
-39
lines changed

4 files changed

+224
-39
lines changed

‎document-store/src/integrationTest/java/org/hypertrace/core/documentstore/mongo/MongoDocStoreTest.java‎

Lines changed: 90 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -781,41 +781,106 @@ public void testLike() {
781781
}
782782

783783
@Test
784-
public void testNotEquals() {
785-
MongoClient mongoClient = MongoClients.create("mongodb://localhost:27017");
784+
public void testNotEquals() throws IOException {
785+
datastore.createCollection(COLLECTION_NAME, null);
786+
Collection collection = datastore.getCollection(COLLECTION_NAME);
786787

787-
MongoDatabase db = mongoClient.getDatabase("default_db");
788-
String collectionName = "myTest2";
789-
MongoCollection<BasicDBObject> myTest2 = db.getCollection(collectionName, BasicDBObject.class);
790-
myTest2.drop();
788+
collection.upsert(new SingleValueKey("default", "testKey1"),
789+
Utils.createDocument(
790+
ImmutablePair.of("key1", "abc1"),
791+
ImmutablePair.of("key2", "xyz1")));
792+
collection.upsert(new SingleValueKey("default", "testKey2"),
793+
Utils.createDocument(
794+
ImmutablePair.of("key1", "abc2"),
795+
ImmutablePair.of("key2", "xyz2")));
796+
collection.upsert(new SingleValueKey("default", "testKey3"),
797+
Utils.createDocument(
798+
ImmutablePair.of("key1", "abc3"),
799+
ImmutablePair.of("key2", "xyz3")));
800+
collection.upsert(new SingleValueKey("default", "testKey4"),
801+
Utils.createDocument(
802+
ImmutablePair.of("key1", "abc4")));
803+
804+
collection.updateSubDoc(new SingleValueKey("default", "testKey1"),
805+
"subdoc", Utils.createDocument("nestedkey1", "pqr1"));
806+
collection.updateSubDoc(new SingleValueKey("default", "testKey2"),
807+
"subdoc", Utils.createDocument("nestedkey1", "pqr2"));
808+
collection.updateSubDoc(new SingleValueKey("default", "testKey3"),
809+
"subdoc", Utils.createDocument("nestedkey1", "pqr3"));
791810

811+
// NEQ on ID
792812
{
793-
BasicDBObject basicDBObject = new BasicDBObject();
794-
basicDBObject.put("testKey1", "abc1");
795-
myTest2.insertOne(basicDBObject);
813+
Query query = new Query();
814+
query.setFilter(new Filter(Op.NEQ, "_id", "default:testKey3"));
815+
Iterator<Document> results = collection.search(query);
816+
List<Document> documents = new ArrayList<>();
817+
while (results.hasNext()) {
818+
documents.add(results.next());
819+
}
820+
821+
assertEquals(3, documents.size());
822+
documents.forEach(document -> {
823+
String jsonStr = document.toJson();
824+
assertTrue(jsonStr.contains("\"key1\":\"abc1\"")
825+
|| document.toJson().contains("\"key1\":\"abc2\"")
826+
|| document.toJson().contains("\"key1\":\"abc4\""));
827+
});
796828
}
829+
830+
831+
// NEQ on document fields
797832
{
798-
BasicDBObject basicDBObject = new BasicDBObject();
799-
basicDBObject.put("testKey1", "xyz1");
800-
myTest2.insertOne(basicDBObject);
833+
Query query = new Query();
834+
query.setFilter(new Filter(Op.NEQ, "key1", "abc3"));
835+
Iterator<Document> results = collection.search(query);
836+
List<Document> documents = new ArrayList<>();
837+
while (results.hasNext()) {
838+
documents.add(results.next());
839+
}
840+
assertEquals(3, documents.size());
841+
documents.forEach(document -> {
842+
String jsonStr = document.toJson();
843+
assertTrue(jsonStr.contains("\"key1\":\"abc1\"")
844+
|| document.toJson().contains("\"key1\":\"abc2\"")
845+
|| document.toJson().contains("\"key1\":\"abc4\""));
846+
});
801847
}
848+
849+
// NEQ on non existing fields
802850
{
803-
BasicDBObject basicDBObject = new BasicDBObject();
804-
basicDBObject.put("testKey2", "abc2");
805-
myTest2.insertOne(basicDBObject);
851+
Query query = new Query();
852+
query.setFilter(new Filter(Op.NEQ, "key2", "xyz2"));
853+
Iterator<Document> results = collection.search(query);
854+
List<Document> documents = new ArrayList<>();
855+
while (results.hasNext()) {
856+
documents.add(results.next());
857+
}
858+
assertEquals(3, documents.size());
859+
documents.forEach(document -> {
860+
String jsonStr = document.toJson();
861+
assertTrue(jsonStr.contains("\"key1\":\"abc1\"")
862+
|| document.toJson().contains("\"key1\":\"abc3\"")
863+
|| document.toJson().contains("\"key1\":\"abc4\""));
864+
});
806865
}
807866

808-
BasicDBObject notEquals = new BasicDBObject();
809-
notEquals.append("$ne", "abc1");
810-
811-
FindIterable<BasicDBObject> result = myTest2.find(new BasicDBObject("testKey1", notEquals));
812-
MongoCursor<BasicDBObject> cursor = result.cursor();
813-
List<DBObject> results = new ArrayList<>();
814-
while (cursor.hasNext()) {
815-
DBObject dbObject = cursor.next();
816-
results.add(dbObject);
867+
// NEQ on nested fields
868+
{
869+
Query query = new Query();
870+
query.setFilter(new Filter(Op.NEQ, "subdoc.nestedkey1", "pqr2"));
871+
Iterator<Document> results = collection.search(query);
872+
List<Document> documents = new ArrayList<>();
873+
while (results.hasNext()) {
874+
documents.add(results.next());
875+
}
876+
assertEquals(3, documents.size());
877+
documents.forEach(document -> {
878+
String jsonStr = document.toJson();
879+
assertTrue(jsonStr.contains("\"key1\":\"abc1\"")
880+
|| document.toJson().contains("\"key1\":\"abc3\"")
881+
|| document.toJson().contains("\"key1\":\"abc4\""));
882+
});
817883
}
818-
assertEquals(2, results.size());
819884
}
820885

821886
}

‎document-store/src/integrationTest/java/org/hypertrace/core/documentstore/postgres/PostgresDocStoreTest.java‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package org.hypertrace.core.documentstore.postgres;
22

3+
import static org.junit.jupiter.api.Assertions.assertEquals;
34
import static org.junit.jupiter.api.Assertions.assertThrows;
5+
import static org.junit.jupiter.api.Assertions.assertTrue;
46

57
import com.fasterxml.jackson.databind.ObjectMapper;
68
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -713,4 +715,108 @@ public void testWithDifferentFieldTypes() throws IOException {
713715

714716
datastore.deleteCollection(COLLECTION_NAME);
715717
}
718+
719+
@Test
720+
public void testNotEquals() throws IOException {
721+
datastore.createCollection(COLLECTION_NAME, null);
722+
Collection collection = datastore.getCollection(COLLECTION_NAME);
723+
724+
collection.upsert(new SingleValueKey("default", "testKey1"),
725+
Utils.createDocument(
726+
ImmutablePair.of("key1", "abc1"),
727+
ImmutablePair.of("key2", "xyz1")));
728+
collection.upsert(new SingleValueKey("default", "testKey2"),
729+
Utils.createDocument(
730+
ImmutablePair.of("key1", "abc2"),
731+
ImmutablePair.of("key2", "xyz2")));
732+
collection.upsert(new SingleValueKey("default", "testKey3"),
733+
Utils.createDocument(
734+
ImmutablePair.of("key1", "abc3"),
735+
ImmutablePair.of("key2", "xyz3")));
736+
collection.upsert(new SingleValueKey("default", "testKey4"),
737+
Utils.createDocument(
738+
ImmutablePair.of("key1", "abc4")));
739+
740+
collection.updateSubDoc(new SingleValueKey("default", "testKey1"),
741+
"subdoc", Utils.createDocument("nestedkey1", "pqr1"));
742+
collection.updateSubDoc(new SingleValueKey("default", "testKey2"),
743+
"subdoc", Utils.createDocument("nestedkey1", "pqr2"));
744+
collection.updateSubDoc(new SingleValueKey("default", "testKey3"),
745+
"subdoc", Utils.createDocument("nestedkey1", "pqr3"));
746+
747+
// NEQ on ID
748+
{
749+
Query query = new Query();
750+
query.setFilter(new Filter(Op.NEQ, "_id", "default:testKey3"));
751+
Iterator<Document> results = collection.search(query);
752+
List<Document> documents = new ArrayList<>();
753+
while (results.hasNext()) {
754+
documents.add(results.next());
755+
}
756+
757+
assertEquals(3, documents.size());
758+
documents.forEach(document -> {
759+
String jsonStr = document.toJson();
760+
assertTrue(jsonStr.contains("\"key1\":\"abc1\"")
761+
|| document.toJson().contains("\"key1\":\"abc2\"")
762+
|| document.toJson().contains("\"key1\":\"abc4\""));
763+
});
764+
}
765+
766+
767+
// NEQ on document fields
768+
{
769+
Query query = new Query();
770+
query.setFilter(new Filter(Op.NEQ, "key1", "abc3"));
771+
Iterator<Document> results = collection.search(query);
772+
List<Document> documents = new ArrayList<>();
773+
while (results.hasNext()) {
774+
documents.add(results.next());
775+
}
776+
assertEquals(3, documents.size());
777+
documents.forEach(document -> {
778+
String jsonStr = document.toJson();
779+
assertTrue(jsonStr.contains("\"key1\":\"abc1\"")
780+
|| document.toJson().contains("\"key1\":\"abc2\"")
781+
|| document.toJson().contains("\"key1\":\"abc4\""));
782+
});
783+
}
784+
785+
// NEQ on non existing fields
786+
{
787+
Query query = new Query();
788+
query.setFilter(new Filter(Op.NEQ, "key2", "xyz2"));
789+
Iterator<Document> results = collection.search(query);
790+
List<Document> documents = new ArrayList<>();
791+
while (results.hasNext()) {
792+
documents.add(results.next());
793+
}
794+
assertEquals(3, documents.size());
795+
documents.forEach(document -> {
796+
String jsonStr = document.toJson();
797+
assertTrue(jsonStr.contains("\"key1\":\"abc1\"")
798+
|| document.toJson().contains("\"key1\":\"abc3\"")
799+
|| document.toJson().contains("\"key1\":\"abc4\""));
800+
});
801+
}
802+
803+
// NEQ on nested fields
804+
{
805+
Query query = new Query();
806+
query.setFilter(new Filter(Op.NEQ, "subdoc.nestedkey1", "pqr2"));
807+
Iterator<Document> results = collection.search(query);
808+
List<Document> documents = new ArrayList<>();
809+
while (results.hasNext()) {
810+
documents.add(results.next());
811+
}
812+
assertEquals(3, documents.size());
813+
documents.forEach(document -> {
814+
String jsonStr = document.toJson();
815+
assertTrue(jsonStr.contains("\"key1\":\"abc1\"")
816+
|| document.toJson().contains("\"key1\":\"abc3\"")
817+
|| document.toJson().contains("\"key1\":\"abc4\""));
818+
});
819+
}
820+
}
821+
716822
}

‎document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ protected String parseNonCompositeFilter(Filter filter, Params.Builder paramsBui
211211
Filter.Op op = filter.getOp();
212212
Object value = filter.getValue();
213213
String fieldName = filter.getFieldName();
214-
String fullFieldName = prepareCast(getFieldPrefix(fieldName), value);
214+
String fullFieldName = prepareCast(prepareFieldDataAccessorExpr(fieldName), value);
215215
StringBuilder filterString = new StringBuilder(fullFieldName);
216216
String sqlOperator;
217217
switch (op) {
@@ -266,8 +266,19 @@ protected String parseNonCompositeFilter(Filter filter, Params.Builder paramsBui
266266
}
267267
break;
268268
case NEQ:
269-
// Disabled for parity with mongo collection API.
270-
// Can be implemented by wrapping predicate for EQ in a NOT()
269+
sqlOperator = " != ";
270+
// https://github.com/hypertrace/document-store/pull/20#discussion_r547101520
271+
// The expected behaviour is to get all documents which either satisfy non equality condition
272+
// or the key doesn't exist in them
273+
// Semantics for handling if key not exists and if it exists, its value
274+
// doesn't equal to the filter for Jsonb document will be done as:
275+
// "document->key IS NULL OR document->key->> != value"
276+
StringBuilder notEquals = prepareFieldAccessorExpr(fieldName);
277+
// For fields inside jsonb
278+
if (notEquals != null) {
279+
filterString = notEquals.append(" IS NULL OR ").append(fullFieldName);
280+
}
281+
break;
271282
case CONTAINS:
272283
// TODO: Matches condition inside an array of documents
273284
default:
@@ -367,7 +378,7 @@ private String getJsonSubDocPath(String subDocPath) {
367378
* keys. Note: It doesn't handle array elements in json document. e.g SELECT * FROM TABLE where
368379
* document ->> 'first' = 'name' and document -> 'address' ->> 'pin' = "00000"
369380
*/
370-
private String getFieldPrefix(String fieldName) {
381+
private String prepareFieldDataAccessorExpr(String fieldName) {
371382
StringBuilder fieldPrefix = new StringBuilder(fieldName);
372383
if (!OUTER_COLUMNS.contains(fieldName)) {
373384
fieldPrefix = new StringBuilder(DOCUMENT);
@@ -385,7 +396,7 @@ private String getFieldPrefix(String fieldName) {
385396
private String parseOrderByQuery(List<OrderBy> orderBys) {
386397
return orderBys
387398
.stream()
388-
.map(orderBy -> getFieldPrefix(orderBy.getField()) + " " + (orderBy.isAsc() ? "ASC" : "DESC"))
399+
.map(orderBy -> prepareFieldDataAccessorExpr(orderBy.getField()) + " " + (orderBy.isAsc() ? "ASC" : "DESC"))
389400
.filter(str -> !StringUtils.isEmpty(str))
390401
.collect(Collectors.joining(" , "));
391402
}

‎document-store/src/test/java/org/hypertrace/core/documentstore/postgres/PostgresCollectionTest.java‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ public void testParseNonCompositeFilter() {
3232
Assertions.assertEquals(ID + " = ?", query);
3333
}
3434

35+
{
36+
Filter filter = new Filter(Filter.Op.NEQ, ID, "val1");
37+
String query = collection.parseNonCompositeFilter(filter, initParams());
38+
Assertions.assertEquals(ID + " != ?", query);
39+
}
40+
3541
{
3642
Filter filter = new Filter(Filter.Op.GT, ID, 5);
3743
String query = collection.parseNonCompositeFilter(filter, initParams());
@@ -77,6 +83,12 @@ public void testParseNonCompositeFilterForJsonField() {
7783
Assertions.assertEquals("document->>'key1' = ?", query);
7884
}
7985

86+
{
87+
Filter filter = new Filter(Filter.Op.NEQ, "key1", "val1");
88+
String query = collection.parseNonCompositeFilter(filter, initParams());
89+
Assertions.assertEquals("document->'key1' IS NULL OR document->>'key1' != ?", query);
90+
}
91+
8092
{
8193
Filter filter = new Filter(Filter.Op.GT, "key1", 5);
8294
String query = collection.parseNonCompositeFilter(filter, initParams());
@@ -136,15 +148,6 @@ public void testParseNonCompositeFilterForJsonField() {
136148
@Test
137149
public void testNonCompositeFilterUnsupportedException() {
138150
String expectedMessage = collection.UNSUPPORTED_QUERY_OPERATION;
139-
{
140-
Filter filter = new Filter(Filter.Op.NEQ, "key1", null);
141-
String expected = String.format(expectedMessage, Filter.Op.NEQ);
142-
Exception exception = assertThrows(UnsupportedOperationException.class,
143-
() -> collection.parseNonCompositeFilter(filter, initParams()));
144-
String actualMessage = exception.getMessage();
145-
Assertions.assertTrue(actualMessage.contains(expected));
146-
}
147-
148151
{
149152
Filter filter = new Filter(Filter.Op.CONTAINS, "key1", null);
150153
String expected = String.format(expectedMessage, Filter.Op.CONTAINS);

0 commit comments

Comments
 (0)