Replace upper bound tick number filter if it exceeds max tick

* Previously we had two filters. One always restricting to max tick. Only having one filter is beneficial performance wise and easier to maintain.

Author: qubicmio

v2/domain/repository/elastic/transaction.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"sort"
+	"strconv"
 	"strings"
 
 	api "github.com/qubic/archive-query-service/v2/api/archive-query-service/v2"
@@ -166,9 +167,17 @@ func createIdentitiesQuery(identity string, filters map[string][]string, ranges
 
 	includeFilters, excludeFilters := splitFilters(filters)
 
+	// Check if there's an upper bound tickNumber range filter (lt/lte) and adjust if needed
+	hasUpperBoundTickFilter, err := modifyUpperBoundTickNumberFilterIfNecessary(ranges, maxTick)
+	if err != nil {
+		return "", err
+	}
+
 	filterStrings := make([]string, 0, len(includeFilters)+len(ranges)+1)
-	// restrict to max tick (we don't care about potential duplicate tickNumber range filter)
-	filterStrings = append(filterStrings, fmt.Sprintf(`{"range":{"tickNumber":{"lte":"%d"}}}`, maxTick))
+	// restrict to max tick only if no upper bound tickNumber filter is present
+	if !hasUpperBoundTickFilter {
+		filterStrings = append(filterStrings, fmt.Sprintf(`{"range":{"tickNumber":{"lte":"%d"}}}`, maxTick))
+	}
 	// normal filters
 	filterStrings = append(filterStrings, getFilterStrings(includeFilters)...)
 	// append range filters
@@ -214,6 +223,34 @@ func createIdentitiesQuery(identity string, filters map[string][]string, ranges
 	return query, nil
 }
 
+func modifyUpperBoundTickNumberFilterIfNecessary(ranges map[string][]*entities.Range, maxTick uint32) (bool, error) {
+	hasUpperBoundTickFilter := false
+	if tickRanges, ok := ranges["tickNumber"]; ok {
+		for _, r := range tickRanges {
+			if r.Operation == "lt" || r.Operation == "lte" {
+				hasUpperBoundTickFilter = true
+				// Parse the value and compare with maxTick
+				tickValue, err := strconv.ParseUint(r.Value, 10, 32)
+				if err != nil {
+					return false, fmt.Errorf("parsing tickNumber range value: %w", err)
+				}
+				maxTickValue := If(r.Operation == "lte", maxTick, maxTick+1)
+				if uint32(tickValue) > maxTickValue {
+					r.Value = fmt.Sprintf("%d", maxTickValue)
+				}
+			}
+		}
+	}
+	return hasUpperBoundTickFilter, nil
+}
+
+func If[T any](cond bool, vtrue, vfalse T) T {
+	if cond {
+		return vtrue
+	}
+	return vfalse
+}
+
 const excludeSuffix = "-exclude"
 
 func splitFilters(filters map[string][]string) (map[string][]string, map[string][]string) {

v2/domain/repository/elastic/transaction_identities_query_test.go

@@ -168,3 +168,201 @@ func Test_createIdentitiesQuery_givenRangesAndFilters_returnQueryWithAllFilters(
 	log.Println(query)
 	require.JSONEq(t, expectedQuery, query)
 }
+
+func Test_modifyUpperBoundTickNumberFilterIfNecessary_noRangeFilter(t *testing.T) {
+	ranges := map[string][]*entities.Range{}
+	maxTick := uint32(1000)
+
+	hasUpperBound, err := modifyUpperBoundTickNumberFilterIfNecessary(ranges, maxTick)
+	require.NoError(t, err)
+	require.False(t, hasUpperBound)
+}
+
+func Test_modifyUpperBoundTickNumberFilterIfNecessary_upperBoundReplacedWithMaxTick(t *testing.T) {
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {
+			{Operation: "lte", Value: "5000"},
+		},
+	}
+	maxTick := uint32(1000)
+
+	hasUpperBound, err := modifyUpperBoundTickNumberFilterIfNecessary(ranges, maxTick)
+	require.NoError(t, err)
+	require.True(t, hasUpperBound)
+	require.Equal(t, "1000", ranges["tickNumber"][0].Value)
+}
+
+func Test_modifyUpperBoundTickNumberFilterIfNecessary_upperBoundNotReplaced(t *testing.T) {
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {
+			{Operation: "lte", Value: "999"},
+		},
+	}
+	maxTick := uint32(1000)
+
+	hasUpperBound, err := modifyUpperBoundTickNumberFilterIfNecessary(ranges, maxTick)
+	require.NoError(t, err)
+	require.True(t, hasUpperBound)
+	require.Equal(t, "999", ranges["tickNumber"][0].Value)
+}
+
+func Test_modifyUpperBoundTickNumberFilterIfNecessary_onlyLowerBound(t *testing.T) {
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {
+			{Operation: "gte", Value: "100"},
+		},
+	}
+	maxTick := uint32(1000)
+
+	hasUpperBound, err := modifyUpperBoundTickNumberFilterIfNecessary(ranges, maxTick)
+	require.NoError(t, err)
+	require.False(t, hasUpperBound)
+	require.Equal(t, "100", ranges["tickNumber"][0].Value)
+}
+
+func Test_modifyUpperBoundTickNumberFilterIfNecessary_ltOperatorNotReplacedWithMaxTick(t *testing.T) {
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {
+			{Operation: "lt", Value: "1001"},
+		},
+	}
+	maxTick := uint32(1000)
+
+	hasUpperBound, err := modifyUpperBoundTickNumberFilterIfNecessary(ranges, maxTick)
+	require.NoError(t, err)
+	require.True(t, hasUpperBound)
+	require.Equal(t, "1001", ranges["tickNumber"][0].Value)
+}
+
+func Test_modifyUpperBoundTickNumberFilterIfNecessary_ltOperatorReplacedWithMaxTick(t *testing.T) {
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {
+			{Operation: "lt", Value: "1002"},
+		},
+	}
+	maxTick := uint32(1000)
+
+	hasUpperBound, err := modifyUpperBoundTickNumberFilterIfNecessary(ranges, maxTick)
+	require.NoError(t, err)
+	require.True(t, hasUpperBound)
+	require.Equal(t, "1001", ranges["tickNumber"][0].Value)
+}
+
+func Test_modifyUpperBoundTickNumberFilterIfNecessary_lteOperatorReplacedWithMaxTick(t *testing.T) {
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {
+			{Operation: "lte", Value: "5000"},
+		},
+	}
+	maxTick := uint32(1000)
+
+	hasUpperBound, err := modifyUpperBoundTickNumberFilterIfNecessary(ranges, maxTick)
+	require.NoError(t, err)
+	require.True(t, hasUpperBound)
+	require.Equal(t, "1000", ranges["tickNumber"][0].Value)
+}
+
+func Test_modifyUpperBoundTickNumberFilterIfNecessary_invalidValueReturnsError(t *testing.T) {
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {
+			{Operation: "lte", Value: "not-a-number"},
+		},
+	}
+	maxTick := uint32(1000)
+
+	hasUpperBound, err := modifyUpperBoundTickNumberFilterIfNecessary(ranges, maxTick)
+	require.Error(t, err)
+	require.False(t, hasUpperBound)
+	require.Contains(t, err.Error(), "parsing tickNumber range value")
+}
+
+func Test_createIdentitiesQuery_withTickNumberUpperBound_omitsMaxTickFilter(t *testing.T) {
+	expectedQuery := `{
+      "query": {
+		"bool": {
+		  "should": [
+			{ "term":{"source":"some-identity"} },
+			{ "term":{"destination":"some-identity"} }
+		  ],
+		  "minimum_should_match": 1,
+		  "filter": [
+			{"range":{"tickNumber":{"lte":"500"}}}
+          ]
+		}
+	  },
+	  "sort": [ {"tickNumber":{"order":"desc"}} ],
+	  "from": 0,
+	  "size": 10,
+	  "track_total_hits": 10000
+	}`
+
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {{Operation: "lte", Value: "500"}},
+	}
+	query, err := createIdentitiesQuery(testIdentity, nil, ranges, 0, 10, 1000)
+	require.NoError(t, err)
+	require.NotEmpty(t, query)
+
+	require.JSONEq(t, expectedQuery, query)
+}
+
+func Test_createIdentitiesQuery_withTickNumberUpperBoundExceedingMaxTick_replacesWithMaxTick(t *testing.T) {
+	expectedQuery := `{
+      "query": {
+		"bool": {
+		  "should": [
+			{ "term":{"source":"some-identity"} },
+			{ "term":{"destination":"some-identity"} }
+		  ],
+		  "minimum_should_match": 1,
+		  "filter": [
+			{"range":{"tickNumber":{"lt":"1001"}}}
+          ]
+		}
+	  },
+	  "sort": [ {"tickNumber":{"order":"desc"}} ],
+	  "from": 0,
+	  "size": 10,
+	  "track_total_hits": 10000
+	}`
+
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {{Operation: "lt", Value: "5000"}},
+	}
+	query, err := createIdentitiesQuery(testIdentity, nil, ranges, 0, 10, 1000)
+	require.NoError(t, err)
+	require.NotEmpty(t, query)
+
+	require.JSONEq(t, expectedQuery, query)
+}
+
+func Test_createIdentitiesQuery_withTickNumberLowerBoundOnly_includesMaxTickFilter(t *testing.T) {
+	expectedQuery := `{
+      "query": {
+		"bool": {
+		  "should": [
+			{ "term":{"source":"some-identity"} },
+			{ "term":{"destination":"some-identity"} }
+		  ],
+		  "minimum_should_match": 1,
+		  "filter": [
+			{"range":{"tickNumber":{"lte":"1000"}}},
+			{"range":{"tickNumber":{"gte":"100"}}}
+          ]
+		}
+	  },
+	  "sort": [ {"tickNumber":{"order":"desc"}} ],
+	  "from": 0,
+	  "size": 10,
+	  "track_total_hits": 10000
+	}`
+
+	ranges := map[string][]*entities.Range{
+		"tickNumber": {{Operation: "gte", Value: "100"}},
+	}
+	query, err := createIdentitiesQuery(testIdentity, nil, ranges, 0, 10, 1000)
+	require.NoError(t, err)
+	require.NotEmpty(t, query)
+
+	require.JSONEq(t, expectedQuery, query)
+}