chore: more accurate ticket expiry

Author: odouglas-es

internal/app/query/cache.go

@@ -147,7 +147,7 @@ func updateTicketCache(store statestore.Service, value interface{}) error {
 
 	t := time.Now()
 	previousCount := len(tickets)
-	currentAll, err := store.GetIndexedIDSet(context.Background())
+	currentAll, err := store.GetIndexedIDSetWithTTL(context.Background())
 	if err != nil {
 		return err
 	}

internal/app/synchronizer/synchronizer_service.go

@@ -303,6 +303,12 @@ Registration:
 	if err != nil {
 		logger.Errorf("Failed to clean up backfills, %s", err.Error())
 	}
+
+	// TODO: this should probably be enabled much later like after a day of running without it for the transition for Prod
+	err = s.store.CleanupTickets(ctx)
+	if err != nil {
+		logger.Errorf("Failed to clean up tickets, %s", err.Error())
+	}
 }
 
 ///////////////////////////////////////

internal/statestore/instrumented.go

@@ -210,8 +210,37 @@ func (is *instrumentedService) DeleteBackfillCompletely(ctx context.Context, id
 	return is.s.DeleteBackfillCompletely(ctx, id)
 }
 
+// GetIndexedTicketCount returns the current count of indexed tickets
 func (is *instrumentedService) GetIndexedTicketCount(ctx context.Context) (int, error) {
 	ctx, span := trace.StartSpan(context.Background(), "statestore/instrumented.GetIndexedTicketCount")
 	defer span.End()
 	return is.s.GetIndexedTicketCount(ctx)
 }
+
+// CleanupTickets removes expired tickets
+func (is *instrumentedService) CleanupTickets(ctx context.Context) error {
+	_, span := trace.StartSpan(context.Background(), "statestore/instrumented.CleanupTickets")
+	defer span.End()
+	return is.s.CleanupTickets(ctx)
+}
+
+// DeleteTicketCompletely performs a set of operations to remove the ticket and all related entries.
+func (is *instrumentedService) DeleteTicketCompletely(ctx context.Context, id string) error {
+	_, span := trace.StartSpan(context.Background(), "statestore/instrumented.DeleteTicketCompletely")
+	defer span.End()
+	return is.s.DeleteTicketCompletely(ctx, id)
+}
+
+// GetExpiredTicketIDs gets all ticket IDs which are expired
+func (is *instrumentedService) GetExpiredTicketIDs(ctx context.Context) ([]string, error) {
+	ctx, span := trace.StartSpan(ctx, "statestore/instrumented.GetExpiredTicketIDs")
+	defer span.End()
+	return is.s.GetExpiredTicketIDs(ctx)
+}
+
+// GetIndexedIDSetWithTTL returns the ids of all tickets currently indexed but within a given TTL.
+func (is *instrumentedService) GetIndexedIDSetWithTTL(ctx context.Context) (map[string]struct{}, error) {
+	ctx, span := trace.StartSpan(ctx, "statestore/instrumented.GetIndexedIDSetWithTTL")
+	defer span.End()
+	return is.s.GetIndexedIDSetWithTTL(ctx)
+}

internal/statestore/public.go

@@ -127,6 +127,18 @@ type Service interface {
 
 	// GetIndexedTicketCount returns the current count of indexed tickets
 	GetIndexedTicketCount(ctx context.Context) (int, error)
+
+	//CleanupTickets removes expired tickets
+	CleanupTickets(ctx context.Context) error
+
+	// DeleteTicketCompletely performs a set of operations to remove the ticket and all related entries.
+	DeleteTicketCompletely(ctx context.Context, id string) error
+
+	// GetExpiredTicketIDs gets all ticket IDs which are expired
+	GetExpiredTicketIDs(ctx context.Context) ([]string, error)
+
+	// GetIndexedIDSetWithTTL returns the ids of all tickets currently indexed but within a given TTL.
+	GetIndexedIDSetWithTTL(ctx context.Context) (map[string]struct{}, error)
 }
 
 // New creates a Service based on the configuration.