All table lists need to be obtained during GC (#22626)

All table lists need to be obtained during GC

Approved by: @XuPeng-SH

Author: LeftHandCold

pkg/vm/engine/tae/db/gc/v3/checkpoint.go

@@ -1193,6 +1193,7 @@ func (c *checkpointCleaner) doGCAgainstGlobalCheckpointLocked(
 		pitrs,
 		c.mutation.snapshotMeta,
 		iscp,
+		c.checkpointCli,
 		memoryBuffer,
 		c.config.canGCCacheSize,
 		c.config.estimateRows,
@@ -1365,6 +1366,7 @@ func (c *checkpointCleaner) DoCheck(ctx context.Context) error {
 		pitr,
 		c.mutation.snapshotMeta,
 		iscp,
+		c.checkpointCli,
 		buffer,
 		c.config.canGCCacheSize,
 		c.config.estimateRows,
@@ -1388,6 +1390,7 @@ func (c *checkpointCleaner) DoCheck(ctx context.Context) error {
 		pitr,
 		c.mutation.snapshotMeta,
 		iscp,
+		c.checkpointCli,
 		buffer,
 		c.config.canGCCacheSize,
 		c.config.estimateRows,

pkg/vm/engine/tae/db/gc/v3/exec_v1.go

@@ -16,10 +16,10 @@ package gc
 
 import (
 	"context"
-
 	"github.com/matrixorigin/matrixone/pkg/common/bloomfilter"
 	"github.com/matrixorigin/matrixone/pkg/common/malloc"
 	"github.com/matrixorigin/matrixone/pkg/logutil"
+	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint"
 	"go.uber.org/zap"
 
 	"github.com/matrixorigin/matrixone/pkg/container/types"
@@ -71,6 +71,7 @@ type CheckpointBasedGCJob struct {
 	ts               *types.TS
 	globalCkpLoc     objectio.Location
 	globalCkpVer     uint32
+	checkpointCli    checkpoint.Runner // Added to access catalog
 
 	result struct {
 		vecToGC    *vector.Vector
@@ -87,6 +88,7 @@ func NewCheckpointBasedGCJob(
 	accountSnapshots map[uint32][]types.TS,
 	iscpTables map[uint64]types.TS,
 	snapshotMeta *logtail.SnapshotMeta,
+	checkpointCli checkpoint.Runner,
 	buffer *containers.OneSchemaBatchBuffer,
 	isOwner bool,
 	mp *mpool.MPool,
@@ -103,6 +105,7 @@ func NewCheckpointBasedGCJob(
 		globalCkpLoc:     globalCkpLoc,
 		globalCkpVer:     gckpVersion,
 		iscpTables:       iscpTables,
+		checkpointCli:    checkpointCli,
 	}
 	for _, opt := range opts {
 		opt(e)
@@ -173,6 +176,7 @@ func (e *CheckpointBasedGCJob) Execute(ctx context.Context) error {
 		e.snapshotMeta,
 		transObjects,
 		e.iscpTables,
+		e.checkpointCli,
 	)
 	if err != nil {
 		return err
@@ -316,17 +320,58 @@ func MakeBloomfilterCoarseFilter(
 	}, nil
 }
 
+// buildTableExistenceMap creates a combined map of table IDs from both snapshotMeta and catalog
+// This avoids the need for locking on every table existence check
+func buildTableExistenceMap(snapshotMeta *logtail.SnapshotMeta, checkpointCli checkpoint.Runner) (map[uint64]bool, error) {
+	// First, copy all table IDs from snapshotMeta
+	tableExistenceMap := snapshotMeta.GetAllTableIDs()
+	catalog := checkpointCli.GetCatalog()
+	count := len(tableExistenceMap)
+	defer func() {
+		logutil.Info("GC-TRACE-TABLE-LIST",
+			zap.Int("gc-table-count", count),
+			zap.Int("all-table-count", len(tableExistenceMap)))
+	}()
+	if catalog == nil {
+		return tableExistenceMap, nil
+	}
+	it := catalog.MakeDBIt(true)
+	for ; it.Valid(); it.Next() {
+		db := it.Get().GetPayload()
+
+		itTable := db.MakeTableIt(true)
+		for itTable.Valid() {
+			table := itTable.Get().GetPayload()
+			drop := table.GetDeleteAtLocked()
+			if drop.IsEmpty() {
+				tableID := table.GetID()
+				tableExistenceMap[tableID] = true
+			}
+			itTable.Next()
+		}
+	}
+
+	return tableExistenceMap, nil
+}
+
 func MakeSnapshotAndPitrFineFilter(
 	ts *types.TS,
 	accountSnapshots map[uint32][]types.TS,
 	pitrs *logtail.PitrInfo,
 	snapshotMeta *logtail.SnapshotMeta,
 	transObjects map[string]map[uint64]*ObjectEntry,
 	iscpTables map[uint64]types.TS,
+	checkpointCli checkpoint.Runner,
 ) (
 	filter FilterFn,
 	err error,
 ) {
+	// Build combined table existence map from both snapshotMeta and catalog
+	tableExistenceMap, err := buildTableExistenceMap(snapshotMeta, checkpointCli)
+	if err != nil {
+		return nil, err
+	}
+
 	tableSnapshots, tablePitrs := snapshotMeta.AccountToTableSnapshots(
 		accountSnapshots,
 		pitrs,
@@ -354,8 +399,8 @@ func MakeSnapshotAndPitrFineFilter(
 			if transObjects[name] != nil {
 				tables := transObjects[name]
 				if entry := tables[tableID]; entry != nil {
-					// Check if the table still exists
-					_, ok := snapshotMeta.GetAccountId(tableID)
+					// Check if the table still exists using the combined map
+					ok := tableExistenceMap[tableID]
 					// The table has not been dropped, and the dropTS is empty, so it cannot be deleted.
 					if entry.dropTS.IsEmpty() && ok {
 						continue

pkg/vm/engine/tae/db/gc/v3/window.go

@@ -124,6 +124,7 @@ func (w *GCWindow) ExecuteGlobalCheckpointBasedGC(
 	pitrs *logtail.PitrInfo,
 	snapshotMeta *logtail.SnapshotMeta,
 	iscpTables map[uint64]types.TS,
+	checkpointCli checkpoint.Runner,
 	buffer *containers.OneSchemaBatchBuffer,
 	cacheSize int,
 	estimateRows int,
@@ -144,6 +145,7 @@ func (w *GCWindow) ExecuteGlobalCheckpointBasedGC(
 		accountSnapshots,
 		iscpTables,
 		snapshotMeta,
+		checkpointCli,
 		buffer,
 		false,
 		mp,

pkg/vm/engine/tae/logtail/snapshot.go

@@ -1875,6 +1875,17 @@ func (sm *SnapshotMeta) GetAccountId(tid uint64) (uint32, bool) {
 	return sm.tableIDIndex[tid].accountID, true
 }
 
+// GetAllTableIDs returns a copy of all table IDs in the snapshot meta
+func (sm *SnapshotMeta) GetAllTableIDs() map[uint64]bool {
+	sm.RLock()
+	defer sm.RUnlock()
+	result := make(map[uint64]bool, len(sm.tableIDIndex))
+	for tableID := range sm.tableIDIndex {
+		result[tableID] = true
+	}
+	return result
+}
+
 // for test
 func (sm *SnapshotMeta) GetTablePK(tid uint64) string {
 	sm.RLock()