JFMIG-13 - Add ability to provide include files pattern to migrate only a part of repository (#1506)

Author: deepakkjfrog

artifactory/commands/transferfiles/filediff_test.go

@@ -1,9 +1,14 @@
 package transferfiles
 
 import (
+	"context"
+	"encoding/json"
+	"net/http"
 	"testing"
 
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
+	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state"
+	commonTests "github.com/jfrog/jfrog-cli-core/v2/common/tests"
 	servicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils"
 	"github.com/stretchr/testify/assert"
 )
@@ -72,3 +77,171 @@ func TestGenerateDockerManifestAqlQuery(t *testing.T) {
 		})
 	}
 }
+
+// TestGetNonDockerTimeFrameFilesDiffWithPatterns tests that getNonDockerTimeFrameFilesDiff uses pattern filtering when patterns are set
+func TestGetNonDockerTimeFrameFilesDiffWithPatterns(t *testing.T) {
+	stateManager, cleanUp := state.InitStateTest(t)
+	defer cleanUp()
+
+	aqlCalled := false
+	receivedQuery := ""
+
+	mockAqlResults := servicesUtils.AqlSearchResult{
+		Results: []servicesUtils.ResultItem{
+			{Repo: "test-repo", Path: "org/company/projectA", Name: "file1.jar", Size: 100, Type: "file"},
+		},
+	}
+
+	testServer, serverDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.RequestURI == "/api/search/aql" {
+			aqlCalled = true
+			// Read the query body
+			buf := make([]byte, 1024)
+			n, _ := r.Body.Read(buf)
+			receivedQuery = string(buf[:n])
+
+			w.WriteHeader(http.StatusOK)
+			response, _ := json.Marshal(mockAqlResults)
+			_, _ = w.Write(response)
+		}
+	})
+	defer testServer.Close()
+
+	assert.NoError(t, stateManager.SetRepoState("test-repo", 0, 0, false, true))
+
+	phase := &filesDiffPhase{
+		phaseBase: phaseBase{
+			context:              context.Background(),
+			stateManager:         stateManager,
+			repoKey:              "test-repo",
+			srcRtDetails:         serverDetails,
+			includeFilesPatterns: []string{"org/company/*"},
+		},
+	}
+
+	// Call getNonDockerTimeFrameFilesDiff with patterns
+	result, err := phase.getNonDockerTimeFrameFilesDiff("2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z", 0)
+	assert.NoError(t, err)
+	assert.True(t, aqlCalled, "AQL should be called")
+	assert.Len(t, result.Results, 1)
+
+	// Verify the query contains pattern filtering
+	assert.Contains(t, receivedQuery, "$or", "Query should contain pattern $or condition")
+	assert.Contains(t, receivedQuery, "org/company", "Query should contain the pattern")
+}
+
+// TestGetNonDockerTimeFrameFilesDiffWithoutPatterns tests that getNonDockerTimeFrameFilesDiff uses default query when no patterns
+func TestGetNonDockerTimeFrameFilesDiffWithoutPatterns(t *testing.T) {
+	stateManager, cleanUp := state.InitStateTest(t)
+	defer cleanUp()
+
+	aqlCalled := false
+	receivedQuery := ""
+
+	mockAqlResults := servicesUtils.AqlSearchResult{
+		Results: []servicesUtils.ResultItem{
+			{Repo: "test-repo", Path: "any/path", Name: "file1.jar", Size: 100, Type: "file"},
+		},
+	}
+
+	testServer, serverDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.RequestURI == "/api/search/aql" {
+			aqlCalled = true
+			buf := make([]byte, 1024)
+			n, _ := r.Body.Read(buf)
+			receivedQuery = string(buf[:n])
+
+			w.WriteHeader(http.StatusOK)
+			response, _ := json.Marshal(mockAqlResults)
+			_, _ = w.Write(response)
+		}
+	})
+	defer testServer.Close()
+
+	assert.NoError(t, stateManager.SetRepoState("test-repo", 0, 0, false, true))
+
+	phase := &filesDiffPhase{
+		phaseBase: phaseBase{
+			context:              context.Background(),
+			stateManager:         stateManager,
+			repoKey:              "test-repo",
+			srcRtDetails:         serverDetails,
+			includeFilesPatterns: []string{}, // No patterns
+		},
+	}
+
+	// Call getNonDockerTimeFrameFilesDiff without patterns
+	result, err := phase.getNonDockerTimeFrameFilesDiff("2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z", 0)
+	assert.NoError(t, err)
+	assert.True(t, aqlCalled, "AQL should be called")
+	assert.Len(t, result.Results, 1)
+
+	// Verify the query does NOT contain pattern filtering $or
+	assert.NotContains(t, receivedQuery, `"$or":[{"path"`, "Query should not contain pattern $or condition when no patterns set")
+}
+
+// TestGetDockerTimeFrameFilesDiffWithPatterns tests that getDockerTimeFrameFilesDiff uses pattern filtering when patterns are set
+func TestGetDockerTimeFrameFilesDiffWithPatterns(t *testing.T) {
+	stateManager, cleanUp := state.InitStateTest(t)
+	defer cleanUp()
+
+	aqlCallCount := 0
+	firstQuery := ""
+
+	// First call returns manifest files, second call returns dir content
+	testServer, serverDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.RequestURI == "/api/search/aql" {
+			aqlCallCount++
+			buf := make([]byte, 2048)
+			n, _ := r.Body.Read(buf)
+			query := string(buf[:n])
+
+			if aqlCallCount == 1 {
+				// First query - capture for assertion
+				firstQuery = query
+				// Return manifest file
+				mockAqlResults := servicesUtils.AqlSearchResult{
+					Results: []servicesUtils.ResultItem{
+						{Repo: "docker-repo", Path: "myapp/1.0", Name: "manifest.json", Size: 100, Type: "file"},
+					},
+				}
+				w.WriteHeader(http.StatusOK)
+				response, _ := json.Marshal(mockAqlResults)
+				_, _ = w.Write(response)
+			} else {
+				// Second query - return dir content
+				mockAqlResults := servicesUtils.AqlSearchResult{
+					Results: []servicesUtils.ResultItem{
+						{Repo: "docker-repo", Path: "myapp/1.0", Name: "layer1.tar", Size: 1000, Type: "file"},
+					},
+				}
+				w.WriteHeader(http.StatusOK)
+				response, _ := json.Marshal(mockAqlResults)
+				_, _ = w.Write(response)
+			}
+		}
+	})
+	defer testServer.Close()
+
+	assert.NoError(t, stateManager.SetRepoState("docker-repo", 0, 0, false, true))
+
+	phase := &filesDiffPhase{
+		phaseBase: phaseBase{
+			context:              context.Background(),
+			stateManager:         stateManager,
+			repoKey:              "docker-repo",
+			srcRtDetails:         serverDetails,
+			includeFilesPatterns: []string{"myapp/*"},
+		},
+	}
+
+	// Call getDockerTimeFrameFilesDiff with patterns
+	result, err := phase.getDockerTimeFrameFilesDiff("2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z", 0)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, aqlCallCount, "Two AQL calls should be made (manifest + dir content)")
+	assert.Len(t, result.Results, 1)
+
+	// Verify the FIRST query contains pattern filtering and docker manifest names
+	assert.Contains(t, firstQuery, "myapp", "First query should contain the pattern")
+	assert.Contains(t, firstQuery, "manifest.json", "First query should contain manifest.json")
+}

artifactory/commands/transferfiles/filesdiff.go

@@ -125,6 +125,7 @@ func (f *filesDiffPhase) handleTimeFrameFilesDiff(pcWrapper *producerConsumerWra
 			break
 		}
 		files := convertResultsToFileRepresentation(result)
+		// Note: Pattern filtering is handled at AQL level for efficiency
 		totalSize := 0
 		for _, r := range files {
 			totalSize += int(r.Size)
@@ -214,7 +215,14 @@ func (f *filesDiffPhase) getTimeFrameFilesDiff(fromTimestamp, toTimestamp string
 }
 
 func (f *filesDiffPhase) getNonDockerTimeFrameFilesDiff(fromTimestamp, toTimestamp string, paginationOffset int) (aqlResult *servicesUtils.AqlSearchResult, err error) {
-	query := generateDiffAqlQuery(f.repoKey, fromTimestamp, toTimestamp, paginationOffset, f.disabledDistinctiveAql)
+	var query string
+	if len(f.includeFilesPatterns) > 0 {
+		// Use AQL with pattern filtering
+		query = generateDiffAqlQueryWithPatterns(f.repoKey, fromTimestamp, toTimestamp, f.includeFilesPatterns, paginationOffset, f.disabledDistinctiveAql)
+	} else {
+		// Use default query without pattern filtering
+		query = generateDiffAqlQuery(f.repoKey, fromTimestamp, toTimestamp, paginationOffset, f.disabledDistinctiveAql)
+	}
 	return runAql(f.context, f.srcRtDetails, query)
 }
 
@@ -225,7 +233,12 @@ func (f *filesDiffPhase) getNonDockerTimeFrameFilesDiff(fromTimestamp, toTimesta
 // to get all artifacts in its path (that includes the "manifest.json" file itself and all its layouts).
 func (f *filesDiffPhase) getDockerTimeFrameFilesDiff(fromTimestamp, toTimestamp string, paginationOffset int) (aqlResult *servicesUtils.AqlSearchResult, err error) {
 	// Get all newly created or modified manifest files ("manifest.json" and "list.manifest.json" files)
-	query := generateDockerManifestAqlQuery(f.repoKey, fromTimestamp, toTimestamp, paginationOffset, f.disabledDistinctiveAql)
+	var query string
+	if len(f.includeFilesPatterns) > 0 {
+		query = generateDockerManifestAqlQueryWithPatterns(f.repoKey, fromTimestamp, toTimestamp, f.includeFilesPatterns, paginationOffset, f.disabledDistinctiveAql)
+	} else {
+		query = generateDockerManifestAqlQuery(f.repoKey, fromTimestamp, toTimestamp, paginationOffset, f.disabledDistinctiveAql)
+	}
 	manifestFilesResult, err := runAql(f.context, f.srcRtDetails, query)
 	if err != nil {
 		return

artifactory/commands/transferfiles/fulltransfer.go

@@ -2,10 +2,11 @@ package transferfiles
 
 import (
 	"fmt"
-	"github.com/jfrog/gofrog/safeconvert"
 	"path"
 	"time"
 
+	"github.com/jfrog/gofrog/safeconvert"
+
 	"github.com/jfrog/gofrog/parallel"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state"
@@ -93,6 +94,19 @@ func (m *fullTransferPhase) skipPhase() {
 
 func (m *fullTransferPhase) run() error {
 	m.transferManager = newTransferManager(m.phaseBase, getDelayUploadComparisonFunctions(m.repoSummary.PackageType))
+
+	// If include patterns are provided, use AQL-based direct query instead of folder traversal
+	if len(m.includeFilesPatterns) > 0 {
+		return m.runWithAqlPatternFiltering()
+	}
+
+	// Default: use folder traversal
+	return m.runWithFolderTraversal()
+}
+
+// runWithFolderTraversal uses the traditional folder-by-folder traversal approach.
+// This is the default behavior when no include patterns are specified.
+func (m *fullTransferPhase) runWithFolderTraversal() error {
 	action := func(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error {
 		if ShouldStop(&m.phaseBase, &delayHelper, errorsChannelMng) {
 			return nil
@@ -119,6 +133,75 @@ func (m *fullTransferPhase) run() error {
 	return m.transferManager.doTransferWithProducerConsumer(action, delayAction)
 }
 
+// runWithAqlPatternFiltering uses a direct AQL query to fetch all files matching the include patterns.
+// This is more efficient than folder traversal when filtering is needed.
+func (m *fullTransferPhase) runWithAqlPatternFiltering() error {
+	log.Info("Using AQL-based pattern filtering for include patterns:", m.includeFilesPatterns)
+
+	action := func(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error {
+		if ShouldStop(&m.phaseBase, &delayHelper, errorsChannelMng) {
+			return nil
+		}
+
+		// Fetch all matching files using AQL with pattern filtering
+		paginationOffset := 0
+		for {
+			if ShouldStop(&m.phaseBase, &delayHelper, errorsChannelMng) {
+				return nil
+			}
+
+			result, lastPage, err := m.getPatternMatchingFiles(paginationOffset)
+			if err != nil {
+				return err
+			}
+
+			if len(result) == 0 {
+				if paginationOffset == 0 {
+					log.Info("No files found matching the include patterns")
+				}
+				break
+			}
+
+			// Convert results to file representations and upload
+			files := convertResultsToFileRepresentation(result)
+			shouldStop, err := uploadByChunks(files, uploadChunkChan, m.phaseBase, delayHelper, errorsChannelMng, pcWrapper)
+			if err != nil || shouldStop {
+				return err
+			}
+
+			if lastPage {
+				break
+			}
+			paginationOffset++
+		}
+		return nil
+	}
+
+	delayAction := func(phase phaseBase, addedDelayFiles []string) error {
+		// Disable repo transfer snapshot as it is not used for delayed files.
+		if err := m.stateManager.SaveStateAndSnapshots(); err != nil {
+			return err
+		}
+		m.stateManager.DisableRepoTransferSnapshot()
+		return consumeDelayFilesIfNoErrors(phase, addedDelayFiles)
+	}
+
+	return m.transferManager.doTransferWithProducerConsumer(action, delayAction)
+}
+
+// getPatternMatchingFiles fetches files from source Artifactory using AQL with pattern filtering.
+func (m *fullTransferPhase) getPatternMatchingFiles(paginationOffset int) (result []servicesUtils.ResultItem, lastPage bool, err error) {
+	query := generatePatternBasedAqlQuery(m.repoKey, m.includeFilesPatterns, paginationOffset, m.disabledDistinctiveAql)
+	aqlResults, err := runAql(m.context, m.srcRtDetails, query)
+	if err != nil {
+		return []servicesUtils.ResultItem{}, false, err
+	}
+
+	lastPage = len(aqlResults.Results) < AqlPaginationLimit
+	result, err = m.locallyGeneratedFilter.FilterLocallyGenerated(aqlResults.Results)
+	return
+}
+
 type folderFullTransferHandlerFunc func(params folderParams) parallel.TaskFunc
 
 type folderParams struct {
@@ -241,6 +324,8 @@ func (m *fullTransferPhase) handleFoundChildFolder(params folderParams, pcWrappe
 	return
 }
 
+// Note: Pattern filtering is handled at AQL level when --include-files is provided.
+// This function is only called during folder traversal (when no patterns are specified).
 func (m *fullTransferPhase) handleFoundFile(pcWrapper *producerConsumerWrapper,
 	uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng,
 	node *reposnapshot.Node, item servicesUtils.ResultItem, curUploadChunk *api.UploadChunk) (err error) {