Merge branch 'dev' into feature/ruby-gems-support

Author: agrasth

artifactory/commands/utils/testdata/transferconfig/virtual_repo_a.json

@@ -31,7 +31,8 @@
   "priorityResolution": false,
   "projectKey": "default",
   "environments": [],
-  "repositories": ["b-virtual"],
+  "repositories": ["b-virtual","a-local"],
+  "defaultDeploymentRepo": "a-local",
   "hideUnauthorizedResources": false,
   "artifactoryRequestsCanRetrieveRemoteArtifacts": false,
   "resolveDockerTagsByTimestamp": false,

artifactory/commands/utils/testdata/transferconfig/virtual_repo_b.json

@@ -31,6 +31,8 @@
   "priorityResolution": false,
   "projectKey": "default",
   "environments": [],
+  "repositories": ["b-local"],
+  "defaultDeploymentRepo": "b-local",
   "hideUnauthorizedResources": false,
   "artifactoryRequestsCanRetrieveRemoteArtifacts": false,
   "resolveDockerTagsByTimestamp": false,

artifactory/commands/utils/transferconfigbase.go

@@ -224,17 +224,22 @@ func (tcb *TransferConfigBase) transferVirtualRepositoriesToTarget(reposToTransf
 			return
 		}
 
-		// Create virtual repository without included repositories
+		// Create virtual repository without included repositories and default deployment repo
 		repositories := singleRepoParamsMap["repositories"]
 		delete(singleRepoParamsMap, "repositories")
+		defaultDeploymentRepo := singleRepoParamsMap["defaultDeploymentRepo"]
+		delete(singleRepoParamsMap, "defaultDeploymentRepo")
 		if err = tcb.createRepositoryAndAssignToProject(singleRepoParamsMap, repoToTransfer); err != nil {
 			return
 		}
 
-		// Restore included repositories to set them later on
+		// Restore included repositories and default deployment repo to set them later on
 		if repositories != nil {
 			singleRepoParamsMap["repositories"] = repositories
 		}
+		if defaultDeploymentRepo != nil {
+			singleRepoParamsMap["defaultDeploymentRepo"] = defaultDeploymentRepo
+		}
 	}
 
 	// Step 2 - Update all virtual repositories with the included repositories

artifactory/commands/utils/transferconfigbase_test.go

@@ -223,6 +223,8 @@ func TestTransferVirtualRepositoriesToTarget(t *testing.T) {
 			if r.Method == http.MethodPut {
 				delete(expectedVirtualRepoAParamsMap, "repositories")
 				delete(expectedVirtualRepoBParamsMap, "repositories")
+				delete(expectedVirtualRepoAParamsMap, "defaultDeploymentRepo")
+				delete(expectedVirtualRepoBParamsMap, "defaultDeploymentRepo")
 			}
 
 			switch r.RequestURI {

artifactory/utils/container/buildinfo.go

@@ -2,6 +2,8 @@ package container
 
 import (
 	"encoding/json"
+	"fmt"
+	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -53,15 +55,28 @@ type buildInfoBuilder struct {
 	imageLayers       []utils.ResultItem
 }
 
+type RepositoryDetails struct {
+	key      string
+	isRemote bool
+	repoType string
+}
+
 // Create instance of docker build info builder.
 func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, project string, serviceManager artifactory.ArtifactoryServicesManager) (*buildInfoBuilder, error) {
 	var err error
 	builder := &buildInfoBuilder{}
 	builder.repositoryDetails.key = repository
-	builder.repositoryDetails.isRemote, err = artutils.IsRemoteRepo(repository, serviceManager)
+
+	// Get repository details in one API call to determine both isRemote and repoType
+	repoDetails := &services.RepositoryDetails{}
+	err = serviceManager.GetRepository(repository, &repoDetails)
 	if err != nil {
-		return nil, err
+		return nil, errorutils.CheckErrorf("failed to get details for repository '" + repository + "'. Error:\n" + err.Error())
 	}
+
+	builder.repositoryDetails.isRemote = repoDetails.GetRepoType() == "remote"
+	builder.repositoryDetails.repoType = repoDetails.GetRepoType()
+
 	builder.image = image
 	builder.buildName = buildName
 	builder.buildNumber = buildNumber
@@ -70,11 +85,6 @@ func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, proje
 	return builder, nil
 }
 
-type RepositoryDetails struct {
-	key      string
-	isRemote bool
-}
-
 func (builder *buildInfoBuilder) setImageSha2(imageSha2 string) {
 	builder.imageSha2 = imageSha2
 }
@@ -91,8 +101,7 @@ func (builder *buildInfoBuilder) getSearchableRepo() string {
 }
 
 // Set build properties on image layers in Artifactory.
-func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager) (err error) {
-	// Skip if no build info is provided
+func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) (err error) {
 	if buildName == "" || buildNumber == "" {
 		log.Debug("Skipping setting properties - build name and build number are required")
 		return nil
@@ -103,13 +112,23 @@ func setBuildProperties(buildName, buildNumber, project string, imageLayers []ut
 		return
 	}
 
-	// Skip if no properties were created
 	if len(props) == 0 {
 		log.Debug("Skipping setting properties - no properties created")
 		return nil
 	}
 
-	pathToFile, err := writeLayersToFile(imageLayers)
+	filteredLayers, err := filterLayersForVirtualRepository(imageLayers, serviceManager, originalRepo, repoDetails)
+	if err != nil {
+		log.Debug("Failed to filter layers for virtual repository, proceeding with all layers:", err.Error())
+		filteredLayers = imageLayers
+	}
+
+	if len(filteredLayers) == 0 {
+		log.Debug("No layers to set properties on, skipping property setting")
+		return nil
+	}
+
+	pathToFile, err := writeLayersToFile(filteredLayers)
 	if err != nil {
 		return
 	}
@@ -119,6 +138,120 @@ func setBuildProperties(buildName, buildNumber, project string, imageLayers []ut
 	return
 }
 
+// filterLayersForVirtualRepository filters image layers to only include those from the default deployment repository
+// when dealing with virtual repositories. For non-virtual repositories, it returns all layers unchanged.
+func filterLayersForVirtualRepository(imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) ([]utils.ResultItem, error) {
+	if len(imageLayers) == 0 {
+		return imageLayers, nil
+	}
+
+	// Optimization: If we already know the repo type and it's not virtual, skip the API call
+	if repoDetails != nil && repoDetails.repoType != "" && repoDetails.repoType != "virtual" {
+		log.Debug("Repository ", originalRepo, "is not virtual (type:", repoDetails.repoType+"), skipping determining default deployment config")
+		return imageLayers, nil
+	}
+
+	// For backwards compatibility or when repoDetails is not available, fall back to API call
+	if repoDetails == nil || repoDetails.repoType == "" {
+		log.Debug("Repository type not cached, making API call to determine repository configuration")
+		repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
+		if err != nil {
+			return imageLayers, errorutils.CheckErrorf("failed to get repository configuration for '%s': %w", originalRepo, err)
+		}
+
+		// If it's not a virtual repository, return all layers unchanged
+		if repoConfig == nil || repoConfig.Rclass != "virtual" {
+			log.Debug("Repository", originalRepo, "is not virtual, proceeding with all layers")
+			return imageLayers, nil
+		}
+
+		// If it's a virtual repository but has no default deployment repo, return all layers
+		if repoConfig.DefaultDeploymentRepo == "" {
+			log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
+			return imageLayers, nil
+		}
+
+		// Filter layers to only include those from the default deployment repository
+		var filteredLayers []utils.ResultItem
+		for _, layer := range imageLayers {
+			if layer.Repo == repoConfig.DefaultDeploymentRepo {
+				filteredLayers = append(filteredLayers, layer)
+			}
+		}
+
+		if len(filteredLayers) == 0 {
+			log.Warn(fmt.Sprintf(`No layers found in default deployment repository '%s' for virtual repository '%s'.
+This may indicate that image layers exist in other repositories but not in the default deployment repository.
+Properties will not be set to maintain consistency with virtual repository configuration.
+To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.`, repoConfig.DefaultDeploymentRepo, originalRepo))
+			return []utils.ResultItem{}, nil
+		}
+		log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
+
+		return filteredLayers, nil
+	}
+
+	log.Info("Determining virtual repository", originalRepo, "config to determine default deployment repository")
+	repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
+	if err != nil {
+		return imageLayers, errorutils.CheckErrorf("failed to get repository configuration for virtual repository '%s': %w", originalRepo, err)
+	}
+
+	// If it's a virtual repository but has no default deployment repo, return all layers
+	if repoConfig.DefaultDeploymentRepo == "" {
+		log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
+		return imageLayers, nil
+	}
+
+	// Filter layers to only include those from the default deployment repository
+	var filteredLayers []utils.ResultItem
+	for _, layer := range imageLayers {
+		if layer.Repo == repoConfig.DefaultDeploymentRepo {
+			filteredLayers = append(filteredLayers, layer)
+		}
+	}
+
+	if len(filteredLayers) == 0 {
+		log.Warn(fmt.Sprintf(`No layers found in default deployment repository '%s' for virtual repository '%s'. 
+This may indicate that image layers exist in other repositories but not in the default deployment repository. 	
+Properties will not be set to maintain consistency with virtual repository configuration. 
+To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.`, repoConfig.DefaultDeploymentRepo, originalRepo))
+		return []utils.ResultItem{}, nil
+	}
+	log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
+
+	return filteredLayers, nil
+}
+
+// repositoryConfig represents the virtual repository configuration
+type repositoryConfig struct {
+	Key                   string `json:"key"`
+	Rclass                string `json:"rclass"`
+	DefaultDeploymentRepo string `json:"defaultDeploymentRepo"`
+}
+
+// getRepositoryConfiguration fetches the repository configuration from Artifactory
+func getRepositoryConfiguration(repoKey string, serviceManager artifactory.ArtifactoryServicesManager) (*repositoryConfig, error) {
+	httpClientDetails := serviceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails()
+
+	baseUrl := serviceManager.GetConfig().GetServiceDetails().GetUrl()
+	endpoint := "api/repositories/" + repoKey
+	url := baseUrl + endpoint
+	resp, body, _, err := serviceManager.Client().SendGet(url, true, &httpClientDetails)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to get repository configuration: HTTP %d", resp.StatusCode)
+	}
+	var config repositoryConfig
+	if err := json.Unmarshal(body, &config); err != nil {
+		return nil, fmt.Errorf("failed to parse repository configuration: %v", err)
+	}
+
+	return &config, nil
+}
+
 // Download the content of layer search result.
 func downloadLayer(searchResult utils.ResultItem, result interface{}, serviceManager artifactory.ArtifactoryServicesManager, repo string) error {
 	// Search results may include artifacts from the remote-cache repository.
@@ -340,7 +473,7 @@ func (builder *buildInfoBuilder) createBuildInfo(commandType CommandType, manife
 			return nil, err
 		}
 		if !builder.skipTaggingLayers {
-			if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager); err != nil {
+			if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails); err != nil {
 				return nil, err
 			}
 		}
@@ -399,7 +532,7 @@ func (builder *buildInfoBuilder) createMultiPlatformBuildInfo(fatManifest *FatMa
 			Parent:    imageLongNameWithoutRepo,
 		})
 	}
-	return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager)
+	return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails)
 }
 
 // Construct the manifest's module ID by its type (attestation) or its platform.

artifactory/utils/utils.go

@@ -29,6 +29,7 @@ import (
 	"github.com/jfrog/jfrog-client-go/jfconnect"
 	"github.com/jfrog/jfrog-client-go/lifecycle"
 	"github.com/jfrog/jfrog-client-go/metadata"
+	"github.com/jfrog/jfrog-client-go/onemodel"
 	clientUtils "github.com/jfrog/jfrog-client-go/utils"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	ioUtils "github.com/jfrog/jfrog-client-go/utils/io"
@@ -263,6 +264,27 @@ func CreateMetadataServiceManager(serviceDetails *config.ServerDetails, isDryRun
 	return metadata.NewManager(serviceConfig)
 }
 
+func CreateOnemodelServiceManager(serviceDetails *config.ServerDetails, isDryRun bool) (onemodel.Manager, error) {
+	certsPath, err := coreutils.GetJfrogCertsDir()
+	if err != nil {
+		return nil, err
+	}
+	mdAuth, err := serviceDetails.CreateOnemodelAuthConfig()
+	if err != nil {
+		return nil, err
+	}
+	serviceConfig, err := clientConfig.NewConfigBuilder().
+		SetServiceDetails(mdAuth).
+		SetCertificatesPath(certsPath).
+		SetInsecureTls(serviceDetails.InsecureTls).
+		SetDryRun(isDryRun).
+		Build()
+	if err != nil {
+		return nil, err
+	}
+	return onemodel.NewManager(serviceConfig)
+}
+
 func CreateJfConnectServiceManager(serverDetails *config.ServerDetails, httpRetries, httpRetryWaitMilliSecs int) (jfconnect.Manager, error) {
 	certsPath, err := coreutils.GetJfrogCertsDir()
 	if err != nil {

common/commands/configfile.go

@@ -2,12 +2,13 @@ package commands
 
 import (
 	"fmt"
-	"github.com/jfrog/jfrog-client-go/artifactory/services"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
+	"github.com/jfrog/jfrog-client-go/artifactory/services"
+
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
 	"github.com/jfrog/jfrog-cli-core/v2/common/project"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
@@ -155,6 +156,8 @@ func handleInteractiveConfigCreation(configFile *ConfigFile, confType project.Pr
 		return
 	}
 	switch confType {
+	case project.Ruby:
+		return configFile.setDeployerResolver()
 	case project.Go:
 		return configFile.setDeployerResolver()
 	case project.Pip, project.Pipenv:

common/commands/configfile_test.go

@@ -2,12 +2,13 @@ package commands
 
 import (
 	"flag"
-	testsutils "github.com/jfrog/jfrog-client-go/utils/tests"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	testsutils "github.com/jfrog/jfrog-client-go/utils/tests"
+
 	"github.com/jfrog/jfrog-cli-core/v2/common/project"
 	"github.com/jfrog/jfrog-cli-core/v2/common/tests"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
@@ -158,6 +159,24 @@ func TestNpmConfigFile(t *testing.T) {
 	assert.Equal(t, "repo-local", config.GetString("deployer.repo"))
 }
 
+func TestRubyConfigFile(t *testing.T) {
+	// Set JFROG_CLI_HOME_DIR environment variable
+	tempDirPath := createTempEnv(t)
+	defer testsutils.RemoveAllAndAssert(t, tempDirPath)
+
+	// Create build config
+	context := createContext(t, resolutionServerId+"=relServer", resolutionRepo+"=repo", deploymentServerId+"=depServer", deploymentRepo+"=repo-local")
+	err := CreateBuildConfig(context, project.Ruby)
+	assert.NoError(t, err)
+
+	// Check configuration
+	config := checkCommonAndGetConfiguration(t, project.Ruby.String(), tempDirPath)
+	assert.Equal(t, "relServer", config.GetString("resolver.serverId"))
+	assert.Equal(t, "repo", config.GetString("resolver.repo"))
+	assert.Equal(t, "depServer", config.GetString("deployer.serverId"))
+	assert.Equal(t, "repo-local", config.GetString("deployer.repo"))
+}
+
 // In case resolver/deployer server-id flags are not provided - the default configured global server will be chosen.
 func TestNpmConfigFileWithDefaultServerId(t *testing.T) {
 	// Set JFROG_CLI_HOME_DIR environment variable

common/project/projectconfig_test.go

@@ -1,8 +1,9 @@
 package project
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestFromString(t *testing.T) {
@@ -15,6 +16,7 @@ func TestFromString(t *testing.T) {
 		{"pip", Pip},
 		{"npm", Npm},
 		{"pnpm", Pnpm},
+		{"ruby", Ruby},
 	}
 
 	for _, testCase := range testCases {