Skip to content

Commit dc4668b

Browse files
agrasthagrasth
committed
Optimize-virtual-repo-layer-filtering
1 parent 60a3b4b commit dc4668b

File tree

1 file changed

+156
-11
lines changed

1 file changed

+156
-11
lines changed

‎artifactory/utils/container/buildinfo.go‎

Lines changed: 156 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package container
22

33
import (
44
"encoding/json"
5+
"fmt"
56
"os"
67
"path"
78
"strings"
@@ -53,15 +54,28 @@ type buildInfoBuilder struct {
5354
imageLayers []utils.ResultItem
5455
}
5556

57+
type RepositoryDetails struct {
58+
key string
59+
isRemote bool
60+
repoType string // Add repo type to avoid additional API calls
61+
}
62+
5663
// Create instance of docker build info builder.
5764
func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, project string, serviceManager artifactory.ArtifactoryServicesManager) (*buildInfoBuilder, error) {
5865
var err error
5966
builder := &buildInfoBuilder{}
6067
builder.repositoryDetails.key = repository
61-
builder.repositoryDetails.isRemote, err = artutils.IsRemoteRepo(repository, serviceManager)
68+
69+
// Get repository details in one API call to determine both isRemote and repoType
70+
repoDetails := &services.RepositoryDetails{}
71+
err = serviceManager.GetRepository(repository, &repoDetails)
6272
if err != nil {
63-
return nil, err
73+
return nil, errorutils.CheckErrorf("failed to get details for repository '" + repository + "'. Error:\n" + err.Error())
6474
}
75+
76+
builder.repositoryDetails.isRemote = repoDetails.GetRepoType() == "remote"
77+
builder.repositoryDetails.repoType = repoDetails.GetRepoType()
78+
6579
builder.image = image
6680
builder.buildName = buildName
6781
builder.buildNumber = buildNumber
@@ -70,11 +84,6 @@ func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, proje
7084
return builder, nil
7185
}
7286

73-
type RepositoryDetails struct {
74-
key string
75-
isRemote bool
76-
}
77-
7887
func (builder *buildInfoBuilder) setImageSha2(imageSha2 string) {
7988
builder.imageSha2 = imageSha2
8089
}
@@ -91,7 +100,7 @@ func (builder *buildInfoBuilder) getSearchableRepo() string {
91100
}
92101

93102
// Set build properties on image layers in Artifactory.
94-
func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager) (err error) {
103+
func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) (err error) {
95104
// Skip if no build info is provided
96105
if buildName == "" || buildNumber == "" {
97106
log.Debug("Skipping setting properties - build name and build number are required")
@@ -109,7 +118,14 @@ func setBuildProperties(buildName, buildNumber, project string, imageLayers []ut
109118
return nil
110119
}
111120

112-
pathToFile, err := writeLayersToFile(imageLayers)
121+
// Filter image layers for virtual repositories to only include layers from default deployment repository
122+
filteredLayers, err := filterLayersForVirtualRepository(imageLayers, serviceManager, originalRepo, repoDetails)
123+
if err != nil {
124+
log.Debug("Failed to filter layers for virtual repository, proceeding with all layers:", err.Error())
125+
filteredLayers = imageLayers
126+
}
127+
128+
pathToFile, err := writeLayersToFile(filteredLayers)
113129
if err != nil {
114130
return
115131
}
@@ -119,6 +135,135 @@ func setBuildProperties(buildName, buildNumber, project string, imageLayers []ut
119135
return
120136
}
121137

138+
// filterLayersForVirtualRepository filters image layers to only include those from the default deployment repository
139+
// when dealing with virtual repositories. For non-virtual repositories, it returns all layers unchanged.
140+
func filterLayersForVirtualRepository(imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) ([]utils.ResultItem, error) {
141+
if len(imageLayers) == 0 {
142+
return imageLayers, nil
143+
}
144+
145+
// Optimization: If we already know the repo type and it's not virtual, skip the API call
146+
if repoDetails != nil && repoDetails.repoType != "" && repoDetails.repoType != "virtual" {
147+
log.Info("OPTIMIZATION: Repository", originalRepo, "is not virtual (type:", repoDetails.repoType+"), skipping API call and proceeding with all layers")
148+
return imageLayers, nil
149+
}
150+
151+
// For backwards compatibility or when repoDetails is not available, fall back to API call
152+
if repoDetails == nil || repoDetails.repoType == "" {
153+
log.Debug("Repository type not cached, making API call to determine repository configuration")
154+
// Check if the original repository is a virtual repository by getting its configuration
155+
repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
156+
if err != nil {
157+
log.Debug("Failed to get repository configuration for", originalRepo, ":", err.Error())
158+
return imageLayers, err
159+
}
160+
161+
// If it's not a virtual repository, return all layers unchanged
162+
if repoConfig == nil || repoConfig.Rclass != "virtual" {
163+
log.Debug("Repository", originalRepo, "is not virtual, proceeding with all layers")
164+
return imageLayers, nil
165+
}
166+
167+
// If it's a virtual repository but has no default deployment repo, return all layers
168+
if repoConfig.DefaultDeploymentRepo == "" {
169+
log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
170+
return imageLayers, nil
171+
}
172+
173+
// Filter layers to only include those from the default deployment repository
174+
var filteredLayers []utils.ResultItem
175+
for _, layer := range imageLayers {
176+
if layer.Repo == repoConfig.DefaultDeploymentRepo {
177+
filteredLayers = append(filteredLayers, layer)
178+
}
179+
}
180+
181+
if len(filteredLayers) > 0 {
182+
log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
183+
} else {
184+
log.Warn("No layers found in default deployment repository", repoConfig.DefaultDeploymentRepo, "for virtual repository", originalRepo)
185+
log.Warn("This may indicate that the image layers exist in other repositories but not in the default deployment repository.")
186+
log.Warn("Properties will not be set to maintain consistency with virtual repository configuration.")
187+
log.Warn("To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.")
188+
return []utils.ResultItem{}, nil // Return empty slice instead of all layers
189+
}
190+
191+
return filteredLayers, nil
192+
}
193+
194+
// We know it's a virtual repository, so we need to get its default deployment repo
195+
// We still need an API call here to get the DefaultDeploymentRepo, but only for virtual repos
196+
log.Info("Making API call for virtual repository", originalRepo, "to get DefaultDeploymentRepo")
197+
repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
198+
if err != nil {
199+
log.Debug("Failed to get repository configuration for virtual repository", originalRepo, ":", err.Error())
200+
return imageLayers, err
201+
}
202+
203+
// If it's a virtual repository but has no default deployment repo, return all layers
204+
if repoConfig.DefaultDeploymentRepo == "" {
205+
log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
206+
return imageLayers, nil
207+
}
208+
209+
// Filter layers to only include those from the default deployment repository
210+
var filteredLayers []utils.ResultItem
211+
for _, layer := range imageLayers {
212+
if layer.Repo == repoConfig.DefaultDeploymentRepo {
213+
filteredLayers = append(filteredLayers, layer)
214+
}
215+
}
216+
217+
if len(filteredLayers) > 0 {
218+
log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
219+
} else {
220+
log.Warn("No layers found in default deployment repository", repoConfig.DefaultDeploymentRepo, "for virtual repository", originalRepo)
221+
log.Warn("This may indicate that the image layers exist in other repositories but not in the default deployment repository.")
222+
log.Warn("Properties will not be set to maintain consistency with virtual repository configuration.")
223+
log.Warn("To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.")
224+
return []utils.ResultItem{}, nil // Return empty slice instead of all layers
225+
}
226+
227+
return filteredLayers, nil
228+
}
229+
230+
// repositoryConfig represents the virtual repository configuration
231+
type repositoryConfig struct {
232+
Key string `json:"key"`
233+
Rclass string `json:"rclass"`
234+
DefaultDeploymentRepo string `json:"defaultDeploymentRepo"`
235+
}
236+
237+
// getRepositoryConfiguration fetches the repository configuration from Artifactory
238+
func getRepositoryConfiguration(repoKey string, serviceManager artifactory.ArtifactoryServicesManager) (*repositoryConfig, error) {
239+
// Create HTTP client details for the request
240+
httpClientDetails := serviceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails()
241+
242+
// Construct the API endpoint URL
243+
baseUrl := serviceManager.GetConfig().GetServiceDetails().GetUrl()
244+
endpoint := "api/repositories/" + repoKey
245+
url := baseUrl + endpoint
246+
247+
// Make the HTTP GET request
248+
resp, body, _, err := serviceManager.Client().SendGet(url, true, &httpClientDetails)
249+
if err != nil {
250+
return nil, err
251+
}
252+
253+
// Check for successful response
254+
if resp.StatusCode != 200 {
255+
return nil, fmt.Errorf("failed to get repository configuration: HTTP %d", resp.StatusCode)
256+
}
257+
258+
// Parse the JSON response
259+
var config repositoryConfig
260+
if err := json.Unmarshal(body, &config); err != nil {
261+
return nil, fmt.Errorf("failed to parse repository configuration: %v", err)
262+
}
263+
264+
return &config, nil
265+
}
266+
122267
// Download the content of layer search result.
123268
func downloadLayer(searchResult utils.ResultItem, result interface{}, serviceManager artifactory.ArtifactoryServicesManager, repo string) error {
124269
// Search results may include artifacts from the remote-cache repository.
@@ -340,7 +485,7 @@ func (builder *buildInfoBuilder) createBuildInfo(commandType CommandType, manife
340485
return nil, err
341486
}
342487
if !builder.skipTaggingLayers {
343-
if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager); err != nil {
488+
if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails); err != nil {
344489
return nil, err
345490
}
346491
}
@@ -399,7 +544,7 @@ func (builder *buildInfoBuilder) createMultiPlatformBuildInfo(fatManifest *FatMa
399544
Parent: imageLongNameWithoutRepo,
400545
})
401546
}
402-
return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager)
547+
return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails)
403548
}
404549

405550
// Construct the manifest's module ID by its type (attestation) or its platform.

0 commit comments

Comments
 (0)