Skip to content

Commit 093be7f

Browse files
agrasthagrasth
committed
Optimize-virtual-repo-layer-filtering
1 parent 60a3b4b commit 093be7f

File tree

1 file changed

+157
-11
lines changed

1 file changed

+157
-11
lines changed

‎artifactory/utils/container/buildinfo.go‎

Lines changed: 157 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package container
22

33
import (
44
"encoding/json"
5+
"fmt"
6+
"net/http"
57
"os"
68
"path"
79
"strings"
@@ -53,15 +55,28 @@ type buildInfoBuilder struct {
5355
imageLayers []utils.ResultItem
5456
}
5557

58+
type RepositoryDetails struct {
59+
key string
60+
isRemote bool
61+
repoType string // Add repo type to avoid additional API calls
62+
}
63+
5664
// Create instance of docker build info builder.
5765
func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, project string, serviceManager artifactory.ArtifactoryServicesManager) (*buildInfoBuilder, error) {
5866
var err error
5967
builder := &buildInfoBuilder{}
6068
builder.repositoryDetails.key = repository
61-
builder.repositoryDetails.isRemote, err = artutils.IsRemoteRepo(repository, serviceManager)
69+
70+
// Get repository details in one API call to determine both isRemote and repoType
71+
repoDetails := &services.RepositoryDetails{}
72+
err = serviceManager.GetRepository(repository, &repoDetails)
6273
if err != nil {
63-
return nil, err
74+
return nil, errorutils.CheckErrorf("failed to get details for repository '" + repository + "'. Error:\n" + err.Error())
6475
}
76+
77+
builder.repositoryDetails.isRemote = repoDetails.GetRepoType() == "remote"
78+
builder.repositoryDetails.repoType = repoDetails.GetRepoType()
79+
6580
builder.image = image
6681
builder.buildName = buildName
6782
builder.buildNumber = buildNumber
@@ -70,11 +85,6 @@ func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, proje
7085
return builder, nil
7186
}
7287

73-
type RepositoryDetails struct {
74-
key string
75-
isRemote bool
76-
}
77-
7888
func (builder *buildInfoBuilder) setImageSha2(imageSha2 string) {
7989
builder.imageSha2 = imageSha2
8090
}
@@ -91,7 +101,7 @@ func (builder *buildInfoBuilder) getSearchableRepo() string {
91101
}
92102

93103
// Set build properties on image layers in Artifactory.
94-
func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager) (err error) {
104+
func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) (err error) {
95105
// Skip if no build info is provided
96106
if buildName == "" || buildNumber == "" {
97107
log.Debug("Skipping setting properties - build name and build number are required")
@@ -109,7 +119,14 @@ func setBuildProperties(buildName, buildNumber, project string, imageLayers []ut
109119
return nil
110120
}
111121

112-
pathToFile, err := writeLayersToFile(imageLayers)
122+
// Filter image layers for virtual repositories to only include layers from default deployment repository
123+
filteredLayers, err := filterLayersForVirtualRepository(imageLayers, serviceManager, originalRepo, repoDetails)
124+
if err != nil {
125+
log.Debug("Failed to filter layers for virtual repository, proceeding with all layers:", err.Error())
126+
filteredLayers = imageLayers
127+
}
128+
129+
pathToFile, err := writeLayersToFile(filteredLayers)
113130
if err != nil {
114131
return
115132
}
@@ -119,6 +136,135 @@ func setBuildProperties(buildName, buildNumber, project string, imageLayers []ut
119136
return
120137
}
121138

139+
// filterLayersForVirtualRepository filters image layers to only include those from the default deployment repository
140+
// when dealing with virtual repositories. For non-virtual repositories, it returns all layers unchanged.
141+
func filterLayersForVirtualRepository(imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) ([]utils.ResultItem, error) {
142+
if len(imageLayers) == 0 {
143+
return imageLayers, nil
144+
}
145+
146+
// Optimization: If we already know the repo type and it's not virtual, skip the API call
147+
if repoDetails != nil && repoDetails.repoType != "" && repoDetails.repoType != "virtual" {
148+
log.Info("OPTIMIZATION: Repository", originalRepo, "is not virtual (type:", repoDetails.repoType+"), skipping API call and proceeding with all layers")
149+
return imageLayers, nil
150+
}
151+
152+
// For backwards compatibility or when repoDetails is not available, fall back to API call
153+
if repoDetails == nil || repoDetails.repoType == "" {
154+
log.Debug("Repository type not cached, making API call to determine repository configuration")
155+
// Check if the original repository is a virtual repository by getting its configuration
156+
repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
157+
if err != nil {
158+
log.Debug("Failed to get repository configuration for", originalRepo, ":", err.Error())
159+
return imageLayers, err
160+
}
161+
162+
// If it's not a virtual repository, return all layers unchanged
163+
if repoConfig == nil || repoConfig.Rclass != "virtual" {
164+
log.Debug("Repository", originalRepo, "is not virtual, proceeding with all layers")
165+
return imageLayers, nil
166+
}
167+
168+
// If it's a virtual repository but has no default deployment repo, return all layers
169+
if repoConfig.DefaultDeploymentRepo == "" {
170+
log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
171+
return imageLayers, nil
172+
}
173+
174+
// Filter layers to only include those from the default deployment repository
175+
var filteredLayers []utils.ResultItem
176+
for _, layer := range imageLayers {
177+
if layer.Repo == repoConfig.DefaultDeploymentRepo {
178+
filteredLayers = append(filteredLayers, layer)
179+
}
180+
}
181+
182+
if len(filteredLayers) > 0 {
183+
log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
184+
} else {
185+
log.Warn("No layers found in default deployment repository", repoConfig.DefaultDeploymentRepo, "for virtual repository", originalRepo)
186+
log.Warn("This may indicate that the image layers exist in other repositories but not in the default deployment repository.")
187+
log.Warn("Properties will not be set to maintain consistency with virtual repository configuration.")
188+
log.Warn("To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.")
189+
return []utils.ResultItem{}, nil // Return empty slice instead of all layers
190+
}
191+
192+
return filteredLayers, nil
193+
}
194+
195+
// We know it's a virtual repository, so we need to get its default deployment repo
196+
// We still need an API call here to get the DefaultDeploymentRepo, but only for virtual repos
197+
log.Info("Making API call for virtual repository", originalRepo, "to get DefaultDeploymentRepo")
198+
repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
199+
if err != nil {
200+
log.Debug("Failed to get repository configuration for virtual repository", originalRepo, ":", err.Error())
201+
return imageLayers, err
202+
}
203+
204+
// If it's a virtual repository but has no default deployment repo, return all layers
205+
if repoConfig.DefaultDeploymentRepo == "" {
206+
log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
207+
return imageLayers, nil
208+
}
209+
210+
// Filter layers to only include those from the default deployment repository
211+
var filteredLayers []utils.ResultItem
212+
for _, layer := range imageLayers {
213+
if layer.Repo == repoConfig.DefaultDeploymentRepo {
214+
filteredLayers = append(filteredLayers, layer)
215+
}
216+
}
217+
218+
if len(filteredLayers) > 0 {
219+
log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
220+
} else {
221+
log.Warn("No layers found in default deployment repository", repoConfig.DefaultDeploymentRepo, "for virtual repository", originalRepo)
222+
log.Warn("This may indicate that the image layers exist in other repositories but not in the default deployment repository.")
223+
log.Warn("Properties will not be set to maintain consistency with virtual repository configuration.")
224+
log.Warn("To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.")
225+
return []utils.ResultItem{}, nil // Return empty slice instead of all layers
226+
}
227+
228+
return filteredLayers, nil
229+
}
230+
231+
// repositoryConfig represents the virtual repository configuration
232+
type repositoryConfig struct {
233+
Key string `json:"key"`
234+
Rclass string `json:"rclass"`
235+
DefaultDeploymentRepo string `json:"defaultDeploymentRepo"`
236+
}
237+
238+
// getRepositoryConfiguration fetches the repository configuration from Artifactory
239+
func getRepositoryConfiguration(repoKey string, serviceManager artifactory.ArtifactoryServicesManager) (*repositoryConfig, error) {
240+
// Create HTTP client details for the request
241+
httpClientDetails := serviceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails()
242+
243+
// Construct the API endpoint URL
244+
baseUrl := serviceManager.GetConfig().GetServiceDetails().GetUrl()
245+
endpoint := "api/repositories/" + repoKey
246+
url := baseUrl + endpoint
247+
248+
// Make the HTTP GET request
249+
resp, body, _, err := serviceManager.Client().SendGet(url, true, &httpClientDetails)
250+
if err != nil {
251+
return nil, err
252+
}
253+
254+
// Check for successful response
255+
if resp.StatusCode != http.StatusOK {
256+
return nil, fmt.Errorf("failed to get repository configuration: HTTP %d", resp.StatusCode)
257+
}
258+
259+
// Parse the JSON response
260+
var config repositoryConfig
261+
if err := json.Unmarshal(body, &config); err != nil {
262+
return nil, fmt.Errorf("failed to parse repository configuration: %v", err)
263+
}
264+
265+
return &config, nil
266+
}
267+
122268
// Download the content of layer search result.
123269
func downloadLayer(searchResult utils.ResultItem, result interface{}, serviceManager artifactory.ArtifactoryServicesManager, repo string) error {
124270
// Search results may include artifacts from the remote-cache repository.
@@ -340,7 +486,7 @@ func (builder *buildInfoBuilder) createBuildInfo(commandType CommandType, manife
340486
return nil, err
341487
}
342488
if !builder.skipTaggingLayers {
343-
if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager); err != nil {
489+
if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails); err != nil {
344490
return nil, err
345491
}
346492
}
@@ -399,7 +545,7 @@ func (builder *buildInfoBuilder) createMultiPlatformBuildInfo(fatManifest *FatMa
399545
Parent: imageLongNameWithoutRepo,
400546
})
401547
}
402-
return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager)
548+
return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails)
403549
}
404550

405551
// Construct the manifest's module ID by its type (attestation) or its platform.

0 commit comments

Comments
 (0)