Skip to content

Commit 05fa697

Browse files
agrasthagrasth
committed
Optimize-virtual-repo-layer-filtering
1 parent 60a3b4b commit 05fa697

File tree

1 file changed

+146
-13
lines changed

1 file changed

+146
-13
lines changed

‎artifactory/utils/container/buildinfo.go‎

Lines changed: 146 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package container
22

33
import (
44
"encoding/json"
5+
"fmt"
6+
"net/http"
57
"os"
68
"path"
79
"strings"
@@ -53,15 +55,28 @@ type buildInfoBuilder struct {
5355
imageLayers []utils.ResultItem
5456
}
5557

58+
type RepositoryDetails struct {
59+
key string
60+
isRemote bool
61+
repoType string
62+
}
63+
5664
// Create instance of docker build info builder.
5765
func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, project string, serviceManager artifactory.ArtifactoryServicesManager) (*buildInfoBuilder, error) {
5866
var err error
5967
builder := &buildInfoBuilder{}
6068
builder.repositoryDetails.key = repository
61-
builder.repositoryDetails.isRemote, err = artutils.IsRemoteRepo(repository, serviceManager)
69+
70+
// Get repository details in one API call to determine both isRemote and repoType
71+
repoDetails := &services.RepositoryDetails{}
72+
err = serviceManager.GetRepository(repository, &repoDetails)
6273
if err != nil {
63-
return nil, err
74+
return nil, errorutils.CheckErrorf("failed to get details for repository '" + repository + "'. Error:\n" + err.Error())
6475
}
76+
77+
builder.repositoryDetails.isRemote = repoDetails.GetRepoType() == "remote"
78+
builder.repositoryDetails.repoType = repoDetails.GetRepoType()
79+
6580
builder.image = image
6681
builder.buildName = buildName
6782
builder.buildNumber = buildNumber
@@ -70,11 +85,6 @@ func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, proje
7085
return builder, nil
7186
}
7287

73-
type RepositoryDetails struct {
74-
key string
75-
isRemote bool
76-
}
77-
7888
func (builder *buildInfoBuilder) setImageSha2(imageSha2 string) {
7989
builder.imageSha2 = imageSha2
8090
}
@@ -91,8 +101,7 @@ func (builder *buildInfoBuilder) getSearchableRepo() string {
91101
}
92102

93103
// Set build properties on image layers in Artifactory.
94-
func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager) (err error) {
95-
// Skip if no build info is provided
104+
func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) (err error) {
96105
if buildName == "" || buildNumber == "" {
97106
log.Debug("Skipping setting properties - build name and build number are required")
98107
return nil
@@ -103,13 +112,23 @@ func setBuildProperties(buildName, buildNumber, project string, imageLayers []ut
103112
return
104113
}
105114

106-
// Skip if no properties were created
107115
if len(props) == 0 {
108116
log.Debug("Skipping setting properties - no properties created")
109117
return nil
110118
}
111119

112-
pathToFile, err := writeLayersToFile(imageLayers)
120+
filteredLayers, err := filterLayersForVirtualRepository(imageLayers, serviceManager, originalRepo, repoDetails)
121+
if err != nil {
122+
log.Debug("Failed to filter layers for virtual repository, proceeding with all layers:", err.Error())
123+
filteredLayers = imageLayers
124+
}
125+
126+
if len(filteredLayers) == 0 {
127+
log.Debug("No layers to set properties on, skipping property setting")
128+
return nil
129+
}
130+
131+
pathToFile, err := writeLayersToFile(filteredLayers)
113132
if err != nil {
114133
return
115134
}
@@ -119,6 +138,120 @@ func setBuildProperties(buildName, buildNumber, project string, imageLayers []ut
119138
return
120139
}
121140

141+
// filterLayersForVirtualRepository filters image layers to only include those from the default deployment repository
142+
// when dealing with virtual repositories. For non-virtual repositories, it returns all layers unchanged.
143+
func filterLayersForVirtualRepository(imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) ([]utils.ResultItem, error) {
144+
if len(imageLayers) == 0 {
145+
return imageLayers, nil
146+
}
147+
148+
// Optimization: If we already know the repo type and it's not virtual, skip the API call
149+
if repoDetails != nil && repoDetails.repoType != "" && repoDetails.repoType != "virtual" {
150+
log.Debug("Repository ", originalRepo, "is not virtual (type:", repoDetails.repoType+"), skipping determining default deployment config")
151+
return imageLayers, nil
152+
}
153+
154+
// For backwards compatibility or when repoDetails is not available, fall back to API call
155+
if repoDetails == nil || repoDetails.repoType == "" {
156+
log.Debug("Repository type not cached, making API call to determine repository configuration")
157+
repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
158+
if err != nil {
159+
return imageLayers, errorutils.CheckErrorf("failed to get repository configuration for '%s': %w", originalRepo, err)
160+
}
161+
162+
// If it's not a virtual repository, return all layers unchanged
163+
if repoConfig == nil || repoConfig.Rclass != "virtual" {
164+
log.Debug("Repository", originalRepo, "is not virtual, proceeding with all layers")
165+
return imageLayers, nil
166+
}
167+
168+
// If it's a virtual repository but has no default deployment repo, return all layers
169+
if repoConfig.DefaultDeploymentRepo == "" {
170+
log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
171+
return imageLayers, nil
172+
}
173+
174+
// Filter layers to only include those from the default deployment repository
175+
var filteredLayers []utils.ResultItem
176+
for _, layer := range imageLayers {
177+
if layer.Repo == repoConfig.DefaultDeploymentRepo {
178+
filteredLayers = append(filteredLayers, layer)
179+
}
180+
}
181+
182+
if len(filteredLayers) == 0 {
183+
log.Warn(fmt.Sprintf(`No layers found in default deployment repository '%s' for virtual repository '%s'.
184+
This may indicate that image layers exist in other repositories but not in the default deployment repository.
185+
Properties will not be set to maintain consistency with virtual repository configuration.
186+
To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.`, repoConfig.DefaultDeploymentRepo, originalRepo))
187+
return []utils.ResultItem{}, nil
188+
}
189+
log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
190+
191+
return filteredLayers, nil
192+
}
193+
194+
log.Info("Determining virtual repository", originalRepo, "config to determine default deployment repository")
195+
repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
196+
if err != nil {
197+
return imageLayers, errorutils.CheckErrorf("failed to get repository configuration for virtual repository '%s': %w", originalRepo, err)
198+
}
199+
200+
// If it's a virtual repository but has no default deployment repo, return all layers
201+
if repoConfig.DefaultDeploymentRepo == "" {
202+
log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
203+
return imageLayers, nil
204+
}
205+
206+
// Filter layers to only include those from the default deployment repository
207+
var filteredLayers []utils.ResultItem
208+
for _, layer := range imageLayers {
209+
if layer.Repo == repoConfig.DefaultDeploymentRepo {
210+
filteredLayers = append(filteredLayers, layer)
211+
}
212+
}
213+
214+
if len(filteredLayers) == 0 {
215+
log.Warn(fmt.Sprintf(`No layers found in default deployment repository '%s' for virtual repository '%s'.
216+
This may indicate that image layers exist in other repositories but not in the default deployment repository.
217+
Properties will not be set to maintain consistency with virtual repository configuration.
218+
To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.`, repoConfig.DefaultDeploymentRepo, originalRepo))
219+
return []utils.ResultItem{}, nil
220+
}
221+
log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
222+
223+
return filteredLayers, nil
224+
}
225+
226+
// repositoryConfig represents the virtual repository configuration
227+
type repositoryConfig struct {
228+
Key string `json:"key"`
229+
Rclass string `json:"rclass"`
230+
DefaultDeploymentRepo string `json:"defaultDeploymentRepo"`
231+
}
232+
233+
// getRepositoryConfiguration fetches the repository configuration from Artifactory
234+
func getRepositoryConfiguration(repoKey string, serviceManager artifactory.ArtifactoryServicesManager) (*repositoryConfig, error) {
235+
httpClientDetails := serviceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails()
236+
237+
baseUrl := serviceManager.GetConfig().GetServiceDetails().GetUrl()
238+
endpoint := "api/repositories/" + repoKey
239+
url := baseUrl + endpoint
240+
resp, body, _, err := serviceManager.Client().SendGet(url, true, &httpClientDetails)
241+
if err != nil {
242+
return nil, err
243+
}
244+
if resp.StatusCode != http.StatusOK {
245+
return nil, fmt.Errorf("failed to get repository configuration: HTTP %d", resp.StatusCode)
246+
}
247+
var config repositoryConfig
248+
if err := json.Unmarshal(body, &config); err != nil {
249+
return nil, fmt.Errorf("failed to parse repository configuration: %v", err)
250+
}
251+
252+
return &config, nil
253+
}
254+
122255
// Download the content of layer search result.
123256
func downloadLayer(searchResult utils.ResultItem, result interface{}, serviceManager artifactory.ArtifactoryServicesManager, repo string) error {
124257
// Search results may include artifacts from the remote-cache repository.
@@ -340,7 +473,7 @@ func (builder *buildInfoBuilder) createBuildInfo(commandType CommandType, manife
340473
return nil, err
341474
}
342475
if !builder.skipTaggingLayers {
343-
if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager); err != nil {
476+
if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails); err != nil {
344477
return nil, err
345478
}
346479
}
@@ -399,7 +532,7 @@ func (builder *buildInfoBuilder) createMultiPlatformBuildInfo(fatManifest *FatMa
399532
Parent: imageLongNameWithoutRepo,
400533
})
401534
}
402-
return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager)
535+
return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails)
403536
}
404537

405538
// Construct the manifest's module ID by its type (attestation) or its platform.

0 commit comments

Comments
 (0)