Skip to content

Commit 780ffe9

Browse files
uibmuibm
committed
added bulk api support
1 parent aead563 commit 780ffe9

File tree

2 files changed

+219
-14
lines changed

2 files changed

+219
-14
lines changed

‎ibm/flex/structures.go‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2725,6 +2725,190 @@ func GetTags(d *schema.ResourceData, meta interface{}) error {
27252725
// return nil
27262726
// }
27272727

2728+
// bulk
2729+
// GetGlobalTagsUsingCRNBulk fetches tags for multiple CRNs in a single API call
2730+
// Handles 1MB API body size limit by dynamically batching requests
2731+
func GetGlobalTagsUsingCRNBulk(meta interface{}, crns []string, resourceType, tagType string) (map[string]*schema.Set, error) {
2732+
if len(crns) == 0 {
2733+
return make(map[string]*schema.Set), nil
2734+
}
2735+
2736+
gsClient, err := meta.(conns.ClientSession).GlobalSearchAPIV2()
2737+
if err != nil {
2738+
return nil, fmt.Errorf("[ERROR] Error getting global search client settings: %s", err)
2739+
}
2740+
2741+
tagsMap := make(map[string]*schema.Set)
2742+
2743+
// Dynamically batch CRNs to stay under 1MB limit
2744+
// Average CRN length ~150 chars, query overhead, leaving buffer
2745+
// Conservative estimate: ~5000 CRNs per batch (depends on CRN length)
2746+
batches := createDynamicBatches(crns, resourceType, 900000) // 900KB to leave buffer
2747+
2748+
for _, batch := range batches {
2749+
batchTags, err := fetchTagsForBatch(gsClient, batch, resourceType, tagType, meta)
2750+
if err != nil {
2751+
// If batch fails, try splitting it further
2752+
if len(batch) > 1 {
2753+
log.Printf("[WARN] Batch of %d CRNs failed, splitting into smaller batches", len(batch))
2754+
halfSize := len(batch) / 2
2755+
subBatch1 := batch[:halfSize]
2756+
subBatch2 := batch[halfSize:]
2757+
2758+
tags1, err1 := fetchTagsForBatch(gsClient, subBatch1, resourceType, tagType, meta)
2759+
if err1 != nil {
2760+
log.Printf("[ERROR] Sub-batch 1 failed: %s", err1)
2761+
} else {
2762+
mergeMaps(tagsMap, tags1)
2763+
}
2764+
2765+
tags2, err2 := fetchTagsForBatch(gsClient, subBatch2, resourceType, tagType, meta)
2766+
if err2 != nil {
2767+
log.Printf("[ERROR] Sub-batch 2 failed: %s", err2)
2768+
} else {
2769+
mergeMaps(tagsMap, tags2)
2770+
}
2771+
} else {
2772+
log.Printf("[ERROR] Single CRN batch failed: %s", err)
2773+
}
2774+
continue
2775+
}
2776+
2777+
mergeMaps(tagsMap, batchTags)
2778+
}
2779+
2780+
return tagsMap, nil
2781+
}
2782+
2783+
// createDynamicBatches splits CRNs into batches that stay under the size limit
2784+
func createDynamicBatches(crns []string, resourceType string, maxSizeBytes int) [][]string {
2785+
var batches [][]string
2786+
var currentBatch []string
2787+
currentSize := 0
2788+
2789+
// Base query overhead
2790+
baseOverhead := 200 // Generous estimate for JSON structure, fields, etc.
2791+
2792+
for _, crn := range crns {
2793+
var itemSize int
2794+
if strings.Contains(resourceType, "SoftLayer_") {
2795+
// "doc.id:CRN OR " format
2796+
itemSize = len("doc.id:") + len(crn) + len(" OR ")
2797+
} else {
2798+
// "crn:\"CRN\" OR " format
2799+
itemSize = len("crn:\"") + len(crn) + len("\" OR ")
2800+
}
2801+
2802+
// Check if adding this CRN would exceed limit
2803+
if currentSize+itemSize+baseOverhead > maxSizeBytes && len(currentBatch) > 0 {
2804+
batches = append(batches, currentBatch)
2805+
currentBatch = []string{crn}
2806+
currentSize = itemSize
2807+
} else {
2808+
currentBatch = append(currentBatch, crn)
2809+
currentSize += itemSize
2810+
}
2811+
}
2812+
2813+
// Add final batch
2814+
if len(currentBatch) > 0 {
2815+
batches = append(batches, currentBatch)
2816+
}
2817+
2818+
return batches
2819+
}
2820+
2821+
// fetchTagsForBatch fetches tags for a single batch of CRNs
2822+
func fetchTagsForBatch(gsClient globalsearchv2.GlobalSearchV2, crns []string, resourceType, tagType string, meta interface{}) (map[string]*schema.Set, error) {
2823+
tagsMap := make(map[string]*schema.Set)
2824+
2825+
options := &globalsearchv2.SearchOptions{}
2826+
var query string
2827+
2828+
// Build bulk query
2829+
if strings.Contains(resourceType, "SoftLayer_") {
2830+
queryParts := make([]string, len(crns))
2831+
for i, crn := range crns {
2832+
queryParts[i] = fmt.Sprintf("doc.id:%s", crn)
2833+
}
2834+
query = fmt.Sprintf("(%s) AND family:ims", strings.Join(queryParts, " OR "))
2835+
} else {
2836+
queryParts := make([]string, len(crns))
2837+
for i, crn := range crns {
2838+
queryParts[i] = fmt.Sprintf("crn:\"%s\"", crn)
2839+
}
2840+
query = strings.Join(queryParts, " OR ")
2841+
}
2842+
2843+
options.SetQuery(query)
2844+
2845+
if tagType == "service" {
2846+
userDetails, err := meta.(conns.ClientSession).BluemixUserDetails()
2847+
if err != nil {
2848+
return nil, err
2849+
}
2850+
options.SetAccountID(userDetails.UserAccount)
2851+
}
2852+
2853+
options.SetFields([]string{"access_tags", "tags", "service_tags", "crn"})
2854+
options.SetLimit(500)
2855+
2856+
// Handle pagination
2857+
for {
2858+
result, resp, err := gsClient.Search(options)
2859+
if err != nil {
2860+
return nil, fmt.Errorf("[ERROR] Error querying tags (batch size: %d): %s %s", len(crns), err, resp)
2861+
}
2862+
2863+
// Process items
2864+
for _, item := range result.Items {
2865+
crnProp := item.GetProperty("crn")
2866+
if crnProp == nil {
2867+
continue
2868+
}
2869+
crn := fmt.Sprintf("%v", crnProp)
2870+
2871+
var t interface{}
2872+
if tagType == "access" {
2873+
t = item.GetProperty("access_tags")
2874+
} else if tagType == "service" {
2875+
t = item.GetProperty("service_tags")
2876+
} else {
2877+
t = item.GetProperty("tags")
2878+
}
2879+
2880+
var taglist []string
2881+
if t != nil {
2882+
switch reflect.TypeOf(t).Kind() {
2883+
case reflect.Slice:
2884+
s := reflect.ValueOf(t)
2885+
for j := 0; j < s.Len(); j++ {
2886+
tag := fmt.Sprintf("%s", s.Index(j))
2887+
taglist = append(taglist, tag)
2888+
}
2889+
}
2890+
}
2891+
2892+
tagsMap[crn] = NewStringSet(ResourceIBMVPCHash, taglist)
2893+
}
2894+
2895+
// Check for pagination
2896+
if result.SearchCursor == nil || *result.SearchCursor == "" {
2897+
break
2898+
}
2899+
options.SetSearchCursor(*result.SearchCursor)
2900+
}
2901+
2902+
return tagsMap, nil
2903+
}
2904+
2905+
// mergeMaps merges source map into destination map
2906+
func mergeMaps(dest, src map[string]*schema.Set) {
2907+
for k, v := range src {
2908+
dest[k] = v
2909+
}
2910+
}
2911+
27282912
func GetGlobalTagsUsingCRN(meta interface{}, resourceID, resourceType, tagType string) (*schema.Set, error) {
27292913
taggingResult, err := GetGlobalTagsUsingSearchAPI(meta, resourceID, resourceType, tagType)
27302914
if err != nil {

‎ibm/service/vpc/data_source_ibm_is_images.go‎

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,7 @@ func dataSourceIBMISImagesRead(context context.Context, d *schema.ResourceData,
373373
}
374374
return nil
375375
}
376+
376377
func imageList(context context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
377378
sess, err := vpcClient(meta)
378379
if err != nil {
@@ -400,17 +401,16 @@ func imageList(context context.Context, d *schema.ResourceData, meta interface{}
400401
// Apply filters that aren't supported by API
401402
filteredImages := filterImages(allImages, d)
402403

403-
// Process images concurrently
404+
// Process images concurrently with bulk tag fetching
404405
imagesInfo, err := processImagesConcurrently(context, filteredImages, meta)
405406
if err != nil {
406407
return diag.FromErr(err)
407408
}
408409

409-
d.SetId(time.Now().UTC().String())
410+
d.SetId(dataSourceIBMISImagesID(d))
410411
if err = d.Set("images", imagesInfo); err != nil {
411412
return flex.DiscriminatedTerraformErrorf(err, fmt.Sprintf("Error setting images %s", err), "(Data) ibm_is_images", "read", "images-set").GetDiag()
412413
}
413-
414414
return nil
415415
}
416416

@@ -478,15 +478,40 @@ func filterImages(images []vpcv1.Image, d *schema.ResourceData) []vpcv1.Image {
478478
}
479479

480480
func processImagesConcurrently(ctx context.Context, images []vpcv1.Image, meta interface{}) ([]map[string]interface{}, error) {
481+
// Collect all CRNs that need tag fetching (non-stock, non-remote images)
482+
var crnsForBulkFetch []string
483+
484+
for _, image := range images {
485+
isStockImage := strings.HasPrefix(strings.ToLower(*image.Name), "ibm")
486+
isRemoteImage := image.Remote != nil
487+
488+
if !isStockImage && !isRemoteImage {
489+
crnsForBulkFetch = append(crnsForBulkFetch, *image.CRN)
490+
}
491+
}
492+
493+
// Bulk fetch all tags in one API call
494+
var bulkTags map[string]*schema.Set
495+
if len(crnsForBulkFetch) > 0 {
496+
var err error
497+
bulkTags, err = flex.GetGlobalTagsUsingCRNBulk(meta, crnsForBulkFetch, "", isImageAccessTagType)
498+
if err != nil {
499+
log.Printf("[ERROR] Bulk tag fetch failed: %s", err)
500+
bulkTags = make(map[string]*schema.Set) // Initialize empty map to avoid nil checks
501+
}
502+
} else {
503+
bulkTags = make(map[string]*schema.Set)
504+
}
505+
506+
// Process images concurrently
481507
type result struct {
482508
index int
483509
data map[string]interface{}
484510
err error
485511
}
486512

487513
results := make(chan result, len(images))
488-
sem := make(chan struct{}, 10) // Limit concurrent goroutines to 10
489-
514+
sem := make(chan struct{}, 10) // Limit to 10 concurrent goroutines
490515
var wg sync.WaitGroup
491516

492517
for idx, image := range images {
@@ -496,7 +521,7 @@ func processImagesConcurrently(ctx context.Context, images []vpcv1.Image, meta i
496521
sem <- struct{}{} // Acquire
497522
defer func() { <-sem }() // Release
498523

499-
data, err := processImage(ctx, img, meta)
524+
data, err := processImageWithBulkTags(ctx, img, bulkTags)
500525
results <- result{index: i, data: data, err: err}
501526
}(idx, image)
502527
}
@@ -527,7 +552,7 @@ func processImagesConcurrently(ctx context.Context, images []vpcv1.Image, meta i
527552
return imagesInfo, nil
528553
}
529554

530-
func processImage(ctx context.Context, image vpcv1.Image, meta interface{}) (map[string]interface{}, error) {
555+
func processImageWithBulkTags(ctx context.Context, image vpcv1.Image, bulkTags map[string]*schema.Set) (map[string]interface{}, error) {
531556
l := map[string]interface{}{
532557
"name": *image.Name,
533558
"id": *image.ID,
@@ -598,16 +623,12 @@ func processImage(ctx context.Context, image vpcv1.Image, meta interface{}) (map
598623
l["allowed_use"] = []map[string]interface{}{modelMap}
599624
}
600625

601-
// Only fetch access tags for user-owned, non-stock images
602-
// Skip if: 1) Remote/provider image, 2) IBM stock image
626+
// Add access tags from bulk fetch (only for non-stock)
603627
isStockImage := strings.HasPrefix(strings.ToLower(*image.Name), "ibm")
604628

605629
if !isStockImage {
606-
accesstags, err := flex.GetGlobalTagsUsingCRN(meta, *image.CRN, "", isImageAccessTagType)
607-
if err != nil {
608-
log.Printf("[WARN] Error fetching access tags for image %s: %s", *image.ID, err)
609-
} else {
610-
l[isImageAccessTags] = accesstags
630+
if tags, exists := bulkTags[*image.CRN]; exists {
631+
l[isImageAccessTags] = tags
611632
}
612633
}
613634

0 commit comments

Comments
 (0)