Merge pull request #536 from SumoLogic/INVS-1408-match-lists

INVS-1408: resource_sumologic_cse_match_list bug fixes

Author: sumovishal

CHANGELOG.md

@@ -5,6 +5,7 @@ FEATURES:
 
 BUG FIXES:
 * Enforce validation of `group_by_fields` in cse_*_rule resources, on non empty string elements. (GH-535)
+* Fixes `resource_sumologic_cse_match_list` to allow for match lists with more than 1000 items to be created. (INVS-1408)
 
 ## 2.23.0 (May 24, 2023)
 FEATURES:

README.md

@@ -53,8 +53,20 @@ Then run `terraform init` to initialize it.
 
 In order to run the full suite of Acceptance tests, run `make testacc`.
 
+To run a specific acceptance test, run `go test -v ./sumologic  -run YourSpecificTestName`
+
 *Note:* 
-- Acceptance tests *create real resources*, and often cost money to run. The environment variables `SUMOLOGIC_ACCESSID`, `SUMOLOGIC_ACCESSKEY`, and `SUMOLOGIC_ENVIRONMENT` must also be set for acceptance tests to work properly.
+- Acceptance tests *create real resources*, and often cost money to run. The environment variables `SUMOLOGIC_ACCESSID`, `SUMOLOGIC_ACCESSKEY`, `SUMOLOGIC_ENVIRONMENT` / `SUMOLOGIC_BASE_URL`,  and `TF_ACC` must also be set for acceptance tests to work properly.
+  - For example, you can generate a personal access key in stag-alpha [here](https://cse-stag-alpha.stag.sumologic.net/ui/#/preferences). Once your test runs, you are then capable of viewing the real resources created by Terraform in the [stag-alpha UI](https://cse-stag-alpha.stag.sumologic.net/sec/hud?hours=24).
+     ```sh
+     $ export SUMOLOGIC_ACCESSID="yourAccessID"
+     $ export SUMOLOGIC_ACCESSKEY="yourAccessKey"
+     $ export SUMOLOGIC_ENVIRONMENT="stag"
+     $ export SUMOLOGIC_BASE_URL="https://stag-api.sumologic.net/api/"
+     $ export TF_ACC=1
+     ```
+  - More information on configuration can be found at the [Terraform Provider codelabs documentation](https://github.com/Sanyaku/codelabs/blob/master/backend/pages/SumoLogicTerraformProvider.md).
+
 - Environment variable `SUMOLOGIC_TEST_GOOGLE_APPLICATION_CREDENTIALS` must be set for gcp metrics acceptance tests to work properly (ex. below).
     - export SUMOLOGIC_TEST_GOOGLE_APPLICATION_CREDENTIALS=`cat /path/to/service_acccount.json`
     - Set Environment variable `SUMOLOGIC_ENABLE_GCP_METRICS_ACC_TESTS` to false, to disable acceptance test for Gcp Metrics. 

sumologic/resource_sumologic_cse_match_list.go

@@ -2,10 +2,11 @@ package sumologic
 
 import (
 	"fmt"
-	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
-	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
 	"log"
 	"time"
+
+	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
+	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
 )
 
 func resourceSumologicCSEMatchList() *schema.Resource {
@@ -56,7 +57,7 @@ func resourceSumologicCSEMatchList() *schema.Resource {
 				Computed: true,
 			},
 			"items": {
-				Type:     schema.TypeList,
+				Type:     schema.TypeSet,
 				Optional: true,
 				Elem: &schema.Resource{
 					Schema: map[string]*schema.Schema{
@@ -114,13 +115,11 @@ func resourceSumologicCSEMatchListRead(d *schema.ResourceData, meta interface{})
 	d.Set("last_updated", CSEMatchList.LastUpdated)
 	d.Set("last_updated_by", CSEMatchList.LastUpdatedBy)
 
-	//items
-	var CSEMatchListItems *CSEMatchListItemsInMatchListGet
-
-	CSEMatchListItems, err2 := c.GetCSEMatchListItemsInMatchList(id)
-	if err2 != nil {
-		log.Printf("[WARN] CSE Match List items not found when looking by match list id: %s, err: %v", id, err2)
+	CSEMatchListItems, err := c.GetCSEMatchListItemsInMatchList(id)
+	if err != nil {
+		log.Printf("[WARN] CSE Match List items not found when looking by match list id: %s, err: %v", id, err)
 	}
+
 	if CSEMatchListItems == nil {
 		d.Set("items", nil)
 	} else {
@@ -152,7 +151,6 @@ func resourceSumologicCSEMatchListDelete(d *schema.ResourceData, meta interface{
 	c := meta.(*Client)
 	err := c.DeleteCSEMatchList(d.Id())
 	return err
-
 }
 
 func resourceSumologicCSEMatchListCreate(d *schema.ResourceData, meta interface{}) error {
@@ -172,19 +170,17 @@ func resourceSumologicCSEMatchListCreate(d *schema.ResourceData, meta interface{
 		}
 		d.SetId(id)
 
-		//Match list items
-		itemsData := d.Get("items").([]interface{})
+		itemsData := d.Get("items").(*schema.Set).List()
 		var items []CSEMatchListItemPost
 		for _, data := range itemsData {
-			item, _ := resourceToCSEMatchListItem([]interface{}{data})
+			item := resourceToCSEMatchListItem([]interface{}{data})
 			items = append(items, item)
-
 		}
 
 		if len(items) > 0 {
-			err2 := c.CreateCSEMatchListItems(items, id)
-			if err2 != nil {
-				log.Printf("[WARN] An error occurred while adding match list items to match list id: %s, err: %v", id, err2)
+			err = c.CreateCSEMatchListItems(items, id)
+			if err != nil {
+				log.Printf("[WARN] An error occurred while adding match list items to match list id: %s, err: %v", id, err)
 			}
 
 		}
@@ -196,6 +192,7 @@ func resourceSumologicCSEMatchListCreate(d *schema.ResourceData, meta interface{
 			Refresh: func() (interface{}, string, error) {
 				resp, err := c.GetCSEMatchListItemsInMatchList(d.Id())
 				if err != nil {
+					log.Printf("[WARN] CSE Match List items not found when looking by match list id: %s, err: %v", d.Id(), err)
 					return 0, "", err
 				}
 				return resp, fmt.Sprint(resp.Total), nil
@@ -216,7 +213,7 @@ func resourceSumologicCSEMatchListCreate(d *schema.ResourceData, meta interface{
 	return resourceSumologicCSEMatchListRead(d, meta)
 }
 
-func resourceToCSEMatchListItem(data interface{}) (CSEMatchListItemPost, string) {
+func resourceToCSEMatchListItem(data interface{}) CSEMatchListItemPost {
 	itemsSlice := data.([]interface{})
 	item := CSEMatchListItemPost{}
 	if len(itemsSlice) > 0 {
@@ -227,7 +224,7 @@ func resourceToCSEMatchListItem(data interface{}) (CSEMatchListItemPost, string)
 		item.Expiration = itemObj["expiration"].(string)
 		item.Value = itemObj["value"].(string)
 	}
-	return item, item.ID
+	return item
 }
 
 func resourceSumologicCSEMatchListUpdate(d *schema.ResourceData, meta interface{}) error {
@@ -241,51 +238,83 @@ func resourceSumologicCSEMatchListUpdate(d *schema.ResourceData, meta interface{
 		return err
 	}
 
-	//Match list items
-	itemsData := d.Get("items").([]interface{})
-	var itemIds []string
-	var items []CSEMatchListItemPost
+	itemsData := d.Get("items").(*schema.Set).List()
+	var newItems []CSEMatchListItemPost
 	for _, data := range itemsData {
-		item, id := resourceToCSEMatchListItem([]interface{}{data})
-		item.ID = ""
-		items = append(items, item)
-		itemIds = append(itemIds, id)
+		item := resourceToCSEMatchListItem([]interface{}{data})
+		newItems = append(newItems, item)
+	}
 
+	CSEMatchListItems, err := c.GetCSEMatchListItemsInMatchList(d.Id())
+	if err != nil {
+		log.Printf("[WARN] CSE Match List items not found when looking by match list id: %s, err: %v", d.Id(), err)
 	}
+	var oldItemIds []string
+	for _, item := range CSEMatchListItems.CSEMatchListItemsGetObjects {
+		oldItemIds = append(oldItemIds, item.ID)
+	}
+
+	oldItemCount := CSEMatchListItems.Total
+	newItemCount := len(itemsData) + oldItemCount
 
-	if len(items) > 0 {
-		err2 := c.CreateCSEMatchListItems(items, d.Id())
-		if err2 != nil {
-			log.Printf("[WARN] An error occurred while adding match list items to match list id: %s, err: %v", d.Id(), err2)
+	//Add new items
+	if len(newItems) > 0 {
+		err = c.CreateCSEMatchListItems(newItems, d.Id())
+		if err != nil {
+			log.Printf("[WARN] An error occurred while adding match list items to match list id: %s, err: %v", d.Id(), err)
 		}
 
 	}
 
-	var CSEMatchListItems *CSEMatchListItemsInMatchListGet
+	CSEMatchListItems, err = c.GetCSEMatchListItemsInMatchList(d.Id())
+	if err != nil {
+		log.Printf("[WARN] CSE Match List items not found when looking by match list id: %s, err: %v", d.Id(), err)
+	}
 
-	CSEMatchListItems, err2 := c.GetCSEMatchListItemsInMatchList(d.Id())
-	if err2 != nil {
-		log.Printf("[WARN] CSE Match List items not found when looking by match list id: %s, err: %v", d.Id(), err2)
+	// Wait for addition to finish
+	createStateConf := &resource.StateChangeConf{
+		Target: []string{
+			fmt.Sprint(newItemCount),
+		},
+		Refresh: func() (interface{}, string, error) {
+			CSEMatchListItems, err = c.GetCSEMatchListItemsInMatchList(d.Id())
+			if err != nil {
+				log.Printf("[WARN] CSE Match List items not found when looking by match list id: %s, err: %v", d.Id(), err)
+				return 0, "", err
+			}
+
+			return CSEMatchListItems, fmt.Sprint(CSEMatchListItems.Total), nil
+		},
+		Timeout:                   d.Timeout(schema.TimeoutUpdate),
+		Delay:                     10 * time.Second,
+		MinTimeout:                5 * time.Second,
+		ContinuousTargetOccurence: 1,
 	}
-	if CSEMatchListItems != nil {
 
-		for _, t := range CSEMatchListItems.CSEMatchListItemsGetObjects {
-			if !contains(itemIds, t.ID) {
-				err3 := c.DeleteCSEMatchListItem(t.ID)
-				if err3 != nil {
-					log.Printf("[WARN] An error occurred deleting match list item with id: %s, err: %v", t.ID, err3)
-				}
+	_, err = createStateConf.WaitForState()
+	if err != nil {
+		return fmt.Errorf("error waiting for match list (%s) to be updated: %s", d.Id(), err)
+	}
+
+	// Delete old items
+	for _, t := range CSEMatchListItems.CSEMatchListItemsGetObjects {
+		if contains(oldItemIds, t.ID) {
+			err = c.DeleteCSEMatchListItem(t.ID)
+			if err != nil {
+				log.Printf("[WARN] An error occurred deleting match list item with id: %s, err: %v", t.ID, err)
 			}
 		}
 	}
 
-	createStateConf := &resource.StateChangeConf{
+	// Wait for deletion to finish
+	createStateConf = &resource.StateChangeConf{
 		Target: []string{
-			fmt.Sprint(len(items)),
+			fmt.Sprint(len(newItems)),
 		},
 		Refresh: func() (interface{}, string, error) {
 			resp, err := c.GetCSEMatchListItemsInMatchList(d.Id())
 			if err != nil {
+				log.Printf("[WARN] CSE Match List items not found when looking by match list id: %s, err: %v", d.Id(), err)
 				return 0, "", err
 			}
 			return resp, fmt.Sprint(resp.Total), nil