feat: support new data expiration rule attribute in online archive (#1528)

* feat: support new data expiration rule attribute in online archive

* addressing docs review

Author: AgustinBettati

examples/online-archives/main.tf

@@ -11,6 +11,10 @@ resource "mongodbatlas_online_archive" "users_archive" {
     expire_after_days = 2
   }
 
+  data_expiration_rule {
+    expire_after_days = 90
+  }
+
   partition_fields {
     field_name = "created"
     order      = 0

mongodbatlas/data_source_mongodbatlas_online_archive.go

@@ -118,6 +118,18 @@ func schemaOnlineArchive() map[string]*schema.Schema {
 				},
 			},
 		},
+		"data_expiration_rule": {
+			Type:     schema.TypeList,
+			Computed: true,
+			Elem: &schema.Resource{
+				Schema: map[string]*schema.Schema{
+					"expire_after_days": {
+						Type:     schema.TypeInt,
+						Computed: true,
+					},
+				},
+			},
+		},
 		"schedule": {
 			Type:     schema.TypeList,
 			Computed: true,

mongodbatlas/resource_mongodbatlas_online_archive.go

@@ -98,6 +98,20 @@ func getMongoDBAtlasOnlineArchiveSchema() map[string]*schema.Schema {
 				},
 			},
 		},
+		"data_expiration_rule": {
+			Type:     schema.TypeList,
+			MinItems: 1,
+			MaxItems: 1,
+			Optional: true,
+			Elem: &schema.Resource{
+				Schema: map[string]*schema.Schema{
+					"expire_after_days": {
+						Type:     schema.TypeInt,
+						Required: true,
+					},
+				},
+			},
+		},
 		"schedule": {
 			Type:     schema.TypeList,
 			Optional: true,
@@ -352,6 +366,7 @@ func mapToArchivePayload(d *schema.ResourceData) admin.BackupOnlineArchiveCreate
 	}
 
 	requestInput.Criteria = mapCriteria(d)
+	requestInput.DataExpirationRule = mapDataExpirationRule(d)
 	requestInput.Schedule = mapSchedule(d)
 
 	if partitions, ok := d.GetOk("partition_fields"); ok {
@@ -391,31 +406,42 @@ func resourceMongoDBAtlasOnlineArchiveUpdate(ctx context.Context, d *schema.Reso
 	projectID := ids["project_id"]
 	clusterName := ids["cluster_name"]
 
-	// if the criteria or the paused is enable then perform an update
-	paused := d.HasChange("paused")
-	criteria := d.HasChange("criteria")
-	schedule := d.HasChange("schedule")
+	// if the criteria or the pausedHasChange is enable then perform an update
+	pausedHasChange := d.HasChange("paused")
+	criteriaHasChange := d.HasChange("criteria")
+	dataExpirationRuleHasChange := d.HasChange("data_expiration_rule")
+	scheduleHasChange := d.HasChange("schedule")
 
 	collectionTypeHasChange := d.HasChange("collection_type")
 
 	// nothing to do, let's go
-	if !paused && !criteria && !collectionTypeHasChange && !schedule {
+	if !pausedHasChange && !criteriaHasChange && !collectionTypeHasChange && !scheduleHasChange && !dataExpirationRuleHasChange {
 		return nil
 	}
 
 	request := admin.BackupOnlineArchive{}
 
 	// reading current value
-	if paused {
+	if pausedHasChange {
 		request.Paused = pointy.Bool(d.Get("paused").(bool))
 	}
 
-	if criteria {
+	if criteriaHasChange {
 		newCriteria := mapCriteria(d)
 		request.Criteria = &newCriteria
 	}
 
-	if schedule {
+	if dataExpirationRuleHasChange {
+		newExpirationRule := mapDataExpirationRule(d)
+		if newExpirationRule == nil {
+			// expiration rule has been removed from tf config, empty dataExpirationRule object needs to be sent in patch request
+			request.DataExpirationRule = &admin.DataExpirationRule{}
+		} else {
+			request.DataExpirationRule = newExpirationRule
+		}
+	}
+
+	if scheduleHasChange {
 		request.Schedule = mapSchedule(d)
 	}
 
@@ -491,6 +517,14 @@ func fromOnlineArchiveToMap(in *admin.BackupOnlineArchive) map[string]interface{
 		schemaVals["schedule"] = []interface{}{schedule}
 	}
 
+	var dataExpirationRule map[string]interface{}
+	if in.DataExpirationRule != nil && in.DataExpirationRule.ExpireAfterDays != nil {
+		dataExpirationRule = map[string]interface{}{
+			"expire_after_days": in.DataExpirationRule.ExpireAfterDays,
+		}
+		schemaVals["data_expiration_rule"] = []interface{}{dataExpirationRule}
+	}
+
 	// partitions fields
 	if len(in.PartitionFields) == 0 {
 		return schemaVals
@@ -511,6 +545,18 @@ func fromOnlineArchiveToMap(in *admin.BackupOnlineArchive) map[string]interface{
 	return schemaVals
 }
 
+func mapDataExpirationRule(d *schema.ResourceData) *admin.DataExpirationRule {
+	if dataExpireRules, ok := d.GetOk("data_expiration_rule"); ok && len(dataExpireRules.([]interface{})) > 0 {
+		dataExpireRule := dataExpireRules.([]interface{})[0].(map[string]interface{})
+		result := admin.DataExpirationRule{}
+		if expireAfterDays, ok := dataExpireRule["expire_after_days"]; ok {
+			result.ExpireAfterDays = pointy.Int(expireAfterDays.(int))
+		}
+		return &result
+	}
+	return nil
+}
+
 func mapCriteria(d *schema.ResourceData) admin.Criteria {
 	criteriaList := d.Get("criteria").([]interface{})
 

mongodbatlas/resource_mongodbatlas_online_archive_test.go

@@ -16,12 +16,14 @@ import (
 
 func TestAccBackupRSOnlineArchive(t *testing.T) {
 	var (
-		cluster                   matlas.Cluster
-		resourceName              = "mongodbatlas_cluster.online_archive_test"
-		onlineArchiveResourceName = "mongodbatlas_online_archive.users_archive"
-		orgID                     = os.Getenv("MONGODB_ATLAS_ORG_ID")
-		projectName               = acctest.RandomWithPrefix("test-acc")
-		name                      = fmt.Sprintf("test-acc-%s", acctest.RandString(10))
+		cluster                      matlas.Cluster
+		resourceName                 = "mongodbatlas_cluster.online_archive_test"
+		onlineArchiveResourceName    = "mongodbatlas_online_archive.users_archive"
+		onlineArchiveDataSourceName  = "data.mongodbatlas_online_archive.read_archive"
+		onlineArchivesDataSourceName = "data.mongodbatlas_online_archives.all"
+		orgID                        = os.Getenv("MONGODB_ATLAS_ORG_ID")
+		projectName                  = acctest.RandomWithPrefix("test-acc")
+		name                         = fmt.Sprintf("test-acc-%s", acctest.RandString(10))
 	)
 
 	resource.ParallelTest(t, resource.TestCase{
@@ -38,7 +40,7 @@ func TestAccBackupRSOnlineArchive(t *testing.T) {
 				),
 			},
 			{
-				Config: testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, name, 1),
+				Config: testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, name, 1, 7),
 				Check: resource.ComposeTestCheckFunc(
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "state"),
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "archive_id"),
@@ -48,10 +50,13 @@ func TestAccBackupRSOnlineArchive(t *testing.T) {
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "schedule.0.end_minute"),
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "schedule.0.start_hour"),
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "schedule.0.start_minute"),
+					resource.TestCheckResourceAttr(onlineArchiveResourceName, "data_expiration_rule.0.expire_after_days", "7"),
+					resource.TestCheckResourceAttr(onlineArchiveDataSourceName, "data_expiration_rule.0.expire_after_days", "7"),
+					resource.TestCheckResourceAttr(onlineArchivesDataSourceName, "results.0.data_expiration_rule.0.expire_after_days", "7"),
 				),
 			},
 			{
-				Config: testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, name, 2),
+				Config: testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, name, 2, 8),
 				Check: resource.ComposeTestCheckFunc(
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "state"),
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "archive_id"),
@@ -61,6 +66,9 @@ func TestAccBackupRSOnlineArchive(t *testing.T) {
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "schedule.0.end_minute"),
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "schedule.0.start_hour"),
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "schedule.0.start_minute"),
+					resource.TestCheckResourceAttr(onlineArchiveResourceName, "data_expiration_rule.0.expire_after_days", "8"),
+					resource.TestCheckResourceAttr(onlineArchiveDataSourceName, "data_expiration_rule.0.expire_after_days", "8"),
+					resource.TestCheckResourceAttr(onlineArchivesDataSourceName, "results.0.data_expiration_rule.0.expire_after_days", "8"),
 				),
 			},
 			{
@@ -140,7 +148,7 @@ func TestAccBackupRSOnlineArchiveBasic(t *testing.T) {
 				),
 			},
 			{
-				Config: testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, name, 1),
+				Config: testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, name, 1, 1),
 				Check: resource.ComposeTestCheckFunc(
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "state"),
 					resource.TestCheckResourceAttrSet(onlineArchiveResourceName, "archive_id"),
@@ -214,9 +222,9 @@ func populateWithSampleData(resourceName string, cluster *matlas.Cluster) resour
 	}
 }
 
-func testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, clusterName string, startHour int) string {
+func testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, clusterName string, startHour, deleteExpirationDays int) string {
 	return fmt.Sprintf(`
-	%s
+	%[1]s
 	resource "mongodbatlas_online_archive" "users_archive" {
 		project_id = mongodbatlas_cluster.online_archive_test.project_id
 		cluster_name = mongodbatlas_cluster.online_archive_test.name
@@ -231,11 +239,15 @@ func testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, clu
 			expire_after_days = 2
 		}
 
+		data_expiration_rule {
+			expire_after_days = %[3]d
+		}
+
 		schedule {
 			type = "DAILY"
 			end_hour = 1
 			end_minute = 1
-			start_hour = %d
+			start_hour = %[2]d
 			start_minute = 1
 		}
 
@@ -267,7 +279,7 @@ func testAccBackupRSOnlineArchiveConfigWithDailySchedule(orgID, projectName, clu
 		project_id =  mongodbatlas_online_archive.users_archive.project_id
 		cluster_name = mongodbatlas_online_archive.users_archive.cluster_name
 	}
-	`, testAccBackupRSOnlineArchiveConfigFirstStep(orgID, projectName, clusterName), startHour)
+	`, testAccBackupRSOnlineArchiveConfigFirstStep(orgID, projectName, clusterName), startHour, deleteExpirationDays)
 }
 
 func testAccBackupRSOnlineArchiveConfigWithoutSchedule(orgID, projectName, clusterName string) string {

website/docs/d/online_archive.html.markdown

@@ -33,7 +33,12 @@ data "mongodbatlas_online_archive" "test" {
 ## Attributes reference
 * `db_name`          -  Name of the database that contains the collection.
 * `coll_name`        -  Name of the collection.
-* `collection_type`  -  Classification of MongoDB database collection that you want to return, "TIMESERIES" or "STANDARD". Default is "STANDARD". 
+* `collection_type`  -  Type of MongoDB collection that you want to return. This value can be "TIMESERIES" or "STANDARD". Default is "STANDARD".
+* `criteria` - Criteria to use for archiving data. See [criteria](#criteria).
+* `data_expiration_rule` - Rule for specifying when data should be deleted from the archive. See [data expiration rule](#data-expiration-rule).
+* `schedule` - Regular frequency and duration when archiving process occurs. See [schedule](#schedule).
+* `partition_fields` - Fields to use to partition data. You can specify up to two frequently queried fields to use for partitioning data. Queries that don’t contain the specified fields require a full collection scan of all archived documents, which takes longer and increases your costs. To learn more about how partition improves query performance, see [Data Structure in S3](https://docs.mongodb.com/datalake/admin/optimize-query-performance/#data-structure-in-s3). The value of a partition field can be up to a maximum of 700 characters. Documents with values exceeding 700 characters are not archived. See [partition fields](#partition).
+* `paused` - State of the online archive. This is required for pausing an active online archive or resuming a paused online archive. If the collection has another active online archive, the resume request fails.
 * `state`    - Status of the online archive. Valid values are: Pending, Archiving, Idle, Pausing, Paused, Orphaned and Deleted
 
 ### Criteria
@@ -43,6 +48,10 @@ data "mongodbatlas_online_archive" "test" {
 * `expire_after_days` - Number of days after the value in the criteria.dateField when MongoDB Cloud archives data in the specified cluster. Set this parameter when `type` is `DATE`.
 * `query` - JSON query to use to select documents for archiving. Atlas uses the specified query with the db.collection.find(query) command. The empty document {} to return all documents is not supported. Set this parameter when `type` is `CUSTOM`.
 
+### Data Expiration Rule
+* `expire_after_days` - Number of days used in the date criteria for nominating documents for deletion. Value must be between 7 and 9215.
+
+
 ### Schedule
 
 * `type`          - Type of schedule. Valid values: `DEFAULT`, `DAILY`, `MONTHLY`, `WEEKLY`.

website/docs/d/online_archives.html.markdown

@@ -36,27 +36,35 @@ In addition to all arguments above, the following attributes are exported:
 
 # Online Archive
 ## Attributes reference
-* `db_name`          -  Name of the database that contains the collection.
-* `coll_name`        -  Name of the collection.
-* `collection_type`  -  Classification of MongoDB database collection that you want to return, "TIMESERIES" or "STANDARD". Default is "STANDARD". 
-* `state`    - Status of the online archive. Valid values are: Pending, Archiving, Idle, Pausing, Paused, Orphaned and Deleted
+* `db_name` - Name of the database that contains the collection.
+* `coll_name` -  Name of the collection.
+* `collection_type` - Type of MongoDB collection that you want to return. This value can be "TIMESERIES" or "STANDARD". Default is "STANDARD". 
+* `criteria` - Criteria to use for archiving data. See [criteria](#criteria).
+* `data_expiration_rule` - Rule for specifying when data should be deleted from the archive. See [data expiration rule](#data-expiration-rule).
+* `schedule` - Regular frequency and duration when archiving process occurs. See [schedule](#schedule).
+* `partition_fields` - Fields to use to partition data. You can specify up to two frequently queried fields to use for partitioning data. Queries that don’t contain the specified fields require a full collection scan of all archived documents, which takes longer and increases your costs. To learn more about how partition improves query performance, see [Data Structure in S3](https://docs.mongodb.com/datalake/admin/optimize-query-performance/#data-structure-in-s3). The value of a partition field can be up to a maximum of 700 characters. Documents with values exceeding 700 characters are not archived. See [partition fields](#partition).
+* `paused` - State of the online archive. This is required for pausing an active online archive or resuming a paused online archive. If the collection has another active online archive, the resume request fails.
+* `state` - Status of the online archive. Valid values are: Pending, Archiving, Idle, Pausing, Paused, Orphaned and Deleted
 
 ### Criteria
-* `type`          - Type of criteria (DATE, CUSTOM)
-* `date_field`   - Indexed database parameter that stores the date that determines when data moves to the online archive. MongoDB Cloud archives the data when the current date exceeds the date in this database parameter plus the number of days specified through the expireAfterDays parameter. Set this parameter when `type` is `DATE`.
-* `date_format`   - Syntax used to write the date after which data moves to the online archive. Date can be expressed as ISO 8601 or Epoch timestamps. The Epoch timestamp can be expressed as nanoseconds, milliseconds, or seconds. Set this parameter when `type` is `DATE`. You must set `type` to `DATE` if `collectionType` is `TIMESERIES`. Valid values:  ISODATE (default), EPOCH_SECONDS, EPOCH_MILLIS, EPOCH_NANOSECONDS.
+* `type` - Type of criteria (DATE, CUSTOM)
+* `date_field` - Indexed database parameter that stores the date that determines when data moves to the online archive. MongoDB Cloud archives the data when the current date exceeds the date in this database parameter plus the number of days specified through the expireAfterDays parameter. Set this parameter when `type` is `DATE`.
+* `date_format` - Syntax used to write the date after which data moves to the online archive. Date can be expressed as ISO 8601 or Epoch timestamps. The Epoch timestamp can be expressed as nanoseconds, milliseconds, or seconds. Set this parameter when `type` is `DATE`. You must set `type` to `DATE` if `collectionType` is `TIMESERIES`. Valid values:  ISODATE (default), EPOCH_SECONDS, EPOCH_MILLIS, EPOCH_NANOSECONDS.
 * `expire_after_days` - Number of days after the value in the criteria.dateField when MongoDB Cloud archives data in the specified cluster. Set this parameter when `type` is `DATE`.
 * `query` - JSON query to use to select documents for archiving. Atlas uses the specified query with the db.collection.find(query) command. The empty document {} to return all documents is not supported. Set this parameter when `type` is `CUSTOM`.
 
+### Data Expiration Rule
+* `expire_after_days` - Number of days used in the date criteria for nominating documents for deletion. Value must be between 7 and 9215.
+
 ### Schedule
 
-* `type`          - Type of schedule (`DAILY`, `MONTHLY`, `WEEKLY`).
-* `start_hour`    - Hour of the day when the when the scheduled window to run one online archive starts.  
-* `end_hour`      - Hour of the day when the scheduled window to run one online archive ends.
-* `start_minute`   - Minute of the hour when the scheduled window to run one online archive starts.
-* `end_minute`     - Minute of the hour when the scheduled window to run one online archive ends.
-* `day_of_month`   - Day of the month when the scheduled archive starts.
-* `day_of_week`     - Day of the week when the scheduled archive starts. The week starts with Monday (1) and ends with Sunday (7).
+* `type` - Type of schedule (`DAILY`, `MONTHLY`, `WEEKLY`).
+* `start_hour` - Hour of the day when the when the scheduled window to run one online archive starts.  
+* `end_hour` - Hour of the day when the scheduled window to run one online archive ends.
+* `start_minute` - Minute of the hour when the scheduled window to run one online archive starts.
+* `end_minute` - Minute of the hour when the scheduled window to run one online archive ends.
+* `day_of_month` - Day of the month when the scheduled archive starts.
+* `day_of_week` - Day of the week when the scheduled archive starts. The week starts with Monday (1) and ends with Sunday (7).
 
 ### Partition
 * `field_name` - Human-readable label that identifies the parameter that MongoDB Cloud uses to partition data. To specify a nested parameter, use the dot notation.