Skip to content

Commit 72edcaf

Browse files
sravani-bobbalasravani-bobbala
authored
Add support for dataproc cluster idle_stop_ttl and auto_stop_time (#15859)
1 parent 42fb617 commit 72edcaf

File tree

4 files changed

+174
-1
lines changed

4 files changed

+174
-1
lines changed

‎mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1571,7 +1571,7 @@ by Dataproc`,
15711571
Optional: true,
15721572
MaxItems: 1,
15731573
AtLeastOneOf: clusterConfigKeys,
1574-
Description: `The settings for auto deletion cluster schedule.`,
1574+
Description: `The settings for auto stop and deletion cluster schedule.`,
15751575
Elem: &schema.Resource{
15761576
Schema: map[string]*schema.Schema{
15771577
"idle_delete_ttl": {
@@ -1581,6 +1581,8 @@ by Dataproc`,
15811581
AtLeastOneOf: []string{
15821582
"cluster_config.0.lifecycle_config.0.idle_delete_ttl",
15831583
"cluster_config.0.lifecycle_config.0.auto_delete_time",
1584+
"cluster_config.0.lifecycle_config.0.idle_stop_ttl",
1585+
"cluster_config.0.lifecycle_config.0.auto_stop_time",
15841586
},
15851587
},
15861588
"idle_start_time": {
@@ -1600,6 +1602,35 @@ by Dataproc`,
16001602
AtLeastOneOf: []string{
16011603
"cluster_config.0.lifecycle_config.0.idle_delete_ttl",
16021604
"cluster_config.0.lifecycle_config.0.auto_delete_time",
1605+
"cluster_config.0.lifecycle_config.0.idle_stop_ttl",
1606+
"cluster_config.0.lifecycle_config.0.auto_stop_time",
1607+
},
1608+
},
1609+
"idle_stop_ttl": {
1610+
Type: schema.TypeString,
1611+
Optional: true,
1612+
Description: `The duration to keep the cluster started while idling (no jobs running). After this TTL, the cluster will be stopped. Valid range: [10m, 14d].`,
1613+
AtLeastOneOf: []string{
1614+
"cluster_config.0.lifecycle_config.0.idle_delete_ttl",
1615+
"cluster_config.0.lifecycle_config.0.auto_delete_time",
1616+
"cluster_config.0.lifecycle_config.0.idle_stop_ttl",
1617+
"cluster_config.0.lifecycle_config.0.auto_stop_time",
1618+
},
1619+
},
1620+
// the API also has the auto_stop_ttl option in its request, however,
1621+
// the value is not returned in the response, rather the auto_stop_time
1622+
// after calculating ttl with the update time is returned, thus, for now
1623+
// we will only allow auto_stop_time to updated.
1624+
"auto_stop_time": {
1625+
Type: schema.TypeString,
1626+
Optional: true,
1627+
Description: `The time when cluster will be auto-stopped. A timestamp in RFC3339 UTC "Zulu" format, accurate to nanoseconds. Example: "2014-10-02T15:01:23.045123456Z".`,
1628+
DiffSuppressFunc: tpgresource.TimestampDiffSuppress(time.RFC3339Nano),
1629+
AtLeastOneOf: []string{
1630+
"cluster_config.0.lifecycle_config.0.idle_delete_ttl",
1631+
"cluster_config.0.lifecycle_config.0.auto_delete_time",
1632+
"cluster_config.0.lifecycle_config.0.idle_stop_ttl",
1633+
"cluster_config.0.lifecycle_config.0.auto_stop_time",
16031634
},
16041635
},
16051636
},
@@ -2462,6 +2493,12 @@ func expandLifecycleConfig(cfg map[string]interface{}) *dataproc.LifecycleConfig
24622493
if v, ok := cfg["auto_delete_time"]; ok {
24632494
conf.AutoDeleteTime = v.(string)
24642495
}
2496+
if v, ok := cfg["idle_stop_ttl"]; ok {
2497+
conf.IdleStopTtl = v.(string)
2498+
}
2499+
if v, ok := cfg["auto_stop_time"]; ok {
2500+
conf.AutoStopTime = v.(string)
2501+
}
24652502
return conf
24662503
}
24672504

@@ -2780,6 +2817,28 @@ func resourceDataprocClusterUpdate(d *schema.ResourceData, meta interface{}) err
27802817
updMask = append(updMask, "config.lifecycle_config.auto_delete_time")
27812818
}
27822819

2820+
if d.HasChange("cluster_config.0.lifecycle_config.0.idle_stop_ttl") {
2821+
idleStopTtl := d.Get("cluster_config.0.lifecycle_config.0.idle_stop_ttl").(string)
2822+
cluster.Config.LifecycleConfig = &dataproc.LifecycleConfig{
2823+
IdleStopTtl: idleStopTtl,
2824+
}
2825+
2826+
updMask = append(updMask, "config.lifecycle_config.idle_stop_ttl")
2827+
}
2828+
2829+
if d.HasChange("cluster_config.0.lifecycle_config.0.auto_stop_time") {
2830+
desiredStopTime := d.Get("cluster_config.0.lifecycle_config.0.auto_stop_time").(string)
2831+
if cluster.Config.LifecycleConfig != nil {
2832+
cluster.Config.LifecycleConfig.AutoStopTime = desiredStopTime
2833+
} else {
2834+
cluster.Config.LifecycleConfig = &dataproc.LifecycleConfig{
2835+
AutoStopTime: desiredStopTime,
2836+
}
2837+
}
2838+
2839+
updMask = append(updMask, "config.lifecycle_config.auto_stop_time")
2840+
}
2841+
27832842
if len(updMask) > 0 {
27842843
gracefulDecommissionTimeout := d.Get("graceful_decommission_timeout").(string)
27852844

@@ -3124,6 +3183,8 @@ func flattenLifecycleConfig(d *schema.ResourceData, lc *dataproc.LifecycleConfig
31243183
data := map[string]interface{}{
31253184
"idle_delete_ttl": lc.IdleDeleteTtl,
31263185
"auto_delete_time": lc.AutoDeleteTime,
3186+
"idle_stop_ttl": lc.IdleStopTtl,
3187+
"auto_stop_time": lc.AutoStopTime,
31273188
}
31283189

31293190
return []map[string]interface{}{data}

‎mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_meta.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ fields:
4545
- field: 'cluster_config.initialization_action.timeout_sec'
4646
- field: 'cluster_config.lifecycle_config.auto_delete_time'
4747
- field: 'cluster_config.lifecycle_config.idle_delete_ttl'
48+
- field: 'cluster_config.lifecycle_config.auto_stop_time'
49+
- field: 'cluster_config.lifecycle_config.idle_stop_ttl'
4850
- field: 'cluster_config.lifecycle_config.idle_start_time'
4951
- field: 'cluster_config.master_config.accelerators.accelerator_count'
5052
- field: 'cluster_config.master_config.accelerators.accelerator_type'

‎mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -968,6 +968,69 @@ func TestAccDataprocCluster_withLifecycleConfigAutoDeletion(t *testing.T) {
968968
})
969969
}
970970

971+
func TestAccDataprocCluster_withLifecycleConfigIdleStopTtl(t *testing.T) {
972+
t.Parallel()
973+
974+
rnd := acctest.RandString(t, 10)
975+
networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster")
976+
subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName)
977+
acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName)
978+
var cluster dataproc.Cluster
979+
980+
acctest.VcrTest(t, resource.TestCase{
981+
PreCheck: func() { acctest.AccTestPreCheck(t) },
982+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
983+
CheckDestroy: testAccCheckDataprocClusterDestroy(t),
984+
Steps: []resource.TestStep{
985+
{
986+
Config: testAccDataprocCluster_withLifecycleConfigIdleStopTtl(rnd, "600s", subnetworkName),
987+
Check: resource.ComposeTestCheckFunc(
988+
testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_lifecycle_config", &cluster),
989+
),
990+
},
991+
{
992+
Config: testAccDataprocCluster_withLifecycleConfigIdleStopTtl(rnd, "610s", subnetworkName),
993+
Check: resource.ComposeTestCheckFunc(
994+
testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_lifecycle_config", &cluster),
995+
),
996+
},
997+
},
998+
})
999+
}
1000+
1001+
func TestAccDataprocCluster_withLifecycleConfigAutoStop(t *testing.T) {
1002+
// Uses time.Now
1003+
acctest.SkipIfVcr(t)
1004+
t.Parallel()
1005+
1006+
rnd := acctest.RandString(t, 10)
1007+
now := time.Now()
1008+
fmtString := "2006-01-02T15:04:05.072Z"
1009+
networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster")
1010+
subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName)
1011+
acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName)
1012+
1013+
var cluster dataproc.Cluster
1014+
acctest.VcrTest(t, resource.TestCase{
1015+
PreCheck: func() { acctest.AccTestPreCheck(t) },
1016+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
1017+
CheckDestroy: testAccCheckDataprocClusterDestroy(t),
1018+
Steps: []resource.TestStep{
1019+
{
1020+
Config: testAccDataprocCluster_withLifecycleConfigAutoStopTime(rnd, now.Add(time.Hour*10).Format(fmtString), subnetworkName),
1021+
Check: resource.ComposeTestCheckFunc(
1022+
testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_lifecycle_config", &cluster),
1023+
),
1024+
},
1025+
{
1026+
Config: testAccDataprocCluster_withLifecycleConfigAutoStopTime(rnd, now.Add(time.Hour*20).Format(fmtString), subnetworkName),
1027+
Check: resource.ComposeTestCheckFunc(
1028+
testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_lifecycle_config", &cluster),
1029+
),
1030+
},
1031+
},
1032+
})
1033+
}
9711034
func TestAccDataprocCluster_withLabels(t *testing.T) {
9721035
t.Parallel()
9731036

@@ -2669,6 +2732,44 @@ resource "google_dataproc_cluster" "with_lifecycle_config" {
26692732
`, rnd, subnetworkName, tm)
26702733
}
26712734

2735+
func testAccDataprocCluster_withLifecycleConfigIdleStopTtl(rnd, tm, subnetworkName string) string {
2736+
return fmt.Sprintf(`
2737+
resource "google_dataproc_cluster" "with_lifecycle_config" {
2738+
name = "tf-test-dproc-%s"
2739+
region = "us-central1"
2740+
2741+
cluster_config {
2742+
gce_cluster_config {
2743+
subnetwork = "%s"
2744+
}
2745+
2746+
lifecycle_config {
2747+
idle_stop_ttl = "%s"
2748+
}
2749+
}
2750+
}
2751+
`, rnd, subnetworkName, tm)
2752+
}
2753+
2754+
func testAccDataprocCluster_withLifecycleConfigAutoStopTime(rnd, tm, subnetworkName string) string {
2755+
return fmt.Sprintf(`
2756+
resource "google_dataproc_cluster" "with_lifecycle_config" {
2757+
name = "tf-test-dproc-%s"
2758+
region = "us-central1"
2759+
2760+
cluster_config {
2761+
gce_cluster_config {
2762+
subnetwork = "%s"
2763+
}
2764+
2765+
lifecycle_config {
2766+
auto_stop_time = "%s"
2767+
}
2768+
}
2769+
}
2770+
`, rnd, subnetworkName, tm)
2771+
}
2772+
26722773
func testAccDataprocCluster_withServiceAcc(sa, rnd, subnetworkName string) string {
26732774
return fmt.Sprintf(`
26742775
data "google_project" "project" {}

‎mmv1/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -944,6 +944,8 @@ cluster_config {
944944
lifecycle_config {
945945
idle_delete_ttl = "10m"
946946
auto_delete_time = "2120-01-01T12:00:00.01Z"
947+
idle_stop_ttl = "10m"
948+
auto_stop_time = "2120-01-01T12:00:00.01Z"
947949
}
948950
}
949951
```
@@ -955,6 +957,13 @@ cluster_config {
955957
A timestamp in RFC3339 UTC "Zulu" format, accurate to nanoseconds.
956958
Example: "2014-10-02T15:01:23.045123456Z".
957959

960+
* `idle_stop_ttl` - (Optional) The duration to keep the cluster alive while idling
961+
(no jobs running). After this TTL, the cluster will be stopped. Valid range: [10m, 14d].
962+
963+
* `auto_stop_time` - (Optional) The time when cluster will be auto-stopped.
964+
A timestamp in RFC3339 UTC "Zulu" format, accurate to nanoseconds.
965+
Example: "2014-10-02T15:01:23.045123456Z".
966+
958967
- - -
959968

960969
<a name="nested_endpoint_config"></a>The `endpoint_config` block (Optional, Computed, Beta) supports:

0 commit comments

Comments
 (0)