Skip to content

Commit 99bb678

Browse files
modular-magicianmodular-magician
authored
Add support for reservation affinity, dataproc oss metric collection and node-group configs, also support 'SPOT' under preemptibility (#6889) (#4974)
* Add support for reservation affinity under dataproc cluster * make consume_reservation_type default as false. * Remove Default for values since Default is not allowed for TypeSet. * space * remove space * Remove Default for key and values entry Added test for reservation_affinity Update dataproc doc * fix test case * fix test * fix test * Added metrics related tags. * fix typo * fix beta version * fix beta version * remove AtLeastOneOf if Required is set * fix atLeastOneOf error * fix build issue * add test for metrics. * disable test on beta * fix test * fix test * add reservation_affinity and test case. * update related html doc. * add zone_uri fix test * fix zone test error * fix test * refactory some entry * fix typo * fix typo * fix typo * update html doc * fix test * update based on comments. * update html * refactory * refactory * fix test * fix test * fix test * fix test * change test to catch error * fix test * change to list * fix tests * update text * add test for SPOT option * skip SPOT test for v1beta * guard SPOT with v1 only * text * text * add valid test for reservation affinity * fix test * add valid test for node affinity * fix test * fix tests * increase node count * fix test * fix test * adjust space Signed-off-by: Modular Magician <[email protected]> Signed-off-by: Modular Magician <[email protected]>
1 parent 45adf0d commit 99bb678

File tree

4 files changed

+258
-0
lines changed

4 files changed

+258
-0
lines changed

‎.changelog/6889.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
```release-note:enhancement
2+
dataproc: added support for `reservation_affinity` in `google_dataproc_cluster`
3+
```
4+
```release-note:enhancement
5+
dataproc: added support for `node_group_affinity` in `google_dataproc_cluster`
6+
```
7+
```release-note:enhancement
8+
dataproc: added support for `dataproc_metric_config` in `google_dataproc_cluster`
9+
```
10+
```release-note:enhancement
11+
dataproc: added support for `SPOT` option for `preemptibility` in `google_dataproc_cluster`
12+
```

‎google-beta/resource_dataproc_cluster.go‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ var (
2828
"cluster_config.0.gce_cluster_config.0.internal_ip_only",
2929
"cluster_config.0.gce_cluster_config.0.shielded_instance_config",
3030
"cluster_config.0.gce_cluster_config.0.metadata",
31+
"cluster_config.0.gce_cluster_config.0.reservation_affinity",
32+
"cluster_config.0.gce_cluster_config.0.node_group_affinity",
3133
}
3234

3335
schieldedInstanceConfigKeys = []string{
@@ -36,6 +38,12 @@ var (
3638
"cluster_config.0.gce_cluster_config.0.shielded_instance_config.0.enable_integrity_monitoring",
3739
}
3840

41+
reservationAffinityKeys = []string{
42+
"cluster_config.0.gce_cluster_config.0.reservation_affinity.0.consume_reservation_type",
43+
"cluster_config.0.gce_cluster_config.0.reservation_affinity.0.key",
44+
"cluster_config.0.gce_cluster_config.0.reservation_affinity.0.values",
45+
}
46+
3947
preemptibleWorkerDiskConfigKeys = []string{
4048
"cluster_config.0.preemptible_worker_config.0.disk_config.0.num_local_ssds",
4149
"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_size_gb",
@@ -320,6 +328,62 @@ func resourceDataprocCluster() *schema.Resource {
320328
},
321329
},
322330
},
331+
332+
"reservation_affinity": {
333+
Type: schema.TypeList,
334+
Optional: true,
335+
AtLeastOneOf: gceClusterConfigKeys,
336+
Computed: true,
337+
MaxItems: 1,
338+
Description: `Reservation Affinity for consuming Zonal reservation.`,
339+
Elem: &schema.Resource{
340+
Schema: map[string]*schema.Schema{
341+
"consume_reservation_type": {
342+
Type: schema.TypeString,
343+
Optional: true,
344+
AtLeastOneOf: reservationAffinityKeys,
345+
ForceNew: true,
346+
ValidateFunc: validation.StringInSlice([]string{"NO_RESERVATION", "ANY_RESERVATION", "SPECIFIC_RESERVATION"}, false),
347+
Description: `Type of reservation to consume.`,
348+
},
349+
"key": {
350+
Type: schema.TypeString,
351+
Optional: true,
352+
AtLeastOneOf: reservationAffinityKeys,
353+
ForceNew: true,
354+
Description: `Corresponds to the label key of reservation resource.`,
355+
},
356+
"values": {
357+
Type: schema.TypeSet,
358+
Elem: &schema.Schema{Type: schema.TypeString},
359+
Optional: true,
360+
AtLeastOneOf: reservationAffinityKeys,
361+
ForceNew: true,
362+
Description: `Corresponds to the label values of reservation resource.`,
363+
},
364+
},
365+
},
366+
},
367+
368+
"node_group_affinity": {
369+
Type: schema.TypeList,
370+
Optional: true,
371+
AtLeastOneOf: gceClusterConfigKeys,
372+
Computed: true,
373+
MaxItems: 1,
374+
Description: `Node Group Affinity for sole-tenant clusters.`,
375+
Elem: &schema.Resource{
376+
Schema: map[string]*schema.Schema{
377+
"node_group_uri": {
378+
Type: schema.TypeString,
379+
ForceNew: true,
380+
Required: true,
381+
Description: `The URI of a sole-tenant that the cluster will be created on.`,
382+
DiffSuppressFunc: compareSelfLinkOrResourceName,
383+
},
384+
},
385+
},
386+
},
323387
},
324388
},
325389
},
@@ -1065,6 +1129,26 @@ func expandGceClusterConfig(d *schema.ResourceData, config *Config) (*dataproc.G
10651129
conf.ShieldedInstanceConfig.EnableVtpm = v.(bool)
10661130
}
10671131
}
1132+
if v, ok := d.GetOk("cluster_config.0.gce_cluster_config.0.reservation_affinity"); ok {
1133+
cfgRa := v.([]interface{})[0].(map[string]interface{})
1134+
conf.ReservationAffinity = &dataproc.ReservationAffinity{}
1135+
if v, ok := cfgRa["consume_reservation_type"]; ok {
1136+
conf.ReservationAffinity.ConsumeReservationType = v.(string)
1137+
}
1138+
if v, ok := cfgRa["key"]; ok {
1139+
conf.ReservationAffinity.Key = v.(string)
1140+
}
1141+
if v, ok := cfgRa["values"]; ok {
1142+
conf.ReservationAffinity.Values = convertStringSet(v.(*schema.Set))
1143+
}
1144+
}
1145+
if v, ok := d.GetOk("cluster_config.0.gce_cluster_config.0.node_group_affinity"); ok {
1146+
cfgNga := v.([]interface{})[0].(map[string]interface{})
1147+
conf.NodeGroupAffinity = &dataproc.NodeGroupAffinity{}
1148+
if v, ok := cfgNga["node_group_uri"]; ok {
1149+
conf.NodeGroupAffinity.NodeGroupUri = v.(string)
1150+
}
1151+
}
10681152
return conf, nil
10691153
}
10701154

@@ -1648,6 +1732,22 @@ func flattenGceClusterConfig(d *schema.ResourceData, gcc *dataproc.GceClusterCon
16481732
},
16491733
}
16501734
}
1735+
if gcc.ReservationAffinity != nil {
1736+
gceConfig["reservation_affinity"] = []map[string]interface{}{
1737+
{
1738+
"consume_reservation_type": gcc.ReservationAffinity.ConsumeReservationType,
1739+
"key": gcc.ReservationAffinity.Key,
1740+
"values": gcc.ReservationAffinity.Values,
1741+
},
1742+
}
1743+
}
1744+
if gcc.NodeGroupAffinity != nil {
1745+
gceConfig["node_group_affinity"] = []map[string]interface{}{
1746+
{
1747+
"node_group_uri": gcc.NodeGroupAffinity.NodeGroupUri,
1748+
},
1749+
}
1750+
}
16511751

16521752
return []map[string]interface{}{gceConfig}
16531753
}

‎google-beta/resource_dataproc_cluster_test.go‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,52 @@ func TestAccDataprocCluster_withMetadataAndTags(t *testing.T) {
328328
})
329329
}
330330

331+
func TestAccDataprocCluster_withReservationAffinity(t *testing.T) {
332+
t.Parallel()
333+
334+
var cluster dataproc.Cluster
335+
rnd := randString(t, 10)
336+
vcrTest(t, resource.TestCase{
337+
PreCheck: func() { testAccPreCheck(t) },
338+
Providers: testAccProviders,
339+
CheckDestroy: testAccCheckDataprocClusterDestroy(t),
340+
Steps: []resource.TestStep{
341+
{
342+
Config: testAccDataprocCluster_withReservationAffinity(rnd),
343+
Check: resource.ComposeTestCheckFunc(
344+
testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.basic", &cluster),
345+
346+
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.reservation_affinity.0.consume_reservation_type", "SPECIFIC_RESERVATION"),
347+
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.reservation_affinity.0.key", "compute.googleapis.com/reservation-name"),
348+
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.reservation_affinity.0.values.#", "1"),
349+
),
350+
},
351+
},
352+
})
353+
}
354+
355+
func TestAccDataprocCluster_withNodeGroupAffinity(t *testing.T) {
356+
t.Parallel()
357+
358+
var cluster dataproc.Cluster
359+
rnd := randString(t, 10)
360+
vcrTest(t, resource.TestCase{
361+
PreCheck: func() { testAccPreCheck(t) },
362+
Providers: testAccProviders,
363+
CheckDestroy: testAccCheckDataprocClusterDestroy(t),
364+
Steps: []resource.TestStep{
365+
{
366+
Config: testAccDataprocCluster_withNodeGroupAffinity(rnd),
367+
Check: resource.ComposeTestCheckFunc(
368+
testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.basic", &cluster),
369+
370+
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.node_group_affinity.0.node_group_uri", "https://www.googleapis.com/compute/v1/projects/ci-test-project-188019/zones/us-central1-f/nodeGroups/test-nodegroup"),
371+
),
372+
},
373+
},
374+
})
375+
}
376+
331377
func TestAccDataprocCluster_singleNodeCluster(t *testing.T) {
332378
t.Parallel()
333379

@@ -1209,6 +1255,89 @@ resource "google_dataproc_cluster" "basic" {
12091255
`, rnd)
12101256
}
12111257

1258+
func testAccDataprocCluster_withReservationAffinity(rnd string) string {
1259+
return fmt.Sprintf(`
1260+
1261+
resource "google_compute_reservation" "reservation" {
1262+
name = "default"
1263+
zone = "us-central1-f"
1264+
1265+
specific_reservation {
1266+
count = 10
1267+
instance_properties {
1268+
machine_type = "n1-standard-2"
1269+
}
1270+
}
1271+
specific_reservation_required = true
1272+
}
1273+
1274+
resource "google_dataproc_cluster" "basic" {
1275+
name = "tf-test-dproc-%s"
1276+
region = "us-central1"
1277+
1278+
cluster_config {
1279+
1280+
master_config {
1281+
machine_type = "n1-standard-2"
1282+
}
1283+
1284+
worker_config {
1285+
machine_type = "n1-standard-2"
1286+
}
1287+
1288+
gce_cluster_config {
1289+
zone = "us-central1-f"
1290+
reservation_affinity {
1291+
consume_reservation_type = "SPECIFIC_RESERVATION"
1292+
key = "compute.googleapis.com/reservation-name"
1293+
values = [google_compute_reservation.reservation.name]
1294+
}
1295+
}
1296+
}
1297+
}
1298+
`, rnd)
1299+
}
1300+
1301+
func testAccDataprocCluster_withNodeGroupAffinity(rnd string) string {
1302+
return fmt.Sprintf(`
1303+
1304+
resource "google_compute_node_template" "nodetmpl" {
1305+
name = "test-nodetmpl"
1306+
region = "us-central1"
1307+
1308+
node_affinity_labels = {
1309+
tfacc = "test"
1310+
}
1311+
1312+
node_type = "n1-node-96-624"
1313+
1314+
cpu_overcommit_type = "ENABLED"
1315+
}
1316+
1317+
resource "google_compute_node_group" "nodes" {
1318+
name = "test-nodegroup"
1319+
zone = "us-central1-f"
1320+
1321+
size = 3
1322+
node_template = google_compute_node_template.nodetmpl.self_link
1323+
}
1324+
1325+
resource "google_dataproc_cluster" "basic" {
1326+
name = "tf-test-dproc-%s"
1327+
region = "us-central1"
1328+
1329+
cluster_config {
1330+
gce_cluster_config {
1331+
zone = "us-central1-f"
1332+
node_group_affinity {
1333+
node_group_uri = google_compute_node_group.nodes.name
1334+
}
1335+
}
1336+
}
1337+
}
1338+
`, rnd)
1339+
}
1340+
12121341
func testAccDataprocCluster_singleNodeCluster(rnd string) string {
12131342
return fmt.Sprintf(`
12141343
resource "google_dataproc_cluster" "single_node_cluster" {

‎website/docs/r/dataproc_cluster.html.markdown‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,14 @@ resource "google_dataproc_cluster" "accelerated_cluster" {
372372
* `endpoint_config` (Optional) The config settings for port access on the cluster.
373373
Structure [defined below](#nested_endpoint_config).
374374

375+
* `dataproc_metric_config` (Optional) The Compute Engine accelerator (GPU) configuration for these instances. Can be specified multiple times.
376+
377+
* `metrics` - (Required) Metrics sources to enable.
378+
379+
* `metric_source` - (Required) A source for the collection of Dataproc OSS metrics (see [available OSS metrics](https://cloud.google.com//dataproc/docs/guides/monitoring#available_oss_metrics)).
380+
381+
* `metric_overrides` - (Optional) One or more [available OSS metrics] (https://cloud.google.com/dataproc/docs/guides/monitoring#available_oss_metrics) to collect for the metric course.
382+
375383
* `metastore_config` (Optional) The config setting for metastore service with the cluster.
376384
Structure [defined below](#nested_metastore_config).
377385
- - -
@@ -428,6 +436,14 @@ resource "google_dataproc_cluster" "accelerated_cluster" {
428436
* `metadata` - (Optional) A map of the Compute Engine metadata entries to add to all instances
429437
(see [Project and instance metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
430438

439+
* `reservation_affinity` - (Optional) Reservation Affinity for consuming Zonal reservation.
440+
* `consume_reservation_type` - (Optional) Metrics sources to enable.
441+
* `key` - (Optional) Corresponds to the label key of reservation resource.
442+
* `values` - (Optional) Corresponds to the label values of reservation resource.
443+
444+
* `node_group_affinity` - (Optional) The name of the sole-tenant node group to create the cluster on.
445+
* `node_group_uri` - (Required) The URI of a sole-tenant node group resource that the cluster will be created on.
446+
431447
* `shielded_instance_config` (Optional) Shielded Instance Config for clusters using [Compute Engine Shielded VMs](https://cloud.google.com/security/shielded-cloud/shielded-vm).
432448

433449
- - -
@@ -604,6 +620,7 @@ will be set for you based on whatever was set for the `worker_config.machine_typ
604620
* PREEMPTIBILITY_UNSPECIFIED
605621
* NON_PREEMPTIBLE
606622
* PREEMPTIBLE
623+
* SPOT
607624

608625
* `disk_config` (Optional) Disk Config
609626

0 commit comments

Comments
 (0)