Skip to content

Commit e340001

Browse files
modular-magicianmodular-magician
authored
Software and workloads configs (#9653) (#6797)
* add dag processor resources * add web_server_plugins_mode * add docs * add tests, replace go client, guard nil PrivateClusterConfig * new api version * add mapping for web server plugins mode * add create test * make beta only * valid cpu in dag_processor test * add 'Computed' to individual workload resources, align. * make workload resources computed * remove computed from required fields [upstream:09a193c942d2b3c1b1fe0a2c857ba1010a149c5c] Signed-off-by: Modular Magician <[email protected]>
1 parent a15d8ec commit e340001

File tree

4 files changed

+227
-2
lines changed

4 files changed

+227
-2
lines changed

‎.changelog/9653.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
google_composer_environment: added support for creation and update for config.software_config.web_server_plugins_mode and config.workloads_config. dag_processor fields.
3+
```

‎google-beta/services/composer/resource_composer_environment.go‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ var (
5252
"config.0.software_config.0.python_version",
5353
"config.0.software_config.0.scheduler_count",
5454
"config.0.software_config.0.cloud_data_lineage_integration",
55+
"config.0.software_config.0.web_server_plugins_mode",
5556
}
5657

5758
composerConfigKeys = []string{
@@ -80,6 +81,7 @@ var (
8081
"config.0.workloads_config.0.triggerer",
8182
"config.0.workloads_config.0.web_server",
8283
"config.0.workloads_config.0.worker",
84+
"config.0.workloads_config.0.dag_processor",
8385
}
8486

8587
composerPrivateEnvironmentConfig = []string{
@@ -466,6 +468,15 @@ func ResourceComposerEnvironment() *schema.Resource {
466468
},
467469
},
468470
},
471+
"web_server_plugins_mode": {
472+
Type: schema.TypeString,
473+
Optional: true,
474+
Computed: true,
475+
ForceNew: false,
476+
AtLeastOneOf: composerSoftwareConfigKeys,
477+
ValidateFunc: validation.StringInSlice([]string{"ENABLED", "DISABLED"}, false),
478+
Description: `Should be either 'ENABLED' or 'DISABLED'. Defaults to 'ENABLED'. Used in Composer 3.`,
479+
},
469480
},
470481
},
471482
},
@@ -669,6 +680,7 @@ func ResourceComposerEnvironment() *schema.Resource {
669680
Optional: true,
670681
AtLeastOneOf: workloadsConfigKeys,
671682
ForceNew: false,
683+
Computed: true,
672684
Description: `Configuration for resources used by Airflow schedulers.`,
673685
MaxItems: 1,
674686
Elem: &schema.Resource{
@@ -677,27 +689,31 @@ func ResourceComposerEnvironment() *schema.Resource {
677689
Type: schema.TypeFloat,
678690
Optional: true,
679691
ForceNew: false,
692+
Computed: true,
680693
ValidateFunc: validation.FloatAtLeast(0),
681694
Description: `CPU request and limit for a single Airflow scheduler replica`,
682695
},
683696
"memory_gb": {
684697
Type: schema.TypeFloat,
685698
Optional: true,
686699
ForceNew: false,
700+
Computed: true,
687701
ValidateFunc: validation.FloatAtLeast(0),
688702
Description: `Memory (GB) request and limit for a single Airflow scheduler replica.`,
689703
},
690704
"storage_gb": {
691705
Type: schema.TypeFloat,
692706
Optional: true,
693707
ForceNew: false,
708+
Computed: true,
694709
ValidateFunc: validation.FloatAtLeast(0),
695710
Description: `Storage (GB) request and limit for a single Airflow scheduler replica.`,
696711
},
697712
"count": {
698713
Type: schema.TypeInt,
699714
Optional: true,
700715
ForceNew: false,
716+
Computed: true,
701717
ValidateFunc: validation.IntAtLeast(0),
702718
Description: `The number of schedulers.`,
703719
},
@@ -708,6 +724,7 @@ func ResourceComposerEnvironment() *schema.Resource {
708724
Type: schema.TypeList,
709725
Optional: true,
710726
AtLeastOneOf: workloadsConfigKeys,
727+
Computed: true,
711728
Description: `Configuration for resources used by Airflow triggerers.`,
712729
MaxItems: 1,
713730
Elem: &schema.Resource{
@@ -738,6 +755,7 @@ func ResourceComposerEnvironment() *schema.Resource {
738755
Optional: true,
739756
AtLeastOneOf: workloadsConfigKeys,
740757
ForceNew: false,
758+
Computed: true,
741759
Description: `Configuration for resources used by Airflow web server.`,
742760
MaxItems: 1,
743761
Elem: &schema.Resource{
@@ -746,20 +764,23 @@ func ResourceComposerEnvironment() *schema.Resource {
746764
Type: schema.TypeFloat,
747765
Optional: true,
748766
ForceNew: false,
767+
Computed: true,
749768
ValidateFunc: validation.FloatAtLeast(0),
750769
Description: `CPU request and limit for Airflow web server.`,
751770
},
752771
"memory_gb": {
753772
Type: schema.TypeFloat,
754773
Optional: true,
755774
ForceNew: false,
775+
Computed: true,
756776
ValidateFunc: validation.FloatAtLeast(0),
757777
Description: `Memory (GB) request and limit for Airflow web server.`,
758778
},
759779
"storage_gb": {
760780
Type: schema.TypeFloat,
761781
Optional: true,
762782
ForceNew: false,
783+
Computed: true,
763784
ValidateFunc: validation.FloatAtLeast(0),
764785
Description: `Storage (GB) request and limit for Airflow web server.`,
765786
},
@@ -771,6 +792,7 @@ func ResourceComposerEnvironment() *schema.Resource {
771792
Optional: true,
772793
AtLeastOneOf: workloadsConfigKeys,
773794
ForceNew: false,
795+
Computed: true,
774796
Description: `Configuration for resources used by Airflow workers.`,
775797
MaxItems: 1,
776798
Elem: &schema.Resource{
@@ -779,40 +801,82 @@ func ResourceComposerEnvironment() *schema.Resource {
779801
Type: schema.TypeFloat,
780802
Optional: true,
781803
ForceNew: false,
804+
Computed: true,
782805
ValidateFunc: validation.FloatAtLeast(0),
783806
Description: `CPU request and limit for a single Airflow worker replica.`,
784807
},
785808
"memory_gb": {
786809
Type: schema.TypeFloat,
787810
Optional: true,
788811
ForceNew: false,
812+
Computed: true,
789813
ValidateFunc: validation.FloatAtLeast(0),
790814
Description: `Memory (GB) request and limit for a single Airflow worker replica.`,
791815
},
792816
"storage_gb": {
793817
Type: schema.TypeFloat,
794818
Optional: true,
795819
ForceNew: false,
820+
Computed: true,
796821
ValidateFunc: validation.FloatAtLeast(0),
797822
Description: `Storage (GB) request and limit for a single Airflow worker replica.`,
798823
},
799824
"min_count": {
800825
Type: schema.TypeInt,
801826
Optional: true,
802827
ForceNew: false,
828+
Computed: true,
803829
ValidateFunc: validation.IntAtLeast(0),
804830
Description: `Minimum number of workers for autoscaling.`,
805831
},
806832
"max_count": {
807833
Type: schema.TypeInt,
808834
Optional: true,
809835
ForceNew: false,
836+
Computed: true,
810837
ValidateFunc: validation.IntAtLeast(0),
811838
Description: `Maximum number of workers for autoscaling.`,
812839
},
813840
},
814841
},
815842
},
843+
"dag_processor": {
844+
Type: schema.TypeList,
845+
Optional: true,
846+
AtLeastOneOf: workloadsConfigKeys,
847+
ForceNew: false,
848+
Computed: true,
849+
Description: `Configuration for resources used by DAG processor.`,
850+
MaxItems: 1,
851+
Elem: &schema.Resource{
852+
Schema: map[string]*schema.Schema{
853+
"cpu": {
854+
Type: schema.TypeFloat,
855+
Optional: true,
856+
ForceNew: false,
857+
Computed: true,
858+
ValidateFunc: validation.FloatAtLeast(0),
859+
Description: `CPU request and limit for DAG processor.`,
860+
},
861+
"memory_gb": {
862+
Type: schema.TypeFloat,
863+
Optional: true,
864+
ForceNew: false,
865+
Computed: true,
866+
ValidateFunc: validation.FloatAtLeast(0),
867+
Description: `Memory (GB) request and limit for DAG processor.`,
868+
},
869+
"storage_gb": {
870+
Type: schema.TypeFloat,
871+
Optional: true,
872+
ForceNew: false,
873+
Computed: true,
874+
ValidateFunc: validation.FloatAtLeast(0),
875+
Description: `Storage (GB) request and limit for DAG processor.`,
876+
},
877+
},
878+
},
879+
},
816880
},
817881
},
818882
},
@@ -1157,6 +1221,21 @@ func resourceComposerEnvironmentUpdate(d *schema.ResourceData, meta interface{})
11571221
}
11581222
}
11591223

1224+
if d.HasChange("config.0.software_config.0.web_server_plugins_mode") {
1225+
patchObj := &composer.Environment{
1226+
Config: &composer.EnvironmentConfig{
1227+
SoftwareConfig: &composer.SoftwareConfig{},
1228+
},
1229+
}
1230+
if config != nil && config.SoftwareConfig != nil {
1231+
patchObj.Config.SoftwareConfig.WebServerPluginsMode = config.SoftwareConfig.WebServerPluginsMode
1232+
}
1233+
err = resourceComposerEnvironmentPatchField("config.softwareConfig.webServerPluginsMode", userAgent, patchObj, d, tfConfig)
1234+
if err != nil {
1235+
return err
1236+
}
1237+
}
1238+
11601239
if d.HasChange("config.0.node_count") {
11611240
patchObj := &composer.Environment{Config: &composer.EnvironmentConfig{}}
11621241
if config != nil {
@@ -1516,11 +1595,13 @@ func flattenComposerEnvironmentConfigWorkloadsConfig(workloadsConfig *composer.W
15161595
transformedTriggerer := make(map[string]interface{})
15171596
transformedWebServer := make(map[string]interface{})
15181597
transformedWorker := make(map[string]interface{})
1598+
transformedDagProcessor := make(map[string]interface{})
15191599

15201600
wlCfgScheduler := workloadsConfig.Scheduler
15211601
wlCfgTriggerer := workloadsConfig.Triggerer
15221602
wlCfgWebServer := workloadsConfig.WebServer
15231603
wlCfgWorker := workloadsConfig.Worker
1604+
wlCfgDagProcessor := workloadsConfig.DagProcessor
15241605

15251606
if wlCfgScheduler == nil {
15261607
transformedScheduler = nil
@@ -1557,12 +1638,21 @@ func flattenComposerEnvironmentConfigWorkloadsConfig(workloadsConfig *composer.W
15571638
transformedWorker["max_count"] = wlCfgWorker.MaxCount
15581639
}
15591640

1641+
if wlCfgDagProcessor == nil {
1642+
transformedDagProcessor = nil
1643+
} else {
1644+
transformedDagProcessor["cpu"] = wlCfgDagProcessor.Cpu
1645+
transformedDagProcessor["memory_gb"] = wlCfgDagProcessor.MemoryGb
1646+
transformedDagProcessor["storage_gb"] = wlCfgDagProcessor.StorageGb
1647+
}
1648+
15601649
transformed["scheduler"] = []interface{}{transformedScheduler}
15611650
if transformedTriggerer != nil {
15621651
transformed["triggerer"] = []interface{}{transformedTriggerer}
15631652
}
15641653
transformed["web_server"] = []interface{}{transformedWebServer}
15651654
transformed["worker"] = []interface{}{transformedWorker}
1655+
transformed["dag_processor"] = []interface{}{transformedDagProcessor}
15661656

15671657
return []interface{}{transformed}
15681658
}
@@ -1649,6 +1739,13 @@ func flattenComposerEnvironmentConfigSoftwareConfig(softwareCfg *composer.Softwa
16491739
transformed["env_variables"] = softwareCfg.EnvVariables
16501740
transformed["scheduler_count"] = softwareCfg.SchedulerCount
16511741
transformed["cloud_data_lineage_integration"] = flattenComposerEnvironmentConfigSoftwareConfigCloudDataLineageIntegration(softwareCfg.CloudDataLineageIntegration)
1742+
if softwareCfg.WebServerPluginsMode == "PLUGINS_DISABLED" {
1743+
transformed["web_server_plugins_mode"] = "DISABLED"
1744+
} else if softwareCfg.WebServerPluginsMode == "PLUGINS_ENABLED" {
1745+
transformed["web_server_plugins_mode"] = "ENABLED"
1746+
} else {
1747+
transformed["web_server_plugins_mode"] = softwareCfg.WebServerPluginsMode
1748+
}
16521749
return []interface{}{transformed}
16531750
}
16541751

@@ -1970,6 +2067,17 @@ func expandComposerEnvironmentConfigWorkloadsConfig(v interface{}, d *schema.Res
19702067
}
19712068
}
19722069

2070+
if v, ok := original["dag_processor"]; ok {
2071+
if len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil {
2072+
transformedDagProcessor := &composer.DagProcessorResource{}
2073+
originalDagProcessorRaw := v.([]interface{})[0].(map[string]interface{})
2074+
transformedDagProcessor.Cpu = originalDagProcessorRaw["cpu"].(float64)
2075+
transformedDagProcessor.MemoryGb = originalDagProcessorRaw["memory_gb"].(float64)
2076+
transformedDagProcessor.StorageGb = originalDagProcessorRaw["storage_gb"].(float64)
2077+
transformed.DagProcessor = transformedDagProcessor
2078+
}
2079+
}
2080+
19732081
return transformed, nil
19742082
}
19752083

@@ -2293,6 +2401,14 @@ func expandComposerEnvironmentConfigSoftwareConfig(v interface{}, d *schema.Reso
22932401
}
22942402
transformed.CloudDataLineageIntegration = transformedCloudDataLineageIntegration
22952403

2404+
if original["web_server_plugins_mode"].(string) == "DISABLED" {
2405+
transformed.WebServerPluginsMode = "PLUGINS_DISABLED"
2406+
} else if original["web_server_plugins_mode"].(string) == "ENABLED" {
2407+
transformed.WebServerPluginsMode = "PLUGINS_ENABLED"
2408+
} else {
2409+
transformed.WebServerPluginsMode = original["web_server_plugins_mode"].(string)
2410+
}
2411+
22962412
return transformed, nil
22972413
}
22982414

0 commit comments

Comments
 (0)