Skip to content

Commit 104bc7d

Browse files
martinzuernmartinzuern
authored
Fixed InitScripts Type to work with GCS and ABFS in databricks_cluster resource (#2067)
* Updated InitSrcipts Type to work with GCS and other sources. * Add Test
1 parent fa0f9f8 commit 104bc7d

File tree

2 files changed

+92
-43
lines changed

2 files changed

+92
-43
lines changed

‎clusters/clusters_api.go‎

Lines changed: 43 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -470,49 +470,49 @@ type ClusterList struct {
470470

471471
// ClusterInfo contains the information when getting cluster info from the get request.
472472
type ClusterInfo struct {
473-
NumWorkers int32 `json:"num_workers,omitempty"`
474-
AutoScale *AutoScale `json:"autoscale,omitempty"`
475-
ClusterID string `json:"cluster_id,omitempty"`
476-
CreatorUserName string `json:"creator_user_name,omitempty"`
477-
Driver *SparkNode `json:"driver,omitempty"`
478-
Executors []SparkNode `json:"executors,omitempty"`
479-
SparkContextID int64 `json:"spark_context_id,omitempty"`
480-
JdbcPort int32 `json:"jdbc_port,omitempty"`
481-
ClusterName string `json:"cluster_name,omitempty"`
482-
SparkVersion string `json:"spark_version"`
483-
SparkConf map[string]string `json:"spark_conf,omitempty"`
484-
AwsAttributes *AwsAttributes `json:"aws_attributes,omitempty"`
485-
AzureAttributes *AzureAttributes `json:"azure_attributes,omitempty"`
486-
GcpAttributes *GcpAttributes `json:"gcp_attributes,omitempty"`
487-
NodeTypeID string `json:"node_type_id,omitempty"`
488-
DriverNodeTypeID string `json:"driver_node_type_id,omitempty"`
489-
SSHPublicKeys []string `json:"ssh_public_keys,omitempty"`
490-
CustomTags map[string]string `json:"custom_tags,omitempty"`
491-
ClusterLogConf *StorageInfo `json:"cluster_log_conf,omitempty"`
492-
InitScripts []StorageInfo `json:"init_scripts,omitempty"`
493-
SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"`
494-
AutoterminationMinutes int32 `json:"autotermination_minutes,omitempty"`
495-
EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"`
496-
EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"`
497-
InstancePoolID string `json:"instance_pool_id,omitempty"`
498-
DriverInstancePoolID string `json:"driver_instance_pool_id,omitempty" tf:"computed"`
499-
PolicyID string `json:"policy_id,omitempty"`
500-
SingleUserName string `json:"single_user_name,omitempty"`
501-
ClusterSource Availability `json:"cluster_source,omitempty"`
502-
DockerImage *DockerImage `json:"docker_image,omitempty"`
503-
State ClusterState `json:"state"`
504-
StateMessage string `json:"state_message,omitempty"`
505-
StartTime int64 `json:"start_time,omitempty"`
506-
TerminateTime int64 `json:"terminate_time,omitempty"`
507-
LastStateLossTime int64 `json:"last_state_loss_time,omitempty"`
508-
LastActivityTime int64 `json:"last_activity_time,omitempty"`
509-
ClusterMemoryMb int64 `json:"cluster_memory_mb,omitempty"`
510-
ClusterCores float64 `json:"cluster_cores,omitempty"`
511-
DefaultTags map[string]string `json:"default_tags"`
512-
ClusterLogStatus *LogSyncStatus `json:"cluster_log_status,omitempty"`
513-
TerminationReason *TerminationReason `json:"termination_reason,omitempty"`
514-
DataSecurityMode string `json:"data_security_mode,omitempty"`
515-
RuntimeEngine string `json:"runtime_engine,omitempty"`
473+
NumWorkers int32 `json:"num_workers,omitempty"`
474+
AutoScale *AutoScale `json:"autoscale,omitempty"`
475+
ClusterID string `json:"cluster_id,omitempty"`
476+
CreatorUserName string `json:"creator_user_name,omitempty"`
477+
Driver *SparkNode `json:"driver,omitempty"`
478+
Executors []SparkNode `json:"executors,omitempty"`
479+
SparkContextID int64 `json:"spark_context_id,omitempty"`
480+
JdbcPort int32 `json:"jdbc_port,omitempty"`
481+
ClusterName string `json:"cluster_name,omitempty"`
482+
SparkVersion string `json:"spark_version"`
483+
SparkConf map[string]string `json:"spark_conf,omitempty"`
484+
AwsAttributes *AwsAttributes `json:"aws_attributes,omitempty"`
485+
AzureAttributes *AzureAttributes `json:"azure_attributes,omitempty"`
486+
GcpAttributes *GcpAttributes `json:"gcp_attributes,omitempty"`
487+
NodeTypeID string `json:"node_type_id,omitempty"`
488+
DriverNodeTypeID string `json:"driver_node_type_id,omitempty"`
489+
SSHPublicKeys []string `json:"ssh_public_keys,omitempty"`
490+
CustomTags map[string]string `json:"custom_tags,omitempty"`
491+
ClusterLogConf *StorageInfo `json:"cluster_log_conf,omitempty"`
492+
InitScripts []InitScriptStorageInfo `json:"init_scripts,omitempty"`
493+
SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"`
494+
AutoterminationMinutes int32 `json:"autotermination_minutes,omitempty"`
495+
EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"`
496+
EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"`
497+
InstancePoolID string `json:"instance_pool_id,omitempty"`
498+
DriverInstancePoolID string `json:"driver_instance_pool_id,omitempty" tf:"computed"`
499+
PolicyID string `json:"policy_id,omitempty"`
500+
SingleUserName string `json:"single_user_name,omitempty"`
501+
ClusterSource Availability `json:"cluster_source,omitempty"`
502+
DockerImage *DockerImage `json:"docker_image,omitempty"`
503+
State ClusterState `json:"state"`
504+
StateMessage string `json:"state_message,omitempty"`
505+
StartTime int64 `json:"start_time,omitempty"`
506+
TerminateTime int64 `json:"terminate_time,omitempty"`
507+
LastStateLossTime int64 `json:"last_state_loss_time,omitempty"`
508+
LastActivityTime int64 `json:"last_activity_time,omitempty"`
509+
ClusterMemoryMb int64 `json:"cluster_memory_mb,omitempty"`
510+
ClusterCores float64 `json:"cluster_cores,omitempty"`
511+
DefaultTags map[string]string `json:"default_tags"`
512+
ClusterLogStatus *LogSyncStatus `json:"cluster_log_status,omitempty"`
513+
TerminationReason *TerminationReason `json:"termination_reason,omitempty"`
514+
DataSecurityMode string `json:"data_security_mode,omitempty"`
515+
RuntimeEngine string `json:"runtime_engine,omitempty"`
516516
}
517517

518518
// IsRunningOrResizing returns true if cluster is running or resizing

‎clusters/clusters_api_test.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,55 @@ func TestStartAndGetInfo_Pending(t *testing.T) {
499499
assert.Equal(t, ClusterStateRunning, string(clusterInfo.State))
500500
}
501501

502+
func TestStartAndGetInfo_InitScript(t *testing.T) {
503+
TestInitScripts := []InitScriptStorageInfo{
504+
{
505+
Dbfs: &DbfsStorageInfo{
506+
Destination: "dbfs:/my_init_script.sh",
507+
},
508+
},
509+
{
510+
Gcs: &GcsStorageInfo{
511+
Destination: "gs://my_bucket/my_init_script.sh",
512+
},
513+
},
514+
{
515+
S3: &S3StorageInfo{
516+
Destination: "s3://my_bucket/my_init_script.sh",
517+
},
518+
},
519+
{
520+
Abfss: &AbfssStorageInfo{
521+
Destination: "abfss://my_bucket/my_init_script.sh",
522+
},
523+
},
524+
{
525+
File: &LocalFileInfo{
526+
Destination: "/my_init_script.sh",
527+
},
528+
},
529+
}
530+
531+
client, server, err := qa.HttpFixtureClient(t, []qa.HTTPFixture{
532+
{
533+
Method: "GET",
534+
Resource: "/api/2.0/clusters/get?cluster_id=abc",
535+
Response: ClusterInfo{
536+
State: ClusterStateRunning,
537+
ClusterID: "abc",
538+
InitScripts: TestInitScripts,
539+
},
540+
},
541+
})
542+
defer server.Close()
543+
require.NoError(t, err)
544+
545+
ctx := context.Background()
546+
clusterInfo, err := NewClustersAPI(ctx, client).StartAndGetInfo("abc")
547+
require.NoError(t, err)
548+
assert.Equal(t, TestInitScripts, clusterInfo.InitScripts)
549+
}
550+
502551
func TestStartAndGetInfo_Terminating(t *testing.T) {
503552
client, server, err := qa.HttpFixtureClient(t, []qa.HTTPFixture{
504553
{

0 commit comments

Comments
 (0)