Update AWS testing environments

Author: nfx

Makefile

@@ -62,10 +62,6 @@ test-gcp-accounts: install
 	@echo "✓ Running acceptance Tests for Multiple Workspace APIs on GCP..."
 	@/bin/bash scripts/run.sh gcp-accounts '^TestGcpAcc' --debug --tee
 
-test-awsst: install
-	@echo "✓ Running Terraform Acceptance Tests for AWS ST..."
-	@/bin/bash scripts/run.sh awsst '^(TestAcc|TestAwsAcc)' --debug --tee
-
 test-awsmt: install
 	@echo "✓ Running Terraform Acceptance Tests for AWS MT..."
 	@/bin/bash scripts/run.sh awsmt '^(TestAcc|TestAwsAcc)' --debug --tee
@@ -74,8 +70,4 @@ test-preview: install
 	@echo "✓ Running acceptance Tests for Preview features..."
 	@/bin/bash scripts/run.sh preview '^TestPreviewAcc' --debug --tee
 
-snapshot:
-	@echo "✓ Making Snapshot ..."
-	@goreleaser release --rm-dist --snapshot
-
 .PHONY: build fmt python-setup docs vendor build fmt coverage test lint

docs/guides/aws-workspace.md

@@ -111,7 +111,8 @@ resource "databricks_mws_credentials" "this" {
 
 ## VPC
 
-The very first step is VPC creation with necessary firewall rules. Please consult [main documetation page](https://docs.databricks.com/administration-guide/cloud-configurations/aws/customer-managed-vpc.html) for **the most complete and up-to-date details on networking**. AWS VPS is registered as [databricks_mws_networks](../resources/mws_networks.md) resource.
+The very first step is VPC creation with necessary firewall rules. Please consult [main documetation page](https://docs.databricks.com/administration-guide/cloud-configurations/aws/customer-managed-vpc.html) for **the most complete and up-to-date details on networking**. AWS VPS is registered as [databricks_mws_networks](../resources/mws_networks.md) resource. For STS, S3 and Kinesis, you can create VPC gateway or interface endpoints such that the relevant in-region traffic from clusters could transit over the secure AWS backbone rather than the public network, for more direct connections and reduced cost compared to AWS global endpoints. For more information, see [Regional endpoints]
+](https://docs.databricks.com/administration-guide/cloud-configurations/aws/customer-managed-vpc.html#regional-endpoints-1).
 
 ```hcl
 data "aws_availability_zones" "available" {}
@@ -146,52 +147,6 @@ module "vpc" {
   }]
 }
 
-resource "databricks_mws_networks" "this" {
-  provider           = databricks.mws
-  account_id         = var.databricks_account_id
-  network_name       = "${local.prefix}-network"
-  security_group_ids = [module.vpc.default_security_group_id]
-  subnet_ids         = module.vpc.private_subnets
-  vpc_id             = module.vpc.vpc_id
-}
-```
-
-## Regional endpoints
-
-For STS, S3 and Kinesis, you can create VPC gateway or interface endpoints such that the relevant in-region traffic from clusters could transit over the secure AWS backbone rather than the public network, for more direct connections and reduced cost compared to AWS global endpoints. See [Regional endpoints]
-](https://docs.databricks.com/administration-guide/cloud-configurations/aws/customer-managed-vpc.html#regional-endpoints-1) for more information:
-
-```hcl
-module "vpc" {
-  source  = "terraform-aws-modules/vpc/aws"
-  version = "3.2.0"
-
-  name = local.prefix
-  cidr = var.cidr_block
-  azs  = data.aws_availability_zones.available.names
-  tags = var.tags
-
-  enable_dns_hostnames = true
-  enable_nat_gateway   = true
-  create_igw           = true
-
-  public_subnets  = [cidrsubnet(var.cidr_block, 3, 0)]
-  private_subnets = [cidrsubnet(var.cidr_block, 3, 1),
-                     cidrsubnet(var.cidr_block, 3, 2)]
-
-  manage_default_security_group = true
-  default_security_group_name = "${local.prefix}-sg"
-
-  default_security_group_egress = [{
-    cidr_blocks = "0.0.0.0/0"
-  }]
-
-  default_security_group_ingress = [{
-    description = "Allow all internal TCP and UDP"
-    self        = true
-  }]
-}
-
 module "vpc_endpoints" {
   source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints"
   version = "3.2.0"
@@ -203,7 +158,9 @@ module "vpc_endpoints" {
     s3 = {
       service         = "s3"
       service_type    = "Gateway"
-      route_table_ids = flatten([module.vpc.private_route_table_ids, module.vpc.public_route_table_ids])
+      route_table_ids = flatten([
+        module.vpc.private_route_table_ids, 
+        module.vpc.public_route_table_ids])
       tags            = {
         Name = "${local.prefix}-s3-vpc-endpoint"
       }
@@ -228,6 +185,15 @@ module "vpc_endpoints" {
 
   tags = var.tags
 }
+
+resource "databricks_mws_networks" "this" {
+  provider           = databricks.mws
+  account_id         = var.databricks_account_id
+  network_name       = "${local.prefix}-network"
+  security_group_ids = [module.vpc.default_security_group_id]
+  subnet_ids         = module.vpc.private_subnets
+  vpc_id             = module.vpc.vpc_id
+}
 ```
 
 ## Root bucket

scripts/awsmt-integration/main.tf

@@ -2,10 +2,6 @@ data "external" "env" {
   program = ["python", "-c", "import sys,os,json;json.dump(dict(os.environ), sys.stdout)"]
 }
 
-provider "aws" {
-  region = data.external.env.result.TEST_REGION
-}
-
 resource "random_string" "naming" {
   special = false
   upper   = false
@@ -14,14 +10,21 @@ resource "random_string" "naming" {
 
 locals {
   // dltp - databricks labs terraform provider
-  prefix = "dltp${random_string.naming.result}"
+  prefix     = "dltp${random_string.naming.result}"
+  cidr_block = data.external.env.result.TEST_CIDR
+  region     = data.external.env.result.TEST_REGION
+  account_id = data.external.env.result.DATABRICKS_ACCOUNT_ID
   tags = {
     Environment = "Testing"
     Owner       = data.external.env.result.OWNER
     Epoch       = random_string.naming.result
   }
 }
 
+provider "aws" {
+  region = local.region
+}
+
 // initialize provider in "MWS" mode to provision new workspace
 provider "databricks" {
   alias = "mws"
@@ -30,7 +33,7 @@ provider "databricks" {
 
 data "databricks_aws_assume_role_policy" "this" {
   provider    = databricks.mws
-  external_id = data.external.env.result.DATABRICKS_ACCOUNT_ID
+  external_id = local.account_id
 }
 
 resource "aws_iam_role" "cross_account_role" {
@@ -53,7 +56,7 @@ resource "aws_iam_role_policy" "this" {
 // register cross-account ARN
 resource "databricks_mws_credentials" "this" {
   provider         = databricks.mws
-  account_id       = data.external.env.result.DATABRICKS_ACCOUNT_ID
+  account_id       = local.account_id
   role_arn         = aws_iam_role.cross_account_role.arn
   credentials_name = "${local.prefix}-creds"
 
@@ -63,35 +66,106 @@ resource "databricks_mws_credentials" "this" {
 
 module "this" {
   source     = "../modules/aws-mws-common"
-  cidr_block = data.external.env.result.TEST_CIDR
-  region     = data.external.env.result.TEST_REGION
+  cidr_block = local.cidr_block
+  region     = local.region
   prefix     = local.prefix
   tags       = local.tags
 }
 
 // register root bucket
 resource "databricks_mws_storage_configurations" "this" {
   provider                   = databricks.mws
-  account_id                 = data.external.env.result.DATABRICKS_ACCOUNT_ID
+  account_id                 = local.account_id
   bucket_name                = module.this.root_bucket
   storage_configuration_name = "${local.prefix}-storage"
 }
 
 // register VPC
+data "aws_availability_zones" "available" {}
+
+module "vpc" {
+  source  = "terraform-aws-modules/vpc/aws"
+  version = "3.2.0"
+
+  name = local.prefix
+  cidr = local.cidr_block
+  azs  = data.aws_availability_zones.available.names
+  tags = local.tags
+
+  enable_dns_hostnames = true
+  enable_nat_gateway   = true
+  create_igw           = true
+
+  public_subnets  = [cidrsubnet(local.cidr_block, 3, 0)]
+  private_subnets = [cidrsubnet(local.cidr_block, 3, 1),
+                     cidrsubnet(local.cidr_block, 3, 2)]
+
+  manage_default_security_group = true
+  default_security_group_name = "${local.prefix}-sg"
+
+  default_security_group_egress = [{
+    cidr_blocks = "0.0.0.0/0"
+  }]
+
+  default_security_group_ingress = [{
+    description = "Allow all internal TCP and UDP"
+    self        = true
+  }]
+}
+
+module "vpc_endpoints" {
+  source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints"
+  version = "3.2.0"
+
+  vpc_id             = module.vpc.vpc_id
+  security_group_ids = [module.vpc.default_security_group_id]
+
+  endpoints = {
+    s3 = {
+      service         = "s3"
+      service_type    = "Gateway"
+      route_table_ids = flatten([
+        module.vpc.private_route_table_ids, 
+        module.vpc.public_route_table_ids])
+      tags            = {
+        Name = "${local.prefix}-s3-vpc-endpoint"
+      }
+    },
+    sts = {
+      service             = "sts"
+      private_dns_enabled = true
+      subnet_ids          = module.vpc.private_subnets
+      tags                = {
+        Name = "${local.prefix}-sts-vpc-endpoint"
+      }
+    },
+    kinesis-streams = {
+      service             = "kinesis-streams"
+      private_dns_enabled = true
+      subnet_ids          = module.vpc.private_subnets
+      tags                = {
+        Name = "${local.prefix}-kinesis-vpc-endpoint"
+      }
+    },    
+  }
+
+  tags = local.tags
+}
+
 resource "databricks_mws_networks" "this" {
   provider           = databricks.mws
-  account_id         = data.external.env.result.DATABRICKS_ACCOUNT_ID
+  account_id         = local.account_id
   network_name       = "${local.prefix}-network"
-  subnet_ids         = [module.this.subnet_public, module.this.subnet_private]
-  vpc_id             = module.this.vpc_id
-  security_group_ids = [module.this.security_group]
+  security_group_ids = [module.vpc.default_security_group_id]
+  subnet_ids         = module.vpc.private_subnets
+  vpc_id             = module.vpc.vpc_id
 }
 
 // create workspace in given VPC with DBFS on root bucket
 resource "databricks_mws_workspaces" "this" {
   provider        = databricks.mws
-  account_id      = data.external.env.result.DATABRICKS_ACCOUNT_ID
-  aws_region      = data.external.env.result.TEST_REGION
+  account_id      = local.account_id
+  aws_region      = local.region
   workspace_name  = local.prefix
   deployment_name = local.prefix
 

scripts/awsst-integration/main.tf

@@ -112,6 +112,10 @@ resource "aws_route_table" "private" {
   })
 }
 
+output "private_rt" {
+  value = aws_route_table.private.id
+}
+
 resource "aws_route_table_association" "private" {
   route_table_id = aws_route_table.private.id
   subnet_id      = aws_subnet.private.id