diff --git a/.github/setup/aws/.terraform.lock.hcl b/.github/setup/aws/.terraform.lock.hcl new file mode 100644 index 0000000..cdc1668 --- /dev/null +++ b/.github/setup/aws/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.0" + hashes = [ + "h1:Ijt7pOlB7Tr7maGQIqtsLFbl7pSMIj06TVdkoSBcYOw=", + "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", + "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", + "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", + "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b", + "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862", + "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93", + "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2", + "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e", + "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421", + "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4", + "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9", + "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9", + "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70", + ] +} diff --git a/.github/setup/aws/backend.tf b/.github/setup/aws/backend.tf new file mode 100644 index 0000000..98222f9 --- /dev/null +++ b/.github/setup/aws/backend.tf @@ -0,0 +1,9 @@ +terraform { + backend "s3" { + bucket = "materialize-terraform-self-managed-state" + key = "github-setup/oidc/aws/terraform.tfstate" + region = "us-east-1" + encrypt = true + # profile = "" # Add your profile name here since backend block doesn't accept variables + } +} diff --git a/.github/setup/aws/main.tf b/.github/setup/aws/main.tf new file mode 100644 index 0000000..3cfa7b0 --- /dev/null +++ b/.github/setup/aws/main.tf @@ -0,0 +1,35 @@ +# IAM Role for GitHub Actions +resource "aws_iam_role" "github_actions" { + name = "mz-self-managed-github-actions-role" + max_session_duration = var.max_session_duration + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRoleWithWebIdentity" + Effect = "Allow" + Principal = { + Federated = var.oidc_provider_arn + } + Condition = { + StringEquals = { + "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" + } + StringLike = { + "token.actions.githubusercontent.com:sub" = "repo:MaterializeInc/materialize-terraform-self-managed:*" + } + } + } + ] + }) + + tags = { + Name = "materialize-terraform-self-managed-github-actions-role" + } +} + +# Admin policy for GitHub Actions (simplified for testing) +resource "aws_iam_role_policy_attachment" "github_actions_admin" { + role = aws_iam_role.github_actions.name + policy_arn = "arn:aws:iam::aws:policy/AdministratorAccess" +} diff --git a/.github/setup/aws/outputs.tf b/.github/setup/aws/outputs.tf new file mode 100644 index 0000000..8e9a616 --- /dev/null +++ b/.github/setup/aws/outputs.tf @@ -0,0 +1,10 @@ +# Outputs for GitHub Actions configuration +output "github_actions_role_arn" { + description = "ARN of the IAM role for GitHub Actions" + value = aws_iam_role.github_actions.arn +} + +output "github_actions_role_name" { + description = "Name of the IAM role for GitHub Actions" + value = aws_iam_role.github_actions.name +} diff --git a/.github/setup/aws/variables.tf b/.github/setup/aws/variables.tf new file mode 100644 index 0000000..5a9bc89 --- /dev/null +++ b/.github/setup/aws/variables.tf @@ -0,0 +1,16 @@ +variable "profile" { + description = "The AWS CLI profile to use for authentication" + type = string + default = "default" +} + +variable "oidc_provider_arn" { + description = "The ARN of the OIDC provider for GitHub Actions" + type = string +} + +variable "max_session_duration" { + description = "The maximum session duration for the IAM role" + type = number + default = 28800 # 8 hours +} diff --git a/.github/setup/aws/versions.tf b/.github/setup/aws/versions.tf new file mode 100644 index 0000000..275ed21 --- /dev/null +++ b/.github/setup/aws/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + profile = var.profile + region = "us-east-1" +} diff --git a/.github/setup/azure/.terraform.lock.hcl b/.github/setup/azure/.terraform.lock.hcl new file mode 100644 index 0000000..51e01ff --- /dev/null +++ b/.github/setup/azure/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "2.53.1" + constraints = "~> 2.0" + hashes = [ + "h1:EZNO8sEtUABuRxujQrDrW1z1QsG0dq6iLbzWtnG7Om4=", + "zh:162916b037e5133f49298b0ffa3e7dcef7d76530a8ca738e7293373980f73c68", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:492931cea4f30887ab5bca36a8556dfcb897288eddd44619c0217fc5da2d57e7", + "zh:4c895e450e18335ad8714cc6d3488fc1a78816ad2851a91b06cb2ef775dd7c66", + "zh:60d92fdaf7235574201f2d8f68f733ee00a822993b3fc95e6952e09e6ec76999", + "zh:67a169119efa41c1fb867ef1a8e79bf03472a2324384c36eb55370c817dcce42", + "zh:9dd4d5ed9233cf9329262200bc5a1aa60942b80dbc611e2ef4b09f47531b39b1", + "zh:a3c160e35b9e40fc1497b83c2f37a8e24565b05a1783c7733609f3695735c2a9", + "zh:a4a221da42b1f46e7c436c7145e5beaadfd9d03f3be6fd526d132c03f18a5979", + "zh:af0d3476a9702d2287e168e3baa670e64daab9c9b01c01e17025a5248f3e28e9", + "zh:e3579bff7894f3d36066b74ec324be6d28f56a42a387a2b8a0eabf33cbff86df", + "zh:f1749ee8ad972ae6424665aa9d2c0ece8c40c51d41ec2f38b863148cb437e865", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.117.1" + constraints = "~> 3.0" + hashes = [ + "h1:j6wnjpHfBcQC4xd3ZYquaIPIIR46xJQs7rxwPdSOZos=", + "zh:0c513676836e3c50d004ece7d2624a8aff6faac14b833b96feeac2e4bc2c1c12", + "zh:50ea01ada95bae2f187db9e926e463f45d860767a85ebc59160414e00e76c35d", + "zh:52c2a9edacc06b3f72153f5ef6daca0761c6292158815961fe37f60bc576a3d7", + "zh:618eed2a06b19b1a025b45b05891846d570a6a1cca4d23f4942f5a99e1f747ae", + "zh:61cde5d3165d7e5ec311d5d89486819cd605c1b2d54611b5c97bd4e97dba2762", + "zh:6a873358d5031fc222f5e05f029d1237f3dce8345c767665f393283dfa2627f6", + "zh:afdd80064b2a04da311856feb4ed45f77ff4df6c356e8c2b10afb51fe7e61c70", + "zh:b09113df7e0e8c8959539bd22bae6c39faeb269ba3c4cd948e742f5cf58c35fb", + "zh:d340db7973109761cfc27d52aa02560363337c908b2c99b3628adc5a70a99d5b", + "zh:d5a577226ebc8c65e8f19384878a86acc4b51ede4b4a82d37c3b331b0efcd4a7", + "zh:e2962b147f9e71732df8dbc74940c10d20906f3c003cbfaa1eb9fabbf601a9f0", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/.github/setup/azure/accessTokenLifeTime.sh b/.github/setup/azure/accessTokenLifeTime.sh new file mode 100755 index 0000000..bfb5e54 --- /dev/null +++ b/.github/setup/azure/accessTokenLifeTime.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Exit early on error/unset var/pipe failure +set -euo pipefail + +# 1. Create a 4-hour token lifetime policy and capture the policy ID +POLICY_RESPONSE=$(az rest --method POST \ + --url "https://graph.microsoft.com/v1.0/policies/tokenLifetimePolicies" \ + --headers "Content-Type=application/json" \ + --body '{ + "definition": ["{\"TokenLifetimePolicy\":{\"Version\":1,\"AccessTokenLifetime\":\"04:00:00\"}}"], + "displayName": "ExtendedAccessTokenPolicy", + "isOrganizationDefault": false + }') + +# 2. Extract policy ID and get application ID +POLICY_ID=$(echo $POLICY_RESPONSE | jq -r '.id') +APP_ID=$(az ad app list --display-name "mz-self-managed-github-actions" --query "[0].id" -o tsv) + +echo "Policy ID: $POLICY_ID" +echo "Application ID: $APP_ID" + +# 3. Assign policy to application +az rest --method POST \ + --url "https://graph.microsoft.com/v1.0/applications/${APP_ID}/tokenLifetimePolicies/\$ref" \ + --headers "Content-Type=application/json" \ + --body "{\"@odata.id\": \"https://graph.microsoft.com/v1.0/policies/tokenLifetimePolicies/${POLICY_ID}\"}" + +echo "โœ… Token lifetime policy applied successfully!" diff --git a/.github/setup/azure/backend.tf b/.github/setup/azure/backend.tf new file mode 100644 index 0000000..4e347c7 --- /dev/null +++ b/.github/setup/azure/backend.tf @@ -0,0 +1,9 @@ +terraform { + backend "s3" { + bucket = "materialize-terraform-self-managed-state" + key = "github-setup/oidc/azure/terraform.tfstate" + region = "us-east-1" + encrypt = true + # profile = "" # Add your profile name here since backend block doesn't accept variables + } +} diff --git a/.github/setup/azure/main.tf b/.github/setup/azure/main.tf new file mode 100644 index 0000000..35bc571 --- /dev/null +++ b/.github/setup/azure/main.tf @@ -0,0 +1,211 @@ +# Azure AD Application and Service Principal for GitHub Actions OIDC +# https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-azure +# +# Note: Token lifetime is controlled by Azure AD's default policy (60 minutes). +# To extend token lifetime, you would need to create a TokenLifetimePolicy using accessTokenLifeTime.sh script after applying this terraform configuration + +# TODO: Fix Azure federated identity credential for merge queue authentication +# - Add environment-based credential or specific merge queue subject pattern +# - Subject pattern needed: repo:MaterializeInc/materialize-terraform-self-managed:environment:production +# - Or alternative: repo:MaterializeInc/materialize-terraform-self-managed:merge_group(verify if this works, if it does then prefer this) + +# TODO: After permissions issue is fixed: +# 1. Test terraform configuration and apply federated identity credential changes +# 2. Update Azure workflow to use environment-based authentication (add environment: production to job) +# 3. Re-enable merge_group trigger in .github/workflows/test-azure.yml +# 4. Validate merge queue authentication works with new federated identity credential + +# TODO: Investigate and resolve Azure permissions issues preventing terraform apply +resource "azuread_application" "github_actions" { + display_name = "mz-self-managed-github-actions" + description = "Application for GitHub Actions CI/CD with OIDC authentication" + + # Optional: Enable service principal creation + owners = [data.azuread_client_config.current.object_id] +} + +# Service Principal for the application +resource "azuread_service_principal" "github_actions" { + client_id = azuread_application.github_actions.client_id + app_role_assignment_required = false + owners = [data.azuread_client_config.current.object_id] + + tags = ["GitHubActions", "MaterializeTerraform", "OIDC"] +} + +# Federated Identity Credentials for GitHub Actions OIDC +# Azure requires specific branch patterns rather than wildcards + +# For main branch +resource "azuread_application_federated_identity_credential" "github_actions_main" { + application_id = azuread_application.github_actions.id + display_name = "mz-github-actions-oidc-main" + description = "Federated credential for GitHub Actions on main branch" + audiences = ["api://AzureADTokenExchange"] + issuer = "https://token.actions.githubusercontent.com" + subject = "repo:${var.github_repository}:ref:refs/heads/main" +} + +# For pull requests +resource "azuread_application_federated_identity_credential" "github_actions_pr" { + application_id = azuread_application.github_actions.id + display_name = "mz-github-actions-oidc-pr" + description = "Federated credential for GitHub Actions on pull requests" + audiences = ["api://AzureADTokenExchange"] + issuer = "https://token.actions.githubusercontent.com" + subject = "repo:${var.github_repository}:pull_request" +} + +# Get current Azure AD configuration +data "azuread_client_config" "current" {} + +# Get current subscription +data "azurerm_client_config" "current" {} + +# Get built-in role definitions for ABAC conditions +data "azurerm_role_definition" "owner" { + name = "Owner" + scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +} + +data "azurerm_role_definition" "user_access_administrator" { + name = "User Access Administrator" + scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +} + +data "azurerm_role_definition" "rbac_administrator" { + name = "Role Based Access Control Administrator" + scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +} + + +# Principle of Least Privilege: Minimal role assignments based on fixture requirements +# Refer roles from https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles to follow the principle of least privilege +# For AKS cluster management +# resource "azurerm_role_assignment" "github_actions_aks_contributor" { +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# role_definition_name = "Azure Kubernetes Service Contributor Role" +# principal_id = azuread_service_principal.github_actions.object_id +# } + +# # For networking (VNets, subnets) +# resource "azurerm_role_assignment" "github_actions_network_contributor" { +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# role_definition_name = "Network Contributor" +# principal_id = azuread_service_principal.github_actions.object_id +# } + +# # For PostgreSQL database management +# resource "azurerm_role_assignment" "github_actions_sql_contributor" { +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# role_definition_name = "SQL DB Contributor" +# principal_id = azuread_service_principal.github_actions.object_id +# } + +# # For storage account management +# resource "azurerm_role_assignment" "github_actions_storage_contributor" { +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# role_definition_name = "Storage Account Contributor" +# principal_id = azuread_service_principal.github_actions.object_id +# } + +# # For blob storage operations +# resource "azurerm_role_assignment" "github_actions_storage_blob" { +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# role_definition_name = "Storage Blob Data Contributor" +# principal_id = azuread_service_principal.github_actions.object_id +# } + +# # For workload identity management (Azure AD managed identities) +# resource "azurerm_role_assignment" "github_actions_managed_identity_contributor" { +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# role_definition_name = "Managed Identity Contributor" +# principal_id = azuread_service_principal.github_actions.object_id +# } + +# For Azure Monitor and Log Analytics (if enabled) +# resource "azurerm_role_assignment" "github_actions_log_analytics_contributor" { +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# role_definition_name = "Log Analytics Contributor" +# principal_id = azuread_service_principal.github_actions.object_id +# } + +# Facing issues with the custom role definition, so using the built-in Contributor role instead. Because I cannot create custom role definitions. +# For resource group creation/management (required by networking fixture) +# Note: No specific "Resource Group Contributor" role exists - using generic Contributor +# resource "azurerm_role_assignment" "github_actions_contributor" { +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# role_definition_name = "Contributor" +# principal_id = azuread_service_principal.github_actions.object_id +# } + +# resource "azurerm_role_definition" "resource_group_manager" { +# name = "GitHub Actions Resource Group Manager" +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# description = "Minimal permissions for resource group creation and management" + +# permissions { +# actions = [ +# # Core resource group operations (absolutely minimal) +# "Microsoft.Resources/subscriptions/resourcegroups/read", +# "Microsoft.Resources/subscriptions/resourcegroups/write", +# "Microsoft.Resources/subscriptions/resourcegroups/delete" +# ] +# not_actions = [] +# } + +# assignable_scopes = [ +# "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# ] +# } + +# # Assign the custom resource group role not able to perform this getting authz error +# resource "azurerm_role_assignment" "github_actions_resource_group_manager" { +# scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" +# role_definition_id = azurerm_role_definition.resource_group_manager.role_definition_resource_id +# principal_id = azuread_service_principal.github_actions.object_id +# } + +resource "azurerm_role_assignment" "github_actions_contributor" { + scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" + role_definition_name = "Contributor" + principal_id = azuread_service_principal.github_actions.object_id +} + +# RBAC Administrator role for AKS role assignments +# AKS modules need to assign network roles to managed identities and subnets +# Commented out due to ABAC restrictions - use more specific roles instead +# Maybe make it more restrictive to only assing +# ---> Network Contributor and Storage Blob Data Contributor since we only assign those in az modules +resource "azurerm_role_assignment" "github_actions_rbac_admin" { + scope = "/subscriptions/${data.azurerm_client_config.current.subscription_id}" + role_definition_name = "Role Based Access Control Administrator" + principal_id = azuread_service_principal.github_actions.object_id + + # ABAC condition to block assignment of high-privilege roles + # Allows assignment of any role EXCEPT: Owner, User Access Administrator, RBAC Administrator + # Using data sources to get role GUIDs dynamically instead of hardcoding + # TODO: Test and validate this configuration, havent tested due to Perms issue. + condition = <<-EOT + ( + (!(ActionMatches{'Microsoft.Authorization/roleAssignments/write'})) + OR + (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAllOfAllValues:GuidNotEquals { + ${data.azurerm_role_definition.owner.id}, + ${data.azurerm_role_definition.user_access_administrator.id}, + ${data.azurerm_role_definition.rbac_administrator.id} + }) + ) + AND + ( + (!(ActionMatches{'Microsoft.Authorization/roleAssignments/delete'})) + OR + (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAllOfAllValues:GuidNotEquals { + ${data.azurerm_role_definition.owner.id}, + ${data.azurerm_role_definition.user_access_administrator.id}, + ${data.azurerm_role_definition.rbac_administrator.id} + }) + ) + EOT + condition_version = "2.0" +} diff --git a/.github/setup/azure/outputs.tf b/.github/setup/azure/outputs.tf new file mode 100644 index 0000000..7ca7e20 --- /dev/null +++ b/.github/setup/azure/outputs.tf @@ -0,0 +1,20 @@ +# Outputs for GitHub Actions configuration +output "client_id" { + description = "Azure AD Application (Client) ID for GitHub Actions" + value = azuread_application.github_actions.client_id +} + +output "service_principal_object_id" { + description = "Object ID of the Service Principal for GitHub Actions" + value = azuread_service_principal.github_actions.object_id +} + +output "tenant_id" { + description = "Azure Tenant ID" + value = data.azurerm_client_config.current.tenant_id +} + +output "subscription_id" { + description = "Azure Subscription ID" + value = data.azurerm_client_config.current.subscription_id +} diff --git a/.github/setup/azure/variables.tf b/.github/setup/azure/variables.tf new file mode 100644 index 0000000..d7ea6e3 --- /dev/null +++ b/.github/setup/azure/variables.tf @@ -0,0 +1,15 @@ +variable "github_repository" { + description = "GitHub repository in format 'owner/repo-name'" + type = string + default = "MaterializeInc/materialize-terraform-self-managed" +} + +variable "subscription_id" { + description = "Azure subscription ID" + type = string +} + +variable "tenant_id" { + description = "Azure tenant ID" + type = string +} diff --git a/.github/setup/azure/versions.tf b/.github/setup/azure/versions.tf new file mode 100644 index 0000000..c38dbe0 --- /dev/null +++ b/.github/setup/azure/versions.tf @@ -0,0 +1,25 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + azuread = { + source = "hashicorp/azuread" + version = "~> 2.0" + } + } +} + +# Configure the Microsoft Azure Provider +provider "azurerm" { + subscription_id = var.subscription_id + features {} +} + +# Configure the Azure Active Directory Provider +provider "azuread" { + tenant_id = var.tenant_id +} diff --git a/.github/setup/gcp/.terraform.lock.hcl b/.github/setup/gcp/.terraform.lock.hcl new file mode 100644 index 0000000..78b8795 --- /dev/null +++ b/.github/setup/gcp/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "6.50.0" + constraints = "~> 6.0" + hashes = [ + "h1:79CwMTsp3Ud1nOl5hFS5mxQHyT0fGVye7pqpU0PPlHI=", + "zh:1f3513fcfcbf7ca53d667a168c5067a4dd91a4d4cccd19743e248ff31065503c", + "zh:3da7db8fc2c51a77dd958ea8baaa05c29cd7f829bd8941c26e2ea9cb3aadc1e5", + "zh:3e09ac3f6ca8111cbb659d38c251771829f4347ab159a12db195e211c76068bb", + "zh:7bb9e41c568df15ccf1a8946037355eefb4dfb4e35e3b190808bb7c4abae547d", + "zh:81e5d78bdec7778e6d67b5c3544777505db40a826b6eb5abe9b86d4ba396866b", + "zh:8d309d020fb321525883f5c4ea864df3d5942b6087f6656d6d8b3a1377f340fc", + "zh:93e112559655ab95a523193158f4a4ac0f2bfed7eeaa712010b85ebb551d5071", + "zh:d3efe589ffd625b300cef5917c4629513f77e3a7b111c9df65075f76a46a63c7", + "zh:d4a4d672bbef756a870d8f32b35925f8ce2ef4f6bbd5b71a3cb764f1b6c85421", + "zh:e13a86bca299ba8a118e80d5f84fbdd708fe600ecdceea1a13d4919c068379fe", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fec30c095647b583a246c39d557704947195a1b7d41f81e369ba377d997faef6", + ] +} diff --git a/.github/setup/gcp/backend.tf b/.github/setup/gcp/backend.tf new file mode 100644 index 0000000..25908c1 --- /dev/null +++ b/.github/setup/gcp/backend.tf @@ -0,0 +1,9 @@ +terraform { + backend "s3" { + bucket = "materialize-terraform-self-managed-state" + key = "github-setup/oidc/gcp/terraform.tfstate" + region = "us-east-1" + encrypt = true + # profile = "" # Add your profile name here since backend block doesn't accept variables + } +} diff --git a/.github/setup/gcp/main.tf b/.github/setup/gcp/main.tf new file mode 100644 index 0000000..1c1238a --- /dev/null +++ b/.github/setup/gcp/main.tf @@ -0,0 +1,120 @@ +# Workload Identity Federation through a Service Account for GitHub Actions +# https://github.com/google-github-actions/auth?tab=readme-ov-file#workload-identity-federation-through-a-service-account +# +# PREREQUISITES: The user running this Terraform must have the following roles: +# 1. Create Workload Identity Pools: +# gcloud projects add-iam-policy-binding PROJECT_ID \ +# --member="user:YOUR_EMAIL" --role="roles/iam.workloadIdentityPoolAdmin" +# 2. Enable/manage services: +# gcloud projects add-iam-policy-binding PROJECT_ID \ +# --member="user:YOUR_EMAIL" --role="roles/serviceusage.serviceUsageAdmin" +# 3. Create and manage service accounts + IAM policies: +# gcloud projects add-iam-policy-binding PROJECT_ID \ +# --member="user:YOUR_EMAIL" --role="roles/iam.serviceAccountAdmin" + +# Workload Identity Pool +resource "google_iam_workload_identity_pool" "github_actions" { + project = var.project_id + workload_identity_pool_id = "github-actions-pool" + display_name = "GitHub Actions Pool" + description = "Workload Identity Pool for GitHub Actions CI/CD" +} + +# Workload Identity Provider for GitHub Actions OIDC +resource "google_iam_workload_identity_pool_provider" "github_actions" { + project = var.project_id + workload_identity_pool_id = google_iam_workload_identity_pool.github_actions.workload_identity_pool_id + workload_identity_pool_provider_id = "mz-github-actions-provider" + display_name = "GA Materialize Provider" + description = "Materialize OIDC provider for GitHub Actions" + + # Attribute mapping from GitHub OIDC token to GCP attributes + attribute_mapping = { + "google.subject" = "assertion.repository + \":\" + assertion.run_id" + "attribute.actor" = "assertion.actor" + "attribute.repository" = "assertion.repository" + "attribute.repository_owner" = "assertion.repository_owner" + "attribute.ref" = "assertion.ref" + "attribute.workflow" = "assertion.workflow" + "attribute.run_id" = "assertion.run_id" + } + + # Security: Only allow tokens from MaterializeInc organization + attribute_condition = "assertion.repository_owner == 'MaterializeInc'" + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +# Create a service account for GitHub Actions +resource "google_service_account" "github_actions" { + project = var.project_id + account_id = "github-actions-materialize" + display_name = "GitHub Actions Materialize Service Account" + description = "Service Account for GitHub Actions CI/CD workflows" +} + +# Grant IAM permissions to the Service Account +# GitHub Actions will impersonate this service account +resource "google_project_iam_member" "github_actions_editor" { + project = var.project_id + role = "roles/editor" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +resource "google_project_iam_member" "github_actions_iam_service_account_admin" { + project = var.project_id + role = "roles/iam.serviceAccountAdmin" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +resource "google_project_iam_member" "github_actions_servicenetworking_networks_admin" { + project = var.project_id + role = "roles/servicenetworking.networksAdmin" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +resource "google_project_iam_member" "github_actions_storage_admin" { + project = var.project_id + role = "roles/storage.admin" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +resource "google_project_iam_member" "github_actions_container_admin" { + project = var.project_id + role = "roles/container.admin" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +resource "google_project_iam_member" "github_actions_compute_admin" { + project = var.project_id + role = "roles/compute.admin" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +# Allow the Workload Identity Pool to impersonate the Service Account +resource "google_service_account_iam_member" "github_actions_workload_identity_user" { + service_account_id = google_service_account.github_actions.name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.github_actions.name}/attribute.repository/${var.github_repository}" +} + +# Enable required APIs for the tests +resource "google_project_service" "required_apis" { + for_each = toset([ + "container.googleapis.com", + "compute.googleapis.com", # Required for GKE node creation + "sqladmin.googleapis.com", + "cloudresourcemanager.googleapis.com", + "servicenetworking.googleapis.com", + "iamcredentials.googleapis.com", + "iam.googleapis.com", # Required for roles/iam.serviceAccountAdmin + "storage.googleapis.com" # Required for roles/storage.admin + ]) + + project = var.project_id + service = each.value + + disable_dependent_services = false +} diff --git a/.github/setup/gcp/outputs.tf b/.github/setup/gcp/outputs.tf new file mode 100644 index 0000000..df706f6 --- /dev/null +++ b/.github/setup/gcp/outputs.tf @@ -0,0 +1,15 @@ +# Outputs for GitHub Actions configuration +output "workload_identity_provider" { + description = "Full resource name of the Workload Identity Provider for GitHub Actions" + value = google_iam_workload_identity_pool_provider.github_actions.name +} + +output "workload_identity_pool" { + description = "Full resource name of the Workload Identity Pool" + value = google_iam_workload_identity_pool.github_actions.name +} + +output "service_account_email" { + description = "Email address of the GitHub Actions Service Account" + value = google_service_account.github_actions.email +} diff --git a/.github/setup/gcp/variables.tf b/.github/setup/gcp/variables.tf new file mode 100644 index 0000000..7175891 --- /dev/null +++ b/.github/setup/gcp/variables.tf @@ -0,0 +1,10 @@ +variable "project_id" { + description = "The GCP project ID" + type = string +} + +variable "github_repository" { + description = "GitHub repository in format 'owner/repo-name'" + type = string + default = "MaterializeInc/materialize-terraform-self-managed" +} diff --git a/.github/setup/gcp/versions.tf b/.github/setup/gcp/versions.tf new file mode 100644 index 0000000..75a3858 --- /dev/null +++ b/.github/setup/gcp/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 6.0" + } + } +} diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..d8c68ec --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,63 @@ +# Infrastructure Testing Workflows + +## ๐Ÿ›ก๏ธ **Merge Queue Integration** + +Infrastructure tests **integrate with GitHub's merge queue** to ensure only approved, tested code reaches `main`. + +### **How It Works** +1. **Create PR** โ†’ Request review +2. **Get approval** โ†’ PR enters merge queue automatically +3. **Tests run** โ†’ Only affected cloud providers tested (smart path filtering) +4. **Auto-merge** โ†’ When tests pass, code merges to `main` +5. **Manual trigger** โ†’ Use `gh workflow run test-.yml` if needed + + +### **What Gets Tested** + +| Path Changes | Tests Triggered | +|-------------|----------------| +| `*/modules/**/*.tf`, `kubernetes/modules/**/*.{tf,yaml,yml}` | โœ… Relevant cloud tests | +| `test/aws/**/*.{go,tf}`, `test/gcp/**/*.{go,tf}`, `test/azure/**/*.{go,tf}` | โœ… Relevant cloud tests | +| `test/utils/**`, `test/shared/**`, `test/*.go` | โœ… **All cloud tests** | +| `*/examples/**`, `README.md`, `.env`, docs | โŒ No tests | + +### **Features** +- โœ… **Granular path filtering** - Only tests infrastructure changes (excludes docs/README) +- โœ… **Smart cloud detection** - Tests only affected clouds, or all clouds for shared changes +- โœ… **Merge queue integration** - Automatic testing on approved PRs +- โœ… **Conflict resolution** - Auto-retests when merge conflicts occur +- โœ… **Parallel cloud testing** - AWS/GCP/Azure run simultaneously when needed + +## **Setup Requirements** + +**Branch Protection + Merge Queue:** +- Enable merge queue for `main` branch +- Require PR approvals (dismisses stale approvals) +- Add required status checks: `AWS Tests`, `GCP Tests`, `Azure Tests` + + +**Repository Secrets:** +``` +MATERIALIZE_LICENSE_KEY +AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID +GCP_WORKLOAD_IDENTITY_PROVIDER, GCP_SERVICE_ACCOUNT_EMAIL +``` + +**Repository Variables:** +``` +GA_AWS_IAM_ROLE +TF_TEST_S3_BUCKET, TF_TEST_S3_REGION, TF_TEST_S3_PREFIX +GOOGLE_PROJECT, AWS_REGION +``` + +## **Manual Testing** + +```bash +# Run individual cloud tests manually (for debugging/testing) +gh workflow run test-aws.yml --ref your-branch +gh workflow run test-gcp.yml --ref your-branch +gh workflow run test-azure.yml --ref your-branch + +# Note: Manual runs bypass merge queue but still require proper authentication +# Production merges should always go through the merge queue process +``` diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1d0fb1d..1bf3ebf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,61 +1,107 @@ -name: Lint and Test +name: Lint and Validate on: + merge_group: pull_request: - branches: [ main ] + branches: [main] jobs: lint: - name: Lint + name: Terraform Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - - uses: hashicorp/setup-terraform@v3 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 with: - terraform_version: "1.8.0" + terraform_version: "1.9.0" - name: Terraform Format Check run: terraform fmt -check -recursive - - uses: terraform-linters/setup-tflint@v4 + - name: Setup TFLint + uses: terraform-linters/setup-tflint@v4 with: tflint_version: v0.58.0 - - name: TFLint - run: | - tflint --init - tflint --recursive --format compact + - name: Initialize TFLint + run: tflint --init + + - name: Run TFLint + run: tflint --recursive --format compact validate: - name: Validate Examples + name: Validate Simple Examples runs-on: ubuntu-latest strategy: matrix: example: # Add new examples here as they are created - "aws/examples/simple" + - "azure/examples/simple" + - "gcp/examples/simple" fail-fast: false steps: - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - - uses: hashicorp/setup-terraform@v3 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 with: - terraform_version: "1.8.0" + terraform_version: "1.9.0" - name: Check if example exists + id: check-dir run: | if [ ! -d "${{ matrix.example }}" ]; then - echo "Example directory ${{ matrix.example }} does not exist, skipping" - exit 0 + echo "exists=false" >> $GITHUB_OUTPUT + echo "โš ๏ธ Example directory ${{ matrix.example }} does not exist, skipping" + else + echo "exists=true" >> $GITHUB_OUTPUT + echo "โœ… Example directory ${{ matrix.example }} exists" fi - name: Terraform Init + if: steps.check-dir.outputs.exists == 'true' run: | cd ${{ matrix.example }} terraform init -backend=false - name: Terraform Validate + if: steps.check-dir.outputs.exists == 'true' run: | cd ${{ matrix.example }} terraform validate + + lint-go: + name: Go Tests Lint + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.23.5" + cache-dependency-path: test/go.sum + + - name: Run go fmt + working-directory: test + run: | + if [ -n "$(gofmt -l .)" ]; then + echo "โŒ Go files are not formatted. Run 'gofmt -w .' to fix:" + gofmt -l . + exit 1 + fi + echo "โœ… All Go files are properly formatted" + + - name: Run go vet + working-directory: test + run: go vet ./... + + - name: Run go mod verify + working-directory: test + run: go mod verify diff --git a/.github/workflows/test-aws.yml b/.github/workflows/test-aws.yml new file mode 100644 index 0000000..e02926b --- /dev/null +++ b/.github/workflows/test-aws.yml @@ -0,0 +1,107 @@ +name: AWS Tests + +on: + # Primary trigger: GitHub merge queue with smart path filtering + merge_group: + paths-ignore: + # Exclude other cloud providers + - "gcp/**" + - "azure/**" + - "test/gcp/**" + - "test/azure/**" + # Exclude setup files from other cloud providers + - ".github/setup/gcp/**" + - ".github/setup/azure/**" + # Exclude examples + - "aws/examples/**" + # Exclude documentation and config + - "**.md" + - "**.env" + - ".gitignore" + - "LICENSE" + # Exclude Azure/GCP specific workflows + - ".github/workflows/test-gcp*.yml" + - ".github/workflows/test-azure*.yml" + - ".github/scripts/**" + # Manual trigger: For testing and debugging purposes + workflow_dispatch: + inputs: + test_stage: + description: "Test stage to run" + required: false + default: "full-test" + type: choice + options: + - network-only + - full-test + + # Future enhancement: Add scheduled runs for nightly testing + # schedule: + # - cron: '30 4 * * *' # 10 AM IST (4:30 AM UTC) for nightly infrastructure validation + +permissions: + id-token: write # Required for OIDC + contents: read # Required to checkout code + +jobs: + test-aws: + name: AWS Infrastructure Tests + runs-on: ubuntu-latest + # Note: Triggered automatically when approved PRs enter GitHub's merge queue + # Smart path filtering ensures tests run only when AWS infrastructure changes + env: + DURATION_SECONDS: 14400 # 4 hours + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure AWS credentials via OIDC + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.GA_AWS_IAM_ROLE }} + role-session-name: GitHubActions-AWS-Tests + aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} + # Max duration of the role is 8 hours as per IAM role configuration in .github/setup/aws/main.tf + role-duration-seconds: ${{ env.DURATION_SECONDS }} + + - name: Verify AWS Authentication + run: | + echo "Testing AWS authentication..." + aws sts get-caller-identity + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.9.0" + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.23.5" + cache-dependency-path: test/go.sum + + - name: Install dependencies + working-directory: test + run: go mod tidy + + - name: Run AWS Tests + working-directory: test/aws + env: + AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + MATERIALIZE_LICENSE_KEY: ${{ secrets.MATERIALIZE_LICENSE_KEY }} + GITHUB_ACTIONS: "true" + # S3 Remote Backend Configuration + TF_TEST_REMOTE_BACKEND: "true" + TF_TEST_S3_BUCKET: ${{ vars.TF_TEST_S3_BUCKET }} + TF_TEST_S3_REGION: ${{ vars.TF_TEST_S3_REGION || vars.AWS_REGION || 'us-east-1' }} + TF_TEST_S3_PREFIX: ${{ vars.TF_TEST_S3_PREFIX || 'github-actions-test-runs' }} + # Note: AWS credentials automatically available via OIDC for both: + # - Terraform AWS provider authentication + # - S3 backend authentication + # Terratest: Set env vars to SKIP stages, unset to RUN stages + SKIP_setup_materialize_disk_enabled: ${{ github.event.inputs.test_stage == 'network-only' && 'true' || '' }} + SKIP_setup_materialize_disk_disabled: ${{ github.event.inputs.test_stage == 'network-only' && 'true' || '' }} + run: | + echo "Running tests" + go test -timeout ${{ env.DURATION_SECONDS }}s -run TestStagedDeploymentSuite -v diff --git a/.github/workflows/test-azure.yml b/.github/workflows/test-azure.yml new file mode 100644 index 0000000..46805f8 --- /dev/null +++ b/.github/workflows/test-azure.yml @@ -0,0 +1,129 @@ +name: Azure Tests + +on: + # DISABLED: Primary trigger commented out until Azure permissions and federated identity issues are resolved + # merge_group: + # paths-ignore: + # # Exclude other cloud providers + # - "aws/**" + # - "gcp/**" + # - "test/aws/**" + # - "test/gcp/**" + # # Exclude setup files from other cloud providers + # - ".github/setup/aws/**" + # - ".github/setup/gcp/**" + # # Exclude examples + # - "azure/examples/**" + # # Exclude documentation and config + # - "**.md" + # - "**.env" + # - ".gitignore" + # - "LICENSE" + # # Exclude AWS/GCP specific workflows + # - ".github/workflows/test-aws*.yml" + # - ".github/workflows/test-gcp*.yml" + # - ".github/scripts/**" + + # Manual trigger: For testing and debugging purposes (ONLY trigger until issues are resolved) + workflow_dispatch: + inputs: + test_stage: + description: "Test stage to run" + required: false + default: "full-test" + type: choice + options: + - network-only + - full-test + + # Future enhancement: Add scheduled runs for nightly testing + # schedule: + # - cron: '30 4 * * *' # 10 AM IST (4:30 AM UTC) for nightly infrastructure validation + +permissions: + id-token: write # Required for OIDC + contents: read # Required to checkout code + +jobs: + test-azure: + name: Azure Infrastructure Tests + runs-on: ubuntu-latest + env: + DURATION_SECONDS: 14400 # 4 hours + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure AWS credentials via OIDC (for S3 backend) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.GA_AWS_IAM_ROLE }} + role-session-name: GitHubActions-Azure-S3Backend + aws-region: ${{ vars.TF_TEST_S3_REGION || 'us-east-1' }} + # Max duration of the role is 8 hours as per IAM role configuration in .github/setup/aws/main.tf + role-duration-seconds: ${{ env.DURATION_SECONDS }} + + - name: Verify AWS Authentication (for S3 backend) + run: | + echo "Testing AWS authentication for S3 backend..." + aws sts get-caller-identity + + - name: Azure Login via OIDC + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Verify Azure Authentication + run: | + echo "Testing Azure authentication..." + az account show + az account list-locations --output table + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.9.0" + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.23.5" + cache-dependency-path: test/go.sum + + - name: Install dependencies + working-directory: test + run: go mod tidy + + - name: Run Azure Tests + working-directory: test/azure + env: + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + MATERIALIZE_LICENSE_KEY: ${{ secrets.MATERIALIZE_LICENSE_KEY }} + TEST_REGION: westus2 + GITHUB_ACTIONS: "true" + # S3 Remote Backend Configuration (for terraform state) + TF_TEST_REMOTE_BACKEND: "true" + TF_TEST_S3_BUCKET: ${{ vars.TF_TEST_S3_BUCKET }} + TF_TEST_S3_REGION: ${{ vars.TF_TEST_S3_REGION || 'us-east-1' }} + TF_TEST_S3_PREFIX: ${{ vars.TF_TEST_S3_PREFIX || 'github-actions-test-runs' }} + # Note: AWS credentials for S3 backend provided via OIDC above + # Azure credentials for Azure provider provided via OIDC above + # Terratest: Set env vars to SKIP stages, unset to RUN stages + SKIP_setup_materialize_disk_enabled: ${{ github.event.inputs.test_stage == 'network-only' && 'true' || '' }} + SKIP_setup_materialize_disk_disabled: ${{ github.event.inputs.test_stage == 'network-only' && 'true' || '' }} + run: | + echo "๐Ÿš€ Running tests for Azure..." + echo "๐Ÿ“ Current working directory: $(pwd)" + echo "๐Ÿ”ง Go version: $(go version)" + echo "" + echo "๐Ÿ“‹ Environment variables:" + env | sort + echo "" + echo "๐Ÿงช Starting Go tests..." + + go test -timeout ${{ env.DURATION_SECONDS }}s -run TestStagedDeploymentSuite -v diff --git a/.github/workflows/test-gcp.yml b/.github/workflows/test-gcp.yml new file mode 100644 index 0000000..ef6c7aa --- /dev/null +++ b/.github/workflows/test-gcp.yml @@ -0,0 +1,127 @@ +name: GCP Tests + +on: + # Primary trigger: GitHub merge queue with smart path filtering + merge_group: + paths-ignore: + # Exclude other cloud providers + - "aws/**" + - "azure/**" + - "test/aws/**" + - "test/azure/**" + # Exclude setup files from other cloud providers + - ".github/setup/aws/**" + - ".github/setup/azure/**" + # Exclude examples + - "gcp/examples/**" + # Exclude documentation and config + - "**.md" + - "**.env" + - ".gitignore" + - "LICENSE" + # Exclude AWS/Azure specific workflows + - ".github/workflows/test-aws*.yml" + - ".github/workflows/test-azure*.yml" + - ".github/scripts/**" + # Manual trigger: For testing and debugging purposes + workflow_dispatch: + inputs: + test_stage: + description: "Test stage to run" + required: false + default: "full-test" + type: choice + options: + - network-only + - full-test + + # Future enhancement: Add scheduled runs for nightly testing + # schedule: + # - cron: '30 4 * * *' # 10 AM IST (4:30 AM UTC) for nightly infrastructure validation + +permissions: + id-token: write # Required for Workload Identity + contents: read # Required to checkout code + +jobs: + test-gcp: + name: GCP Infrastructure Tests + runs-on: ubuntu-latest + env: + DURATION_SECONDS: 14400 # 4 hours + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure AWS credentials via OIDC (for S3 backend) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.GA_AWS_IAM_ROLE }} + role-session-name: GitHubActions-GCP-S3Backend + aws-region: ${{ vars.TF_TEST_S3_REGION || 'us-east-1' }} + # Max duration of the role is 8 hours as per IAM role configuration in .github/setup/aws/main.tf + role-duration-seconds: ${{ env.DURATION_SECONDS }} + + - name: Authenticate to Google Cloud (Service Account Impersonation) + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} + access_token_lifetime: ${{ env.DURATION_SECONDS }}s # 4 hours for long-running tests + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + + - name: Verify AWS Authentication (for S3 backend) + run: | + echo "Testing AWS authentication for S3 backend..." + aws sts get-caller-identity + + - name: Verify GCP Authentication + run: | + echo "Testing GCP authentication..." + gcloud auth list + gcloud config list + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.9.0" + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.23.5" + cache-dependency-path: test/go.sum + + - name: Install dependencies + working-directory: test + run: go mod tidy + + - name: Run GCP Tests + working-directory: test/gcp + env: + GOOGLE_PROJECT: ${{ vars.GOOGLE_PROJECT }} + MATERIALIZE_LICENSE_KEY: ${{ secrets.MATERIALIZE_LICENSE_KEY }} + GITHUB_ACTIONS: "true" + # S3 Remote Backend Configuration (for terraform state) + TF_TEST_REMOTE_BACKEND: "true" + TF_TEST_S3_BUCKET: ${{ vars.TF_TEST_S3_BUCKET }} + TF_TEST_S3_REGION: ${{ vars.TF_TEST_S3_REGION || 'us-east-1' }} + TF_TEST_S3_PREFIX: ${{ vars.TF_TEST_S3_PREFIX || 'github-actions-test-runs' }} + # Note: AWS credentials for S3 backend provided via OIDC above + # Terratest: Set env vars to SKIP stages, unset to RUN stages + SKIP_setup_materialize_disk_enabled: ${{ github.event.inputs.test_stage == 'network-only' && 'true' || '' }} + SKIP_setup_materialize_disk_disabled: ${{ github.event.inputs.test_stage == 'network-only' && 'true' || '' }} + run: | + echo "๐Ÿš€ Running tests for GCP..." + echo "๐Ÿ“ Current working directory: $(pwd)" + echo "๐Ÿ”ง Go version: $(go version)" + echo "" + echo "๐Ÿ“‹ Environment variables:" + env | sort + echo "" + echo "๐Ÿงช Starting Go tests..." + + go test -timeout ${{ env.DURATION_SECONDS }}s -run TestStagedDeploymentSuite -v diff --git a/.gitignore b/.gitignore index 8731ac2..9f50d0d 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ tests/.terraform.lock.hcl # Test run directories (timestamped directories in test/{cloud}/) test/*/t[0-9]*-*/ +.github/setup/*/terraform.tfstate +.github/setup/*/backend-override.tf diff --git a/azure/examples/simple/main.tf b/azure/examples/simple/main.tf index f20b8ec..8049d79 100644 --- a/azure/examples/simple/main.tf +++ b/azure/examples/simple/main.tf @@ -1,5 +1,5 @@ provider "azurerm" { - # Set the Azure subscription ID here or use the ARM_SUBSCRIPTION_ID environment variable + # Set the Azure subscription ID here or use the AZURE_SUBSCRIPTION_ID environment variable subscription_id = var.subscription_id features { diff --git a/test/aws/fixtures/materialize/main.tf b/test/aws/fixtures/materialize/main.tf index 8cebae1..15e27da 100644 --- a/test/aws/fixtures/materialize/main.tf +++ b/test/aws/fixtures/materialize/main.tf @@ -1,6 +1,6 @@ provider "aws" { region = var.region - profile = var.profile + profile = var.profile != null ? var.profile : null } @@ -139,6 +139,7 @@ module "cert_manager" { depends_on = [ module.eks_node_group, module.eks, + module.aws_lbc, ] } @@ -213,10 +214,13 @@ module "materialize_instance" { } depends_on = [ + module.eks, + module.database, module.storage, module.self_signed_cluster_issuer, module.operator, - module.database, + module.aws_lbc, + module.eks_node_group, ] } diff --git a/test/aws/fixtures/materialize/variables.tf b/test/aws/fixtures/materialize/variables.tf index a12537b..53535f0 100644 --- a/test/aws/fixtures/materialize/variables.tf +++ b/test/aws/fixtures/materialize/variables.tf @@ -5,8 +5,9 @@ variable "region" { } variable "profile" { - description = "AWS profile to use for authentication" + description = "AWS profile to use for authentication (optional for OIDC)" type = string + default = null } # Network Variables diff --git a/test/aws/fixtures/networking/main.tf b/test/aws/fixtures/networking/main.tf index 0e94d24..45a0ae7 100644 --- a/test/aws/fixtures/networking/main.tf +++ b/test/aws/fixtures/networking/main.tf @@ -1,6 +1,6 @@ provider "aws" { region = var.region # Replace with your desired AWS region - profile = var.profile + profile = var.profile != null ? var.profile : null } module "networking" { diff --git a/test/aws/fixtures/networking/variables.tf b/test/aws/fixtures/networking/variables.tf index b6a3fb5..b3c77ed 100644 --- a/test/aws/fixtures/networking/variables.tf +++ b/test/aws/fixtures/networking/variables.tf @@ -1,6 +1,7 @@ variable "profile" { - description = "AWS profile to use for authentication" + description = "AWS profile to use for authentication (optional for OIDC)" type = string + default = null } variable "region" { diff --git a/test/aws/staged_deployment_test.go b/test/aws/staged_deployment_test.go index 6d8d357..aaf537b 100644 --- a/test/aws/staged_deployment_test.go +++ b/test/aws/staged_deployment_test.go @@ -46,7 +46,7 @@ func (suite *StagedDeploymentTestSuite) TearDownSuite() { test_structure.RunTestStage(t, "cleanup_network", func() { // Cleanup network if it was created in this test run networkStageDir := filepath.Join(suite.workingDir, utils.NetworkingDir) - if networkOptions := test_structure.LoadTerraformOptions(t, networkStageDir); networkOptions != nil { + if networkOptions := helpers.SafeLoadTerraformOptions(t, networkStageDir); networkOptions != nil { t.Logf("๐Ÿ—‘๏ธ Cleaning up network...") // TODO: fix cleanup when Destroy errors out because Terraform init was not successful during Terraform InitAndApply terraform.Destroy(t, networkOptions) @@ -97,7 +97,7 @@ func (suite *StagedDeploymentTestSuite) cleanupStage(stageName, stageDir string) t := suite.T() t.Logf("๐Ÿ—‘๏ธ Cleaning up %s stage: %s", stageName, stageDir) - options := test_structure.LoadTerraformOptions(t, filepath.Join(suite.workingDir, stageDir)) + options := helpers.SafeLoadTerraformOptions(t, filepath.Join(suite.workingDir, stageDir)) if options == nil { t.Logf("โ™ป๏ธ No %s stage to cleanup (was not created in this test)", stageName) return @@ -115,7 +115,7 @@ func (suite *StagedDeploymentTestSuite) cleanupStage(stageName, stageDir string) func (suite *StagedDeploymentTestSuite) TestFullDeployment() { t := suite.T() awsRegion := os.Getenv("AWS_REGION") - awsProfile := os.Getenv("AWS_PROFILE") + awsProfile := getAWSProfileForTerraform() // Use helper function for OIDC compatibility // Stage 1: Network Setup test_structure.RunTestStage(t, "setup_network", func() { @@ -148,11 +148,10 @@ func (suite *StagedDeploymentTestSuite) TestFullDeployment() { // Create terraform.tfvars.json file for network stage networkTfvarsPath := filepath.Join(networkingPath, "terraform.tfvars.json") networkVariables := map[string]interface{}{ - "profile": awsProfile, "region": awsRegion, "name_prefix": fmt.Sprintf("%s-net", uniqueId), "vpc_cidr": TestVPCCIDR, - "availability_zones": []string{TestAvailabilityZoneA, TestAvailabilityZoneB}, + "availability_zones": getAvailabilityZones(awsRegion), "private_subnet_cidrs": []string{TestPrivateSubnetCIDRA, TestPrivateSubnetCIDRB}, "public_subnet_cidrs": []string{TestPublicSubnetCIDRA, TestPublicSubnetCIDRB}, "single_nat_gateway": true, @@ -163,6 +162,11 @@ func (suite *StagedDeploymentTestSuite) TestFullDeployment() { "test-run": uniqueId, }, } + + // Add profile only if it's not empty (for local testing) + if awsProfile != "" { + networkVariables["profile"] = awsProfile + } helpers.CreateTfvarsFile(t, networkTfvarsPath, networkVariables) networkOptions := &terraform.Options{ @@ -282,8 +286,7 @@ func (suite *StagedDeploymentTestSuite) setupMaterializeConsolidatedStage(stage, // Build variables map for the generic tfvars creation function variables := map[string]interface{}{ // AWS Configuration - "profile": profile, - "region": region, + "region": region, // Network Configuration "vpc_id": vpcId, @@ -356,6 +359,11 @@ func (suite *StagedDeploymentTestSuite) setupMaterializeConsolidatedStage(stage, }, } + // Add profile only if it's not empty (for local testing) + if profile != "" { + variables["profile"] = profile + } + helpers.CreateTfvarsFile(t, tfvarsPath, variables) materializeOptions := &terraform.Options{ TerraformDir: materializePath, diff --git a/test/aws/test_constants.go b/test/aws/test_constants.go index 787dcbd..9c80841 100644 --- a/test/aws/test_constants.go +++ b/test/aws/test_constants.go @@ -4,20 +4,6 @@ package test // and avoid flakiness from random selections or regional variations const ( - // Fixed test region to avoid quota/availability issues with random regions - // Using us-west-2 for reliable availability and quota - TestRegion = "us-west-2" - - // Availability zones in us-west-2 - TestAvailabilityZoneA = "us-west-2a" - TestAvailabilityZoneB = "us-west-2b" - TestAvailabilityZoneC = "us-west-2c" - - // EC2 instance types that are reliably available in us-west-2 - TestInstanceTypeSmall = "t3.medium" // 2 vCPU, 4 GB RAM - TestInstanceTypeMedium = "t3.large" // 2 vCPU, 8 GB RAM - TestInstanceTypeLarge = "t3.xlarge" // 4 vCPU, 16 GB RAM - // EKS-specific instance types TestEKSDiskEnabledInstanceType = "r7gd.2xlarge" TestEKSDiskDisabledInstanceType = "r7g.2xlarge" @@ -26,27 +12,19 @@ const ( TestKubernetesVersion = "1.32" // RDS instance classes that are reliably available - TestRDSInstanceClassSmall = "db.t3.micro" // 2 vCPU, 1 GB RAM - TestRDSInstanceClassMedium = "db.t3.small" // 2 vCPU, 2 GB RAM - TestRDSInstanceClassLarge = "db.t3.medium" // 2 vCPU, 4 GB RAM + TestRDSInstanceClassSmall = "db.t3.micro" // 2 vCPU, 1 GB RAM // PostgreSQL version that's widely available TestPostgreSQLVersion = "15" // Storage sizes for testing (smaller to reduce costs) - TestAllocatedStorageSmall = 20 - TestAllocatedStorageMedium = 50 - TestAllocatedStorageLarge = 100 + TestAllocatedStorageSmall = 20 // Max allocated storage - TestMaxAllocatedStorageSmall = 40 - TestMaxAllocatedStorageMedium = 100 - TestMaxAllocatedStorageLarge = 200 + TestMaxAllocatedStorageSmall = 40 // Network CIDR blocks that don't conflict - TestVPCCIDR = "10.0.0.0/16" - TestPrivateSubnetCIDR = "10.0.1.0/24" - TestPublicSubnetCIDR = "10.0.101.0/24" + TestVPCCIDR = "10.0.0.0/16" // Custom CIDR blocks for multi-AZ tests TestPrivateSubnetCIDRA = "10.0.1.0/24" @@ -54,17 +32,9 @@ const ( TestPublicSubnetCIDRA = "10.0.101.0/24" TestPublicSubnetCIDRB = "10.0.102.0/24" - // Test timeouts (in seconds) - TestTimeoutShort = 300 // 5 minutes - TestTimeoutMedium = 1800 // 30 minutes - TestTimeoutLong = 3600 // 1 hour - // Retry configuration - TestMaxRetries = 1 - TestRetryDelay = 10 // seconds - TestParallelism = 10 - - // TestRuns directory, this will be created in the directory where the tests are run + TestMaxRetries = 1 + TestRetryDelay = 10 // seconds // Test environment variables TestPassword = "test-password-123!" @@ -76,11 +46,5 @@ const ( TestBackupWindow = "04:00-05:00" TestBackupRetentionPeriod = 7 - // Resource naming format for AWS compatibility - // Format: t{YYMMDDHHMMSS}-{random5} - TestResourceIDFormat = "t%s-%s" - TestRandomIDLength = 5 - - TestOpenEbsVersion = "4.2.0" TestCertManagerVersion = "v1.18.0" ) diff --git a/test/aws/test_helpers.go b/test/aws/test_helpers.go index 66f7b21..d04e01a 100644 --- a/test/aws/test_helpers.go +++ b/test/aws/test_helpers.go @@ -3,6 +3,7 @@ package test import ( "fmt" "math/rand" + "os" "time" "github.com/MaterializeInc/materialize-terraform-self-managed/test/utils/config" @@ -41,8 +42,7 @@ func getRequiredAWSConfigurations() []config.Configuration { Type: config.Critical, }, { - Key: "AWS_PROFILE", - Type: config.Critical, + Key: "AWS_PROFILE", }, { Key: "MATERIALIZE_LICENSE_KEY", @@ -70,3 +70,26 @@ func getRequiredAWSConfigurations() []config.Configuration { }, } } + +// getAWSProfileForTerraform returns the AWS profile for terraform configuration +// Returns empty string when running in GitHub Actions (OIDC environment) +func getAWSProfileForTerraform() string { + profile := os.Getenv("AWS_PROFILE") + + // Check if we're running in GitHub Actions + if os.Getenv("GITHUB_ACTIONS") == "true" { + // In GitHub Actions, credentials are provided via OIDC, no profile needed + return "" + } + + return profile +} + +// getAvailabilityZones returns availability zones for the given AWS region +// Returns the first two AZs (typically 'a' and 'b') for any region +func getAvailabilityZones(awsRegion string) []string { + return []string{ + awsRegion + "a", + awsRegion + "b", + } +} diff --git a/test/azure/.env b/test/azure/.env index 750730f..e5c1d28 100644 --- a/test/azure/.env +++ b/test/azure/.env @@ -1,6 +1,6 @@ # Azure Authentication # Set your Azure subscription ID (required) -ARM_SUBSCRIPTION_ID=your-azure-subscription-id +AZURE_SUBSCRIPTION_ID=your-azure-subscription-id # Test Configuration TEST_REGION=westus2 diff --git a/test/azure/README.md b/test/azure/README.md index 6fdad85..7bad365 100644 --- a/test/azure/README.md +++ b/test/azure/README.md @@ -19,7 +19,7 @@ Terratest tests for Azure Materialize Terraform modules using staged deployment. - Go 1.23+, Terraform 1.0+ - Azure subscription with required services - Service Principal with `Contributor` and `User Access Administrator` roles -- Set `ARM_SUBSCRIPTION_ID=your-subscription-id` +- Set `AZURE_SUBSCRIPTION_ID=your-subscription-id` ## Running Tests @@ -73,7 +73,7 @@ rm -rf test/azure/{uniqueId} **Environment Variables:** - Loaded from `local.env` (if exists) or `.env` -- Can be set manually: `export ARM_SUBSCRIPTION_ID=your-subscription-id` +- Can be set manually: `export AZURE_SUBSCRIPTION_ID=your-subscription-id` ## Troubleshooting diff --git a/test/azure/fixtures/materialize/main.tf b/test/azure/fixtures/materialize/main.tf index cf35cd8..7a2ad6f 100644 --- a/test/azure/fixtures/materialize/main.tf +++ b/test/azure/fixtures/materialize/main.tf @@ -202,10 +202,12 @@ module "materialize_instance" { } depends_on = [ - module.operator, - module.storage, + module.aks, module.database, + module.storage, module.self_signed_cluster_issuer, + module.operator, + module.nodepool, ] } diff --git a/test/azure/staged_deployment_test.go b/test/azure/staged_deployment_test.go index 34bfab3..4e60226 100644 --- a/test/azure/staged_deployment_test.go +++ b/test/azure/staged_deployment_test.go @@ -47,7 +47,7 @@ func (suite *StagedDeploymentSuite) TearDownSuite() { test_structure.RunTestStage(t, "cleanup_network", func() { // Cleanup network if it was created in this test run networkStageDir := filepath.Join(suite.workingDir, utils.NetworkingDir) - if networkOptions := test_structure.LoadTerraformOptions(t, networkStageDir); networkOptions != nil { + if networkOptions := helpers.SafeLoadTerraformOptions(t, networkStageDir); networkOptions != nil { t.Logf("๐Ÿ—‘๏ธ Cleaning up network...") terraform.Destroy(t, networkOptions) t.Logf("โœ… Network cleanup completed") @@ -97,7 +97,7 @@ func (suite *StagedDeploymentSuite) cleanupStage(stageName, stageDir string) { t := suite.T() t.Logf("๐Ÿ—‘๏ธ Cleaning up %s stage: %s", stageName, stageDir) - options := test_structure.LoadTerraformOptions(t, filepath.Join(suite.workingDir, stageDir)) + options := helpers.SafeLoadTerraformOptions(t, filepath.Join(suite.workingDir, stageDir)) if options == nil { t.Logf("โ™ป๏ธ No %s stage to cleanup (was not created in this test)", stageName) return @@ -114,7 +114,7 @@ func (suite *StagedDeploymentSuite) cleanupStage(stageName, stageDir string) { // Stages: Network โ†’ (disk-enabled-setup) โ†’ (disk-disabled-setup) func (suite *StagedDeploymentSuite) TestFullDeployment() { t := suite.T() - subscriptionID := os.Getenv("ARM_SUBSCRIPTION_ID") + subscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") testRegion := os.Getenv("TEST_REGION") if testRegion == "" { testRegion = TestRegion diff --git a/test/azure/test_constants.go b/test/azure/test_constants.go index 53d42e2..844fd19 100644 --- a/test/azure/test_constants.go +++ b/test/azure/test_constants.go @@ -55,7 +55,6 @@ const ( TestRetryDelay = 10 // seconds TestParallelism = 10 - // Test environment variables TestPassword = "test-password-123!" TestDBName = "materialize_test" diff --git a/test/azure/test_helpers.go b/test/azure/test_helpers.go index d08b6a4..3fff571 100644 --- a/test/azure/test_helpers.go +++ b/test/azure/test_helpers.go @@ -41,7 +41,7 @@ func generateAzureCompliantID() string { func getRequiredAzureConfigurations() []config.Configuration { return []config.Configuration{ { - Key: "ARM_SUBSCRIPTION_ID", + Key: "AZURE_SUBSCRIPTION_ID", Type: config.Critical, }, { diff --git a/test/gcp/fixtures/materialize/main.tf b/test/gcp/fixtures/materialize/main.tf index e40ed5f..7790f9f 100644 --- a/test/gcp/fixtures/materialize/main.tf +++ b/test/gcp/fixtures/materialize/main.tf @@ -159,10 +159,12 @@ module "materialize_instance" { } depends_on = [ - module.operator, - module.storage, + module.gke, + module.nodepool, module.database, + module.storage, module.self_signed_cluster_issuer, + module.operator, ] } diff --git a/test/gcp/staged_deployment_test.go b/test/gcp/staged_deployment_test.go index 38c3ecd..c3f38e5 100644 --- a/test/gcp/staged_deployment_test.go +++ b/test/gcp/staged_deployment_test.go @@ -47,7 +47,7 @@ func (suite *StagedDeploymentSuite) TearDownSuite() { test_structure.RunTestStage(t, "cleanup_network", func() { // Cleanup network if it was created in this test run networkStageDir := filepath.Join(suite.workingDir, utils.NetworkingDir) - if networkOptions := test_structure.LoadTerraformOptions(t, networkStageDir); networkOptions != nil { + if networkOptions := helpers.SafeLoadTerraformOptions(t, networkStageDir); networkOptions != nil { t.Logf("๐Ÿ—‘๏ธ Cleaning up network...") terraform.Destroy(t, networkOptions) t.Logf("โœ… Network cleanup completed") @@ -97,7 +97,7 @@ func (suite *StagedDeploymentSuite) cleanupStage(stageName, stageDir string) { t := suite.T() t.Logf("๐Ÿ—‘๏ธ Cleaning up %s stage: %s", stageName, stageDir) - options := test_structure.LoadTerraformOptions(t, filepath.Join(suite.workingDir, stageDir)) + options := helpers.SafeLoadTerraformOptions(t, filepath.Join(suite.workingDir, stageDir)) if options == nil { t.Logf("โ™ป๏ธ No %s stage to cleanup (was not created in this test)", stageName) return @@ -332,7 +332,6 @@ func (suite *StagedDeploymentSuite) setupMaterializeConsolidatedStage(stage, sta // GKE Configuration "namespace": TestGKENamespace, - "skip_nodepool": false, "materialize_node_type": machineType, "min_nodes": TestGKEMinNodes, "max_nodes": TestGKEMaxNodes, @@ -367,7 +366,6 @@ func (suite *StagedDeploymentSuite) setupMaterializeConsolidatedStage(stage, sta // Storage Configuration "storage_bucket_versioning": TestStorageBucketVersioning, "storage_bucket_version_ttl": TestStorageBucketVersionTTL, - "enable_bucket_encryption": true, // Cert Manager Configuration "cert_manager_install_timeout": TestCertManagerInstallTimeout, @@ -378,8 +376,8 @@ func (suite *StagedDeploymentSuite) setupMaterializeConsolidatedStage(stage, sta "operator_namespace": expectedOperatorNamespace, // Materialize Instance Configuration - "instance_name": TestMaterializeInstanceName, - "instance_namespace": expectedInstanceNamespace, + "instance_name": TestMaterializeInstanceName, + "instance_namespace": expectedInstanceNamespace, "user": map[string]interface{}{ "name": TestDBUsername, "password": TestPassword, diff --git a/test/utils/basesuite/baseSuite.go b/test/utils/basesuite/baseSuite.go index b91716d..0996b8c 100644 --- a/test/utils/basesuite/baseSuite.go +++ b/test/utils/basesuite/baseSuite.go @@ -31,7 +31,7 @@ func (suite *BaseTestSuite) SetupBaseSuite(suiteName string, cloud string, confi suite.OriginalEnv[config.Key] = value } } - + suite.loadEnvironmentFiles(cloud) // Log current configuration @@ -58,6 +58,13 @@ func (suite *BaseTestSuite) TearDownBaseSuite() { } func (suite *BaseTestSuite) loadEnvironmentFiles(cloudDir string) { + // Skip loading .env files in GitHub Actions - use repository variables instead + if os.Getenv("GITHUB_ACTIONS") == "true" { + suite.T().Logf("๐Ÿค– Running in GitHub Actions - skipping .env file loading") + suite.T().Logf("๐Ÿ“‹ Configuration will be loaded from repository variables and secrets") + return + } + // First load envs from local.env, // if local env file exists the exit without loading other env files envFiles := []string{"local.env", ".env"} diff --git a/test/utils/helpers/terraform.go b/test/utils/helpers/terraform.go index a51e19f..96ded46 100644 --- a/test/utils/helpers/terraform.go +++ b/test/utils/helpers/terraform.go @@ -4,6 +4,9 @@ import ( "encoding/json" "os" "testing" + + "github.com/gruntwork-io/terratest/modules/terraform" + test_structure "github.com/gruntwork-io/terratest/modules/test-structure" ) // CreateTfvarsFile creates a terraform.tfvars.json file using JSON format @@ -22,3 +25,13 @@ func CreateTfvarsFile(t *testing.T, tfvarsPath string, variables map[string]inte t.Logf("๐Ÿ“ Created terraform.tfvars.json file: %s", tfvarsPath) } + +func SafeLoadTerraformOptions(t *testing.T, testDataDir string) *terraform.Options { + terraformOptionsPath := test_structure.FormatTestDataPath(testDataDir, "TerraformOptions.json") + + if !test_structure.IsTestDataPresent(t, terraformOptionsPath) { + return nil + } + + return test_structure.LoadTerraformOptions(t, testDataDir) +}