[remote-state] Improve backend compatibility (#105)

Author: Nuru

modules/remote-state/README.md

@@ -2,7 +2,58 @@
 
 Terraform module that accepts a component and a stack name and returns remote state outputs for the component.
 
-The module supports `s3` and `remote` (Terraform Cloud) backends.
+The module supports all backends supported by Terraform and OpenTofu, plus the Atmos-specific `static` backend.
+
+
+### Errors
+
+> [!NOTE]
+>
+>  If you experience an error from the `terraform_remote_state` data source, 
+>  this is most likely not an error in this module, but rather an error in the
+>  `remote_state` configuration in the referenced stack. This module performs 
+>  no validation on the remote state configuration, and only modifies the configuration
+>  for the `remote` backend (to set the workspace name) and, 
+>  _only when `var.privileged` is set to `true`_, the `s3` configuration (to remove
+>  settings for assuming a role). If `var.privileged` is left at the default value of `false`
+>  and you are not using the `remote` backend, then this module will not modify the backend
+>  configuration in any way.
+
+### "Local" Backend
+
+> [!IMPORTANT]
+> 
+>  If the local backend has a relative path, it will be resolved
+>  relative to the current working directory, which is usually a root module
+>  referencing the remote state. However, when the local backend is created,
+>  the current working directory is the directory where the target root module
+>  is defined. This can cause the lookup to fail if the source is not reachable
+>  from the client directory as `../source`.
+
+For example, if your directory structure looks like this:
+
+```text
+project
+├── components
+│   ├── client
+│   │   └── main.tf
+│   └── complex
+│       └── source
+│           └── main.tf
+└── local-state
+    └── complex
+        └── source
+            └── terraform.tfstate
+```
+
+Terraform code in `project/components/complex/source` can create its local state 
+file (`terraform.tfstate`) in the `local-state/complex/source`
+directory using `path = "../../../local-state/complex/source/terraform.tfstate"`. 
+However, Terraform code in `project/components/client` that references the same
+local state using the same backend configuration will fail because the current
+working directory is `project/components/client` and the relative path will not
+resolve correctly.
+
 
 ## Usage
 

modules/remote-state/data-source.tf

@@ -1,130 +1,54 @@
 locals {
-  data_source_backends   = ["local", "remote", "s3", "azurerm", "gcs"]
-  is_data_source_backend = contains(local.data_source_backends, local.backend_type)
+  custom_backends        = ["none", "bypass", "static"]
+  is_data_source_backend = !contains(local.custom_backends, local.backend_type)
 
   remote_workspace = var.workspace != null ? var.workspace : local.workspace
   ds_backend       = local.is_data_source_backend ? local.backend_type : "none"
   ds_workspace     = local.ds_backend == "none" ? null : local.remote_workspace
 
+  # The `privileged` flag is no longer used in the Cloud Posse reference architecture, but is maintained for compatibility.
+  # This was and is only supported for the S3 backend.
+  #
+  # When the `privileged` flag is set to `true`, the user running Terraform is considered privileged and therefore
+  # does not need to assume a different role to access the S3 backend.
+  #
+  # This is accomplished by removing any profile or role ARN settings from the configuration.
+  s3_privileged_backend = { for k, v in local.backend : k => v if !contains(["profile", "role_arn", "assume_role", "assume_role_with_web_identity"], k) }
+
+  # Workaround for the fact that the 2 different backends can be different types,
+  # but both results of a conditional must be the same type.
+  s3_backend = {
+    # normal, not privileged
+    false = local.backend
+    # privileged
+    true = local.s3_privileged_backend
+  }
+
+  # Customize certain configurations. Otherwise we will just use whatever was configured in the stack.
   ds_configurations = {
     # If no valid configuration is found for the backend datasource, provide a dummy one.
     none = {
       path = "${path.module}/dummy-remote-state.json"
     }
 
-    # Note: If the local backend has a relative path, it will be resolved
-    # relative to the current working directory, which is usually a root module
-    # referencing the remote state. However, when the local backend is created,
-    # the current working directory is the directory where the target root module
-    # is defined. This will likely cause the lookup to fail unless the current
-    # and target root module directories are in the same directory.
-    #
-    # Both path and workspace_dir are optional.
-    local = local.ds_backend != "local" ? null : merge({},
-      try(length(lookup(local.backend, "path", "")), 0) > 0 ? {
-        path = lookup(local.backend, "path", "")
-      } : {},
-      try(length(lookup(local.backend, "workspace_dir", "")), 0) > 0 ? {
-        workspace_dir = lookup(local.backend, "workspace_dir", "")
-      } : {}
-    )
-
-    remote = local.ds_backend != "remote" ? null : {
-      organization = local.backend.organization
-
+    remote = merge(local.backend, {
       workspaces = {
         name = local.remote_workspace
       }
-    }
-
-    s3 = local.ds_backend != "s3" ? null : {
-      encrypt        = local.backend.encrypt
-      bucket         = local.backend.bucket
-      key            = local.backend.key
-      dynamodb_table = local.backend.dynamodb_table
-      region         = local.backend.region
-
-      # NOTE: component types
-      # Privileged components are those that require elevated (root-level) permissions to provision and access their remote state.
-      # For example: `tfstate-backend`, `account`, `account-map`, `account-settings`, `iam-primary`.
-      # Privileged components are usually provisioned during cold-start (when we don't have any IAM roles provisioned yet) by using an admin user credentials.
-      # To access the remote state of privileged components, the caller needs to have permissions to access the backend and the remote state without assuming roles.
-      # Regular components, on the other hand, don't require root-level permissions and are provisioned and their remote state is accessed by assuming IAM roles (or using profiles).
-      # For example: `vpc`, `eks`, `rds`
-
-      # NOTE: global `backend` config
-      # The global `backend` config should be declared in a global YAML stack config file (e.g. `globals.yaml`)
-      # where all stacks can import it and have access to it (note that the global `backend` config is organization-wide and will not change after cold-start).
-      # The global `backend` config in the global config file should always have the `role_arn` or `profile` specified (added after the cold-start).
-
-      # NOTE: components `backend` config
-      # The `backend` portion for each individual component should be declared in a catalog file (e.g. `stacks/catalog/<component>.yaml`)
-      # along with all the default values for a component.
-      # The `privileged` attribute should always be declared in the `backend` portion for each individual component in the catalog.
-      # Top-level stacks where a component is provisioned import the component's catalog (the default values and the component's backend config portion) and can override the default values.
-
-      # NOTE: `cold-start`
-      # During cold-start we don't have any IAM roles provisioned yet, so we use an admin user credentials to provision the privileged components.
-      # The `privileged` attribute for the privileged components should be set to `true` in the components' catalog,
-      # and the privileged components should be provisioned using an admin user credentials.
-
-      # NOTE: after `cold-start`
-      # After the privileged components (including the primary IAM roles) are provisioned, we update the global `backend` config in the global config file
-      # to add the IAM role or profile to access the backend (after this, the global `backend` config should never change).
-      # For some privileged components we can change the `privileged` attribute in the YAML config from `true` to `false`
-      # to allow the regular components to access their remote state (e.g. we set the `privileged` attribute to `false` in the `account-map` component
-      # since we use `account-map` in almost all regular components.
-      # For each regular component, set the `privileged` attribute to `false` in the components'  portion of `backend` config (in `stacks/catalog/<component>.yaml`)
-
-      # Advantages:
-      # The global `backend` config is specified just once in the global config file, IAM role or profile is added to it after the cold start,
-      # and after that the global `backend` config never changed.
-      # We can make a component privileged or not any time by just updating its `privileged` attribute in the component's catalog file.
-      # We can change a component's `backend` portion any time without touching/affection the backend configs of all other components (e.g. when we add a new
-      # component, we don't touch the `globals.yaml` file at all, and we don't update the component's `role_arn` and `profile` settings).
-
-      # Use the role to access the remote state if the component is not privileged and `role_arn` is specified
-      role_arn = !coalesce(try(local.backend.privileged, null), var.privileged) && contains(keys(local.backend), "role_arn") ? local.backend.role_arn : null
-
-      # Use the profile to access the remote state if the component is not privileged and `profile` is specified
-      profile = !coalesce(try(local.backend.privileged, null), var.privileged) && contains(keys(local.backend), "profile") ? local.backend.profile : null
-
-      workspace_key_prefix = local.workspace_key_prefix
-
-      # S3-compatible backend for Oracle
-      # source: https://docs.oracle.com/en-us/iaas/Content/API/SDKDocs/terraformUsingObjectStore.htm#s3
-      skip_region_validation      = try(local.backend.skip_region_validation, null)
-      skip_credentials_validation = try(local.backend.skip_credentials_validation, null)
-      skip_requesting_account_id  = try(local.backend.skip_requesting_account_id, null)
-      use_path_style              = try(local.backend.use_path_style, null)
-      force_path_style            = try(local.backend.force_path_style, null)
-      skip_metadata_api_check     = try(local.backend.skip_metadata_api_check, null)
-      skip_s3_checksum            = try(local.backend.skip_s3_checksum, null)
-      endpoints                   = try(local.backend.endpoints, null)
-      endpoint                    = try(local.backend.endpoint, null)
-    }
-
-    azurerm = local.ds_backend != "azurerm" ? null : {
-      resource_group_name  = local.backend.resource_group_name
-      storage_account_name = local.backend.storage_account_name
-      container_name       = local.backend.container_name
-      key                  = local.backend.key
-    }
-
-    gcs = local.ds_backend != "gcs" ? null : {
-      bucket = local.backend.bucket
-      prefix = local.backend.prefix
-    }
+    })
 
+    s3 = local.s3_backend[var.privileged]
   } # ds_configurations
 
 }
 
 data "terraform_remote_state" "data_source" {
   count = var.bypass ? 0 : 1
 
+  # Use a dummy local backend when the real backend is not supported by the data source
   backend   = local.ds_backend == "none" ? "local" : local.ds_backend
   workspace = local.ds_workspace
-  config    = local.ds_configurations[local.ds_backend]
-  defaults  = var.defaults
+  # If nothing needs to be customized, just use whatever was configured in the stack
+  config   = lookup(local.ds_configurations, local.ds_backend, local.backend)
+  defaults = var.defaults
 }

modules/remote-state/main.tf

@@ -17,7 +17,7 @@ locals {
   config = try(yamldecode(data.utils_component_config.config[0].output), {})
 
   remote_state_backend_type = try(local.config.remote_state_backend_type, "")
-  backend_type              = try(coalesce(local.remote_state_backend_type, local.config.backend_type), "")
+  backend_type              = try(coalesce(local.remote_state_backend_type, local.config.backend_type), "none")
 
   # If `config.remote_state_backend` is not declared in YAML config, the default value will be an empty map `{}`
   backend_config_key = try(local.config.remote_state_backend, null) != null && try(length(local.config.remote_state_backend), 0) > 0 ? "remote_state_backend" : "backend"
@@ -30,12 +30,10 @@ locals {
 
   backend = local.backend_configs[local.backend_config_key]
 
-  workspace            = lookup(local.config, "workspace", "")
-  workspace_key_prefix = lookup(local.backend, "workspace_key_prefix", null)
+  workspace = lookup(local.config, "workspace", "")
+  # workspace_key_prefix = lookup(local.backend, "workspace_key_prefix", null)
 
   remote_states = {
-    # s3     = data.terraform_remote_state.s3
-    # remote = data.terraform_remote_state.remote
     data_source = try(data.terraform_remote_state.data_source[0].outputs, var.defaults)
     bypass      = var.defaults
     static      = local.backend