Break terraform to modules

.github/workflows/apply.yml → .github/workflows/members-apply.yml

@@ -1,21 +1,21 @@
-name: "Apply org changes"
+name: "Apply org membership changes"
 
 on:
   push:
     branches:
       - main
     paths:
-      - 'terraform/production/*.tfvars'
-      - 'terraform/*.tf'
-      - '.github/workflows/apply.yml'
-      - '.github/workflows/plan.yml'
+      - 'terraform/production/org.tfvars'
+      - 'terraform/members/*.tf'
+      - '.github/workflows/members-apply.yml'
+      - '.github/workflows/members-plan.yml'
 
 concurrency:
-  group: terraform-actions
+  group: terraform-actions-members
 
 jobs:
-  apply-changes:
-    name: "Org changes apply"
+  org-apply-changes:
+    name: "Apply org membership changes"
     runs-on: ubuntu-latest
 
     permissions:
@@ -30,23 +30,21 @@ jobs:
         with:
           persist-credentials: false
       - name: terraform apply
-        # v1.44.0
-        uses: dflook/terraform-apply@8f47d0ad9f3cb9e50fd6b3595c0cb98f00c518df
+        uses: dflook/terraform-apply@8f47d0ad9f3cb9e50fd6b3595c0cb98f00c518df # v2.2.2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TERRAFORM_ACTIONS_GITHUB_TOKEN: ${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}
         with:
-          path: "terraform"
+          path: "terraform/members"
+          label: 'members'
           variables: |
             github_token = "${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}"
           var_file: |
             terraform/production/org.tfvars
-            terraform/production/repositories.tfvars
 
       - name: Commit changes
         if: ${{ always() }}
-        # v0.10.0
-        uses: devops-infra/action-commit-push@8a2d9d73c3f506468129be2e4409e60dbed70357
+        uses: devops-infra/action-commit-push@8a2d9d73c3f506468129be2e4409e60dbed70357 # v1.0.3
         with:
           github_token: ${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}
           commit_prefix: "[AUTO]"

.github/workflows/members-plan.yml

@@ -0,0 +1,58 @@
+name: "Plan org membership changes and list them in a PR"
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'terraform/production/org.tfvars'
+      - 'terraform/members/*.tf'
+      - '.github/workflows/members-apply.yml'
+      # Do not trigger the plan action when it's been changed since this action has write permissions
+
+concurrency:
+  group: terraform-actions-members
+
+jobs:
+  format-terraform-code:
+    name: "Check Terraform code formatting"
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6.0.1
+        with:
+          persist-credentials: false
+
+      - name: terraform fmt check
+        uses: dflook/terraform-fmt-check@10eaa13fa61437aa51be2d12fafe95f152e3512d  # v2.2.2
+        with:
+          path: "terraform/members"
+
+  org-plan-changes:
+    name: "Plan org membership changes and list them in a PR"
+    runs-on: ubuntu-latest
+    needs: ["format-terraform-code"]
+    permissions:
+      pull-requests: write
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6.0.1
+        with:
+          persist-credentials: false
+
+      - name: terraform plan
+        uses: dflook/terraform-plan@dc251c444763eed5defd065b866874b6343017ca  # v2.2.2
+        env:
+          TERRAFORM_ACTIONS_GITHUB_TOKEN: ${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          add_github_comment: true
+          path: "terraform/members"
+          label: 'members'
+          variables: |
+            github_token = "${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}"
+          var_file: |
+            terraform/production/org.tfvars

.github/workflows/repos-apply.yml

@@ -0,0 +1,50 @@
+name: "Apply org-repositories changes"
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'terraform/production/org.tfvars'
+      - 'terraform/repos/*.tf'
+      - '.github/workflows/repos-apply.yml'
+      - '.github/workflows/repos-plan.yml'
+
+concurrency:
+  group: terraform-actions-repos
+
+jobs:
+  repos-apply-changes:
+    name: "Apply org-repositories changes"
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      pull-requests: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6.0.1
+        with:
+          persist-credentials: false
+      - name: terraform apply
+        uses: dflook/terraform-apply@8f47d0ad9f3cb9e50fd6b3595c0cb98f00c518df # v2.2.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TERRAFORM_ACTIONS_GITHUB_TOKEN: ${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}
+        with:
+          path: "terraform/repos"
+          label: 'repos'
+          variables: |
+            github_token = "${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}"
+          var_file: |
+            terraform/production/org.tfvars
+
+      - name: Commit changes
+        if: ${{ always() }}
+        uses: devops-infra/action-commit-push@8a2d9d73c3f506468129be2e4409e60dbed70357 # v1.0.3
+        with:
+          github_token: ${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}
+          commit_prefix: "[AUTO]"
+          commit_message: "State changes after apply"
+          force: false

.github/workflows/plan.yml → .github/workflows/repos-plan.yml

@@ -1,16 +1,16 @@
-name: "Plan org changes and list them in a PR"
+name: "Plan org-repositories changes and list them in a PR"
 on:
   pull_request:
     branches:
       - main
     paths:
       - 'terraform/production/*.tfvars'
-      - 'terraform/*.tf'
-      - '.github/workflows/apply.yml'
+      - 'terraform/repos/*.tf'
+      - '.github/workflows/repos-apply.yml'
       # Do not trigger the plan action when it's been changed since this action has write permissions
 
 concurrency:
-  group: terraform-actions
+  group: terraform-actions-repos
 
 jobs:
   format-terraform-code:
@@ -27,15 +27,14 @@ jobs:
           persist-credentials: false
 
       - name: terraform fmt check
-        # v2.2.2
-        uses: dflook/terraform-fmt-check@10eaa13fa61437aa51be2d12fafe95f152e3512d
+        uses: dflook/terraform-fmt-check@10eaa13fa61437aa51be2d12fafe95f152e3512d  # v2.2.2
         with:
-          path: "terraform"
+          path: "terraform/repos"
 
-  plan-changes:
-    name: "Org changes plan"
+  repos-plan-changes:
+    name: "Plan org-repositories changes and list them in a PR"
     runs-on: ubuntu-latest
-    needs: [ "format-terraform-code" ]
+    needs: ["format-terraform-code"]
     permissions:
       pull-requests: write
       contents: read
@@ -46,14 +45,14 @@ jobs:
           persist-credentials: false
 
       - name: terraform plan
-        # v1.44.0
-        uses: dflook/terraform-plan@dc251c444763eed5defd065b866874b6343017ca
+        uses: dflook/terraform-plan@dc251c444763eed5defd065b866874b6343017ca  # v2.2.2
         env:
           TERRAFORM_ACTIONS_GITHUB_TOKEN: ${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           add_github_comment: true
-          path: "terraform"
+          path: "terraform/repos"
+          label: 'repos'
           variables: |
             github_token = "${{ secrets.TERRAFORM_MANAGEMENT_GITHUB_TOKEN }}"
           var_file: |

terraform/members/locals.tf

@@ -0,0 +1,3 @@
+locals {
+
+}

terraform/members/variables.tf

@@ -0,0 +1,42 @@
+# Input Variables
+# https://www.terraform.io/language/values/variables
+
+variable "admins" {
+  description = "A set of admins to add to the organization"
+  type        = set(string)
+}
+
+variable "super_admins" {
+  description = "A set of users who have operational permissions to add to the organization"
+  type        = set(string)
+}
+
+variable "github_token" {
+  description = "The GitHub token used for managing the organization"
+  type        = string
+  sensitive   = true
+}
+
+variable "members" {
+  description = "A set of members to add to the organization"
+  type        = set(string)
+  default     = []
+}
+
+variable "designers" {
+  description = "A set of designers to add to the organization"
+  type        = set(string)
+  default     = []
+}
+
+variable "organization_teams" {
+  description = "Map of Django Commons organization teams to manage"
+  type = map(object({
+    description               = string
+    maintainers               = optional(set(string), [])
+    members                   = optional(set(string), [])
+    permission                = optional(string, null)
+    privacy                   = optional(string, "closed")
+    review_request_delegation = optional(bool, false)
+  }))
+}

terraform/production/org.tfvars

@@ -148,12 +148,3 @@ organization_teams = {
     ]
   }
 }
-
-################ GitHub Organization Secrets, not used at the moment #############
-
-organization_secrets = {
-  #     "GPG_PASSPHRASE" = {
-  #       description = "GPG Passphrase used to encrypt plan.out files"
-  #       visibility  = "all"
-  #     }
-}

terraform/repos/README.md

@@ -0,0 +1,101 @@
+GitHub Organization as Terraform
+================================
+
+# Structure
+
+- `variables.tf` - define variable types (classes?), notice there is `variable "repositories" {...` there which has a
+  few variables marked as optional with default values. Why I chose to have `has_discussions` as a repo variable
+  while `has_issues` as a constant - I am embarrassed to say I don't have a better answer than laziness :smile: - I just
+  figured if this is the path we want to take, we can continue adding to it.
+- `production/*.tfvars` - instances, should strictly follow the types in `variables.tf`.
+- `main.tf` - build configuration based on instances values from `production.tfvars` (or, if not defined explicitly,
+  then default value from `variables.tf`)
+- `resources-*.tf` - define resources, like `github_repository`, `github_team`, etc.
+- `tfstate.json` - Current state file, pulled using `terraform import ..`
+
+# Why Terraform?
+
+We can define our "desired/default" repository configuration, and within this configuration:
+
+- What is enforced from day one (i.e., constant in `resource "github_repository" "this"`)
+- What is recommended but can be changed by users (i.e., variable with a default value in `variables.tf` that can be
+  updated in `production.tfvars`) => Note this can also help us review outliers, you can see all repos which have
+  non-default values in the `production.tfvars` file.
+- What is determined by users (i.e., variables without default value, like `description`)
+- What is not configured in the infra-as-code (currently, for example, repo-labels).
+
+# What changes can be made
+
+All changes should be made in `production/*.tfvars`:
+
+- Add/Remove organization admins by editing the `admins` list.
+- Add/Remove organization members by editing the `members` list.
+- Add/Remove/Update repositories by editing the `repositories`. A repository can have the following variables:
+    ```terraform
+    repositories = {
+      "repo-name" = {
+        description = "repo description"
+        homepage_url = "" # optional, default is ""
+        allow_auto_merge = false # optional, default is false
+        allow_merge_commit = false # optional, default is false
+        allow_rebase_merge = false # optional, default is false
+        allow_squash_merge = true # optional, default is true
+        allow_update_branch = true # optional, default is true
+        delete_branch_on_merge = true # optional, default is true
+        has_discussions = true # optional, default is true
+        has_downloads = true # optional, default is true
+        has_wiki = false # optional, default is false
+        is_template = false # optional, default is false
+        push_allowances = []
+        template = "" # optional, default is ""
+        topics = []
+        visibility = "public" # optional, default is "public"
+        is_django_commons_repo = optional(bool, false) # Do not create teams for repository
+        required_status_checks_contexts = [] # optional, default is []
+        admins = [] # Members of the repository's admin and repository teams. Have admin permissions
+        committers = [] # Members of the repository's committers and repository teams. Have write permissions
+        members = [] # Members of the repository team. Have triage permissions
+      }
+     # ...
+    }
+    ```
+
+# How to use locally
+
+You might want to try new settings locally before applying them to the repository automation.
+To do so, you can use the following steps:
+
+1. Clone the repository.
+2. From the `terraform/` directory, run `terraform init`.
+3. Create a github-token with the necessary permissions on the organization (see [permissions documentation][1]).
+    - The `repo` permission for full control of private repositories.
+    - The `admin:org` permission for full control of orgs and teams, read and write org projects
+    - The `delete_repo` permission to delete repositories
+
+4. Make changes to `production/*.tfvars` to reflect the desired state (add/update users, repositories, teams, etc.)
+5. To see what changes between the current state of the GitHub organization and the plan
+   run:  `terraform plan -var-file=production/org.tfvars -var-file=production/repositories.tfvars -var github_token=...`
+6. To apply the changes,
+   run: `terraform apply -var-file=production/org.tfvars -var-file=production/repositories.tfvars -var github_token=...`
+
+# Integration with GitHub Actions
+
+The repository is configured to run `terraform plan` on every new pull-request as well as an update to a pull-request
+and list the expected changes as a comment on the pull-request.
+Once the pull-request is merged to the `main` branch, `terraform apply` applies the changes to the GitHub organization, and
+the updated current state is committed to the `main` branch.
+To achieve this, the workflows use `TERRAFORM_MANAGEMENT_GITHUB_TOKEN` secret to plan/apply terraform changes.
+
+`TERRAFORM_MANAGEMENT_GITHUB_TOKEN` is a fine-grained personal access token with permissions the following permissions
+required (see documentation [here][2]):
+
+- The `repo` permission for full control of private repositories
+- The `admin:org` permission for full control of orgs and teams, read and write org projects
+- The `delete_repo` permission to delete repositories
+- Additionally, the token should have permissions to write content to the repository (see, [here][3])
+
+[1]: https://developer.hashicorp.com/terraform/tutorials/it-saas/github-user-teams#configure-your-credentials
+
+[2]: https://developer.hashicorp.com/terraform/tutorials/it-saas/github-user-teams#configure-your-credentials
+
+[3]: https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository