initalized aws s3 terraform via nix (#44)

* tf

* s3

* backend state

* backend

* clean up

* fixed PR

* anti hack

* update name of bucket

* fixed build input

* cleam

Author: dzmitry-lahoda

.github/mergify.yml

@@ -5,24 +5,6 @@ pull_request_rules:
       - "#approved-reviews-by>=1"
       - check-success=build-and-publish-draft
       - base=main
-    actions:
-      merge:
-        method: merge
-  - name: automatic merge documentation when CI passes and 1 reviews
-    conditions:
-      - files=(\.md|\.MD)$
-      - "#approved-reviews-by>=1"
-      - check-success=lint-documentation
-      - base=main
-    actions:
-      merge:
-        method: merge
-  - name: automatic merge code changes with documentation when CI passes and 1 reviews
-    conditions:
-      - "#approved-reviews-by>=1"
-      - check-success=build-runtime
-      - check-success=lint-documentation
-      - base=main
     actions:
       merge:
         method: merge

.github/workflows/pull-request-documentation.yml

@@ -3,14 +3,8 @@ on:
   push:
     branches:
       - main
-    paths-ignore:
-      - "**/*.md"
-      - "**/*.MD"
 
   pull_request:
-    paths-ignore:
-      - "**/*.md"
-      - "**/*.MD"
 
 permissions: 
   pull-requests: write
@@ -20,6 +14,7 @@ env:
   NIX_VERSION: nix-2.13.2
   NIXPKGS_CHANNEL: nixos-22.11
   CACHIX_NAME: golden-gate-ggx
+  OCTOLYTICS_DIMENSION_REPOSITORY_ID: 590614152
 
 jobs:
   build-and-publish-draft:
@@ -64,10 +59,13 @@ jobs:
           nix build --print-build-logs --show-trace --no-update-lock-file
           cp ./result/lib/golden_gate_runtime.compact.compressed.wasm ./out
           nix build .#golden-gate-node --print-build-logs --show-trace --no-update-lock-file
-          cp ./result/bin/golden-gate-node ./out  
-          # can add here contracts and oci image, etc                  
+          cp ./result/bin/golden-gate-node ./out
+          # can add here contracts and oci image, etc
+          nix run .#lint                  
           
       - name: action-gh-release
+        # so we do not allow non team members to do releases
+        if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.id != env.OCTOLYTICS_DIMENSION_REPOSITORY_ID) || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.id == env.OCTOLYTICS_DIMENSION_REPOSITORY_ID) }}
         uses: softprops/action-gh-release@v1
         with:
           draft: true

.github/workflows/pull-request-write.yml

@@ -29,4 +29,42 @@ result
 
 # devops
 *.log
-out
+out
+.secret
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version 
+# control as they are data points which are potentially sensitive and subject 
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+
+!terraform.tfstate.sops

.gitignore

@@ -27,6 +27,12 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
+    # terraform generator to manage clouds/managed services
+    terranix = {
+      url = "github:terranix/terranix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
   };
 
   nixConfig = {
@@ -36,7 +42,7 @@
   };
 
   # inputs and systems are know ahead of time -> we can evalute all nix -> flake make nix """statically typed"""
-  outputs = { self, nixpkgs, devenv, rust-overlay, crane, flake-utils, ... } @ inputs:
+  outputs = { self, nixpkgs, devenv, rust-overlay, crane, flake-utils, terranix, ... } @ inputs:
     flake-utils.lib.eachDefaultSystem (system:
       let
 
@@ -114,6 +120,10 @@
                 == false
                 &&
                 (type == "directory" && ".github" == name) == false
+                && (type == "directory" && "terraform" == name) == false
+
+                # risky, until we move code into separate repo as rust can do include_str! as doc, but good optimization
+                && (type == "regular" && pkgs.lib.strings.hasSuffix ".md" name) == false
               )
             )
 
@@ -168,11 +178,63 @@
           '';
         };
 
+        lint = pkgs.writeShellApplication rec {
+          name = "lint";
+          text = ''
+            ${pkgs.lib.meta.getExe pkgs.nodePackages.markdownlint-cli2} "**/*.md" "#.devenv" "#target"
+          '';
+        };
+
+
+        tf-init = pkgs.writeShellApplication rec {
+          name = "tf-init";
+          text = ''
+            # here you manually obtain login key
+            aws configure
+          '';
+        };
+
+        # can use envvars override to allow run non shared "cloud" for tests
+        age-pub = "age1a8k02z579lr0qr79pjhlneffjw3dvy3a8j5r4fw3zlphd6cyaf5qukkat5";
+        cloud-tools = with pkgs; [
+          awscli2
+          terraform
+          sops
+          age
+        ];
+        tf-apply = pkgs.writeShellApplication rec {
+          name = "tf-apply";
+          runtimeInputs = cloud-tools;
+          text = ''
+            cd ./terraform  
+            # generate terraform input from nix
+            cp --force ${tf-config} config.tf.json
+            terraform init --upgrade
+
+            # decrypt secret state (should run only on CI eventually for safety)
+            # if there is encrypted state, decrypt it
+            if [[ -f terraform.tfstate.sops ]]; then
+              # uses age, so can use any of many providers (including aws)
+              sops --decrypt --age ${age-pub} terraform.tfstate.sops > terraform.tfstate          
+            fi
+          
+            # apply state to cloud, eventually should manually approve in CI
+            terraform apply -auto-approve
+            # encrypt update state back and push it (later in CI special job)
+            sops --encrypt --age ${age-pub} terraform.tfstate > terraform.tfstate.sops
+            # seems good idea to encrypt backup here too
+          '';
+        };
+
 
+        tf-config = terranix.lib.terranixConfiguration {
+          inherit system;
+          modules = [ ./flake/terraform.nix ];
+        };
       in
       rec {
         packages = flake-utils.lib.flattenTree {
-          inherit golden-gate-runtime golden-gate-node single-fast multi-fast;
+          inherit golden-gate-runtime golden-gate-node single-fast multi-fast tf-config tf-apply lint;
           node = golden-gate-node;
           runtime = golden-gate-runtime;
           default = golden-gate-runtime;
@@ -231,7 +293,17 @@
               in
               [
                 {
-                  packages = with pkgs;[ rust-toolchain binaryen llvmPackages.bintools dylint-link ] ++ rust-native-build-inputs ++ darwin;
+                  packages = with pkgs;
+                    [
+                      rust-toolchain
+                      binaryen
+                      llvmPackages.bintools
+                      dylint-link
+                      nodejs-18_x
+                      nodePackages.markdownlint-cli2
+
+                    ]
+                    ++ rust-native-build-inputs ++ darwin ++ cloud-tools;
                   env = rust-env;
                   # can do systemd/docker stuff here
                   enterShell = ''
@@ -243,19 +315,6 @@
                 }
               ];
           };
-          doc-linter = devenv.lib.mkShell {
-            inherit inputs pkgs;
-            modules = [
-              {
-                packages = with pkgs; [ nodejs-18_x nodePackages.markdownlint-cli2 ];
-                env = rust-env;
-                enterShell = ''
-                    markdownlint-cli2 "**/*.md" "#.devenv" "#target"
-                    exit
-                '';
-              }
-            ];
-          };
         };
       }
     );