Add E2E tests (#341)

wata727 · web-flow · commit 5fbdc92e468e · 2022-05-05T21:01:42.000+09:00
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -0,0 +1,42 @@
+name: e2e
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  e2e:
+    name: ${{ matrix.os }} (${{ matrix.version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: [v0.35.0, latest]
+    env:
+      TFLINT_VERSION: ${{ matrix.version }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Install TFLint
+      run: curl -sL https://raw.githubusercontent.com/terraform-linters/tflint/master/install_linux.sh | bash
+    - name: Install plugin (Linux)
+      if: runner.os == 'Linux'
+      run: make install
+    - name: Install plugin (Windows)
+      if: runner.os == 'Windows'
+      run: |
+        mkdir -p ~/.tflint.d/plugins
+        go build -o ~/.tflint.d/plugins/tflint-ruleset-aws.exe
+      shell: bash
+    - name: Run E2E tests
+      run: make e2e
diff --git a/Makefile b/Makefile
@@ -1,7 +1,10 @@
 default: build
 
 test:
-	go test ./...
+	go test $$(go list ./... | grep -v integration)
+
+e2e: 
+	cd integration && go test && cd ../
 
 build:
 	go build
diff --git a/integration/basic/.tflint.hcl b/integration/basic/.tflint.hcl
@@ -0,0 +1,3 @@
+plugin "aws" {
+  enabled = true
+}
diff --git a/integration/basic/result.json b/integration/basic/result.json
@@ -0,0 +1,25 @@
+{
+  "issues": [
+    {
+      "rule": {
+        "name": "aws_instance_invalid_type",
+        "severity": "error",
+        "link": ""
+      },
+      "message": "\"x2.2xlarge\" is an invalid value as instance_type",
+      "range": {
+        "filename": "template.tf",
+        "start": {
+        "line": 2,
+        "column": 19
+        },
+        "end": {
+        "line": 2,
+        "column": 31
+        }
+      },
+      "callers": []
+    }
+  ],
+  "errors": []
+}
diff --git a/integration/basic/template.tf b/integration/basic/template.tf
@@ -0,0 +1,3 @@
+resource "aws_instance" "foo" {
+  instance_type = "x2.2xlarge"
+}
diff --git a/integration/cty-based-eval/.tflint.hcl b/integration/cty-based-eval/.tflint.hcl
@@ -0,0 +1,8 @@
+plugin "aws" {
+  enabled = true
+}
+
+rule "aws_resource_missing_tags" {
+  enabled = true
+  tags = ["Environment", "Name", "Type"]
+}
diff --git a/integration/cty-based-eval/result.json b/integration/cty-based-eval/result.json
@@ -0,0 +1,25 @@
+{
+  "issues": [
+    {
+      "rule": {
+        "name": "aws_resource_missing_tags",
+        "severity": "info",
+        "link": "https://github.com/terraform-linters/tflint-ruleset-aws/blob/v0.13.3/docs/rules/aws_resource_missing_tags.md"
+      },
+      "message": "The resource is missing the following tags: \"Environment\", \"Name\", \"Type\".",
+      "range": {
+        "filename": "template.tf",
+        "start": {
+          "line": 5,
+          "column": 1
+        },
+        "end": {
+          "line": 5,
+          "column": 41
+        }
+      },
+      "callers": []
+    }
+  ],
+  "errors": []
+}
diff --git a/integration/cty-based-eval/template.tf b/integration/cty-based-eval/template.tf
@@ -0,0 +1,7 @@
+variable "tags" {
+  default = []
+}
+
+resource "aws_autoscaling_group" "group" {
+  tags = var.tags
+}
diff --git a/integration/deep-checking-rule-config/.tflint.hcl b/integration/deep-checking-rule-config/.tflint.hcl
@@ -0,0 +1,7 @@
+plugin "aws" {
+  enabled = true
+}
+
+rule "aws_instance_invalid_ami" {
+  enabled = false
+}
diff --git a/integration/deep-checking-rule-config/result.json b/integration/deep-checking-rule-config/result.json
@@ -0,0 +1,4 @@
+{
+  "issues": [],
+  "errors": []
+}
diff --git a/integration/disabled-rules/.tflint.hcl b/integration/disabled-rules/.tflint.hcl
@@ -0,0 +1,11 @@
+plugin "aws" {
+  enabled = true
+}
+
+rule "aws_instance_invalid_type" {
+  enabled = true
+}
+
+rule "aws_instance_previous_type" {
+  enabled = false
+}
diff --git a/integration/disabled-rules/result.json b/integration/disabled-rules/result.json
@@ -0,0 +1,25 @@
+{
+  "issues": [
+    {
+      "rule": {
+        "name": "aws_instance_invalid_type",
+        "severity": "error",
+        "link": ""
+      },
+      "message": "\"x2.2xlarge\" is an invalid value as instance_type",
+      "range": {
+        "filename": "template.tf",
+        "start": {
+          "line": 2,
+          "column": 19
+        },
+        "end": {
+          "line": 2,
+          "column": 31
+        }
+      },
+      "callers": []
+    }
+  ],
+  "errors": []
+}
diff --git a/integration/disabled-rules/template.tf b/integration/disabled-rules/template.tf
@@ -0,0 +1,7 @@
+resource "aws_instance" "invalid" {
+  instance_type = "x2.2xlarge"
+}
+
+resource "aws_instance" "previous" {
+  instance_type = "t1.micro"
+}
diff --git a/integration/heredoc/.tflint.hcl b/integration/heredoc/.tflint.hcl
@@ -0,0 +1,3 @@
+plugin "aws" {
+  enabled = true
+}
diff --git a/integration/heredoc/result.json b/integration/heredoc/result.json
@@ -0,0 +1,25 @@
+{
+  "issues": [
+    {
+      "rule": {
+        "name": "aws_ecr_lifecycle_policy_invalid_policy",
+        "severity": "error",
+        "link": ""
+      },
+      "message": "policy must be 100 characters or higher",
+      "range": {
+        "filename": "template.tf",
+        "start": {
+          "line": 2,
+          "column": 16
+        },
+        "end": {
+          "line": 4,
+          "column": 4
+        }
+      },
+      "callers": []
+    }
+  ],
+  "errors": []
+}
diff --git a/integration/heredoc/template.tf b/integration/heredoc/template.tf
@@ -0,0 +1,5 @@
+resource "aws_ecr_lifecycle_policy" "test" {
+  policy     = <<EOF
+{}
+EOF
+}
diff --git a/integration/integration_test.go b/integration/integration_test.go
@@ -0,0 +1,130 @@
+package integration
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestIntegration(t *testing.T) {
+	cases := []struct {
+		Name    string
+		Command *exec.Cmd
+		Dir     string
+	}{
+		{
+			Name:    "basic",
+			Command: exec.Command("tflint", "--format", "json", "--force"),
+			Dir:     "basic",
+		},
+		{
+			// Regression: https://github.com/terraform-linters/tflint-ruleset-aws/issues/41
+			Name:    "rule config",
+			Command: exec.Command("tflint", "--format", "json", "--force"),
+			Dir:     "rule-config",
+		},
+		{
+			// Regression: https://github.com/terraform-linters/tflint/issues/1028
+			Name:    "deep checking rule config",
+			Command: exec.Command("tflint", "--format", "json", "--force"),
+			Dir:     "deep-checking-rule-config",
+		},
+		{
+			// Regression: https://github.com/terraform-linters/tflint/issues/1029
+			Name:    "heredoc",
+			Command: exec.Command("tflint", "--format", "json", "--force"),
+			Dir:     "heredoc",
+		},
+		{
+			// Regression: https://github.com/terraform-linters/tflint/issues/1054
+			Name:    "disabled-rules",
+			Command: exec.Command("tflint", "--format", "json", "--force"),
+			Dir:     "disabled-rules",
+		},
+		{
+			// Regression: https://github.com/terraform-linters/tflint-ruleset-aws/issues/48
+			Name:    "cty-based-eval",
+			Command: exec.Command("tflint", "--format", "json", "--force"),
+			Dir:     "cty-based-eval",
+		},
+		{
+			// Regression: https://github.com/terraform-linters/tflint/issues/1102
+			Name:    "map-attribute",
+			Command: exec.Command("tflint", "--format", "json", "--force"),
+			Dir:     "map-attribute",
+		},
+		{
+			// Regression: https://github.com/terraform-linters/tflint/issues/1103
+			Name:    "rule config with --enable-rule",
+			Command: exec.Command("tflint", "--enable-rule", "aws_s3_bucket_name", "--format", "json", "--force"),
+			Dir:     "rule-config",
+		},
+		{
+			Name:    "rule config with --only",
+			Command: exec.Command("tflint", "--only", "aws_s3_bucket_name", "--format", "json", "--force"),
+			Dir:     "rule-config",
+		},
+	}
+
+	dir, _ := os.Getwd()
+	defer os.Chdir(dir)
+
+	for _, tc := range cases {
+		t.Run(tc.Name, func(t *testing.T) {
+			testDir := filepath.Join(dir, tc.Dir)
+
+			t.Cleanup(func() {
+				if err := os.Chdir(dir); err != nil {
+					t.Fatal(err)
+				}
+			})
+
+			if err := os.Chdir(testDir); err != nil {
+				t.Fatal(err)
+			}
+
+			var stdout, stderr bytes.Buffer
+			tc.Command.Stdout = &stdout
+			tc.Command.Stderr = &stderr
+			if err := tc.Command.Run(); err != nil {
+				t.Fatalf("%s, stdout=%s stderr=%s", err, stdout.String(), stderr.String())
+			}
+
+			var b []byte
+			var err error
+			if runtime.GOOS == "windows" && IsWindowsResultExist() {
+				b, err = os.ReadFile(filepath.Join(testDir, "result_windows.json"))
+			} else {
+				b, err = os.ReadFile(filepath.Join(testDir, "result.json"))
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var expected interface{}
+			if err := json.Unmarshal(b, &expected); err != nil {
+				t.Fatal(err)
+			}
+
+			var got interface{}
+			if err := json.Unmarshal(stdout.Bytes(), &got); err != nil {
+				t.Fatal(err)
+			}
+
+			if diff := cmp.Diff(got, expected); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func IsWindowsResultExist() bool {
+	_, err := os.Stat("result_windows.json")
+	return !os.IsNotExist(err)
+}
diff --git a/integration/map-attribute/.tflint.hcl b/integration/map-attribute/.tflint.hcl
@@ -0,0 +1,8 @@
+plugin "aws" {
+  enabled = true
+}
+
+rule "aws_resource_missing_tags" {
+  enabled = true
+  tags = ["Environment", "Name", "Type"]
+}
diff --git a/integration/map-attribute/result.json b/integration/map-attribute/result.json
@@ -0,0 +1,25 @@
+{
+  "issues": [
+    {
+      "rule": {
+        "name": "aws_resource_missing_tags",
+        "severity": "info",
+        "link": "https://github.com/terraform-linters/tflint-ruleset-aws/blob/v0.13.3/docs/rules/aws_resource_missing_tags.md"
+      },
+      "message": "The resource is missing the following tags: \"Environment\", \"Name\", \"Type\".",
+      "range": {
+        "filename": "template.tf",
+        "start": {
+          "line": 2,
+          "column": 10
+        },
+        "end": {
+          "line": 2,
+          "column": 25
+        }
+      },
+      "callers": []
+    }
+  ],
+  "errors": []
+}
diff --git a/integration/map-attribute/template.tf b/integration/map-attribute/template.tf
@@ -0,0 +1,3 @@
+resource "aws_instance" "intance" {
+  tags = { foo = "bar" }
+}
diff --git a/integration/rule-config/.tflint.hcl b/integration/rule-config/.tflint.hcl
diff --git a/integration/rule-config/result.json b/integration/rule-config/result.json
diff --git a/integration/rule-config/template.tf b/integration/rule-config/template.tf

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+plugin "aws" {`
	`2`	`+ enabled = true`
	`3`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+resource "aws_instance" "foo" {`
	`2`	`+ instance_type = "x2.2xlarge"`
	`3`	`+}`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +{
 +  "issues": [],
 +  "errors": []
 +}