fix(analyzer): add failed utf conversions to unwanted files

e2e/fixtures/E2E_CLI_105_PAYLOAD.json

@@ -0,0 +1,69 @@
+{
+  "document": [
+    {
+      "file": "file",
+      "id": "0",
+      "playbooks": [
+        {
+          "community.aws.efs": {
+            "encrypt": "no",
+            "name": "myTestEFS",
+            "state": "present",
+            "tags": {
+              "Name": "myTestNameTag",
+              "purpose": "file-storage"
+            },
+            "targets": [
+              {
+                "security_groups": [
+                  "sg-1a2b3c4d"
+                ],
+                "subnet_id": "subnet-748c5d03"
+              }
+            ]
+          },
+          "name": "foo"
+        },
+        {
+          "community.aws.efs": {
+            "encrypt": false,
+            "name": "myTestEFS",
+            "state": "present",
+            "tags": {
+              "Name": "myTestNameTag",
+              "purpose": "file-storage"
+            },
+            "targets": [
+              {
+                "security_groups": [
+                  "sg-1a2b3c4d"
+                ],
+                "subnet_id": "subnet-748c5d03"
+              }
+            ]
+          },
+          "name": "foo2"
+        },
+        {
+          "community.aws.efs": {
+            "name": "myTestEFS",
+            "state": "present",
+            "tags": {
+              "Name": "myTestNameTag",
+              "purpose": "file-storage"
+            },
+            "targets": [
+              {
+                "security_groups": [
+                  "sg-1a2b3c4d"
+                ],
+                "subnet_id": "subnet-748c5d03"
+              }
+            ]
+          },
+          "name": "foo3"
+        }
+      ]
+    }
+  ]
+}

e2e/testcases/e2e-cli-105_ensure_bad_utf_conversion_are_not_scanned.go

@@ -0,0 +1,23 @@
+package testcases
+
+// E2E-CLI-105 - KICS should ignore files with bad UTF-8 conversion and not include them in the scanned files
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should ignore files with bad UTF-8 conversion and not include them in the scanned files [E2E-CLI-105]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{
+					"scan",
+					"-p", "/path/test/fixtures/mix_utf8_and_non_utf/",
+					"--payload-path", "/path/e2e/output/E2E_CLI_105_PAYLOAD",
+				},
+			},
+			ExpectedPayload: []string{
+				"E2E_CLI_105_PAYLOAD.json",
+			},
+		},
+		WantStatus: []int{50},
+	}
+
+	Tests = append(Tests, testSample)
+}

pkg/analyzer/analyzer.go

@@ -537,6 +537,7 @@ func (a *analyzerInfo) checkContent(
 	content, err := utils.ReadFileToUTF8(a.filePath)
 	if err != nil {
 		log.Warn().Msgf("failed to analyze file: %s", err)
+		unwanted <- a.filePath
 		return
 	}
 

pkg/analyzer/analyzer_test.go

@@ -557,6 +557,23 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			excludeGitIgnore:     false,
 			MaxFileSize:          -1,
 		},
+		{
+			name: "analyze_test_non_utf8_is_set_as_unwanted",
+			paths: []string{
+				filepath.FromSlash("../../test/fixtures/mix_utf8_and_non_utf"),
+			},
+			wantTypes: []string{"ansible"},
+			wantExclude: []string{
+				filepath.FromSlash("../../test/fixtures/mix_utf8_and_non_utf/non_utf.json"),
+			},
+			typesFromFlag:        []string{""},
+			excludeTypesFromFlag: []string{""},
+			wantLOC:              57,
+			wantErr:              false,
+			gitIgnoreFileName:    "",
+			excludeGitIgnore:     false,
+			MaxFileSize:          -1,
+		},
 	}
 
 	for _, tt := range tests {

test/fixtures/mix_utf8_and_non_utf/nginx.conf

@@ -0,0 +1,24 @@
+server {
+    listen 80;
+    server_name example.com;
+
+    location / {
+        root /var/www/html;
+        index index.html index.htm;
+    }
+
+    location /api {
+        proxy_pass http://backend:8080;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+    }
+
+    # Security headers
+    add_header X-Frame-Options "SAMEORIGIN" always;
+    add_header X-Content-Type-Options "nosniff" always;
+    add_header X-XSS-Protection "1; mode=block" always;
+
+    error_page 404 /404.html;
+    error_page 500 502 503 504 /50x.html;
+}
+

test/fixtures/mix_utf8_and_non_utf/non_utf.json

@@ -0,0 +1,4 @@
+{
+  "name": "Invalid UTF-8 ÿþ test",
+  "data": "€‚"
+}

test/fixtures/mix_utf8_and_non_utf/positive.yaml

@@ -0,0 +1,33 @@
+---
+- name: foo
+  community.aws.efs:
+    state: present
+    name: myTestEFS
+    encrypt: no
+    tags:
+      Name: myTestNameTag
+      purpose: file-storage
+    targets:
+      - subnet_id: subnet-748c5d03
+        security_groups: ["sg-1a2b3c4d"]
+- name: foo2
+  community.aws.efs:
+    state: present
+    name: myTestEFS
+    encrypt: false
+    tags:
+      Name: myTestNameTag
+      purpose: file-storage
+    targets:
+      - subnet_id: subnet-748c5d03
+        security_groups: ["sg-1a2b3c4d"]
+- name: foo3
+  community.aws.efs:
+    state: present
+    name: myTestEFS
+    tags:
+      Name: myTestNameTag
+      purpose: file-storage
+    targets:
+      - subnet_id: subnet-748c5d03
+        security_groups: ["sg-1a2b3c4d"]