Merge pull request #1402 from k1LoW/structured-error-output

feat: structure error field in JSON output for programmatic error analysis

Author: k1LoW

result.go

@@ -86,11 +86,17 @@ type stepResultSimplified struct {
 	RunnerType         RunnerType             `json:"runner_type,omitempty"`
 	RunnerKey          string                 `json:"runner_key,omitempty"`
 	Result             result                 `json:"result"`
-	Error              string                 `json:"error,omitempty"`
+	Error              *stepErrorSimplified   `json:"error,omitempty"`
 	IncludedRunResults []*runResultSimplified `json:"included_run_result,omitempty"`
 	Elapsed            time.Duration          `json:"elapsed,omitempty"`
 }
 
+type stepErrorSimplified struct {
+	Message   string `json:"message"`
+	Condition string `json:"condition,omitempty"`
+	ExprTrace string `json:"expr_trace,omitempty"`
+}
+
 func newRunResult(desc string, labels []string, path string, included bool, store *store.Store) *RunResult {
 	return &RunResult{
 		Desc:     desc,
@@ -315,7 +321,15 @@ func simplifyStepResults(stepResults []*StepResult) []*stepResultSimplified {
 			Elapsed: sr.Elapsed,
 		}
 		if sr.Err != nil {
-			s.Error = sr.Err.Error()
+			se := &stepErrorSimplified{
+				Message: sr.Err.Error(),
+			}
+			var cfe *condFalseError
+			if errors.As(sr.Err, &cfe) {
+				se.Condition = cfe.cond
+				se.ExprTrace = cfe.tree
+			}
+			s.Error = se
 		}
 		simplified = append(simplified, s)
 	}

result_test.go

@@ -207,6 +207,21 @@ func TestResultOutJSON(t *testing.T) {
 				}}}},
 			},
 		})},
+		{newRunNResult(t, 1, []*RunResult{
+			{
+				ID:   "ab13ba1e546838ceafa17f91ab3220102f397b2e",
+				Desc: "cond false runbook",
+				Path: "testdata/book/runn_cond_false.yml",
+				Err:  newCondFalseError("current.res.status == 200", "current.res.status == 200\n=> 404 == 200\n=> false"),
+				StepResults: []*StepResult{{
+					ID:         "ab13ba1e546838ceafa17f91ab3220102f397b2e?step=0",
+					Key:        "0",
+					RunnerType: RunnerTypeTest,
+					RunnerKey:  "test",
+					Err:        newCondFalseError("current.res.status == 200", "current.res.status == 200\n=> 404 == 200\n=> false"),
+				}},
+			},
+		})},
 	}
 	for i, tt := range tests {
 		key := fmt.Sprintf("result_out_json_%d", i)

testdata/result_out_json_0.golden

@@ -34,7 +34,9 @@
           "runner_type": "http",
           "runner_key": "req",
           "result": "failure",
-          "error": "dummy"
+          "error": {
+            "message": "dummy"
+          }
         }
       ]
     },

testdata/result_out_json_1.golden

@@ -34,7 +34,9 @@
           "runner_type": "http",
           "runner_key": "req",
           "result": "failure",
-          "error": "dummy"
+          "error": {
+            "message": "dummy"
+          }
         }
       ]
     },
@@ -74,7 +76,9 @@
           "index": 0,
           "key": "0",
           "result": "failure",
-          "error": "dummy"
+          "error": {
+            "message": "dummy"
+          }
         }
       ]
     }

testdata/result_out_json_2.golden

@@ -27,7 +27,9 @@
           "index": 0,
           "key": "0",
           "result": "failure",
-          "error": "dummy"
+          "error": {
+            "message": "dummy"
+          }
         }
       ]
     }

testdata/result_out_json_3.golden

@@ -27,7 +27,9 @@
           "index": 0,
           "key": "0",
           "result": "failure",
-          "error": "dummy",
+          "error": {
+            "message": "dummy"
+          },
           "included_run_result": [
             {
               "id": "ab13ba1e546838ceafa17f91ab3220102f397b2e?step=0",
@@ -39,7 +41,9 @@
                   "index": 0,
                   "key": "0",
                   "result": "failure",
-                  "error": "dummy"
+                  "error": {
+                    "message": "dummy"
+                  }
                 }
               ]
             }

testdata/result_out_json_4.golden

@@ -0,0 +1,29 @@
+{
+  "total": 1,
+  "success": 0,
+  "failure": 1,
+  "skipped": 0,
+  "results": [
+    {
+      "id": "ab13ba1e546838ceafa17f91ab3220102f397b2e",
+      "desc": "cond false runbook",
+      "path": "testdata/book/runn_cond_false.yml",
+      "result": "failure",
+      "steps": [
+        {
+          "id": "ab13ba1e546838ceafa17f91ab3220102f397b2e?step=0",
+          "index": 0,
+          "key": "0",
+          "runner_type": "test",
+          "runner_key": "test",
+          "result": "failure",
+          "error": {
+            "message": "condition is not true\n\nCondition:\n  current.res.status == 200\n  =\u003e 404 == 200\n  =\u003e false\n",
+            "condition": "current.res.status == 200",
+            "expr_trace": "current.res.status == 200\n=\u003e 404 == 200\n=\u003e false"
+          }
+        }
+      ]
+    }
+  ]
+}