feat: support to show the error count as summary in report (#305)

LinuxSuRen · web-flow · commit 8e34dba43d77 · 2023-12-05T13:59:16.000+08:00
Co-authored-by: rick &lt;LinuxSuRen@users.noreply.github.com&gt;
diff --git a/.github/testing/trpc.yaml b/.github/testing/trpc.yaml
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -13,7 +13,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
       - name: Unit Test
         run: |
           make test-all-backend test-operator
@@ -45,7 +45,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
       - name: API Test
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -70,7 +70,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         env:
@@ -91,7 +91,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
       - name: Set output
         id: vars
         run: echo "tag=$(git describe --tags)" >> $GITHUB_OUTPUT
@@ -126,7 +126,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
       - name: Use Node.js
         uses: actions/setup-node@v3
         with:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -24,7 +24,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19
+          go-version: 1.20
       - name: Use Node.js
         uses: actions/setup-node@v3
         with:
@@ -46,7 +46,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
       - name: Unit Test
         run: |
           make test-all-backend test-operator
diff --git a/Dockerfile b/Dockerfile
@@ -5,7 +5,7 @@ COPY console/atest-ui .
 RUN npm install --ignore-scripts --registry=https://registry.npmmirror.com
 RUN npm run build-only
 
-FROM docker.io/golang:1.19 AS builder
+FROM docker.io/golang:1.20 AS builder
 
 ARG VERSION
 ARG GOPROXY
diff --git a/cmd/run.go b/cmd/run.go
@@ -26,6 +26,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -41,6 +42,7 @@ import (
 	"github.com/linuxsuren/api-testing/pkg/runner"
 	"github.com/linuxsuren/api-testing/pkg/runner/monitor"
 	"github.com/linuxsuren/api-testing/pkg/testing"
+	"github.com/linuxsuren/api-testing/pkg/util"
 	fakeruntime "github.com/linuxsuren/go-fake-runtime"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -320,6 +322,7 @@ func (o *runOption) runSuite(loader testing.Loader, dataContext map[string]inter
 		return
 	}
 
+	var errs []error
 	suiteRunner := runner.GetTestSuiteRunner(testSuite)
 	suiteRunner.WithTestReporter(o.reporter)
 	suiteRunner.WithSecure(testSuite.Spec.Secure)
@@ -343,15 +346,21 @@ func (o *runOption) runSuite(loader testing.Loader, dataContext map[string]inter
 			ctxWithTimeout, _ := context.WithTimeout(ctx, o.requestTimeout)
 			ctxWithTimeout = context.WithValue(ctxWithTimeout, runner.ContextKey("").ParentDir(), loader.GetContext())
 
-			if output, err = suiteRunner.RunTestCase(&testCase, dataContext, ctxWithTimeout); err != nil && !o.requestIgnoreError {
-				err = fmt.Errorf("failed to run '%s', %v", testCase.Name, err)
-				return
-			} else {
-				err = nil
+			output, err = suiteRunner.RunTestCase(&testCase, dataContext, ctxWithTimeout)
+			if err = util.ErrorWrap(err, "failed to run '%s', %v", testCase.Name, err); err != nil {
+				if o.requestIgnoreError {
+					errs = append(errs, err)
+				} else {
+					return
+				}
 			}
 		}
 		dataContext[testCase.Name] = output
 	}
+
+	if len(errs) > 0 {
+		err = errors.Join(errs...)
+	}
 	return
 }
 
diff --git a/cmd/run_test.go b/cmd/run_test.go
@@ -171,6 +171,13 @@ func TestRunCommand(t *testing.T) {
 		prepare: fooPrepare,
 		args:    []string{"-p", "[]]$#%*^&()"},
 		hasErr:  true,
+	}, {
+		name: "request-ignore-error",
+		prepare: func() {
+			gock.New(urlFoo).Get("/bar").Reply(http.StatusBadRequest)
+		},
+		args:   []string{"-p", simpleSuite, "--request-ignore-error"},
+		hasErr: true,
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
diff --git a/extensions/collector/go.mod b/extensions/collector/go.mod
@@ -1,6 +1,6 @@
 module github.com/linuxsuren/api-testing/extensions/collector
 
-go 1.19
+go 1.20
 
 require (
 	github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819
diff --git a/extensions/monitor-docker/go.mod b/extensions/monitor-docker/go.mod
@@ -1,6 +1,6 @@
 module github.com/linuxsuren/api-testing/extensions/monitor-docker
 
-go 1.19
+go 1.20
 
 require (
 	github.com/docker/cli v24.0.7+incompatible
diff --git a/extensions/store-etcd/go.mod b/extensions/store-etcd/go.mod
@@ -1,6 +1,6 @@
 module github.com/linuxsuren/api-testing/extensions/store-etcd
 
-go 1.19
+go 1.20
 
 require (
 	github.com/linuxsuren/api-testing v0.0.0-00010101000000-000000000000
diff --git a/extensions/store-git/go.mod b/extensions/store-git/go.mod
@@ -1,6 +1,6 @@
 module github.com/linuxsuren/api-testing/extensions/store-git
 
-go 1.19
+go 1.20
 
 require (
 	github.com/go-git/go-git/v5 v5.8.1
diff --git a/extensions/store-mongodb/go.mod b/extensions/store-mongodb/go.mod
@@ -1,6 +1,6 @@
 module github.com/linuxsuren/api-testing/extensions/store-mongodb
 
-go 1.19
+go 1.20
 
 require (
 	github.com/linuxsuren/api-testing v0.0.11
diff --git a/extensions/store-orm/go.mod b/extensions/store-orm/go.mod
@@ -1,6 +1,6 @@
 module github.com/linuxsuren/api-testing/extensions/store-orm
 
-go 1.19
+go 1.20
 
 require (
 	github.com/linuxsuren/api-testing v0.0.11
diff --git a/extensions/store-s3/go.mod b/extensions/store-s3/go.mod
@@ -1,6 +1,6 @@
 module github.com/linuxsuren/api-testing/extensions/store-s3
 
-go 1.19
+go 1.20
 
 require (
 	github.com/aws/aws-sdk-go v1.44.312
diff --git a/go.mod b/go.mod
@@ -1,6 +1,6 @@
 module github.com/linuxsuren/api-testing
 
-go 1.19
+go 1.20
 
 require (
 	github.com/Masterminds/sprig/v3 v3.2.3
diff --git a/go.work b/go.work
@@ -1,4 +1,4 @@
-go 1.19
+go 1.20
 
 use ./
 use ./extensions/collector
diff --git a/operator/go.mod b/operator/go.mod
@@ -1,6 +1,6 @@
 module github.com/linuxsuren/api-testing/operator
 
-go 1.19
+go 1.20
 
 require (
 	github.com/onsi/ginkgo/v2 v2.6.0
diff --git a/pkg/runner/data/report.md b/pkg/runner/data/report.md
@@ -1,4 +1,4 @@
-There are {{ .Total }} test cases:
+There are {{ .Total }} test cases, failed count {{ .Error }}:
  
 {{- if gt .Total 6 }}
 {{- if gt .Error 0 }}
diff --git a/pkg/runner/writer_markdown_test.go b/pkg/runner/writer_markdown_test.go
@@ -57,7 +57,7 @@ func TestMarkdownWriter(t *testing.T) {
 		writer.WithAPIConverage(nil)
 		err := writer.Output(createSlice(sample, 2))
 		assert.Nil(t, err)
-		assert.Equal(t, `There are 2 test cases:
+		assert.Equal(t, `There are 2 test cases, failed count 0:
 
 | API | Average | Max | Min | Count | Error |
 |---|---|---|---|---|---|
@@ -71,7 +71,7 @@ func TestMarkdownWriter(t *testing.T) {
 		writer.WithAPIConverage(nil)
 		err := writer.Output(createSlice(sample, 8))
 		assert.Nil(t, err)
-		assert.Equal(t, `There are 8 test cases:
+		assert.Equal(t, `There are 8 test cases, failed count 0:
 
 <details>
   <summary><b>See all test records</b></summary>
@@ -95,7 +95,7 @@ func TestMarkdownWriter(t *testing.T) {
 		writer.WithAPIConverage(nil)
 		err := writer.Output(append(createSlice(sample, 8), errSample))
 		assert.Nil(t, err)
-		assert.Equal(t, `There are 9 test cases:
+		assert.Equal(t, `There are 9 test cases, failed count 1:
 
 | API | Average | Max | Min | Count | Error |
 |---|---|---|---|---|---|
@@ -128,7 +128,7 @@ func TestMarkdownWriter(t *testing.T) {
 		}})
 		err := writer.Output(createSlice(sample, 2))
 		assert.Nil(t, err)
-		assert.Equal(t, `There are 2 test cases:
+		assert.Equal(t, `There are 2 test cases, failed count 0:
 
 | API | Average | Max | Min | Count | Error |
 |---|---|---|---|---|---|
@@ -148,7 +148,7 @@ Resource usage:
 		result.LastErrorMessage = "error happend"
 		err := writer.Output(createSlice(result, 2))
 		assert.Nil(t, err)
-		assert.Equal(t, `There are 2 test cases:
+		assert.Equal(t, `There are 2 test cases, failed count 0:
 
 | API | Average | Max | Min | Count | Error |
 |---|---|---|---|---|---|
@@ -170,7 +170,7 @@ Resource usage:
 		}}))
 		err := writer.Output(createSlice(sample, 2))
 		assert.Nil(t, err)
-		assert.Equal(t, `There are 2 test cases:
+		assert.Equal(t, `There are 2 test cases, failed count 0:
 
 | API | Average | Max | Min | Count | Error |
 |---|---|---|---|---|---|
diff --git a/pkg/server/remote_server.go b/pkg/server/remote_server.go
@@ -515,8 +515,12 @@ func convertConditionalVerify(verify []*ConditionalVerify) (result []testing.Con
 
 func (s *server) CreateTestCase(ctx context.Context, in *TestCaseWithSuite) (reply *HelloReply, err error) {
 	reply = &HelloReply{}
-	loader := s.getLoader(ctx)
-	err = loader.CreateTestCase(in.SuiteName, ToNormalTestCase(in.Data))
+	if in.Data == nil {
+		err = errors.New("data is required")
+	} else {
+		loader := s.getLoader(ctx)
+		err = loader.CreateTestCase(in.SuiteName, ToNormalTestCase(in.Data))
+	}
 	return
 }
 
diff --git a/pkg/util/error.go b/pkg/util/error.go
@@ -24,7 +24,10 @@ SOFTWARE.
 
 package util
 
-import "net/http"
+import (
+	"fmt"
+	"net/http"
+)
 
 // OKOrErrorMessage returns OK or error message
 func OKOrErrorMessage(err error) string {
@@ -39,6 +42,13 @@ func OrErrorMessage(err error, message string) string {
 	return message
 }
 
+func ErrorWrap(err error, msg string, args ...interface{}) error {
+	if err != nil {
+		err = fmt.Errorf(msg, args...)
+	}
+	return err
+}
+
 // IgnoreErrServerClosed ignores ErrServerClosed
 func IgnoreErrServerClosed(err error) error {
 	if err == http.ErrServerClosed {
diff --git a/sample/answer.yaml b/sample/answer.yaml
