feat: support to calculate the API coverage (#89)

* feat: support to calculate the API coverage

* feat: add brace expansion support

Author: LinuxSuRen

README.md

@@ -39,11 +39,13 @@ Usage:
 
 Available Commands:
   completion  Generate the autocompletion script for the specified shell
+  func        Print all the supported functions
   help        Help about any command
   json        Print the JSON schema of the test suites struct
   run         Run the test suite
   sample      Generate a sample test case YAML file
   server      Run as a server mode
+  service     Install atest as a Linux service
 
 Flags:
   -h, --help      help for atest

cmd/function.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"fmt"
 	"go/ast"
 	"go/doc"
 	"go/parser"
@@ -26,7 +25,7 @@ func createFunctionCmd() (c *cobra.Command) {
 					cmd.Println(reflect.TypeOf(fn))
 					desc := FuncDescription(fn)
 					if desc != "" {
-						fmt.Println(desc)
+						cmd.Println(desc)
 					}
 				} else {
 					cmd.Println("No such function")

cmd/run.go

@@ -11,10 +11,12 @@ import (
 	"sync"
 	"time"
 
+	"github.com/linuxsuren/api-testing/pkg/apispec"
 	"github.com/linuxsuren/api-testing/pkg/limit"
 	"github.com/linuxsuren/api-testing/pkg/render"
 	"github.com/linuxsuren/api-testing/pkg/runner"
 	"github.com/linuxsuren/api-testing/pkg/testing"
+	"github.com/linuxsuren/api-testing/pkg/util"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/semaphore"
 )
@@ -35,6 +37,7 @@ type runOption struct {
 	reportWriter       runner.ReportResultWriter
 	report             string
 	reportIgnore       bool
+	swaggerURL         string
 	level              string
 	caseItems          []string
 }
@@ -69,14 +72,15 @@ See also https://github.com/LinuxSuRen/api-testing/tree/master/sample`,
 	// set flags
 	flags := cmd.Flags()
 	flags.StringVarP(&opt.pattern, "pattern", "p", "test-suite-*.yaml",
-		"The file pattern which try to execute the test cases")
+		"The file pattern which try to execute the test cases. Brace expansion is supported, such as: test-suite-{1,2}.yaml")
 	flags.StringVarP(&opt.level, "level", "l", "info", "Set the output log level")
 	flags.DurationVarP(&opt.duration, "duration", "", 0, "Running duration")
 	flags.DurationVarP(&opt.requestTimeout, "request-timeout", "", time.Minute, "Timeout for per request")
 	flags.BoolVarP(&opt.requestIgnoreError, "request-ignore-error", "", false, "Indicate if ignore the request error")
 	flags.StringVarP(&opt.report, "report", "", "", "The type of target report. Supported: markdown, md, html, discard, std")
 	flags.StringVarP(&opt.reportFile, "report-file", "", "", "The file path of the report")
 	flags.BoolVarP(&opt.reportIgnore, "report-ignore", "", false, "Indicate if ignore the report output")
+	flags.StringVarP(&opt.swaggerURL, "swagger-url", "", "", "The URL of swagger")
 	flags.Int64VarP(&opt.thread, "thread", "", 1, "Threads of the execution")
 	flags.Int32VarP(&opt.qps, "qps", "", 5, "QPS")
 	flags.Int32VarP(&opt.burst, "burst", "", 5, "burst")
@@ -108,12 +112,20 @@ func (o *runOption) preRunE(cmd *cobra.Command, args []string) (err error) {
 		err = fmt.Errorf("not supported report type: '%s'", o.report)
 	}
 
+	if err == nil {
+		var swaggerAPI apispec.APIConverage
+		if o.swaggerURL != "" {
+			if swaggerAPI, err = apispec.ParseURLToSwagger(o.swaggerURL); err == nil {
+				o.reportWriter.WithAPIConverage(swaggerAPI)
+			}
+		}
+	}
+
 	o.caseItems = args
 	return
 }
 
 func (o *runOption) runE(cmd *cobra.Command, args []string) (err error) {
-	var files []string
 	o.startTime = time.Now()
 	o.context = cmd.Context()
 	o.limiter = limit.NewDefaultRateLimiter(o.qps, o.burst)
@@ -122,12 +134,20 @@ func (o *runOption) runE(cmd *cobra.Command, args []string) (err error) {
 		o.limiter.Stop()
 	}()
 
-	if files, err = filepath.Glob(o.pattern); err == nil {
-		for i := range files {
-			item := files[i]
-			if err = o.runSuiteWithDuration(item); err != nil {
-				break
-			}
+	var suites []string
+	for _, pattern := range util.Expand(o.pattern) {
+		var files []string
+		if files, err = filepath.Glob(pattern); err == nil {
+			suites = append(suites, files...)
+		}
+	}
+
+	cmd.Println("found suites:", len(suites))
+	for i := range suites {
+		item := suites[i]
+		cmd.Println("run suite:", item)
+		if err = o.runSuiteWithDuration(item); err != nil {
+			break
 		}
 	}
 

cmd/run_test.go

@@ -115,6 +115,14 @@ func TestRunCommand(t *testing.T) {
 		prepare: fooPrepare,
 		args:    []string{"-p", simpleSuite, "--report", "md", "--report-file", tmpFile.Name()},
 		hasErr:  false,
+	}, {
+		name: "report with swagger URL",
+		prepare: func() {
+			fooPrepare()
+			fooPrepare()
+		},
+		args:   []string{"-p", simpleSuite, "--swagger-url", urlFoo + "/bar"},
+		hasErr: false,
 	}, {
 		name:    "report file with error",
 		prepare: fooPrepare,
@@ -124,9 +132,10 @@ func TestRunCommand(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			defer gock.Clean()
+			buf := new(bytes.Buffer)
 			util.MakeSureNotNil(tt.prepare)()
 			root := &cobra.Command{Use: "root"}
-			root.SetOut(&bytes.Buffer{})
+			root.SetOut(buf)
 			root.AddCommand(createRunCommand())
 
 			root.SetArgs(append([]string{"run"}, tt.args...))
@@ -184,6 +193,15 @@ func TestPreRunE(t *testing.T) {
 			assert.Nil(t, err)
 			assert.NotNil(t, ro.reportWriter)
 		},
+	}, {
+		name: "html report",
+		opt: &runOption{
+			report: "html",
+		},
+		verify: func(t *testing.T, ro *runOption, err error) {
+			assert.Nil(t, err)
+			assert.NotNil(t, ro.reportWriter)
+		},
 	}, {
 		name: "empty report",
 		opt: &runOption{

pkg/apispec/swagger.go

@@ -0,0 +1,98 @@
+package apispec
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+)
+
+type Swagger struct {
+	Swagger string                           `json:"swagger"`
+	Paths   map[string]map[string]SwaggerAPI `json:"paths"`
+	Info    SwaggerInfo                      `json:"info"`
+}
+
+type SwaggerAPI struct {
+	OperationId string `json:"operationId"`
+	Summary     string `json:"summary"`
+}
+
+type SwaggerInfo struct {
+	Description string `json:"description"`
+	Title       string `json:"title"`
+	Version     string `json:"version"`
+}
+
+type APIConverage interface {
+	HaveAPI(path, method string) (exist bool)
+	APICount() (count int)
+}
+
+// HaveAPI check if the swagger has the API.
+// If the path is /api/v1/names/linuxsuren, then will match /api/v1/names/{name}
+func (s *Swagger) HaveAPI(path, method string) (exist bool) {
+	method = strings.ToLower(method)
+	for item := range s.Paths {
+		if matchAPI(path, item) {
+			for m := range s.Paths[item] {
+				if strings.ToLower(m) == method {
+					exist = true
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
+func matchAPI(particularAPI, swaggerAPI string) (matched bool) {
+	result := swaggerAPIConvert(swaggerAPI)
+	reg, err := regexp.Compile(result)
+	if err == nil {
+		matched = reg.MatchString(particularAPI)
+	}
+	return
+}
+
+func swaggerAPIConvert(text string) (result string) {
+	result = text
+	reg, err := regexp.Compile("{.*}")
+	if err == nil {
+		result = reg.ReplaceAllString(text, ".*")
+	}
+	return
+}
+
+// APICount return the count of APIs
+func (s *Swagger) APICount() (count int) {
+	for path := range s.Paths {
+		for range s.Paths[path] {
+			count++
+		}
+	}
+	return
+}
+
+func ParseToSwagger(data []byte) (swagger *Swagger, err error) {
+	swagger = &Swagger{}
+	err = json.Unmarshal(data, swagger)
+	return
+}
+
+func ParseURLToSwagger(swaggerURL string) (swagger *Swagger, err error) {
+	var resp *http.Response
+	if resp, err = http.Get(swaggerURL); err == nil && resp != nil && resp.StatusCode == http.StatusOK {
+		swagger, err = ParseStreamToSwagger(resp.Body)
+	}
+	return
+}
+
+func ParseStreamToSwagger(stream io.Reader) (swagger *Swagger, err error) {
+	var data []byte
+	if data, err = io.ReadAll(stream); err == nil {
+		swagger, err = ParseToSwagger(data)
+	}
+	return
+}

pkg/apispec/swagger_test.go

@@ -0,0 +1,111 @@
+package apispec_test
+
+import (
+	"net/http"
+	"testing"
+
+	_ "embed"
+
+	"github.com/h2non/gock"
+	"github.com/linuxsuren/api-testing/pkg/apispec"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseURLToSwagger(t *testing.T) {
+	tests := []struct {
+		name       string
+		swaggerURL string
+		verify     func(t *testing.T, swagger *apispec.Swagger, err error)
+	}{{
+		name:       "normal",
+		swaggerURL: "http://foo",
+		verify: func(t *testing.T, swagger *apispec.Swagger, err error) {
+			assert.NoError(t, err)
+			assert.Equal(t, "2.0", swagger.Swagger)
+			assert.Equal(t, apispec.SwaggerInfo{
+				Description: "sample",
+				Title:       "sample",
+				Version:     "1.0.0",
+			}, swagger.Info)
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gock.New(tt.swaggerURL).Get("/").Reply(200).BodyString(testdataSwaggerJSON)
+			defer gock.Off()
+
+			s, err := apispec.ParseURLToSwagger(tt.swaggerURL)
+			tt.verify(t, s, err)
+		})
+	}
+}
+
+func TestHaveAPI(t *testing.T) {
+	tests := []struct {
+		name         string
+		swaggerURL   string
+		path, method string
+		expectExist  bool
+	}{{
+		name:        "normal, exist",
+		swaggerURL:  "http://foo",
+		path:        "/api/v1/users",
+		method:      http.MethodGet,
+		expectExist: true,
+	}, {
+		name:        "create user, exist",
+		swaggerURL:  "http://foo",
+		path:        "/api/v1/users",
+		method:      http.MethodPost,
+		expectExist: true,
+	}, {
+		name:        "get a user, exist",
+		swaggerURL:  "http://foo",
+		path:        "/api/v1/users/linuxsuren",
+		method:      http.MethodGet,
+		expectExist: true,
+	}, {
+		name:        "normal, not exist",
+		swaggerURL:  "http://foo",
+		path:        "/api/v1/users",
+		method:      http.MethodDelete,
+		expectExist: false,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gock.New(tt.swaggerURL).Get("/").Reply(200).BodyString(testdataSwaggerJSON)
+			defer gock.Off()
+
+			swagger, err := apispec.ParseURLToSwagger(tt.swaggerURL)
+			assert.NoError(t, err)
+			exist := swagger.HaveAPI(tt.path, tt.method)
+			assert.Equal(t, tt.expectExist, exist)
+		})
+	}
+}
+
+func TestAPICount(t *testing.T) {
+	tests := []struct {
+		name        string
+		swaggerURL  string
+		expectCount int
+	}{{
+		name:        "normal",
+		swaggerURL:  "http://foo",
+		expectCount: 5,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gock.New(tt.swaggerURL).Get("/").Reply(200).BodyString(testdataSwaggerJSON)
+			defer gock.Off()
+
+			swagger, err := apispec.ParseURLToSwagger(tt.swaggerURL)
+			assert.NoError(t, err)
+			count := swagger.APICount()
+			assert.Equal(t, tt.expectCount, count)
+		})
+	}
+}
+
+//go:embed testdata/swagger.json
+var testdataSwaggerJSON string