feat: support to run test suite via http (#478)

* feat: support to run test suite via http

* improve the extension download

* download extenion in the e2e

* add extension-registry into the helm values.yaml file

* update helm install in e2e test

* fix the k8s helm chart testing

---------

Co-authored-by: rick <[email protected]>

Author: LinuxSuRen

cmd/extension.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"path/filepath"
 	"runtime"
+	"time"
 
 	"github.com/linuxsuren/api-testing/pkg/downloader"
 	"github.com/spf13/cobra"
@@ -33,6 +34,7 @@ type extensionOption struct {
 	tag           string
 	os            string
 	arch          string
+	timeout       time.Duration
 }
 
 func createExtensionCommand(ociDownloader downloader.PlatformAwareOCIDownloader) (c *cobra.Command) {
@@ -52,12 +54,16 @@ func createExtensionCommand(ociDownloader downloader.PlatformAwareOCIDownloader)
 	flags.StringVarP(&opt.registry, "registry", "", "", "The target extension image registry, supported: docker.io, ghcr.io")
 	flags.StringVarP(&opt.os, "os", "", runtime.GOOS, "The OS")
 	flags.StringVarP(&opt.arch, "arch", "", runtime.GOARCH, "The architecture")
+	flags.DurationVarP(&opt.timeout, "timeout", "", time.Minute, "The timeout of downloading")
 	return
 }
 
 func (o *extensionOption) runE(cmd *cobra.Command, args []string) (err error) {
 	o.ociDownloader.WithOS(o.os)
 	o.ociDownloader.WithArch(o.arch)
+	o.ociDownloader.WithRegistry(o.registry)
+	o.ociDownloader.WithTimeout(o.timeout)
+	o.ociDownloader.WithContext(cmd.Context())
 
 	for _, arg := range args {
 		var reader io.Reader

cmd/extension_test.go

@@ -43,7 +43,8 @@ func TestExtensionCmd(t *testing.T) {
 			server.Stop()
 		}()
 
-		d.WithRegistry(fmt.Sprintf("127.0.0.1:%s", server.GetPort()))
+		registry := fmt.Sprintf("127.0.0.1:%s", server.GetPort())
+		d.WithRegistry(registry)
 		d.WithInsecure(true)
 		d.WithBasicAuth("", "")
 		d.WithOS("linux")
@@ -56,12 +57,12 @@ func TestExtensionCmd(t *testing.T) {
 		assert.NoError(t, err)
 
 		command := createExtensionCommand(d)
-		command.SetArgs([]string{"git", "--output", tmpDownloadDir})
+		command.SetArgs([]string{"git", "--output", tmpDownloadDir, "--registry", registry})
 		err = command.Execute()
 		assert.NoError(t, err)
 
 		// not found
-		command.SetArgs([]string{"orm", "--output", tmpDownloadDir})
+		command.SetArgs([]string{"orm", "--output", tmpDownloadDir, "--registry", registry})
 		err = command.Execute()
 		assert.Error(t, err)
 	})

cmd/server.go

@@ -96,6 +96,7 @@ func createServerCmd(execer fakeruntime.Execer, httpServer server.HTTPServer) (c
 	flags.BoolVarP(&opt.dryRun, "dry-run", "", false, "Do not really start a gRPC server")
 	flags.StringArrayVarP(&opt.mockConfig, "mock-config", "", nil, "The mock config files")
 	flags.StringVarP(&opt.mockPrefix, "mock-prefix", "", "/mock", "The mock server API prefix")
+	flags.StringVarP(&opt.extensionRegistry, "extension-registry", "", "docker.io", "The extension registry URL")
 
 	// gc related flags
 	flags.IntVarP(&opt.gcPercent, "gc-percent", "", 100, "The GC percent of Go")
@@ -114,14 +115,15 @@ type serverOption struct {
 	httpServer server.HTTPServer
 	execer     fakeruntime.Execer
 
-	port         int
-	httpPort     int
-	printProto   bool
-	localStorage []string
-	consolePath  string
-	secretServer string
-	configDir    string
-	skyWalking   string
+	port              int
+	httpPort          int
+	printProto        bool
+	localStorage      []string
+	consolePath       string
+	secretServer      string
+	configDir         string
+	skyWalking        string
+	extensionRegistry string
 
 	auth          string
 	oauthProvider string
@@ -241,8 +243,10 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
 		template.SetSecretGetter(remote.NewGRPCSecretGetter(secretServer))
 	}
 
+	extDownloader := downloader.NewStoreDownloader()
+	extDownloader.WithRegistry(o.extensionRegistry)
 	storeExtMgr := server.NewStoreExtManager(o.execer)
-	storeExtMgr.WithDownloader(downloader.NewStoreDownloader())
+	storeExtMgr.WithDownloader(extDownloader)
 	remoteServer := server.NewRemoteServer(loader, remote.NewGRPCloaderFromStore(), secretServer, storeExtMgr, o.configDir, o.grpcMaxRecvMsgSize)
 	kinds, storeKindsErr := remoteServer.GetStoreKinds(ctx, nil)
 	if storeKindsErr != nil {

e2e/compose-k8s.yaml

@@ -35,8 +35,8 @@ services:
     - k8s:/output
     # This is just so that we get the kubeconfig file out
     # - .:/output
-    ports:
-    - 30000:30000
+    # ports:
+    # - 30000:30000
     # - 6443:6443  # Kubernetes API Server
     # - 80:80      # Ingress controller port 80
     # - 443:443    # Ingress controller port 443

e2e/entrypoint.sh

@@ -4,28 +4,35 @@ set -e
 SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
 mkdir -p /root/.config/atest
 mkdir -p /var/data
-cd "/var/data"
+
 # Generate private key
 openssl genrsa -out server.key 2048
 # Generate self-signed certificate
 openssl req -new -x509 -key server.key -out server.crt -days 36500 \
--subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
+    -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
 # Generate Certificate Signing Request (CSR)
 openssl req -new -key server.key -out server.csr \
--subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
+    -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
 # Generate a new private key
 openssl genpkey -algorithm RSA -out test.key
 # Generate a new CSR
 openssl req -new -nodes -key test.key -out test.csr -days 3650 \
--subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com" \
--config "$SCRIPT_DIR/openssl.cnf" -extensions v3_req
+    -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com" \
+    -config "openssl.cnf" -extensions v3_req
 # Sign the new CSR with the self-signed certificate
 openssl x509 -req -days 365 -in test.csr \
--out test.pem -CA server.crt -CAkey server.key \
--CAcreateserial -extfile "$SCRIPT_DIR/openssl.cnf" -extensions v3_req
+    -out test.pem -CA server.crt -CAkey server.key \
+    -CAcreateserial -extfile "openssl.cnf" -extensions v3_req
+
+echo "start to download extenions"
+atest extension --output /usr/local/bin --registry ghcr.io git
+atest extension --output /usr/local/bin --registry ghcr.io orm
+atest extension --output /usr/local/bin --registry ghcr.io etcd
+atest extension --output /usr/local/bin --registry ghcr.io mongodb
 
-nohup atest server --tls-grpc --cert-file /var/data/test.pem --key-file /var/data/test.key&
-cmd="atest run -p test-suite-common.yaml --report github --report-github-identity e2e-testing --report-file /var/data/report.json --report-github-repo linuxsuren/api-testing --report-github-pr ${PULL_REQUEST:-0}"
+echo "start to run server"
+nohup atest server --tls-grpc --cert-file test.pem --key-file test.key&
+cmd="atest run -p test-suite-common.yaml"
 
 echo "start to run testing: $cmd"
 kind=orm target=mysql:3306 driver=mysql $cmd
@@ -47,4 +54,3 @@ kind=orm target=postgres:5432 driver=postgres $cmd
 # kind=s3 target=minio:9000 atest run -p test-suite-common.yaml
 
 cat /root/.config/atest/stores.yaml
-exit 0

e2e/k8s.sh

@@ -3,13 +3,21 @@ set -e
 
 sleep 6
 echo "Running k8s.sh"
+
+ls -hal
+cd api-testing
+echo "build helm dependency"
+helm dependency build
+
+echo "install helm chart"
 helm install --kube-apiserver https://server:6443 --kube-token abcd --kube-insecure-skip-tls-verify \
-    api-testing ./api-testing \
+    api-testing . \
     --set service.type=NodePort \
     --set service.nodePort=30000 \
     --set persistence.enabled=false \
-    --set image.registry=linuxsuren.docker.scarf.sh \
+    --set image.registry=ghcr.io \
     --set image.repository=linuxsuren/api-testing \
-    --set image.tag=master
+    --set image.tag=master \
+    --set extension.registry=ghcr.io
 
 SERVER=http://server:30000 atest run -p git.yaml

e2e/start.sh

@@ -7,25 +7,5 @@ then
 fi
 
 docker-compose version
-docker-compose -f "$file" up --build -d
-
-while true
-do
-    docker-compose -f "$file" ps | grep testing
-    if [ $? -eq 1 ]
-    then
-        code=-1
-        docker-compose -f "$file" logs | grep testing
-        docker-compose -f "$file" logs | grep testing | grep Usage
-        if [ $? -eq 1 ]
-        then
-            code=0
-            echo "successed"
-        fi
-
-        docker-compose -f "$file" down
-        set -e
-        exit $code
-    fi
-    sleep 1
-done
+docker-compose -f "$file" down
+docker-compose -f "$file" up --build testing --exit-code-from testing --remove-orphans

e2e/test-suite-common.yaml

@@ -275,6 +275,16 @@ items:
         "suite": "{{.param.gRPCSuiteName}}",
         "testcase": "{{.param.gRPCCaseName}}"
       }
+- name: runTestSuite
+  request:
+    api: /RunTestSuite
+    method: POST
+    header:
+      X-Store-Name: "{{.param.store}}"
+    body: |
+      {
+        "name": "{{.param.gRPCSuiteName}}"
+      }
 - name: version
   request:
     api: /GetVersion

helm/api-testing/templates/deployment.yaml

@@ -37,6 +37,7 @@ spec:
           command:
           - atest
           - server
+          - --extension-registry={{ .Values.extension.registry }}
           - --local-storage=/root/.atest/data/*.yaml
           {{- if .Values.skywalking.endpoint.http }}
           - --skywalking={{ .Values.skywalking.endpoint.http }}

helm/api-testing/values.yaml

@@ -93,6 +93,9 @@ persistence:
       size: 500Mi
       volumeMode: Filesystem
 
+extension:
+  registry: docker.io
+
 skywalking:
   endpoint:
     http: "" # http://skywalking-skywalking-helm-oap.skywalking.svc:12800