feat: support to print the progress when oci download (#544)

Co-authored-by: rick <[email protected]>

Author: LinuxSuRen

atest-store-git

@@ -112,7 +112,7 @@ Magic.Keys(addStore, ['Alt+KeyN'])
 const rules = reactive<FormRules<Store>>({
   name: [{ required: true, message: 'Name is required', trigger: 'blur' }],
   url: [{ required: true, message: 'URL is required', trigger: 'blur' }],
-  "kind.name": [{ required: true, message: 'Plugin is required', trigger: 'blur' }]
+  pluginName: [{ required: true, message: 'Plugin is required', trigger: 'blur' }]
 })
 const submitForm = async (formEl: FormInstance | undefined) => {
   if (!formEl) return
@@ -185,7 +185,7 @@ function storeVerify(formEl: FormInstance | undefined) {
       ElMessage.error(e.message)
     }
   }, (e) => {
-    ElMessage.error('Oops, ' + e)
+    ElMessage.error(e.message)
   })
 }
 

console/atest-ui/src/views/StoreManager.vue

@@ -47,10 +47,8 @@ function CreateOrUpdateStore(payload: any, create: boolean,
     })
 }
 
-function ErrorTip(e: {
-    statusText:''
-}) {
-    ElMessage.error('Oops, ' + e.statusText)
+const ErrorTip = (e: any) => {
+    ElMessage.error(e.message)
 }
 
 export const UIAPI = {

console/atest-ui/src/views/net-vue.ts

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/blang/semver/v4"
+	"github.com/linuxsuren/api-testing/pkg/util"
 )
 
 type OCIDownloader interface {
@@ -95,7 +96,7 @@ func (d *defaultOCIDownloader) Download(image, tag, file string) (reader io.Read
 		return
 	}
 
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authStr))
+	req.Header.Set(util.Authorization, fmt.Sprintf("Bearer %s", authStr))
 	req.Header.Set("Accept", "application/vnd.oci.image.manifest.v1+json")
 
 	var resp *http.Response
@@ -106,8 +107,12 @@ func (d *defaultOCIDownloader) Download(image, tag, file string) (reader io.Read
 		err = fmt.Errorf("failed to get manifest from %q, status code: %d", api, resp.StatusCode)
 		return
 	} else {
+		progressReader := NewProgressReader(resp.Body)
+		progressReader.SetLength(resp.ContentLength)
+		progressReader.SetTitle("Fetching manifest:")
+
 		var data []byte
-		data, err = io.ReadAll(resp.Body)
+		data, err = io.ReadAll(progressReader)
 		if err != nil {
 			return
 		}
@@ -161,7 +166,7 @@ func (d *defaultOCIDownloader) getLatestTag(image, authToken string) (tag string
 		return
 	}
 
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken))
+	req.Header.Set(util.Authorization, fmt.Sprintf("Bearer %s", authToken))
 	var resp *http.Response
 	if resp, err = d.getHTTPClient().Do(req); err != nil {
 		err = fmt.Errorf("failed to get image tags from %q, error: %v", req.URL, err)
@@ -170,8 +175,12 @@ func (d *defaultOCIDownloader) getLatestTag(image, authToken string) (tag string
 	} else {
 		defer resp.Body.Close()
 
+		progressReader := NewProgressReader(resp.Body)
+		progressReader.SetLength(resp.ContentLength)
+		progressReader.SetTitle("Fetching tags:")
+
 		var data []byte
-		if data, err = io.ReadAll(resp.Body); err != nil {
+		if data, err = io.ReadAll(progressReader); err != nil {
 			return
 		}
 
@@ -214,13 +223,17 @@ func (d *defaultOCIDownloader) downloadLayer(image, digest, authToken string) (r
 		return
 	}
 
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken))
+	req.Header.Set(util.Authorization, fmt.Sprintf("Bearer %s", authToken))
 	var resp *http.Response
 	if resp, err = d.getHTTPClient().Do(req); err != nil {
 		err = fmt.Errorf("failed to get layer from %q, error: %v", req.URL.String(), err)
 	} else {
+		progressReader := NewProgressReader(resp.Body)
+		progressReader.SetLength(resp.ContentLength)
+		progressReader.SetTitle("Fetching Layer:")
+
 		var data []byte
-		if data, err = io.ReadAll(resp.Body); err != nil {
+		if data, err = io.ReadAll(progressReader); err != nil {
 			return
 		}
 

pkg/downloader/oci.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2024 API Testing Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package downloader
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+type ProgressReader interface {
+	io.Reader
+	Withoutput(io.Writer)
+	SetLength(int64)
+	SetTitle(string)
+}
+
+type defaultProgressReader struct {
+	reader      io.Reader
+	w           io.Writer
+	total       int64
+	current     int
+	lastPercent int64
+	title       string
+}
+
+var _ io.Reader = (*defaultProgressReader)(nil)
+
+func NewProgressReader(r io.Reader) ProgressReader {
+	return &defaultProgressReader{
+		reader: r,
+		w:      os.Stdout,
+	}
+}
+
+func (r *defaultProgressReader) SetTitle(title string) {
+	r.title = title
+}
+
+func (r *defaultProgressReader) Read(p []byte) (count int, err error) {
+	count, err = r.reader.Read(p)
+	if r.total > 0 {
+		if count > 0 {
+			r.current += count
+			newPercent := int64(r.current*100) / r.total
+			if newPercent != int64(r.lastPercent) {
+				r.lastPercent = newPercent
+				fmt.Fprintf(r.w, "%s\t%d%%\n", r.title, newPercent)
+			}
+		}
+	} else {
+		fmt.Fprintf(r.w, "%d\n", count)
+	}
+	return
+}
+
+func (r *defaultProgressReader) Withoutput(w io.Writer) {
+	r.w = w
+}
+
+func (r *defaultProgressReader) SetLength(len int64) {
+	r.total = len
+}

pkg/downloader/progress.go

@@ -51,7 +51,7 @@ func (r *defualtReader) request(api string) (result map[string]interface{}, err
 	client := GetClient()
 	var req *http.Request
 	if req, err = http.NewRequest(http.MethodGet, api, nil); err == nil {
-		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.token))
+		req.Header.Add(util.Authorization, fmt.Sprintf("Bearer %s", r.token))
 		var resp *http.Response
 		if resp, err = client.Do(req); err == nil && resp.StatusCode == http.StatusOK {
 			var data []byte

pkg/runner/kubernetes/client.go

@@ -100,39 +100,39 @@ func TestKubernetesValidatorFunc(t *testing.T) {
 func preparePod() {
 	gock.New(urlFoo).
 		Get("/api/v1/namespaces/ns/pods/foo").
-		MatchHeader("Authorization", defaultToken).
+		MatchHeader(util.Authorization, defaultToken).
 		Reply(http.StatusOK).
 		JSON(`{"kind":"pod"}`)
 }
 
 func prepareDeploy() {
 	gock.New(urlFoo).
 		Get("/apis/apps/v1/namespaces/ns/deployments/foo").
-		MatchHeader("Authorization", defaultToken).
+		MatchHeader(util.Authorization, defaultToken).
 		Reply(http.StatusOK).
 		JSON(`{"kind":"deploy"}`)
 }
 
 func prepareStatefulset() {
 	gock.New(urlFoo).
 		Get("/apis/apps/v1/namespaces/ns/statefulsets/foo").
-		MatchHeader("Authorization", defaultToken).
+		MatchHeader(util.Authorization, defaultToken).
 		Reply(http.StatusOK).
 		JSON(`{"kind":"statefulset"}`)
 }
 
 func prepareDaemonset() {
 	gock.New(urlFoo).
 		Get("/apis/apps/v1/namespaces/ns/daemonsets/foo").
-		MatchHeader("Authorization", defaultToken).
+		MatchHeader(util.Authorization, defaultToken).
 		Reply(http.StatusOK).
 		JSON(`{"kind":"daemonset","items":[]}`)
 }
 
 func prepareCRDVM() {
 	gock.New(urlFoo).
 		Get("/apis/bar/v2/namespaces/ns/vms/foo").
-		MatchHeader("Authorization", defaultToken).
+		MatchHeader(util.Authorization, defaultToken).
 		Reply(http.StatusOK).
 		JSON(`{"kind":"vm"}`)
 }

pkg/runner/kubernetes/verify_test.go

@@ -180,7 +180,7 @@ func (w *githubPRCommentWriter) sendRequest(req *http.Request) (resp *http.Respo
 }
 
 func (w *githubPRCommentWriter) setHeader(req *http.Request) {
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", w.Token))
+	req.Header.Set(util.Authorization, fmt.Sprintf("Bearer %s", w.Token))
 	req.Header.Set("Accept", "application/vnd.github+json")
 	req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
 }

pkg/runner/writer_github_pr_comment.go

@@ -188,7 +188,7 @@ func TestFindParentTestCases(t *testing.T) {
 		testcase: &atest.TestCase{
 			Request: atest.Request{
 				Header: map[string]string{
-					"Authorization": BearerToken,
+					util.Authorization: BearerToken,
 				},
 			},
 		},
@@ -252,7 +252,7 @@ func TestFindParentTestCases(t *testing.T) {
 			Request: atest.Request{
 				API: "/users/{{(index .users 0).name}}{{randomKubernetesName}}",
 				Header: map[string]string{
-					"Authorization": BearerToken,
+					util.Authorization: BearerToken,
 				},
 			},
 		},
@@ -269,7 +269,7 @@ func TestFindParentTestCases(t *testing.T) {
 				Request: atest.Request{
 					API: "/users/{{(index .users 0).name}}",
 					Header: map[string]string{
-						"Authorization": BearerToken,
+						util.Authorization: BearerToken,
 					},
 				},
 			}},

pkg/server/remote_server_test.go

@@ -18,6 +18,7 @@ package server
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -98,6 +99,7 @@ func (s *storeExtManager) Start(name, socket string) (err error) {
 	} else {
 		binaryPath, err = s.execer.LookPath(name)
 		if err != nil {
+			err = fmt.Errorf("not found extension, try to download it.")
 			go func() {
 				reader, dErr := s.ociDownloader.Download(name, "", "")
 				if dErr != nil {
@@ -125,6 +127,8 @@ func (s *storeExtManager) Start(name, socket string) (err error) {
 
 func (s *storeExtManager) startPlugin(socketURL, plugin, pluginName string) (err error) {
 	socketFile := strings.TrimPrefix(socketURL, s.socketPrefix)
+	_ = os.RemoveAll(socketFile) // always deleting the socket file to avoid start failing
+
 	s.filesNeedToBeRemoved = append(s.filesNeedToBeRemoved, socketFile)
 	s.extStatusMap[pluginName] = true
 	if err = s.execer.RunCommandWithIO(plugin, "", os.Stdout, os.Stderr, s.processChan, "--socket", socketFile); err != nil {