diff --git a/cmd/extension.go b/cmd/extension.go index 6d04e6c7..b61ca027 100644 --- a/cmd/extension.go +++ b/cmd/extension.go @@ -31,6 +31,7 @@ type extensionOption struct { ociDownloader downloader.PlatformAwareOCIDownloader output string registry string + kind string tag string os string arch string @@ -53,6 +54,7 @@ func createExtensionCommand(ociDownloader downloader.PlatformAwareOCIDownloader) flags.StringVarP(&opt.output, "output", "", ".", "The target directory") flags.StringVarP(&opt.tag, "tag", "", "", "The extension image tag, try to find the latest one if this is empty") flags.StringVarP(&opt.registry, "registry", "", "", "The target extension image registry, supported: docker.io, ghcr.io") + flags.StringVarP(&opt.kind, "kind", "", "store", "The extension kind") flags.StringVarP(&opt.os, "os", "", runtime.GOOS, "The OS") flags.StringVarP(&opt.arch, "arch", "", runtime.GOARCH, "The architecture") flags.DurationVarP(&opt.timeout, "timeout", "", time.Minute, "The timeout of downloading") @@ -66,6 +68,7 @@ func (o *extensionOption) runE(cmd *cobra.Command, args []string) (err error) { o.ociDownloader.WithRegistry(o.registry) o.ociDownloader.WithImagePrefix(o.imagePrefix) o.ociDownloader.WithTimeout(o.timeout) + o.ociDownloader.WithKind(o.kind) o.ociDownloader.WithContext(cmd.Context()) for _, arg := range args { diff --git a/cmd/server.go b/cmd/server.go index 8a904d1c..35f63744 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "github.com/linuxsuren/api-testing/pkg/apispec" "net" "net/http" "os" @@ -302,6 +303,15 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) { _ = o.httpServer.Shutdown(ctx) }() + go func() { + err := apispec.DownloadSwaggerData("", extDownloader) + if err != nil { + fmt.Println("failed to download swagger data", err) + } else { + fmt.Println("success to download swagger data") + } + }() + mux := runtime.NewServeMux(runtime.WithMetadata(server.MetadataStoreFunc), runtime.WithMarshalerOption("application/json+pretty", &runtime.JSONPb{ MarshalOptions: protojson.MarshalOptions{ @@ -342,6 +352,7 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) { mux.HandlePath(http.MethodGet, "/get", o.getAtestBinary) mux.HandlePath(http.MethodPost, "/runner/{suite}/{case}", service.WebRunnerHandler) mux.HandlePath(http.MethodGet, "/api/v1/sbom", service.SBomHandler) + mux.HandlePath(http.MethodGet, "/api/v1/swaggers", apispec.SwaggersHandler) postRequestProxyFunc := postRequestProxy(o.skyWalking) mux.HandlePath(http.MethodPost, "/browser/{app}", postRequestProxyFunc) diff --git a/console/atest-ui/src/views/TestSuite.vue b/console/atest-ui/src/views/TestSuite.vue index d4c9806b..3489d8c3 100644 --- a/console/atest-ui/src/views/TestSuite.vue +++ b/console/atest-ui/src/views/TestSuite.vue @@ -4,7 +4,7 @@ import { reactive, ref, watch } from 'vue' import { Edit, CopyDocument, Delete } from '@element-plus/icons-vue' import type { FormInstance, FormRules } from 'element-plus' import type { Suite, TestCase, Pair } from './types' -import { NewSuggestedAPIsQuery, GetHTTPMethods } from './types' +import { NewSuggestedAPIsQuery, GetHTTPMethods, SwaggerSuggestion } from './types' import EditButton from '../components/EditButton.vue' import { Cache } from './cache' import { useI18n } from 'vue-i18n' @@ -20,6 +20,7 @@ const props = defineProps({ }) const emit = defineEmits(['updated']) let querySuggestedAPIs = NewSuggestedAPIsQuery(Cache.GetCurrentStore().name, props.name!) +const querySwaggers = SwaggerSuggestion() const suite = ref({ name: '', @@ -325,7 +326,10 @@ const renameTestSuite = (name: string) => { - + diff --git a/console/atest-ui/src/views/net.ts b/console/atest-ui/src/views/net.ts index 6553e27a..ee632ebf 100644 --- a/console/atest-ui/src/views/net.ts +++ b/console/atest-ui/src/views/net.ts @@ -620,6 +620,12 @@ function GetSuggestedAPIs(name: string, .then(callback) } +function GetSwaggers(callback: (d: any) => void) { + fetch(`/api/v1/swaggers`, {}) + .then(DefaultResponseProcess) + .then(callback) +} + function ReloadMockServer(config: any) { const requestOptions = { method: 'POST', @@ -812,7 +818,7 @@ export const API = { CreateOrUpdateStore, GetStores, DeleteStore, VerifyStore, FunctionsQuery, GetSecrets, DeleteSecret, CreateOrUpdateSecret, - GetSuggestedAPIs, + GetSuggestedAPIs, GetSwaggers, ReloadMockServer, GetMockConfig, SBOM, DataQuery, getToken } diff --git a/console/atest-ui/src/views/types.ts b/console/atest-ui/src/views/types.ts index dd97ff7b..66afe234 100644 --- a/console/atest-ui/src/views/types.ts +++ b/console/atest-ui/src/views/types.ts @@ -102,6 +102,28 @@ export function NewSuggestedAPIsQuery(store: string, suite: string) { }) } } + +interface SwaggerItem { + value: string +} + +export function SwaggerSuggestion() { + return function (queryString: string, cb: (arg: any) => void) { + API.GetSwaggers((e) => { + var swaggers = [] as SwaggerItem[] + e.forEach((item: string) => { + swaggers.push({ + "value": `atest://${item}` + }) + }) + + const results = queryString ? swaggers.filter((item: SwaggerItem) => { + return item.value.toLowerCase().indexOf(queryString.toLowerCase()) != -1 + }) : swaggers + cb(results.slice(0, 10)) + }) + } +} export function CreateFilter(queryString: string) { return (v: Pair) => { return v.value.toLowerCase().indexOf(queryString.toLowerCase()) !== -1 diff --git a/docs/site/content/zh/latest/tasks/extension.md b/docs/site/content/zh/latest/tasks/extension.md index 0820d86f..acb0e72a 100644 --- a/docs/site/content/zh/latest/tasks/extension.md +++ b/docs/site/content/zh/latest/tasks/extension.md @@ -31,3 +31,9 @@ atest extension orm ```shell atest extension orm --registry ghcr.io --timeout 2ms ``` + +想要下载其他类型的插件的话，可以使用下面的命令： + +```shell +atest extension --kind data swagger +``` diff --git a/extensions/README.md b/extensions/README.md index b94dd1ae..f6ca271f 100644 --- a/extensions/README.md +++ b/extensions/README.md @@ -11,6 +11,7 @@ Ports in extensions: | Monitor | [docker-monitor](https://github.com/LinuxSuRen/atest-ext-monitor-docker) | | | Agent | [collector](https://github.com/LinuxSuRen/atest-ext-collector) | | | Secret | [Vault](https://github.com/LinuxSuRen/api-testing-vault-extension) | | +| Data | [Swagger](https://github.com/LinuxSuRen/atest-ext-data-swagger) | | ## Contribute a new extension diff --git a/pkg/apispec/remote_swagger.go b/pkg/apispec/remote_swagger.go new file mode 100644 index 00000000..4504e785 --- /dev/null +++ b/pkg/apispec/remote_swagger.go @@ -0,0 +1,134 @@ +/* +Copyright 2025 API Testing Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apispec + +import ( + "archive/tar" + "compress/gzip" + "encoding/json" + "fmt" + "github.com/linuxsuren/api-testing/pkg/downloader" + "github.com/linuxsuren/api-testing/pkg/util/home" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +func DownloadSwaggerData(output string, dw downloader.PlatformAwareOCIDownloader) (err error) { + dw.WithKind("data") + dw.WithOS("") + + var reader io.Reader + if reader, err = dw.Download("swagger", "", ""); err != nil { + return + } + + extFile := dw.GetTargetFile() + + if output == "" { + output = home.GetUserDataDir() + } + if err = os.MkdirAll(filepath.Dir(output), 0755); err != nil { + return + } + + targetFile := filepath.Base(extFile) + fmt.Println("start to save", filepath.Join(output, targetFile)) + if err = downloader.WriteTo(reader, output, targetFile); err == nil { + err = decompressData(filepath.Join(output, targetFile)) + } + return +} + +func SwaggersHandler(w http.ResponseWriter, _ *http.Request, + _ map[string]string) { + swaggers := GetSwaggerList() + if data, err := json.Marshal(swaggers); err == nil { + _, _ = w.Write(data) + } else { + w.WriteHeader(http.StatusInternalServerError) + } +} + +func GetSwaggerList() (swaggers []string) { + dataDir := home.GetUserDataDir() + _ = filepath.WalkDir(dataDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + + if !d.IsDir() && filepath.Ext(path) == ".json" { + swaggers = append(swaggers, filepath.Base(path)) + } + return nil + }) + return +} + +func decompressData(dataFile string) (err error) { + var file *os.File + file, err = os.Open(dataFile) + if err != nil { + return + } + defer file.Close() + + var gzipReader *gzip.Reader + gzipReader, err = gzip.NewReader(file) + if err != nil { + return + } + defer gzipReader.Close() + + tarReader := tar.NewReader(gzipReader) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break // 退出循环 + } + if err != nil { + panic(err) + } + + // Ensure the file path does not contain directory traversal sequences + if strings.Contains(header.Name, "..") { + fmt.Printf("Skipping entry with unsafe path: %s\n", header.Name) + continue + } + + destPath := filepath.Join(filepath.Dir(dataFile), filepath.Base(header.Name)) + + switch header.Typeflag { + case tar.TypeReg: + destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY, os.FileMode(header.Mode)) + if err != nil { + panic(err) + } + defer destFile.Close() + + if _, err := io.Copy(destFile, tarReader); err != nil { + panic(err) + } + default: + fmt.Printf("Skipping entry type %c: %s\n", header.Typeflag, header.Name) + } + } + return +} diff --git a/pkg/apispec/swagger.go b/pkg/apispec/swagger.go index 2b305365..958a8755 100644 --- a/pkg/apispec/swagger.go +++ b/pkg/apispec/swagger.go @@ -18,8 +18,11 @@ package apispec import ( "github.com/go-openapi/spec" + "github.com/linuxsuren/api-testing/pkg/util/home" "io" "net/http" + "os" + "path/filepath" "regexp" "strings" ) @@ -123,6 +126,12 @@ func ParseToSwagger(data []byte) (swagger *spec.Swagger, err error) { } func ParseURLToSwagger(swaggerURL string) (swagger *spec.Swagger, err error) { + if strings.HasPrefix(swaggerURL, "atest://") { + swaggerURL = strings.ReplaceAll(swaggerURL, "atest://", "") + swagger, err = ParseFileToSwagger(filepath.Join(home.GetUserDataDir(), swaggerURL)) + return + } + var resp *http.Response if resp, err = http.Get(swaggerURL); err == nil && resp != nil && resp.StatusCode == http.StatusOK { swagger, err = ParseStreamToSwagger(resp.Body) @@ -130,6 +139,14 @@ func ParseURLToSwagger(swaggerURL string) (swagger *spec.Swagger, err error) { return } +func ParseFileToSwagger(dataFile string) (swagger *spec.Swagger, err error) { + var data []byte + if data, err = os.ReadFile(dataFile); err == nil { + swagger, err = ParseToSwagger(data) + } + return +} + func ParseStreamToSwagger(stream io.Reader) (swagger *spec.Swagger, err error) { var data []byte if data, err = io.ReadAll(stream); err == nil { diff --git a/pkg/downloader/store.go b/pkg/downloader/extension.go similarity index 59% rename from pkg/downloader/store.go rename to pkg/downloader/extension.go index 2d4256e6..62b30b68 100644 --- a/pkg/downloader/store.go +++ b/pkg/downloader/extension.go @@ -1,5 +1,5 @@ /* -Copyright 2024 API Testing Authors. +Copyright 2024-2025 API Testing Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,30 +25,37 @@ import ( "strings" ) -type storeDownloader struct { +type extensionDownloader struct { OCIDownloader os, arch string + kind string extFile string imagePrefix string } func NewStoreDownloader() PlatformAwareOCIDownloader { - ociDownloader := &storeDownloader{ + ociDownloader := &extensionDownloader{ OCIDownloader: NewDefaultOCIDownloader(), } ociDownloader.WithOS(runtime.GOOS) ociDownloader.WithArch(runtime.GOARCH) ociDownloader.WithImagePrefix("linuxsuren") + ociDownloader.WithKind("store") return ociDownloader } -func (d *storeDownloader) Download(name, tag, _ string) (reader io.Reader, err error) { - name = strings.TrimPrefix(name, "atest-store-") - d.extFile = fmt.Sprintf("atest-store-%s_%s_%s/atest-store-%s", name, d.os, d.arch, name) - if d.os == "windows" { - d.extFile = fmt.Sprintf("%s.exe", d.extFile) +func (d *extensionDownloader) Download(name, tag, _ string) (reader io.Reader, err error) { + name = strings.TrimPrefix(name, fmt.Sprintf("atest-%s-", d.kind)) + if d.os == "" { + d.extFile = fmt.Sprintf("atest-%s-%s.tar.gz", d.kind, name) + } else { + d.extFile = fmt.Sprintf("atest-%s-%s_%s_%s/atest-%s-%s", d.kind, name, d.os, d.arch, d.kind, name) + if d.os == "windows" { + d.extFile = fmt.Sprintf("%s.exe", d.extFile) + } } - image := fmt.Sprintf("%s/atest-ext-store-%s", d.imagePrefix, name) + + image := fmt.Sprintf("%s/atest-ext-%s-%s", d.imagePrefix, d.kind, name) reader, err = d.OCIDownloader.Download(image, tag, d.extFile) return } @@ -64,21 +71,25 @@ func WriteTo(reader io.Reader, dir, file string) (err error) { return } -func (d *storeDownloader) GetTargetFile() string { +func (d *extensionDownloader) GetTargetFile() string { return d.extFile } -func (d *storeDownloader) WithOS(os string) { +func (d *extensionDownloader) WithOS(os string) { d.os = os } -func (d *storeDownloader) WithImagePrefix(imagePrefix string) { +func (d *extensionDownloader) WithImagePrefix(imagePrefix string) { d.imagePrefix = imagePrefix } -func (d *storeDownloader) WithArch(arch string) { +func (d *extensionDownloader) WithArch(arch string) { d.arch = arch if d.arch == "amd64" { d.arch = "amd64_v1" } } + +func (d *extensionDownloader) WithKind(kind string) { + d.kind = kind +} diff --git a/pkg/downloader/oci.go b/pkg/downloader/oci.go index 558c024d..304f8cfc 100644 --- a/pkg/downloader/oci.go +++ b/pkg/downloader/oci.go @@ -45,6 +45,7 @@ type PlatformAwareOCIDownloader interface { WithOS(string) WithArch(string) GetTargetFile() string + WithKind(string) WithImagePrefix(string) } @@ -71,6 +72,8 @@ func (d *defaultOCIDownloader) WithBasicAuth(username string, password string) { } func (d *defaultOCIDownloader) Download(image, tag, file string) (reader io.Reader, err error) { + fmt.Println("start to download", image) + if d.registry == "" { d.registry = getRegistry(image) } diff --git a/pkg/server/store_ext_manager.go b/pkg/server/store_ext_manager.go index c6148de2..516aba09 100644 --- a/pkg/server/store_ext_manager.go +++ b/pkg/server/store_ext_manager.go @@ -176,6 +176,8 @@ var ErrDownloadNotSupport = errors.New("no support") type nonDownloader struct{} +var _ downloader.PlatformAwareOCIDownloader = &nonDownloader{} + func (n *nonDownloader) WithBasicAuth(username string, password string) { // Do nothing because this is an empty implementation } @@ -197,6 +199,10 @@ func (n *nonDownloader) WithRegistry(string) { // Do nothing because this is an empty implementation } +func (n *nonDownloader) WithKind(string) { + // Do nothing because this is an empty implementation +} + func (n *nonDownloader) WithImagePrefix(imagePrefix string) { // Do nothing because this is an empty implementation } diff --git a/pkg/util/home/common.go b/pkg/util/home/common.go index 15962f3d..ab6608d1 100644 --- a/pkg/util/home/common.go +++ b/pkg/util/home/common.go @@ -1,5 +1,5 @@ /* -Copyright 2024 API Testing Authors. +Copyright 2024-2025 API Testing Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -33,6 +33,10 @@ func GetUserBinDir() string { return filepath.Join(GetUserConfigDir(), "bin") } +func GetUserDataDir() string { + return filepath.Join(GetUserConfigDir(), "data") +} + func GetExtensionSocketPath(name string) string { return filepath.Join(GetUserConfigDir(), fmt.Sprintf("%s.sock", name)) } diff --git a/pkg/util/home/common_test.go b/pkg/util/home/common_test.go new file mode 100644 index 00000000..d0022940 --- /dev/null +++ b/pkg/util/home/common_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2025 API Testing Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package home + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGetUserBinDir(t *testing.T) { + assert.Contains(t, GetUserConfigDir(), "atest") + assert.Contains(t, GetUserBinDir(), "bin") + assert.Contains(t, GetUserDataDir(), "data") + assert.Contains(t, GetExtensionSocketPath("fake"), "fake.sock") +}