add mysql/etcd query support (#624)

* feat: add data query service

---------

Co-authored-by: rick <[email protected]>

Author: LinuxSuRen

.gitattributes

@@ -1 +1,9 @@
-* text eol=lf
+*.go text eol=lf
+*.vue text eol=lf
+*.js text eol=lf
+*.css text eol=lf
+*.html text eol=lf
+*.md text eol=lf
+*.yaml text eol=lf
+*.json text eol=lf
+*.yml text eol=lf

.gitignore

@@ -19,3 +19,4 @@ console/atest-desktop/node_modules
 console/atest-desktop/atest
 console/atest-desktop/atest.exe
 console/atest-desktop/coverage
+atest-store-git

README.md

@@ -21,6 +21,7 @@ This is an awesome API testing tool. 🚀
 * Pre and post handle with the API request
 * Run in server mode, and provide the [gRPC](pkg/server/server.proto) and HTTP endpoint
 * [VS Code extension](https://github.com/LinuxSuRen/vscode-api-testing) support
+* Simple Database query support
 * Multiple storage backends supported(Local, ORM Database, S3, Git, Etcd, etc.)
 * [HTTP API record](https://github.com/LinuxSuRen/atest-ext-collector)
 * Install in multiple use cases(CLI, Container, Native-Service, [Operator](https://github.com/LinuxSuRen/atest-operator), Helm, etc.)

atest-store-git

@@ -102,7 +102,7 @@ func createServerCmd(execer fakeruntime.Execer, httpServer server.HTTPServer) (c
 	flags.StringArrayVarP(&opt.mockConfig, "mock-config", "", nil, "The mock config files")
 	flags.StringVarP(&opt.mockPrefix, "mock-prefix", "", "/mock", "The mock server API prefix")
 	flags.StringVarP(&opt.extensionRegistry, "extension-registry", "", "docker.io", "The extension registry URL")
-	flags.DurationVarP(&opt.downloadTimeout, "download-timeout", "", time.Second*10, "The timeout of extension download")
+	flags.DurationVarP(&opt.downloadTimeout, "download-timeout", "", time.Minute, "The timeout of extension download")
 
 	// gc related flags
 	flags.IntVarP(&opt.gcPercent, "gc-percent", "", 100, "The GC percent of Go")
@@ -288,6 +288,7 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
 		}
 		server.RegisterRunnerServer(s, remoteServer)
 		server.RegisterMockServer(s, mockServerController)
+		server.RegisterDataServerServer(s, remoteServer.(server.DataServerServer))
 		serverLogger.Info("gRPC server listening at", "addr", lis.Addr())
 		s.Serve(lis)
 	}()
@@ -322,11 +323,14 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
 		}
 		err = errors.Join(
 			server.RegisterRunnerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}),
-			server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}))
+			server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}),
+			server.RegisterDataServerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}))
 	} else {
+		dialOption := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
 		err = errors.Join(
-			server.RegisterRunnerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}),
-			server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}))
+			server.RegisterRunnerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, dialOption),
+			server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, dialOption),
+			server.RegisterDataServerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, dialOption))
 	}
 
 	if err == nil {

cmd/server.go

@@ -7,6 +7,7 @@ import {
   Share,
   ArrowDown,
   Guide,
+  DataAnalysis
 } from '@element-plus/icons-vue'
 import { ref, watch } from 'vue'
 import { API } from './views/net'
@@ -17,6 +18,7 @@ import MockManager from './views/MockManager.vue'
 import StoreManager from './views/StoreManager.vue'
 import SecretManager from './views/SecretManager.vue'
 import WelcomePage from './views/WelcomePage.vue'
+import DataManager from './views/DataManager.vue'
 import { useI18n } from 'vue-i18n'
 
 const { t, locale: i18nLocale } = useI18n()
@@ -114,6 +116,10 @@ const toHistoryPanel = ({ ID: selectID, panelName: historyPanelName }) => {
           <el-icon><Guide /></el-icon>
           <template #title>{{ t('title.mock' )}}</template>
         </el-menu-item>
+        <el-menu-item index="data" test-id="data-menu">
+          <el-icon><DataAnalysis /></el-icon>
+          <template #title>{{ t('title.data' )}}</template>
+        </el-menu-item>
         <el-menu-item index="secret">
           <el-icon><document /></el-icon>
           <template #title>{{ t('title.secrets') }}</template>
@@ -142,6 +148,7 @@ const toHistoryPanel = ({ ID: selectID, panelName: historyPanelName }) => {
       </div>
       <TestingPanel v-if="panelName === 'testing'" @toHistoryPanel="toHistoryPanel"/>
       <TestingHistoryPanel v-else-if="panelName === 'history'" :ID="ID"/>
+      <DataManager v-else-if="panelName === 'data'" />
       <MockManager v-else-if="panelName === 'mock'" />
       <StoreManager v-else-if="panelName === 'store'" />
       <SecretManager v-else-if="panelName === 'secret'" />

console/atest-ui/src/App.vue

@@ -50,7 +50,8 @@
         "functionQuery": "Functions Query",
         "output": "Output",
         "proxy": "Proxy",
-        "secure": "Secure"
+        "secure": "Secure",
+        "data": "Data"
     },
     "tip": {
         "filter": "Filter Keyword",

console/atest-ui/src/locales/en.json

@@ -45,7 +45,8 @@
         "functionQuery": "函数查询",
         "output": "输出",
         "proxy": "代理",
-        "secure": "安全"
+        "secure": "安全",
+        "data": "数据"
     },
     "tip": {
         "filter": "过滤",

console/atest-ui/src/locales/zh.json

@@ -0,0 +1,132 @@
+<script setup lang="ts">
+import { ref, watch } from 'vue'
+import { API } from './net'
+import { ElMessage } from 'element-plus'
+
+const stores = ref([])
+const kind = ref('')
+const store = ref('')
+const sqlQuery = ref('')
+const queryResult = ref([])
+const columns = ref([])
+const queryTip = ref('')
+
+watch(store, (s) => {
+    stores.value.forEach((e: Store) => {
+        if (e.name === s) {
+            kind.value = e.kind.name
+            return
+        }
+    })
+})
+watch(kind, (k) => {
+    switch (k) {
+        case 'atest-store-orm':
+            sqlQuery.value = 'show tables'
+            queryTip.value = 'Enter SQL query'
+            break;
+        case 'atest-store-etcd':
+            sqlQuery.value = ''
+            queryTip.value = 'Enter key'
+            break;
+    }
+})
+
+API.GetStores((data) => {
+  stores.value = data.data
+}, (e) => {
+  ElMessage({
+    showClose: true,
+    message: e.message,
+    type: 'error'
+  });
+})
+
+const ormDataHandler = (data) => {
+    const result = []
+    const cols = new Set()
+
+    data.items.forEach(e => {
+        const obj = {}
+        e.data.forEach(item => {
+            obj[item.key] = item.value
+            cols.add(item.key)
+        })
+        result.push(obj)
+    })
+
+    queryResult.value = result
+    columns.value = Array.from(cols).sort((a, b) => {
+        if (a === 'id') return -1;
+        if (b === 'id') return 1;
+        return a.localeCompare(b);
+    })
+}
+
+const keyValueDataHandler = (data) => {
+    queryResult.value = []
+    data.data.forEach(e => {
+        const obj = {}
+        obj['key'] = e.key
+        obj['value'] = e.value
+        queryResult.value.push(obj)
+
+        columns.value = ['key', 'value']
+    })
+}
+
+const executeQuery = async () => {
+    API.DataQuery(store.value, kind.value, sqlQuery.value, (data) => {
+        switch (kind.value) {
+            case 'atest-store-orm':
+                ormDataHandler(data)
+                break;
+            case 'atest-store-etcd':
+                keyValueDataHandler(data)
+                break;
+            default:
+                ElMessage({
+                    showClose: true,
+                    message: 'Unsupported store kind',
+                    type: 'error'
+                });
+        }
+    }, (e) => {
+        ElMessage({
+            showClose: true,
+            message: e.message,
+            type: 'error'
+        });
+    })
+}
+</script>
+
+<template>
+  <div>
+    <el-form @submit.prevent="executeQuery">
+        <el-row :gutter="10">
+          <el-col :span="2">
+            <el-form-item>
+              <el-select v-model="store" placeholder="Select store">
+                <el-option v-for="item in stores" :key="item.name" :label="item.name"
+                           :value="item.name" :disabled="!item.ready" :kind="item.kind.name"></el-option>
+              </el-select>
+            </el-form-item>
+          </el-col>
+          <el-col :span="18">
+            <el-form-item>
+              <el-input v-model="sqlQuery" :placeholder="queryTip" @keyup.enter="executeQuery"></el-input>
+            </el-form-item>
+          </el-col>
+          <el-col :span="2">
+            <el-form-item>
+              <el-button type="primary" @click="executeQuery">Execute</el-button>
+            </el-form-item>
+          </el-col>
+        </el-row>
+    </el-form>
+    <el-table :data="queryResult">
+      <el-table-column v-for="col in columns" :key="col" :prop="col" :label="col"></el-table-column>
+    </el-table>
+  </div>
+</template>

console/atest-ui/src/views/DataManager.vue

@@ -773,6 +773,29 @@ var SBOM = (callback: (d: any) => void) => {
       .then(callback)
 }
 
+var DataQuery = (store: string, kind: string, query: string, callback: (d: any) => void, errHandler: (d: any) => void) => {
+    const queryObj = {}
+    switch (kind) {
+        case 'atest-store-orm':
+            queryObj['sql'] = query;
+            break;
+        case 'atest-store-etcd':
+            queryObj['key'] = query;
+            break;
+    }
+    const requestOptions = {
+        method: 'POST',
+        headers: {
+            'X-Store-Name': store
+        },
+        body: JSON.stringify(queryObj)
+    }
+    fetch(`/api/v1/data/query`, requestOptions)
+        .then(DefaultResponseProcess)
+        .then(callback)
+        .catch(errHandler)
+}
+
 export const API = {
   DefaultResponseProcess,
   GetVersion,
@@ -785,6 +808,6 @@ export const API = {
   FunctionsQuery,
   GetSecrets, DeleteSecret, CreateOrUpdateSecret,
   GetSuggestedAPIs,
-  ReloadMockServer, GetMockConfig, SBOM,
+  ReloadMockServer, GetMockConfig, SBOM, DataQuery,
   getToken
 }