Skip to content

Commit 0cc0bb8

Browse files
authored
feat: support to download extensions automatically (#471)
* feat: add the extension sub-commad to download it * do not put store-ext in the main image * support to fetch the latest image tag * finish the basic feature of extension downloading * add some unit tests * support different image registry * add more unit tests * remove useless files * fix the unit test case * simplify the apt install in the dockerfile * add explanation for the empty methods --------- Co-authored-by: rick <[email protected]>
1 parent 1735b8f commit 0cc0bb8

File tree

15 files changed

+802
-60
lines changed

15 files changed

+802
-60
lines changed

Dockerfile

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,6 @@ COPY --from=ui /workspace/dist/assets/*.css cmd/data/index.css
3232
RUN CGO_ENABLED=0 go build -v -a -ldflags "-w -s -X github.com/linuxsuren/api-testing/pkg/version.version=${VERSION}\
3333
-X github.com/linuxsuren/api-testing/pkg/version.date=$(date +%Y-%m-%d)" -o atest .
3434

35-
FROM ghcr.io/linuxsuren/atest-ext-store-mongodb:master as mango
36-
FROM ghcr.io/linuxsuren/atest-ext-store-git:master as git
37-
FROM ghcr.io/linuxsuren/atest-ext-store-s3:master as s3
38-
FROM ghcr.io/linuxsuren/atest-ext-store-etcd:master as etcd
39-
FROM ghcr.io/linuxsuren/atest-ext-store-orm:master as orm
40-
FROM ghcr.io/linuxsuren/atest-ext-monitor-docker:master as docker
41-
FROM ghcr.io/linuxsuren/atest-ext-collector:master as collector
42-
FROM ghcr.io/linuxsuren/api-testing-vault-extension:v0.0.1 as vault
43-
4435
FROM docker.io/library/ubuntu:23.10
4536

4637
LABEL "com.github.actions.name"="API testing"
@@ -56,21 +47,13 @@ LABEL "maintainer"="Rick <[email protected]>"
5647
LABEL "Name"="API testing"
5748

5849
COPY --from=builder /workspace/atest /usr/local/bin/atest
59-
COPY --from=collector /usr/local/bin/atest-collector /usr/local/bin/atest-collector
60-
COPY --from=orm /usr/local/bin/atest-store-orm /usr/local/bin/atest-store-orm
61-
COPY --from=s3 /usr/local/bin/atest-store-s3 /usr/local/bin/atest-store-s3
62-
COPY --from=etcd /usr/local/bin/atest-store-etcd /usr/local/bin/atest-store-etcd
63-
COPY --from=git /usr/local/bin/atest-store-git /usr/local/bin/atest-store-git
64-
COPY --from=mango /usr/local/bin/atest-store-mongodb /usr/local/bin/atest-store-mongodb
65-
COPY --from=docker /usr/local/bin/atest-monitor-docker /usr/local/bin/atest-monitor-docker
66-
COPY --from=vault /usr/local/bin/atest-vault-ext /usr/local/bin
6750
COPY --from=builder /workspace/LICENSE /LICENSE
6851
COPY --from=builder /workspace/README.md /README.md
6952

7053
RUN apt update -y && \
7154
# required for atest-store-git
72-
apt install -y --no-install-recommends ssh-client ca-certificates && \
73-
apt install -y curl
55+
apt install -y --no-install-recommends ssh-client ca-certificates curl && \
56+
rm -rf /var/lib/apt/lists/*
7457

7558
EXPOSE 8080
7659
CMD ["atest", "server", "--local-storage=/var/data/api-testing/*.yaml"]

cmd/extension.go

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
Copyright 2024 API Testing Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cmd
18+
19+
import (
20+
"fmt"
21+
"io"
22+
"path/filepath"
23+
"runtime"
24+
25+
"github.com/linuxsuren/api-testing/pkg/downloader"
26+
"github.com/spf13/cobra"
27+
)
28+
29+
type extensionOption struct {
30+
ociDownloader downloader.PlatformAwareOCIDownloader
31+
output string
32+
registry string
33+
tag string
34+
os string
35+
arch string
36+
}
37+
38+
func createExtensionCommand(ociDownloader downloader.PlatformAwareOCIDownloader) (c *cobra.Command) {
39+
opt := &extensionOption{
40+
ociDownloader: ociDownloader,
41+
}
42+
c = &cobra.Command{
43+
Use: "extension",
44+
Short: "Download extension binary files",
45+
Long: "Download the store extension files",
46+
Args: cobra.MinimumNArgs(1),
47+
RunE: opt.runE,
48+
}
49+
flags := c.Flags()
50+
flags.StringVarP(&opt.output, "output", "", ".", "The target directory")
51+
flags.StringVarP(&opt.tag, "tag", "", "", "The extension image tag, try to find the latest one if this is empty")
52+
flags.StringVarP(&opt.registry, "registry", "", "", "The target extension image registry, supported: docker.io, ghcr.io")
53+
flags.StringVarP(&opt.os, "os", "", runtime.GOOS, "The OS")
54+
flags.StringVarP(&opt.arch, "arch", "", runtime.GOARCH, "The architecture")
55+
return
56+
}
57+
58+
func (o *extensionOption) runE(cmd *cobra.Command, args []string) (err error) {
59+
o.ociDownloader.WithOS(o.os)
60+
o.ociDownloader.WithArch(o.arch)
61+
62+
for _, arg := range args {
63+
var reader io.Reader
64+
if reader, err = o.ociDownloader.Download(arg, o.tag, ""); err != nil {
65+
return
66+
} else if reader == nil {
67+
err = fmt.Errorf("cannot find %s", arg)
68+
return
69+
}
70+
extFile := o.ociDownloader.GetTargetFile()
71+
cmd.Println("found target file", extFile)
72+
73+
targetFile := filepath.Base(extFile)
74+
if err = downloader.WriteTo(reader, o.output, targetFile); err == nil {
75+
cmd.Println("downloaded", targetFile)
76+
}
77+
}
78+
return
79+
}

cmd/extension_test.go

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
Copyright 2024 API Testing Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cmd
18+
19+
import (
20+
"fmt"
21+
"os"
22+
"testing"
23+
24+
"github.com/linuxsuren/api-testing/pkg/downloader"
25+
"github.com/linuxsuren/api-testing/pkg/mock"
26+
"github.com/stretchr/testify/assert"
27+
)
28+
29+
func TestExtensionCmd(t *testing.T) {
30+
t.Run("minimum one arg", func(t *testing.T) {
31+
command := createExtensionCommand(nil)
32+
err := command.Execute()
33+
assert.Error(t, err)
34+
})
35+
36+
t.Run("normal", func(t *testing.T) {
37+
d := downloader.NewStoreDownloader()
38+
server := mock.NewInMemoryServer(0)
39+
40+
err := server.Start(mock.NewLocalFileReader("../pkg/downloader/testdata/registry.yaml"), "/v2")
41+
assert.NoError(t, err)
42+
defer func() {
43+
server.Stop()
44+
}()
45+
46+
d.WithRegistry(fmt.Sprintf("127.0.0.1:%s", server.GetPort()))
47+
d.WithInsecure(true)
48+
d.WithBasicAuth("", "")
49+
d.WithOS("linux")
50+
d.WithArch("amd64")
51+
d.WithRoundTripper(nil)
52+
53+
var tmpDownloadDir string
54+
tmpDownloadDir, err = os.MkdirTemp(os.TempDir(), "download")
55+
defer os.RemoveAll(tmpDownloadDir)
56+
assert.NoError(t, err)
57+
58+
command := createExtensionCommand(d)
59+
command.SetArgs([]string{"git", "--output", tmpDownloadDir})
60+
err = command.Execute()
61+
assert.NoError(t, err)
62+
63+
// not found
64+
command.SetArgs([]string{"orm", "--output", tmpDownloadDir})
65+
err = command.Execute()
66+
assert.Error(t, err)
67+
})
68+
}

cmd/root.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package cmd
33
import (
44
"os"
55

6+
"github.com/linuxsuren/api-testing/pkg/downloader"
7+
68
"github.com/linuxsuren/api-testing/pkg/server"
79
"github.com/linuxsuren/api-testing/pkg/version"
810
fakeruntime "github.com/linuxsuren/go-fake-runtime"
@@ -21,7 +23,7 @@ func NewRootCmd(execer fakeruntime.Execer, httpServer server.HTTPServer) (c *cob
2123
createRunCommand(), createSampleCmd(),
2224
createServerCmd(execer, httpServer), createJSONSchemaCmd(),
2325
createServiceCommand(execer), createFunctionCmd(), createConvertCommand(),
24-
createMockCmd())
26+
createMockCmd(), createExtensionCommand(downloader.NewStoreDownloader()))
2527
return
2628
}
2729

cmd/server.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636
pprof "net/http/pprof"
3737

3838
"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
39+
"github.com/linuxsuren/api-testing/pkg/downloader"
3940
"github.com/linuxsuren/api-testing/pkg/logging"
4041
"github.com/linuxsuren/api-testing/pkg/mock"
4142
"github.com/linuxsuren/api-testing/pkg/oauth"
@@ -224,6 +225,7 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
224225
}
225226

226227
storeExtMgr := server.NewStoreExtManager(o.execer)
228+
storeExtMgr.WithDownloader(downloader.NewStoreDownloader())
227229
remoteServer := server.NewRemoteServer(loader, remote.NewGRPCloaderFromStore(), secretServer, storeExtMgr, o.configDir, o.grpcMaxRecvMsgSize)
228230
kinds, storeKindsErr := remoteServer.GetStoreKinds(ctx, nil)
229231
if storeKindsErr != nil {

console/atest-desktop/main.js

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,6 @@ app.whenReady().then(() => {
127127
ipcMain.on('startServer', startServer)
128128
ipcMain.on('stopServer', stopServer)
129129
ipcMain.on('control', (e, okCallback, errCallback) => {
130-
console.log(e + "==" + okCallback + "==" + errCallback)
131130
server.control(okCallback, errCallback)
132131
})
133132
ipcMain.handle('getHomePage', server.getHomePage)
@@ -161,24 +160,20 @@ const startServer = () => {
161160
const serverFile = process.platform === "win32" ? "atest.exe" : "atest"
162161
const atestFromHome = path.join(homeBin, serverFile)
163162
const atestFromPkg = path.join(__dirname, serverFile)
163+
164+
const data = fs.readFileSync(atestFromPkg)
165+
log.info('start to write file with length', data.length)
164166

165-
if (!fs.existsSync(atestFromHome)) {
166-
log.info('cannot find from %s', atestFromHome)
167-
168-
const data = fs.readFileSync(atestFromPkg)
169-
log.info('start to write file with length %d', data.length)
170-
171-
try {
172-
if (process.platform === "win32") {
173-
const file = fs.openSync(atestFromHome, 'w');
174-
fs.writeSync(file, data, 0, data.length, 0);
175-
fs.closeSync(file);
176-
}else{
177-
fs.writeFileSync(atestFromHome, data);
178-
}
179-
} catch (e) {
180-
log.error('Error Code: %s', e.code);
167+
try {
168+
if (process.platform === "win32") {
169+
const file = fs.openSync(atestFromHome, 'w');
170+
fs.writeSync(file, data, 0, data.length, 0);
171+
fs.closeSync(file);
172+
}else{
173+
fs.writeFileSync(atestFromHome, data);
181174
}
175+
} catch (e) {
176+
log.error('Error Code:', e.code);
182177
}
183178
fs.chmodSync(atestFromHome, 0o755);
184179

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ require (
4242
github.com/Masterminds/goutils v1.1.1 // indirect
4343
github.com/Masterminds/semver/v3 v3.2.1 // indirect
4444
github.com/beorn7/perks v1.0.1 // indirect
45+
github.com/blang/semver/v4 v4.0.0 // indirect
4546
github.com/cespare/xxhash/v2 v2.2.0 // indirect
4647
github.com/cucumber/gherkin-go/v19 v19.0.3 // indirect
4748
github.com/cucumber/messages-go/v16 v16.0.1 // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg
99
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
1010
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
1111
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
12+
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
13+
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
1214
github.com/bool64/dev v0.2.34 h1:P9n315P8LdpxusnYQ0X7MP1CZXwBK5ae5RZrd+GdSZE=
1315
github.com/bool64/dev v0.2.34/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg=
1416
github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E=

0 commit comments

Comments
 (0)