Skip to content

Commit 7d9afa5

Browse files
orangedengJacieChao
authored andcommitted
feat: Use socket to do IPC request to kube-explorer
1 parent f548372 commit 7d9afa5

File tree

11 files changed

+179
-98
lines changed

11 files changed

+179
-98
lines changed

Dockerfile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1+
ARG KUBE_EXPLORER_VERSION=latest
2+
FROM cnrancher/kube-explorer:${KUBE_EXPLORER_VERSION} as kube-explorer
13
FROM registry.suse.com/bci/bci-base:15.5
24
ARG TARGETPLATFORM
35
ARG TARGETARCH
46
ARG TARGETOS
57

68
ENV TARGETPLATFORM=${TARGETPLATFORM:-"linux/amd64"} ARCH=${TARGETARCH:-"amd64"} OS=${TARGETOS:-"linux"}
7-
ENV KUBE_EXPLORER_VERSION=v0.5.0
89
ENV HELM_DASHBOARD_VERSION=1.3.3
910

1011
RUN zypper -n install curl ca-certificates tar gzip
@@ -14,8 +15,7 @@ RUN mkdir /home/shell && \
1415
echo 'source <(kubectl completion bash)' >> /home/shell/.bashrc && \
1516
echo 'PS1="> "' >> /home/shell/.bashrc
1617

17-
RUN curl -sSL https://github.com/cnrancher/kube-explorer/releases/download/${KUBE_EXPLORER_VERSION}/kube-explorer-${OS}-${ARCH} > /usr/local/bin/kube-explorer && \
18-
chmod +x /usr/local/bin/kube-explorer
18+
COPY --from=kube-explorer /usr/bin/kube-explorer /usr/local/bin/kube-explorer
1919

2020
RUN curl -sLf https://github.com/komodorio/helm-dashboard/releases/download/v${HELM_DASHBOARD_VERSION}/helm-dashboard_${HELM_DASHBOARD_VERSION}_Linux_x86_64.tar.gz | tar xvzf - -C /usr/local/bin && \
2121
chmod +x /usr/local/bin/helm-dashboard

cmd/explorer.go

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
package cmd
22

33
import (
4+
"context"
5+
"fmt"
6+
"net"
7+
"net/http"
8+
49
"github.com/cnrancher/autok3s/pkg/common"
10+
"github.com/cnrancher/autok3s/pkg/server/proxy"
511

6-
k3dutil "github.com/k3d-io/k3d/v5/cmd/util"
712
"github.com/sirupsen/logrus"
813
"github.com/spf13/cobra"
914
)
@@ -15,7 +20,7 @@ var (
1520
Example: "autok3s explorer --context myk3s",
1621
}
1722
clusterID = ""
18-
explorerPort = 0
23+
explorerPort = 8080
1924
)
2025

2126
func init() {
@@ -31,18 +36,29 @@ func ExplorerCommand() *cobra.Command {
3136
}
3237
return nil
3338
}
34-
explorerCmd.Run = func(_ *cobra.Command, _ []string) {
39+
explorerCmd.Run = func(cmd *cobra.Command, _ []string) {
3540
if err := common.CheckCommandExist(common.KubeExplorerCommand); err != nil {
3641
logrus.Fatalln(err)
3742
}
38-
if explorerPort == 0 {
39-
port, err := k3dutil.GetFreePort()
40-
if err != nil {
41-
logrus.Fatalf("failed to get free port for kube-explorer: %v", err)
42-
}
43-
explorerPort = port
43+
44+
wait, err := common.StartKubeExplorer(cmd.Context(), clusterID)
45+
if err != nil {
46+
logrus.Fatalln(err)
47+
}
48+
49+
server := http.Server{
50+
Addr: fmt.Sprintf(":%d", explorerPort),
51+
Handler: proxy.DynamicPrefixProxy(clusterID),
52+
BaseContext: func(_ net.Listener) context.Context {
53+
return cmd.Context()
54+
},
4455
}
45-
_ = common.StartKubeExplorer(explorerCmd.Context(), clusterID, explorerPort)
56+
go func() {
57+
logrus.Infof("autok3s serving kube-explorer on %s", server.Addr)
58+
_ = server.ListenAndServe()
59+
}()
60+
<-wait
61+
_ = server.Shutdown(context.Background())
4662
}
4763

4864
return explorerCmd

go.mod

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ require (
5959
)
6060

6161
require (
62+
github.com/Microsoft/go-winio v0.6.2
6263
github.com/moby/sys/signal v0.7.0
6364
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00
6465
sigs.k8s.io/yaml v1.4.0
@@ -71,7 +72,6 @@ require (
7172
github.com/MakeNowJust/heredoc v1.0.0 // indirect
7273
github.com/Masterminds/goutils v1.1.1 // indirect
7374
github.com/Masterminds/semver/v3 v3.2.1 // indirect
74-
github.com/Microsoft/go-winio v0.6.1 // indirect
7575
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
7676
github.com/beorn7/perks v1.0.1 // indirect
7777
github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -190,11 +190,9 @@ require (
190190
go.uber.org/multierr v1.11.0 // indirect
191191
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
192192
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
193-
golang.org/x/mod v0.14.0 // indirect
194193
golang.org/x/sys v0.19.0 // indirect
195194
golang.org/x/text v0.14.0 // indirect
196195
golang.org/x/time v0.5.0 // indirect
197-
golang.org/x/tools v0.16.1 // indirect
198196
google.golang.org/appengine v1.6.7 // indirect
199197
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
200198
google.golang.org/grpc v1.59.0 // indirect

go.sum

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
2222
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
2323
github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
2424
github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
25-
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
26-
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
25+
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
26+
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
2727
github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
2828
github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
2929
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
@@ -594,8 +594,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
594594
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
595595
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
596596
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
597-
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
598-
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
599597
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
600598
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
601599
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

hack/make-rules/autok3s.sh

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,8 @@ function build() {
6666
go_generate
6767
autok3s::log::info "building autok3s(${GIT_VERSION},${GIT_COMMIT},${GIT_TREE_STATE},${BUILD_DATE})..."
6868
# TODO default k3s version in k3d should also get from k3d in
69-
local K3D_VERSION=`go list -m all | grep k3d/v5 | awk '{print $2}'`
70-
local K3S_TAG=`curl --silent --retry 3 "https://update.k3s.io/v1-release/channels/stable" | egrep -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/'`
71-
69+
local K3D_VERSION=$(go list -m github.com/k3d-io/k3d/v5 | awk '{print $2}')
70+
local K3S_TAG=$(curl --silent --retry 3 'https://update.k3s.io/v1-release/channels/stable' | grep -E -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/')
7271
local version_flags="
7372
-X main.gitVersion=${GIT_VERSION}
7473
-X main.gitCommit=${GIT_COMMIT}
@@ -81,7 +80,7 @@ function build() {
8180
-X k8s.io/component-base/version.gitCommit=${GIT_COMMIT}
8281
-X k8s.io/component-base/version.gitTreeState=${GIT_TREE_STATE}
8382
-X k8s.io/component-base/version.buildDate=${BUILD_DATE}
84-
-X github.com/k3d-io/k3d/v5/version.Version=${K3D_VERSION:-v5.4.4}
83+
-X github.com/k3d-io/k3d/v5/version.Version=${K3D_VERSION:-v5.4.6}
8584
-X github.com/k3d-io/k3d/v5/version.K3sVersion=${K3S_TAG}"
8685
local flags="
8786
-w -s"

pkg/cluster/cluster.go

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,13 +156,10 @@ func (p *ProviderBase) InitK3sCluster(cluster *types.Cluster, deployCCM func() [
156156
for plugin := range enabledPlugins {
157157
if plugin == "explorer" {
158158
// start kube-explorer
159-
port, err := common.EnableExplorer(context.Background(), cluster.ContextName)
159+
err := common.EnableExplorer(context.Background(), cluster.ContextName)
160160
if err != nil {
161161
p.Logger.Errorf("[%s] failed to start kube-explorer for cluster %s: %v", p.Provider, cluster.ContextName, err)
162162
}
163-
if port != 0 {
164-
p.Logger.Infof("[%s] kube-explorer for cluster %s will listen on 127.0.0.1:%d...", p.Provider, cluster.Name, port)
165-
}
166163
}
167164
}
168165

pkg/common/explorer.go

Lines changed: 39 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import (
66
"os"
77
"os/exec"
88
"path/filepath"
9+
"time"
910

10-
k3dutil "github.com/k3d-io/k3d/v5/cmd/util"
1111
"github.com/sirupsen/logrus"
1212
)
1313

@@ -16,87 +16,73 @@ const (
1616
)
1717

1818
// EnableExplorer will start kube-explorer with random port for specified K3s cluster
19-
func EnableExplorer(ctx context.Context, config string) (int, error) {
19+
func EnableExplorer(ctx context.Context, config string) error {
2020
if _, ok := ExplorerWatchers[config]; ok {
21-
return 0, fmt.Errorf("kube-explorer for cluster %s has already started", config)
21+
return fmt.Errorf("kube-explorer for cluster %s has already started", config)
2222
}
2323
if err := CheckCommandExist(KubeExplorerCommand); err != nil {
24-
return 0, err
24+
return err
2525
}
2626

2727
// command execution validate
2828
if err := checkExplorerCmd(); err != nil {
29-
return 0, err
29+
return err
3030
}
3131

3232
// save config for kube-explorer
3333
exp, err := DefaultDB.GetExplorer(config)
3434
if err != nil {
35-
return 0, err
35+
return err
3636
}
3737

3838
if exp == nil || !exp.Enabled {
39-
var port int
40-
if exp == nil {
41-
port, err = k3dutil.GetFreePort()
42-
if err != nil {
43-
return 0, err
44-
}
45-
} else {
46-
port = exp.Port
47-
}
4839
exp = &Explorer{
4940
ContextName: config,
50-
Port: port,
5141
Enabled: true,
5242
}
5343
if err = DefaultDB.SaveExplorer(exp); err != nil {
54-
return 0, err
44+
return err
5545
}
5646
}
5747

5848
// start kube-explorer
5949
explorerCtx, cancel := context.WithCancel(ctx)
6050
ExplorerWatchers[config] = cancel
61-
go func(ctx context.Context, config string, port int) {
62-
_ = StartKubeExplorer(ctx, config, port)
63-
}(explorerCtx, config, exp.Port)
64-
return exp.Port, nil
51+
52+
if _, err := StartKubeExplorer(explorerCtx, config); err != nil {
53+
return err
54+
}
55+
56+
return nil
6557
}
6658

6759
// DisableExplorer will stop kube-explorer server for specified K3s cluster
6860
func DisableExplorer(config string) error {
69-
if _, ok := ExplorerWatchers[config]; !ok {
61+
cancelFunc, ok := ExplorerWatchers[config]
62+
if !ok {
7063
return fmt.Errorf("cann't disable unactive kube-explorer for cluster %s", config)
7164
}
65+
7266
// update kube-explorer settings
7367
exp, err := DefaultDB.GetExplorer(config)
7468
if err != nil {
7569
return err
7670
}
71+
if exp != nil && exp.Enabled {
72+
// stop kube-explorer
73+
cancelFunc()
74+
delete(ExplorerWatchers, config)
75+
}
7776
if exp == nil || exp.Enabled {
78-
var port int
79-
if exp == nil {
80-
port, err = k3dutil.GetFreePort()
81-
if err != nil {
82-
return err
83-
}
84-
} else {
85-
port = exp.Port
86-
}
8777
err = DefaultDB.SaveExplorer(&Explorer{
8878
ContextName: config,
89-
Port: port,
9079
Enabled: false,
9180
})
9281
if err != nil {
9382
return err
9483
}
9584
}
9685

97-
// stop kube-explorer
98-
ExplorerWatchers[config]()
99-
delete(ExplorerWatchers, config)
10086
return nil
10187
}
10288

@@ -110,26 +96,34 @@ func InitExplorer(ctx context.Context) {
11096
for _, exp := range expList {
11197
if exp.Enabled {
11298
logrus.Infof("start kube-explorer for cluster %s", exp.ContextName)
113-
go func(ctx context.Context, name string) {
114-
if _, err = EnableExplorer(ctx, name); err != nil {
115-
logrus.Errorf("failed to start kube-explorer for cluster %s: %v", name, err)
116-
}
117-
}(ctx, exp.ContextName)
99+
if err = EnableExplorer(ctx, exp.ContextName); err != nil {
100+
logrus.Errorf("failed to start kube-explorer for cluster %s: %v", exp.ContextName, err)
101+
}
118102
}
119103
}
120104
}
121105

122106
// StartKubeExplorer start kube-explorer server listen on specified port
123-
func StartKubeExplorer(ctx context.Context, config string, port int) error {
107+
func StartKubeExplorer(ctx context.Context, clusterID string) (chan int, error) {
108+
socketName := GetSocketName(clusterID)
124109
explorer := exec.CommandContext(ctx, KubeExplorerCommand, fmt.Sprintf("--kubeconfig=%s", filepath.Join(CfgPath, KubeCfgFile)),
125-
fmt.Sprintf("--context=%s", config), fmt.Sprintf("--http-listen-port=%d", port), "--https-listen-port=0")
110+
fmt.Sprintf("--context=%s", clusterID), fmt.Sprintf("--bind-address=%s", socketName))
126111
explorer.Stdout = os.Stdout
127112
explorer.Stderr = os.Stderr
113+
explorer.Cancel = func() error {
114+
return explorer.Process.Signal(os.Interrupt)
115+
}
116+
explorer.WaitDelay = 10 * time.Second
128117
if err := explorer.Start(); err != nil {
129-
logrus.Errorf("fail to start kube-explorer for cluster %s: %v", config, err)
118+
logrus.Errorf("fail to start kube-explorer for cluster %s: %v", clusterID, err)
130119
}
131-
logrus.Infof("kube-explorer for %s K3s cluster will listen on 127.0.0.1:%d ...", config, port)
132-
return explorer.Wait()
120+
logrus.Infof("kube-explorer for %s K3s cluster will listen on %s ...", clusterID, socketName)
121+
stopChan := make(chan int)
122+
go func() {
123+
_ = explorer.Wait()
124+
close(stopChan)
125+
}()
126+
return stopChan, nil
133127
}
134128

135129
func CheckCommandExist(cmd string) error {

pkg/common/explorer_unix.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//go:build unix
2+
// +build unix
3+
4+
package common
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"net"
10+
"net/url"
11+
"path/filepath"
12+
)
13+
14+
const (
15+
socketFileName = "socket.sock"
16+
)
17+
18+
func GetSocketName(clusterID string) string {
19+
return fmt.Sprintf("unix://%s", filepath.Join(GetClusterContextPath(clusterID), socketFileName))
20+
}
21+
22+
func GetSocketDialer() func(context.Context, string, string) (net.Conn, error) {
23+
return func(ctx context.Context, _, addr string) (net.Conn, error) {
24+
clusterID, _, _ := net.SplitHostPort(addr)
25+
u, _ := url.Parse(GetSocketName(clusterID))
26+
var d net.Dialer
27+
return d.DialContext(ctx, "unix", u.Path)
28+
}
29+
}

pkg/common/explorer_windows.go

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
//go:build windows
2+
// +build windows
3+
4+
package common
5+
6+
import (
7+
"context"
8+
"crypto/md5"
9+
"encoding/hex"
10+
"fmt"
11+
"net"
12+
"net/url"
13+
14+
"github.com/Microsoft/go-winio"
15+
)
16+
17+
func GetSocketName(clusterID string) string {
18+
return fmt.Sprintf("namedpipe:/\\.\\pipe\\autok3s-%s", md5hash(clusterID))
19+
}
20+
21+
func GetSocketDialer() func(context.Context, string, string) (net.Conn, error) {
22+
return func(ctx context.Context, _, addr string) (net.Conn, error) {
23+
clusterID, _, _ := net.SplitHostPort(addr)
24+
u, _ := url.Parse(GetSocketName(clusterID))
25+
return winio.DialPipeContext(ctx, u.Path)
26+
}
27+
}
28+
29+
func md5hash(s string) string {
30+
hash := md5.Sum([]byte(s))
31+
hexStr := hex.EncodeToString(hash[:])
32+
return hexStr[:16]
33+
}

0 commit comments

Comments
 (0)