feat: support setting kubeconfig in a stack dimension (#593)

liu-hm19 · web-flow · commit ebc034acee30 · 2023-11-09T07:54:04.000-06:00
diff --git a/pkg/engine/operation/apply.go b/pkg/engine/operation/apply.go
@@ -88,7 +88,7 @@ func (ao *ApplyOperation) Apply(request *ApplyRequest) (rsp *ApplyResponse, st s
 
 	resources := request.Spec.Resources
 	resources = append(resources, priorState.Resources...)
-	runtimesMap, s := runtimeinit.Runtimes(resources)
+	runtimesMap, s := runtimeinit.Runtimes(resources, o.Stack)
 	if status.IsErr(s) {
 		return nil, s
 	}
diff --git a/pkg/engine/operation/apply_test.go b/pkg/engine/operation/apply_test.go
@@ -176,7 +176,7 @@ func TestOperation_Apply(t *testing.T) {
 				o.ResultState = rs
 				return nil
 			}).Build()
-			mockey.Mock(runtimeinit.Runtimes).To(func(resources models.Resources) (map[models.Type]runtime.Runtime, status.Status) {
+			mockey.Mock(runtimeinit.Runtimes).To(func(resources models.Resources, stack *projectstack.Stack) (map[models.Type]runtime.Runtime, status.Status) {
 				return map[models.Type]runtime.Runtime{runtime.Kubernetes: &kubernetes.KubernetesRuntime{}}, nil
 			}).Build()
 
diff --git a/pkg/engine/operation/destory.go b/pkg/engine/operation/destory.go
@@ -54,7 +54,7 @@ func (do *DestroyOperation) Destroy(request *DestroyRequest) (st status.Status)
 
 	// only destroy resources we have recorded
 	resources := priorState.Resources
-	runtimesMap, s := runtimeinit.Runtimes(resources)
+	runtimesMap, s := runtimeinit.Runtimes(resources, o.Stack)
 	if status.IsErr(s) {
 		return s
 	}
diff --git a/pkg/engine/operation/preview.go b/pkg/engine/operation/preview.go
@@ -64,7 +64,7 @@ func (po *PreviewOperation) Preview(request *PreviewRequest) (rsp *PreviewRespon
 	// Kusion is a multi-runtime system. We initialize runtimes dynamically by resource types
 	resources := request.Spec.Resources
 	resources = append(resources, priorState.Resources...)
-	runtimesMap, s := runtimeinit.Runtimes(resources)
+	runtimesMap, s := runtimeinit.Runtimes(resources, o.Stack)
 	if status.IsErr(s) {
 		return nil, s
 	}
diff --git a/pkg/engine/operation/preview_test.go b/pkg/engine/operation/preview_test.go
@@ -264,7 +264,7 @@ func TestOperation_Preview(t *testing.T) {
 				},
 			}
 
-			mockey.Mock(runtimeinit.Runtimes).To(func(resources models.Resources) (map[models.Type]runtime.Runtime, status.Status) {
+			mockey.Mock(runtimeinit.Runtimes).To(func(resources models.Resources, stack *projectstack.Stack) (map[models.Type]runtime.Runtime, status.Status) {
 				return map[models.Type]runtime.Runtime{runtime.Kubernetes: &fakePreviewRuntime{}}, nil
 			}).Build()
 			gotRsp, gotS := o.Preview(tt.args.request)
diff --git a/pkg/engine/operation/watch.go b/pkg/engine/operation/watch.go
@@ -36,7 +36,7 @@ func (wo *WatchOperation) Watch(req *WatchRequest) error {
 
 	// init runtimes
 	resources := req.Spec.Resources
-	runtimes, s := runtimeinit.Runtimes(resources)
+	runtimes, s := runtimeinit.Runtimes(resources, wo.Stack)
 	if status.IsErr(s) {
 		return errors.New(s.Message())
 	}
diff --git a/pkg/engine/operation/watch_test.go b/pkg/engine/operation/watch_test.go
@@ -13,6 +13,7 @@ import (
 	"kusionstack.io/kusion/pkg/engine/runtime"
 	runtimeinit "kusionstack.io/kusion/pkg/engine/runtime/init"
 	"kusionstack.io/kusion/pkg/models"
+	"kusionstack.io/kusion/pkg/projectstack"
 	"kusionstack.io/kusion/pkg/status"
 )
 
@@ -31,7 +32,7 @@ func TestWatchOperation_Watch(t *testing.T) {
 				},
 			},
 		}
-		mockey.Mock(runtimeinit.Runtimes).To(func(resources models.Resources) (map[models.Type]runtime.Runtime, status.Status) {
+		mockey.Mock(runtimeinit.Runtimes).To(func(resources models.Resources, stack *projectstack.Stack) (map[models.Type]runtime.Runtime, status.Status) {
 			return map[models.Type]runtime.Runtime{runtime.Kubernetes: fooRuntime}, nil
 		}).Build()
 		wo := &WatchOperation{opsmodels.Operation{RuntimeMap: map[models.Type]runtime.Runtime{runtime.Kubernetes: fooRuntime}}}
diff --git a/pkg/engine/runtime/init/init.go b/pkg/engine/runtime/init/init.go
@@ -8,6 +8,7 @@ import (
 	"kusionstack.io/kusion/pkg/engine/runtime/kubernetes"
 	"kusionstack.io/kusion/pkg/engine/runtime/terraform"
 	"kusionstack.io/kusion/pkg/models"
+	"kusionstack.io/kusion/pkg/projectstack"
 	"kusionstack.io/kusion/pkg/status"
 )
 
@@ -17,9 +18,9 @@ var SupportRuntimes = map[models.Type]InitFn{
 }
 
 // InitFn runtime init func
-type InitFn func() (runtime.Runtime, error)
+type InitFn func(stack *projectstack.Stack) (runtime.Runtime, error)
 
-func Runtimes(resources models.Resources) (map[models.Type]runtime.Runtime, status.Status) {
+func Runtimes(resources models.Resources, stack *projectstack.Stack) (map[models.Type]runtime.Runtime, status.Status) {
 	runtimesMap := map[models.Type]runtime.Runtime{}
 	if resources == nil {
 		return runtimesMap, nil
@@ -35,7 +36,7 @@ func Runtimes(resources models.Resources) (map[models.Type]runtime.Runtime, stat
 			return nil, status.NewErrorStatusWithCode(status.IllegalManifest, fmt.Errorf("unknow resource type: %s. Currently supported resource types are: %v",
 				rt, reflect.ValueOf(SupportRuntimes).MapKeys()))
 		} else if runtimesMap[rt] == nil {
-			r, err := SupportRuntimes[rt]()
+			r, err := SupportRuntimes[rt](stack)
 			if err != nil {
 				return nil, status.NewErrorStatus(fmt.Errorf("init %s runtime failed. %w", rt, err))
 			}
diff --git a/pkg/engine/runtime/kubernetes/kubernetes_runtime.go b/pkg/engine/runtime/kubernetes/kubernetes_runtime.go
@@ -30,6 +30,7 @@ import (
 	"kusionstack.io/kusion/pkg/engine/runtime"
 	"kusionstack.io/kusion/pkg/log"
 	"kusionstack.io/kusion/pkg/models"
+	"kusionstack.io/kusion/pkg/projectstack"
 	"kusionstack.io/kusion/pkg/status"
 	jsonutil "kusionstack.io/kusion/pkg/util/json"
 	"kusionstack.io/kusion/pkg/util/kube/config"
@@ -43,8 +44,8 @@ type KubernetesRuntime struct {
 }
 
 // NewKubernetesRuntime create a new KubernetesRuntime
-func NewKubernetesRuntime() (runtime.Runtime, error) {
-	client, mapper, err := getKubernetesClient()
+func NewKubernetesRuntime(stack *projectstack.Stack) (runtime.Runtime, error) {
+	client, mapper, err := getKubernetesClient(stack)
 	if err != nil {
 		return nil, err
 	}
@@ -375,9 +376,9 @@ func (k *KubernetesRuntime) Watch(ctx context.Context, request *runtime.WatchReq
 }
 
 // getKubernetesClient get kubernetes client
-func getKubernetesClient() (dynamic.Interface, meta.RESTMapper, error) {
+func getKubernetesClient(stack *projectstack.Stack) (dynamic.Interface, meta.RESTMapper, error) {
 	// build config
-	cfg, err := clientcmd.BuildConfigFromFlags("", config.GetKubeConfig())
+	cfg, err := clientcmd.BuildConfigFromFlags("", config.GetKubeConfig(stack))
 	if err != nil {
 		return nil, nil, err
 	}
diff --git a/pkg/engine/runtime/terraform/terraform_runtime.go b/pkg/engine/runtime/terraform/terraform_runtime.go
@@ -12,6 +12,7 @@ import (
 	"kusionstack.io/kusion/pkg/engine/runtime/terraform/tfops"
 	"kusionstack.io/kusion/pkg/log"
 	"kusionstack.io/kusion/pkg/models"
+	"kusionstack.io/kusion/pkg/projectstack"
 	"kusionstack.io/kusion/pkg/status"
 )
 
@@ -22,7 +23,7 @@ type TerraformRuntime struct {
 	mu *sync.Mutex
 }
 
-func NewTerraformRuntime() (runtime.Runtime, error) {
+func NewTerraformRuntime(_ *projectstack.Stack) (runtime.Runtime, error) {
 	fs := afero.Afero{Fs: afero.NewOsFs()}
 	ws := tfops.NewWorkSpace(fs)
 	TFRuntime := &TerraformRuntime{
diff --git a/pkg/projectstack/types.go b/pkg/projectstack/types.go
@@ -131,7 +131,8 @@ func (p *Project) TableReport() string {
 
 // StackConfiguration is the stack configuration
 type StackConfiguration struct {
-	Name string `json:"name" yaml:"name"` // Stack name
+	Name       string `json:"name" yaml:"name"`             // Stack name
+	KubeConfig string `json:"kubeConfig" yaml:"kubeConfig"` // KubeConfig file path for this stack
 }
 
 type Stack struct {
diff --git a/pkg/util/kube/config/config.go b/pkg/util/kube/config/config.go
@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	"k8s.io/client-go/util/homedir"
+	"kusionstack.io/kusion/pkg/projectstack"
 )
 
 const (
@@ -18,12 +19,17 @@ var (
 	RecommendedKubeConfigFile = filepath.Join(RecommendedConfigDir, RecommendedKubeConfigFileName)
 )
 
-// 1. If $KUBECONFIG environment variable is set, then it is used it.
-// 2. Otherwise, ${HOME}/.kube/config is used.
-func GetKubeConfig() string {
+// 1. If $KUBECONFIG environment variable is set, then it is used.
+// 2. If not, and the `kubeConfig` in stack configuration is set, then it is used.
+// 3. Otherwise, ${HOME}/.kube/config is used.
+func GetKubeConfig(stack *projectstack.Stack) string {
 	if kubeConfigFile := os.Getenv(RecommendedConfigPathEnvVar); kubeConfigFile != "" {
 		return kubeConfigFile
-	} else {
-		return RecommendedKubeConfigFile
+	} else if stack.KubeConfig != "" {
+		kubeConfigFile, _ := filepath.Abs(stack.KubeConfig)
+		if kubeConfigFile != "" {
+			return kubeConfigFile
+		}
 	}
+	return RecommendedKubeConfigFile
 }
diff --git a/pkg/util/kube/config/config_test.go b/pkg/util/kube/config/config_test.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/bytedance/mockey"
 	"github.com/stretchr/testify/assert"
+	"kusionstack.io/kusion/pkg/projectstack"
 )
 
 func mockGetenv(result string) {
@@ -18,13 +19,24 @@ func mockGetenv(result string) {
 }
 
 func TestGetKubeConfig(t *testing.T) {
+	stack := &projectstack.Stack{
+		StackConfiguration: projectstack.StackConfiguration{
+			KubeConfig: "",
+		},
+	}
+
 	// Mock
 	mockey.PatchConvey("test null env config", t, func() {
 		mockGetenv("")
-		assert.Equal(t, RecommendedKubeConfigFile, GetKubeConfig())
+		assert.Equal(t, RecommendedKubeConfigFile, GetKubeConfig(stack))
 	})
 	mockey.PatchConvey("test env config", t, func() {
 		mockGetenv("test")
-		assert.Equal(t, "test", GetKubeConfig())
+		assert.Equal(t, "test", GetKubeConfig(stack))
+	})
+	mockey.PatchConvey("test stack config", t, func() {
+		mockGetenv("")
+		stack.KubeConfig = "/home/test/kubeconfig"
+		assert.Equal(t, "/home/test/kubeconfig", GetKubeConfig(stack))
 	})
 }

Original file line number	Diff line number	Diff line change
`@@ -88,7 +88,7 @@ func (ao ApplyOperation) Apply(request ApplyRequest) (rsp *ApplyResponse, st s`
`88`	`88`
`89`	`89`	`resources := request.Spec.Resources`
`90`	`90`	`resources = append(resources, priorState.Resources...)`
`91`		`- runtimesMap, s := runtimeinit.Runtimes(resources)`
	`91`	`+ runtimesMap, s := runtimeinit.Runtimes(resources, o.Stack)`
`92`	`92`	`if status.IsErr(s) {`
`93`	`93`	`return nil, s`
`94`	`94`	`}`
Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ func (do DestroyOperation) Destroy(request DestroyRequest) (st status.Status)`
`54`	`54`
`55`	`55`	`// only destroy resources we have recorded`
`56`	`56`	`resources := priorState.Resources`
`57`		`- runtimesMap, s := runtimeinit.Runtimes(resources)`
	`57`	`+ runtimesMap, s := runtimeinit.Runtimes(resources, o.Stack)`
`58`	`58`	`if status.IsErr(s) {`
`59`	`59`	`return s`
`60`	`60`	`}`
Original file line number	Diff line number	Diff line change
`@@ -64,7 +64,7 @@ func (po PreviewOperation) Preview(request PreviewRequest) (rsp *PreviewRespon`
`64`	`64`	`// Kusion is a multi-runtime system. We initialize runtimes dynamically by resource types`
`65`	`65`	`resources := request.Spec.Resources`
`66`	`66`	`resources = append(resources, priorState.Resources...)`
`67`		`- runtimesMap, s := runtimeinit.Runtimes(resources)`
	`67`	`+ runtimesMap, s := runtimeinit.Runtimes(resources, o.Stack)`
`68`	`68`	`if status.IsErr(s) {`
`69`	`69`	`return nil, s`
`70`	`70`	`}`
Original file line number	Diff line number	Diff line change
`@@ -264,7 +264,7 @@ func TestOperation_Preview(t *testing.T) {`
`264`	`264`	`},`
`265`	`265`	`}`
`266`	`266`
`267`		`- mockey.Mock(runtimeinit.Runtimes).To(func(resources models.Resources) (map[models.Type]runtime.Runtime, status.Status) {`
	`267`	`+ mockey.Mock(runtimeinit.Runtimes).To(func(resources models.Resources, stack *projectstack.Stack) (map[models.Type]runtime.Runtime, status.Status) {`
`268`	`268`	`return map[models.Type]runtime.Runtime{runtime.Kubernetes: &fakePreviewRuntime{}}, nil`
`269`	`269`	`}).Build()`
`270`	`270`	`gotRsp, gotS := o.Preview(tt.args.request)`
Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ func (wo WatchOperation) Watch(req WatchRequest) error {`
`36`	`36`
`37`	`37`	`// init runtimes`
`38`	`38`	`resources := req.Spec.Resources`
`39`		`- runtimes, s := runtimeinit.Runtimes(resources)`
	`39`	`+ runtimes, s := runtimeinit.Runtimes(resources, wo.Stack)`
`40`	`40`	`if status.IsErr(s) {`
`41`	`41`	`return errors.New(s.Message())`
`42`	`42`	`}`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@ import (`
`12`	`12`	`"kusionstack.io/kusion/pkg/engine/runtime/terraform/tfops"`
`13`	`13`	`"kusionstack.io/kusion/pkg/log"`
`14`	`14`	`"kusionstack.io/kusion/pkg/models"`
	`15`	`+ "kusionstack.io/kusion/pkg/projectstack"`
`15`	`16`	`"kusionstack.io/kusion/pkg/status"`
`16`	`17`	`)`
`17`	`18`
`@@ -22,7 +23,7 @@ type TerraformRuntime struct {`
`22`	`23`	`mu *sync.Mutex`
`23`	`24`	`}`
`24`	`25`
`25`		`-func NewTerraformRuntime() (runtime.Runtime, error) {`
	`26`	`+func NewTerraformRuntime(_ *projectstack.Stack) (runtime.Runtime, error) {`
`26`	`27`	`fs := afero.Afero{Fs: afero.NewOsFs()}`
`27`	`28`	`ws := tfops.NewWorkSpace(fs)`
`28`	`29`	`TFRuntime := &TerraformRuntime{`