Skip to content

Commit ea15241

Browse files
alexsomesanalexsomesan
authored
Defer client initialization to improve resilience (#767)
* Lazy client initialization and client accessor functions * Don't fail ConfigureFunc on invalid configuration. Print warnings about future failures. * Fix credentials validation for acceptance tests. * Propagate client setup errors * Document credential interpolation issues
1 parent 789b0be commit ea15241

File tree

59 files changed

+895
-237
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+895
-237
lines changed

‎kubernetes/provider.go‎

Lines changed: 50 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
apimachineryschema "k8s.io/apimachinery/pkg/runtime/schema"
1414
kubernetes "k8s.io/client-go/kubernetes"
1515
_ "k8s.io/client-go/plugin/pkg/client/auth"
16+
"k8s.io/client-go/rest"
1617
restclient "k8s.io/client-go/rest"
1718

1819
"k8s.io/client-go/tools/clientcmd"
@@ -186,9 +187,43 @@ func Provider() terraform.ResourceProvider {
186187
return p
187188
}
188189

189-
type KubeClientsets struct {
190-
MainClientset *kubernetes.Clientset
191-
AggregatorClientset *aggregator.Clientset
190+
type KubeClientsets interface {
191+
MainClientset() (*kubernetes.Clientset, error)
192+
AggregatorClientset() (*aggregator.Clientset, error)
193+
}
194+
195+
type kubeClientsets struct {
196+
config *rest.Config
197+
mainClientset *kubernetes.Clientset
198+
aggregatorClientset *aggregator.Clientset
199+
}
200+
201+
func (k kubeClientsets) MainClientset() (*kubernetes.Clientset, error) {
202+
if k.mainClientset != nil {
203+
return k.mainClientset, nil
204+
}
205+
if k.config != nil {
206+
kc, err := kubernetes.NewForConfig(k.config)
207+
if err != nil {
208+
return nil, fmt.Errorf("Failed to configure client: %s", err)
209+
}
210+
k.mainClientset = kc
211+
}
212+
return k.mainClientset, nil
213+
}
214+
215+
func (k kubeClientsets) AggregatorClientset() (*aggregator.Clientset, error) {
216+
if k.aggregatorClientset != nil {
217+
return k.aggregatorClientset, nil
218+
}
219+
if k.config != nil {
220+
ac, err := aggregator.NewForConfig(k.config)
221+
if err != nil {
222+
return nil, fmt.Errorf("Failed to configure client: %s", err)
223+
}
224+
k.aggregatorClientset = ac
225+
}
226+
return k.aggregatorClientset, nil
192227
}
193228

194229
func providerConfigure(d *schema.ResourceData, terraformVersion string) (interface{}, error) {
@@ -199,7 +234,11 @@ func providerConfigure(d *schema.ResourceData, terraformVersion string) (interfa
199234
return nil, err
200235
}
201236
if cfg == nil {
202-
return nil, fmt.Errorf("Failed to initialize config")
237+
// This is a TEMPORARY measure to work around https://github.com/hashicorp/terraform/issues/24055
238+
// IMPORTANT: this will NOT enable a workaround of issue: https://github.com/hashicorp/terraform/issues/4149
239+
// IMPORTANT: if the supplied configuration is incomplete or invalid
240+
///IMPORTANT: provider operations will fail or attempt to connect to localhost endpoints
241+
cfg = &restclient.Config{}
203242
}
204243

205244
cfg.UserAgent = fmt.Sprintf("HashiCorp/1.0 Terraform/%s", terraformVersion)
@@ -211,17 +250,12 @@ func providerConfigure(d *schema.ResourceData, terraformVersion string) (interfa
211250
}
212251
}
213252

214-
k, err := kubernetes.NewForConfig(cfg)
215-
if err != nil {
216-
return nil, fmt.Errorf("Failed to configure: %s", err)
253+
m := kubeClientsets{
254+
config: cfg,
255+
mainClientset: nil,
256+
aggregatorClientset: nil,
217257
}
218-
219-
a, err := aggregator.NewForConfig(cfg)
220-
if err != nil {
221-
return nil, fmt.Errorf("Failed to configure: %s", err)
222-
}
223-
224-
return &KubeClientsets{k, a}, nil
258+
return m, nil
225259
}
226260

227261
func initializeConfiguration(d *schema.ResourceData) (*restclient.Config, error) {
@@ -321,7 +355,8 @@ func initializeConfiguration(d *schema.ResourceData) (*restclient.Config, error)
321355
cc := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loader, overrides)
322356
cfg, err := cc.ClientConfig()
323357
if err != nil {
324-
return nil, fmt.Errorf("Failed to initialize config: %s", err)
358+
log.Printf("[WARN] Invalid provider configuration was supplied. Provider operations likely to fail.")
359+
return nil, nil
325360
}
326361

327362
log.Printf("[INFO] Successfully initialized config")

‎kubernetes/provider_test.go‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -158,12 +158,11 @@ func testAccPreCheck(t *testing.T) {
158158
os.Getenv("KUBE_CTX") != "" ||
159159
os.Getenv("KUBECONFIG") != "" ||
160160
os.Getenv("KUBE_CONFIG") != ""
161+
hasUserCredentials := os.Getenv("KUBE_USER") != "" && os.Getenv("KUBE_PASSWORD") != ""
162+
hasClientCert := os.Getenv("KUBE_CLIENT_CERT_DATA") != "" && os.Getenv("KUBE_CLIENT_KEY_DATA") != ""
161163
hasStaticCfg := (os.Getenv("KUBE_HOST") != "" &&
162-
os.Getenv("KUBE_USER") != "" &&
163-
os.Getenv("KUBE_PASSWORD") != "" &&
164-
os.Getenv("KUBE_CLIENT_CERT_DATA") != "" &&
165-
os.Getenv("KUBE_CLIENT_KEY_DATA") != "" &&
166-
os.Getenv("KUBE_CLUSTER_CA_CERT_DATA") != "")
164+
os.Getenv("KUBE_CLUSTER_CA_CERT_DATA") != "") &&
165+
(hasUserCredentials || hasClientCert || os.Getenv("KUBE_TOKEN") != "")
167166

168167
if !hasFileCfg && !hasStaticCfg {
169168
t.Fatalf("File config (KUBE_CTX_AUTH_INFO and KUBE_CTX_CLUSTER) or static configuration"+
@@ -205,7 +204,10 @@ func getClusterVersion() (*gversion.Version, error) {
205204
return nil, fmt.Errorf("Provider not initialized, unable to check cluster version")
206205
}
207206

208-
conn := meta.(*KubeClientsets).MainClientset
207+
conn, err := meta.(KubeClientsets).MainClientset()
208+
if err != nil {
209+
return nil, err
210+
}
209211
serverVersion, err := conn.ServerVersion()
210212

211213
if err != nil {
@@ -306,7 +308,11 @@ func getFirstNode() (api.Node, error) {
306308
if meta == nil {
307309
return api.Node{}, errors.New("Provider not initialized, unable to get cluster node")
308310
}
309-
conn := meta.(*KubeClientsets).MainClientset
311+
conn, err := meta.(KubeClientsets).MainClientset()
312+
if err != nil {
313+
return api.Node{}, err
314+
}
315+
310316
resp, err := conn.CoreV1().Nodes().List(metav1.ListOptions{})
311317
if err != nil {
312318
return api.Node{}, err

‎kubernetes/resource_kubernetes_api_service.go‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,10 @@ func resourceKubernetesAPIService() *schema.Resource {
101101
}
102102

103103
func resourceKubernetesAPIServiceCreate(d *schema.ResourceData, meta interface{}) error {
104-
conn := meta.(*KubeClientsets).AggregatorClientset
104+
conn, err := meta.(KubeClientsets).AggregatorClientset()
105+
if err != nil {
106+
return err
107+
}
105108

106109
metadata := expandMetadata(d.Get("metadata").([]interface{}))
107110
svc := v1.APIService{
@@ -120,7 +123,10 @@ func resourceKubernetesAPIServiceCreate(d *schema.ResourceData, meta interface{}
120123
}
121124

122125
func resourceKubernetesAPIServiceRead(d *schema.ResourceData, meta interface{}) error {
123-
conn := meta.(*KubeClientsets).AggregatorClientset
126+
conn, err := meta.(KubeClientsets).AggregatorClientset()
127+
if err != nil {
128+
return err
129+
}
124130

125131
name := d.Id()
126132

@@ -147,7 +153,10 @@ func resourceKubernetesAPIServiceRead(d *schema.ResourceData, meta interface{})
147153
}
148154

149155
func resourceKubernetesAPIServiceUpdate(d *schema.ResourceData, meta interface{}) error {
150-
conn := meta.(*KubeClientsets).AggregatorClientset
156+
conn, err := meta.(KubeClientsets).AggregatorClientset()
157+
if err != nil {
158+
return err
159+
}
151160

152161
name := d.Id()
153162

@@ -174,12 +183,15 @@ func resourceKubernetesAPIServiceUpdate(d *schema.ResourceData, meta interface{}
174183
}
175184

176185
func resourceKubernetesAPIServiceDelete(d *schema.ResourceData, meta interface{}) error {
177-
conn := meta.(*KubeClientsets).AggregatorClientset
186+
conn, err := meta.(KubeClientsets).AggregatorClientset()
187+
if err != nil {
188+
return err
189+
}
178190

179191
name := d.Id()
180192

181193
log.Printf("[INFO] Deleting API service: %#v", name)
182-
err := conn.ApiregistrationV1().APIServices().Delete(name, &meta_v1.DeleteOptions{})
194+
err = conn.ApiregistrationV1().APIServices().Delete(name, &meta_v1.DeleteOptions{})
183195
if err != nil {
184196
return err
185197
}
@@ -191,12 +203,15 @@ func resourceKubernetesAPIServiceDelete(d *schema.ResourceData, meta interface{}
191203
}
192204

193205
func resourceKubernetesAPIServiceExists(d *schema.ResourceData, meta interface{}) (bool, error) {
194-
conn := meta.(*KubeClientsets).AggregatorClientset
206+
conn, err := meta.(KubeClientsets).AggregatorClientset()
207+
if err != nil {
208+
return false, err
209+
}
195210

196211
name := d.Id()
197212

198213
log.Printf("[INFO] Checking API service %s", name)
199-
_, err := conn.ApiregistrationV1().APIServices().Get(name, meta_v1.GetOptions{})
214+
_, err = conn.ApiregistrationV1().APIServices().Get(name, meta_v1.GetOptions{})
200215
if err != nil {
201216
if statusErr, ok := err.(*errors.StatusError); ok && statusErr.ErrStatus.Code == 404 {
202217
return false, nil

‎kubernetes/resource_kubernetes_api_service_test.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,10 @@ func TestAccKubernetesAPIService_importBasic(t *testing.T) {
136136
}
137137

138138
func testAccCheckKubernetesAPIServiceDestroy(s *terraform.State) error {
139-
conn := testAccProvider.Meta().(*KubeClientsets).AggregatorClientset
139+
conn, err := testAccProvider.Meta().(KubeClientsets).AggregatorClientset()
140+
if err != nil {
141+
return err
142+
}
140143

141144
for _, rs := range s.RootModule().Resources {
142145
if rs.Type != "kubernetes_api_service" {
@@ -163,11 +166,14 @@ func testAccCheckKubernetesAPIServiceExists(n string) resource.TestCheckFunc {
163166
return fmt.Errorf("Not found: %s", n)
164167
}
165168

166-
conn := testAccProvider.Meta().(*KubeClientsets).AggregatorClientset
169+
conn, err := testAccProvider.Meta().(KubeClientsets).AggregatorClientset()
170+
if err != nil {
171+
return err
172+
}
167173

168174
name := rs.Primary.ID
169175

170-
_, err := conn.ApiregistrationV1().APIServices().Get(name, meta_v1.GetOptions{})
176+
_, err = conn.ApiregistrationV1().APIServices().Get(name, meta_v1.GetOptions{})
171177
if err != nil {
172178
return err
173179
}

‎kubernetes/resource_kubernetes_cluster_role.go‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ func resourceKubernetesClusterRole() *schema.Resource {
3838
}
3939

4040
func resourceKubernetesClusterRoleCreate(d *schema.ResourceData, meta interface{}) error {
41-
conn := meta.(*KubeClientsets).MainClientset
41+
conn, err := meta.(KubeClientsets).MainClientset()
42+
if err != nil {
43+
return err
44+
}
4245

4346
metadata := expandMetadata(d.Get("metadata").([]interface{}))
4447
cRole := api.ClusterRole{
@@ -57,7 +60,10 @@ func resourceKubernetesClusterRoleCreate(d *schema.ResourceData, meta interface{
5760
}
5861

5962
func resourceKubernetesClusterRoleUpdate(d *schema.ResourceData, meta interface{}) error {
60-
conn := meta.(*KubeClientsets).MainClientset
63+
conn, err := meta.(KubeClientsets).MainClientset()
64+
if err != nil {
65+
return err
66+
}
6167

6268
name := d.Id()
6369
ops := patchMetadata("metadata.0.", "/metadata/", d)
@@ -81,7 +87,10 @@ func resourceKubernetesClusterRoleUpdate(d *schema.ResourceData, meta interface{
8187
}
8288

8389
func resourceKubernetesClusterRoleRead(d *schema.ResourceData, meta interface{}) error {
84-
conn := meta.(*KubeClientsets).MainClientset
90+
conn, err := meta.(KubeClientsets).MainClientset()
91+
if err != nil {
92+
return err
93+
}
8594

8695
name := d.Id()
8796
log.Printf("[INFO] Reading cluster role %s", name)
@@ -105,11 +114,14 @@ func resourceKubernetesClusterRoleRead(d *schema.ResourceData, meta interface{})
105114
}
106115

107116
func resourceKubernetesClusterRoleDelete(d *schema.ResourceData, meta interface{}) error {
108-
conn := meta.(*KubeClientsets).MainClientset
117+
conn, err := meta.(KubeClientsets).MainClientset()
118+
if err != nil {
119+
return err
120+
}
109121

110122
name := d.Id()
111123
log.Printf("[INFO] Deleting cluster role: %#v", name)
112-
err := conn.RbacV1().ClusterRoles().Delete(name, &metav1.DeleteOptions{})
124+
err = conn.RbacV1().ClusterRoles().Delete(name, &metav1.DeleteOptions{})
113125
if err != nil {
114126
return err
115127
}
@@ -120,11 +132,14 @@ func resourceKubernetesClusterRoleDelete(d *schema.ResourceData, meta interface{
120132
}
121133

122134
func resourceKubernetesClusterRoleExists(d *schema.ResourceData, meta interface{}) (bool, error) {
123-
conn := meta.(*KubeClientsets).MainClientset
135+
conn, err := meta.(KubeClientsets).MainClientset()
136+
if err != nil {
137+
return false, err
138+
}
124139

125140
name := d.Id()
126141
log.Printf("[INFO] Checking cluster role %s", name)
127-
_, err := conn.RbacV1().ClusterRoles().Get(name, metav1.GetOptions{})
142+
_, err = conn.RbacV1().ClusterRoles().Get(name, metav1.GetOptions{})
128143
if err != nil {
129144
if errors.IsNotFound(err) {
130145
return false, nil

‎kubernetes/resource_kubernetes_cluster_role_binding.go‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,10 @@ func resourceKubernetesClusterRoleBinding() *schema.Resource {
4848
}
4949

5050
func resourceKubernetesClusterRoleBindingCreate(d *schema.ResourceData, meta interface{}) error {
51-
conn := meta.(*KubeClientsets).MainClientset
51+
conn, err := meta.(KubeClientsets).MainClientset()
52+
if err != nil {
53+
return err
54+
}
5255

5356
metadata := expandMetadata(d.Get("metadata").([]interface{}))
5457
binding := &api.ClusterRoleBinding{
@@ -57,7 +60,7 @@ func resourceKubernetesClusterRoleBindingCreate(d *schema.ResourceData, meta int
5760
Subjects: expandRBACSubjects(d.Get("subject").([]interface{})),
5861
}
5962
log.Printf("[INFO] Creating new ClusterRoleBinding: %#v", binding)
60-
binding, err := conn.RbacV1().ClusterRoleBindings().Create(binding)
63+
binding, err = conn.RbacV1().ClusterRoleBindings().Create(binding)
6164

6265
if err != nil {
6366
return err
@@ -69,7 +72,10 @@ func resourceKubernetesClusterRoleBindingCreate(d *schema.ResourceData, meta int
6972
}
7073

7174
func resourceKubernetesClusterRoleBindingRead(d *schema.ResourceData, meta interface{}) error {
72-
conn := meta.(*KubeClientsets).MainClientset
75+
conn, err := meta.(KubeClientsets).MainClientset()
76+
if err != nil {
77+
return err
78+
}
7379

7480
name := d.Id()
7581
log.Printf("[INFO] Reading ClusterRoleBinding %s", name)
@@ -103,7 +109,10 @@ func resourceKubernetesClusterRoleBindingRead(d *schema.ResourceData, meta inter
103109
}
104110

105111
func resourceKubernetesClusterRoleBindingUpdate(d *schema.ResourceData, meta interface{}) error {
106-
conn := meta.(*KubeClientsets).MainClientset
112+
conn, err := meta.(KubeClientsets).MainClientset()
113+
if err != nil {
114+
return err
115+
}
107116

108117
name := d.Id()
109118

@@ -128,11 +137,14 @@ func resourceKubernetesClusterRoleBindingUpdate(d *schema.ResourceData, meta int
128137
}
129138

130139
func resourceKubernetesClusterRoleBindingDelete(d *schema.ResourceData, meta interface{}) error {
131-
conn := meta.(*KubeClientsets).MainClientset
140+
conn, err := meta.(KubeClientsets).MainClientset()
141+
if err != nil {
142+
return err
143+
}
132144

133145
name := d.Id()
134146
log.Printf("[INFO] Deleting ClusterRoleBinding: %#v", name)
135-
err := conn.RbacV1().ClusterRoleBindings().Delete(name, &meta_v1.DeleteOptions{})
147+
err = conn.RbacV1().ClusterRoleBindings().Delete(name, &meta_v1.DeleteOptions{})
136148
if err != nil {
137149
return err
138150
}
@@ -143,11 +155,14 @@ func resourceKubernetesClusterRoleBindingDelete(d *schema.ResourceData, meta int
143155
}
144156

145157
func resourceKubernetesClusterRoleBindingExists(d *schema.ResourceData, meta interface{}) (bool, error) {
146-
conn := meta.(*KubeClientsets).MainClientset
158+
conn, err := meta.(KubeClientsets).MainClientset()
159+
if err != nil {
160+
return false, err
161+
}
147162

148163
name := d.Id()
149164
log.Printf("[INFO] Checking ClusterRoleBinding %s", name)
150-
_, err := conn.RbacV1().ClusterRoleBindings().Get(name, meta_v1.GetOptions{})
165+
_, err = conn.RbacV1().ClusterRoleBindings().Get(name, meta_v1.GetOptions{})
151166
if err != nil {
152167
if statusErr, ok := err.(*errors.StatusError); ok && statusErr.ErrStatus.Code == 404 {
153168
return false, nil

0 commit comments

Comments
 (0)