Skip to content

Commit abaea70

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #274 from verult/cloudconfig
Integration with GCE cloud config
2 parents da81807 + 06bd754 commit abaea70

File tree

21 files changed

+2476
-37
lines changed

21 files changed

+2476
-37
lines changed

‎Gopkg.lock

Lines changed: 24 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cmd/main.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@ func init() {
3333
}
3434

3535
var (
36-
endpoint = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
37-
vendorVersion string
36+
endpoint = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
37+
gceConfigFilePath = flag.String("cloud-config", "", "Path to GCE cloud provider config")
38+
vendorVersion string
3839
)
3940

4041
const (
@@ -57,7 +58,7 @@ func handle() {
5758
gceDriver := driver.GetGCEDriver()
5859

5960
//Initialize GCE Driver (Move setup to main?)
60-
cloudProvider, err := gce.CreateCloudProvider(vendorVersion)
61+
cloudProvider, err := gce.CreateCloudProvider(vendorVersion, *gceConfigFilePath)
6162
if err != nil {
6263
glog.Fatalf("Failed to get cloud provider: %v", err)
6364
}

‎pkg/gce-cloud-provider/compute/gce.go

Lines changed: 106 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,10 @@ limitations under the License.
1515
package gcecloudprovider
1616

1717
import (
18+
"context"
1819
"fmt"
20+
"golang.org/x/oauth2/google"
21+
"gopkg.in/gcfg.v1"
1922
"net/http"
2023
"os"
2124
"runtime"
@@ -24,9 +27,8 @@ import (
2427
"cloud.google.com/go/compute/metadata"
2528
"github.com/golang/glog"
2629
"golang.org/x/oauth2"
27-
"golang.org/x/oauth2/google"
2830
beta "google.golang.org/api/compute/v0.beta"
29-
compute "google.golang.org/api/compute/v1"
31+
"google.golang.org/api/compute/v1"
3032
"google.golang.org/api/googleapi"
3133
"k8s.io/apimachinery/pkg/util/wait"
3234
)
@@ -57,20 +59,44 @@ type CloudProvider struct {
5759

5860
var _ GCECompute = &CloudProvider{}
5961

60-
func CreateCloudProvider(vendorVersion string) (*CloudProvider, error) {
61-
svc, err := createCloudService(vendorVersion)
62+
type ConfigFile struct {
63+
Global ConfigGlobal `gcfg:"global"`
64+
}
65+
66+
type ConfigGlobal struct {
67+
TokenURL string `gcfg:"token-url"`
68+
TokenBody string `gcfg:"token-body"`
69+
ProjectId string `gcfg:"project-id"`
70+
}
71+
72+
func CreateCloudProvider(vendorVersion string, configPath string) (*CloudProvider, error) {
73+
configFile, err := readConfig(configPath)
74+
if err != nil {
75+
return nil, err
76+
}
77+
// At this point configFile could still be nil.
78+
// Any following code that uses configFile should handle nil pointer gracefully.
79+
80+
glog.V(1).Infof("Using GCE provider config %+v", configFile)
81+
82+
tokenSource, err := generateTokenSource(configFile)
83+
if err != nil {
84+
return nil, err
85+
}
86+
87+
svc, err := createCloudService(vendorVersion, tokenSource)
6288
if err != nil {
6389
return nil, err
6490
}
6591

66-
betasvc, err := createBetaCloudService(vendorVersion)
92+
betasvc, err := createBetaCloudService(vendorVersion, tokenSource)
6793
if err != nil {
6894
return nil, err
6995
}
7096

71-
project, zone, err := getProjectAndZoneFromMetadata()
97+
project, zone, err := getProjectAndZone(configFile)
7298
if err != nil {
73-
return nil, fmt.Errorf("Failed getting Project and Zone from Metadata server: %v", err)
99+
return nil, fmt.Errorf("Failed getting Project and Zone: %v", err)
74100
}
75101

76102
return &CloudProvider{
@@ -83,8 +109,55 @@ func CreateCloudProvider(vendorVersion string) (*CloudProvider, error) {
83109

84110
}
85111

86-
func createBetaCloudService(vendorVersion string) (*beta.Service, error) {
87-
client, err := newDefaultOauthClient()
112+
func generateTokenSource(configFile *ConfigFile) (oauth2.TokenSource, error) {
113+
114+
if configFile != nil && configFile.Global.TokenURL != "" && configFile.Global.TokenURL != "nil" {
115+
// configFile.Global.TokenURL is defined
116+
// Use AltTokenSource
117+
118+
tokenSource := NewAltTokenSource(configFile.Global.TokenURL, configFile.Global.TokenBody)
119+
glog.V(4).Infof("Using AltTokenSource %#v", tokenSource)
120+
return tokenSource, nil
121+
}
122+
123+
// Use DefaultTokenSource
124+
125+
tokenSource, err := google.DefaultTokenSource(
126+
context.Background(),
127+
compute.CloudPlatformScope,
128+
compute.ComputeScope)
129+
130+
// DefaultTokenSource relies on GOOGLE_APPLICATION_CREDENTIALS env var being set.
131+
if gac, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); ok {
132+
glog.V(4).Infof("GOOGLE_APPLICATION_CREDENTIALS env var set %v", gac)
133+
} else {
134+
glog.Warningf("GOOGLE_APPLICATION_CREDENTIALS env var not set")
135+
}
136+
glog.V(4).Infof("Using DefaultTokenSource %#v", tokenSource)
137+
138+
return tokenSource, err
139+
}
140+
141+
func readConfig(configPath string) (*ConfigFile, error) {
142+
if configPath == "" {
143+
return nil, nil
144+
}
145+
146+
reader, err := os.Open(configPath)
147+
if err != nil {
148+
return nil, fmt.Errorf("couldn't open cloud provider configuration at %s: %v", configPath, err)
149+
}
150+
defer reader.Close()
151+
152+
cfg := &ConfigFile{}
153+
if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil {
154+
return nil, fmt.Errorf("couldn't read cloud provider configuration at %s: %v", configPath, err)
155+
}
156+
return cfg, nil
157+
}
158+
159+
func createBetaCloudService(vendorVersion string, tokenSource oauth2.TokenSource) (*beta.Service, error) {
160+
client, err := newOauthClient(tokenSource)
88161
if err != nil {
89162
return nil, err
90163
}
@@ -96,13 +169,13 @@ func createBetaCloudService(vendorVersion string) (*beta.Service, error) {
96169
return service, nil
97170
}
98171

99-
func createCloudService(vendorVersion string) (*compute.Service, error) {
100-
svc, err := createCloudServiceWithDefaultServiceAccount(vendorVersion)
172+
func createCloudService(vendorVersion string, tokenSource oauth2.TokenSource) (*compute.Service, error) {
173+
svc, err := createCloudServiceWithDefaultServiceAccount(vendorVersion, tokenSource)
101174
return svc, err
102175
}
103176

104-
func createCloudServiceWithDefaultServiceAccount(vendorVersion string) (*compute.Service, error) {
105-
client, err := newDefaultOauthClient()
177+
func createCloudServiceWithDefaultServiceAccount(vendorVersion string, tokenSource oauth2.TokenSource) (*compute.Service, error) {
178+
client, err := newOauthClient(tokenSource)
106179
if err != nil {
107180
return nil, err
108181
}
@@ -114,22 +187,7 @@ func createCloudServiceWithDefaultServiceAccount(vendorVersion string) (*compute
114187
return service, nil
115188
}
116189

117-
func newDefaultOauthClient() (*http.Client, error) {
118-
// No compute token source, fallback on default
119-
tokenSource, err := google.DefaultTokenSource(
120-
oauth2.NoContext,
121-
compute.CloudPlatformScope,
122-
compute.ComputeScope)
123-
if gac, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); ok {
124-
glog.V(4).Infof("GOOGLE_APPLICATION_CREDENTIALS env var set %v", gac)
125-
} else {
126-
glog.Warningf("GOOGLE_APPLICATION_CREDENTIALS env var not set")
127-
}
128-
glog.V(4).Infof("Using DefaultTokenSource %#v", tokenSource)
129-
if err != nil {
130-
return nil, err
131-
}
132-
190+
func newOauthClient(tokenSource oauth2.TokenSource) (*http.Client, error) {
133191
if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
134192
if _, err := tokenSource.Token(); err != nil {
135193
glog.Errorf("error fetching initial token: %v", err)
@@ -140,18 +198,32 @@ func newDefaultOauthClient() (*http.Client, error) {
140198
return nil, err
141199
}
142200

143-
return oauth2.NewClient(oauth2.NoContext, tokenSource), nil
201+
return oauth2.NewClient(context.Background(), tokenSource), nil
144202
}
145203

146-
func getProjectAndZoneFromMetadata() (string, string, error) {
204+
func getProjectAndZone(config *ConfigFile) (string, string, error) {
205+
var err error
206+
147207
zone, err := metadata.Zone()
148208
if err != nil {
149209
return "", "", err
150210
}
151-
projectID, err := metadata.ProjectID()
152-
if err != nil {
153-
return "", "", err
211+
212+
var projectID string
213+
if config == nil || config.Global.ProjectId == "" {
214+
// Project ID is not available from the local GCE cloud provider config file.
215+
// This could happen if the driver is not running in the master VM.
216+
// Defaulting to project ID from the Metadata server.
217+
projectID, err = metadata.ProjectID()
218+
if err != nil {
219+
return "", "", err
220+
}
221+
glog.V(4).Infof("Using GCP project ID from the Metadata server: %q", projectID)
222+
} else {
223+
projectID = config.Global.ProjectId
224+
glog.V(4).Infof("Using GCP project ID from the local GCE cloud provider config file: %q", projectID)
154225
}
226+
155227
return projectID, zone, nil
156228
}
157229

‎pkg/gce-cloud-provider/compute/token_source.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package gcecloudprovider
18+
19+
import (
20+
"encoding/json"
21+
"net/http"
22+
"strings"
23+
"time"
24+
25+
"k8s.io/client-go/util/flowcontrol"
26+
27+
"golang.org/x/oauth2"
28+
"golang.org/x/oauth2/google"
29+
"google.golang.org/api/googleapi"
30+
)
31+
32+
const (
33+
// Max QPS to allow through to the token URL.
34+
tokenURLQPS = .05 // back off to once every 20 seconds when failing
35+
// Maximum burst of requests to token URL before limiting.
36+
tokenURLBurst = 3
37+
)
38+
39+
// TODO(#276) add metrics around token requests once the driver integrates with Prometheus.
40+
41+
// AltTokenSource is the structure holding the data for the functionality needed to generates tokens
42+
type AltTokenSource struct {
43+
oauthClient *http.Client
44+
tokenURL string
45+
tokenBody string
46+
throttle flowcontrol.RateLimiter
47+
}
48+
49+
// Token returns a token which may be used for authentication
50+
func (a *AltTokenSource) Token() (*oauth2.Token, error) {
51+
a.throttle.Accept()
52+
return a.token()
53+
}
54+
55+
func (a *AltTokenSource) token() (*oauth2.Token, error) {
56+
req, err := http.NewRequest("POST", a.tokenURL, strings.NewReader(a.tokenBody))
57+
if err != nil {
58+
return nil, err
59+
}
60+
res, err := a.oauthClient.Do(req)
61+
if err != nil {
62+
return nil, err
63+
}
64+
defer res.Body.Close()
65+
if err := googleapi.CheckResponse(res); err != nil {
66+
return nil, err
67+
}
68+
var tok struct {
69+
AccessToken string `json:"accessToken"`
70+
ExpireTime time.Time `json:"expireTime"`
71+
}
72+
if err := json.NewDecoder(res.Body).Decode(&tok); err != nil {
73+
return nil, err
74+
}
75+
return &oauth2.Token{
76+
AccessToken: tok.AccessToken,
77+
Expiry: tok.ExpireTime,
78+
}, nil
79+
}
80+
81+
// NewAltTokenSource constructs a new alternate token source for generating tokens.
82+
func NewAltTokenSource(tokenURL, tokenBody string) oauth2.TokenSource {
83+
client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
84+
a := &AltTokenSource{
85+
oauthClient: client,
86+
tokenURL: tokenURL,
87+
tokenBody: tokenBody,
88+
throttle: flowcontrol.NewTokenBucketRateLimiter(tokenURLQPS, tokenURLBurst),
89+
}
90+
return oauth2.ReuseTokenSource(nil, a)
91+
}

0 commit comments

Comments
 (0)