PBM-1484 gcp sdk fix (#1099)

* add client email to config
* add equal
* remove non-mandatory projectId

Author: veceraj

pbm/config/config.go

@@ -150,6 +150,9 @@ func (c *Config) String() string {
 		if c.Storage.GCS.Credentials.PrivateKey != "" {
 			c.Storage.GCS.Credentials.PrivateKey = "***"
 		}
+		if c.Storage.GCS.Credentials.ClientEmail != "" {
+			c.Storage.GCS.Credentials.ClientEmail = "***"
+		}
 	}
 
 	b, err := yaml.Marshal(c)
@@ -254,6 +257,8 @@ func (s *StorageConf) Equal(other *StorageConf) bool {
 		return s.S3.Equal(other.S3)
 	case storage.Azure:
 		return s.Azure.Equal(other.Azure)
+	case storage.GCS:
+		return s.GCS.Equal(other.GCS)
 	case storage.Filesystem:
 		return s.Filesystem.Equal(other.Filesystem)
 	case storage.Blackhole:
@@ -286,6 +291,8 @@ func (s *StorageConf) Typ() string {
 		return "S3"
 	case storage.Azure:
 		return "Azure"
+	case storage.GCS:
+		return "GCS"
 	case storage.Filesystem:
 		return "FS"
 	case storage.Blackhole:
@@ -318,6 +325,11 @@ func (s *StorageConf) Path() string {
 		if s.Azure.Prefix != "" {
 			path += "/" + s.Azure.Prefix
 		}
+	case storage.GCS:
+		path = "gs://" + s.GCS.Bucket
+		if s.GCS.Prefix != "" {
+			path += "/" + s.GCS.Prefix
+		}
 	case storage.Filesystem:
 		path = s.Filesystem.Path
 	}

pbm/storage/gcs/gcs.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"reflect"
 	"strings"
 	"time"
 
@@ -32,8 +33,8 @@ type Config struct {
 }
 
 type Credentials struct {
-	ProjectID  string `bson:"projectId" json:"projectId,omitempty" yaml:"projectId,omitempty"`
-	PrivateKey string `bson:"privateKey" json:"privateKey,omitempty" yaml:"privateKey,omitempty"`
+	ClientEmail string `bson:"clientEmail" json:"clientEmail,omitempty" yaml:"clientEmail,omitempty"`
+	PrivateKey  string `bson:"privateKey" json:"privateKey,omitempty" yaml:"privateKey,omitempty"`
 }
 
 type Retryer struct {
@@ -52,7 +53,6 @@ type Retryer struct {
 
 type ServiceAccountCredentials struct {
 	Type                string `json:"type"`
-	ProjectID           string `json:"project_id"`
 	PrivateKey          string `json:"private_key"`
 	ClientEmail         string `json:"client_email"`
 	AuthURI             string `json:"auth_uri"`
@@ -77,6 +77,28 @@ func (cfg *Config) Clone() *Config {
 	return &rv
 }
 
+func (cfg *Config) Equal(other *Config) bool {
+	if cfg == nil || other == nil {
+		return cfg == other
+	}
+
+	if cfg.Bucket != other.Bucket {
+		return false
+	}
+	if cfg.Prefix != other.Prefix {
+		return false
+	}
+	if cfg.ChunkSize != other.ChunkSize {
+		return false
+	}
+
+	if !reflect.DeepEqual(cfg.Credentials, other.Credentials) {
+		return false
+	}
+
+	return true
+}
+
 func New(opts *Config, node string, l log.LogEvent) (*GCS, error) {
 	g := &GCS{
 		opts: opts,
@@ -241,22 +263,21 @@ func (g *GCS) Copy(src, dst string) error {
 func (g *GCS) gcsClient() (*gcs.Client, error) {
 	ctx := context.Background()
 
-	if g.opts.Credentials.ProjectID == "" || g.opts.Credentials.PrivateKey == "" {
-		return nil, errors.New("projectID and privateKey are required for GCS credentials")
+	if g.opts.Credentials.PrivateKey == "" || g.opts.Credentials.ClientEmail == "" {
+		return nil, errors.New("clientEmail and privateKey are required for GCS credentials")
 	}
 
 	creds, err := json.Marshal(ServiceAccountCredentials{
 		Type:                "service_account",
-		ProjectID:           g.opts.Credentials.ProjectID,
 		PrivateKey:          g.opts.Credentials.PrivateKey,
-		ClientEmail:         fmt.Sprintf("service@%s.iam.gserviceaccount.com", g.opts.Credentials.ProjectID),
+		ClientEmail:         g.opts.Credentials.ClientEmail,
 		AuthURI:             "https://accounts.google.com/o/oauth2/auth",
 		TokenURI:            "https://oauth2.googleapis.com/token",
 		UniverseDomain:      "googleapis.com",
 		AuthProviderCertURL: "https://www.googleapis.com/oauth2/v1/certs",
 		ClientCertURL: fmt.Sprintf(
-			"https://www.googleapis.com/robot/v1/metadata/x509/%s.iam.gserviceaccount.com",
-			g.opts.Credentials.ProjectID,
+			"https://www.googleapis.com/robot/v1/metadata/x509/%s",
+			g.opts.Credentials.ClientEmail,
 		),
 	})
 	if err != nil {