FEATURE: Unique client id for telemetry events (#529)

* telemetry api added for dashboard to get ucid

* code refactored

* code refacored for client id api for posthog

* review changes on posthog client id api for dashboard

* cache implemented for configmap value for cient id

Author: vikramdevtron

Wire.go

@@ -652,6 +652,11 @@ func InitializeApp() (*App, error) {
 		wire.Bind(new(router.SsoLoginRouter), new(*router.SsoLoginRouterImpl)),
 		restHandler.NewSsoLoginRestHandlerImpl,
 		wire.Bind(new(restHandler.SsoLoginRestHandler), new(*restHandler.SsoLoginRestHandlerImpl)),
+
+		router.NewTelemetryRouterImpl,
+		wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)),
+		restHandler.NewTelemetryRestHandlerImpl,
+		wire.Bind(new(restHandler.TelemetryRestHandler), new(*restHandler.TelemetryRestHandlerImpl)),
 		telemetry.NewPosthogClient,
 
 		telemetry.NewTelemetryEventClientImpl,

api/restHandler/TelemetryRestHandler.go

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2020 Devtron Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package restHandler
+
+import (
+	"github.com/devtron-labs/devtron/client/telemetry"
+	"go.uber.org/zap"
+	"net/http"
+)
+
+type TelemetryRestHandler interface {
+	GetClientPlatformIdAndTelemetryUrl(w http.ResponseWriter, r *http.Request)
+}
+
+type TelemetryRestHandlerImpl struct {
+	logger               *zap.SugaredLogger
+	telemetryEventClient telemetry.TelemetryEventClient
+}
+
+func NewTelemetryRestHandlerImpl(logger *zap.SugaredLogger,
+	telemetryEventClient telemetry.TelemetryEventClient) *TelemetryRestHandlerImpl {
+	handler := &TelemetryRestHandlerImpl{logger: logger, telemetryEventClient: telemetryEventClient}
+	return handler
+}
+
+func (handler TelemetryRestHandlerImpl) GetClientPlatformIdAndTelemetryUrl(w http.ResponseWriter, r *http.Request) {
+	res, err := handler.telemetryEventClient.GetClientPlatformIdAndTelemetryUrl()
+	if err != nil {
+		handler.logger.Errorw("service err, GetClientPlatformIdAndTelemetryUrl", "err", err)
+		writeJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
+	}
+	writeJsonResp(w, nil, res, http.StatusOK)
+}

api/router/TelemetryRouter.go

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2020 Devtron Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package router
+
+import (
+	"github.com/devtron-labs/devtron/api/restHandler"
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+)
+
+type TelemetryRouter interface {
+	initTelemetryRouter(router *mux.Router)
+}
+
+type TelemetryRouterImpl struct {
+	logger  *zap.SugaredLogger
+	handler restHandler.TelemetryRestHandler
+}
+
+func NewTelemetryRouterImpl(logger *zap.SugaredLogger, handler restHandler.TelemetryRestHandler) *TelemetryRouterImpl {
+	router := &TelemetryRouterImpl{
+		handler: handler,
+	}
+	return router
+}
+
+func (router TelemetryRouterImpl) initTelemetryRouter(telemetryRouter *mux.Router) {
+	telemetryRouter.Path("/ucid").
+		HandlerFunc(router.handler.GetClientPlatformIdAndTelemetryUrl).Methods("GET")
+}

api/router/router.go

@@ -73,6 +73,7 @@ type MuxRouter struct {
 	commonRouter                     CommonRouter
 	grafanaRouter                    GrafanaRouter
 	ssoLoginRouter                   SsoLoginRouter
+	telemetryRouter                  TelemetryRouter
 	telemetryWatcher                 telemetry.TelemetryEventClient
 }
 
@@ -92,7 +93,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter HelmRouter, PipelineConf
 	ReleaseMetricsRouter ReleaseMetricsRouter, deploymentGroupRouter DeploymentGroupRouter, batchOperationRouter BatchOperationRouter,
 	chartGroupRouter ChartGroupRouter, testSuitRouter TestSuitRouter, imageScanRouter ImageScanRouter,
 	policyRouter PolicyRouter, gitOpsConfigRouter GitOpsConfigRouter, dashboardRouter DashboardRouter, attributesRouter AttributesRouter,
-	commonRouter CommonRouter, grafanaRouter GrafanaRouter, ssoLoginRouter SsoLoginRouter, telemetryWatcher telemetry.TelemetryEventClient) *MuxRouter {
+	commonRouter CommonRouter, grafanaRouter GrafanaRouter, ssoLoginRouter SsoLoginRouter, telemetryRouter TelemetryRouter, telemetryWatcher telemetry.TelemetryEventClient) *MuxRouter {
 	r := &MuxRouter{
 		Router:                           mux.NewRouter(),
 		HelmRouter:                       HelmRouter,
@@ -136,6 +137,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter HelmRouter, PipelineConf
 		commonRouter:                     commonRouter,
 		grafanaRouter:                    grafanaRouter,
 		ssoLoginRouter:                   ssoLoginRouter,
+		telemetryRouter:                  telemetryRouter,
 		telemetryWatcher:                 telemetryWatcher,
 	}
 	return r
@@ -259,4 +261,7 @@ func (r MuxRouter) Init() {
 
 	ssoLoginRouter := r.Router.PathPrefix("/orchestrator/sso").Subrouter()
 	r.ssoLoginRouter.initSsoLoginRouter(ssoLoginRouter)
+
+	telemetryRouter := r.Router.PathPrefix("/orchestrator/telemetry").Subrouter()
+	r.telemetryRouter.initTelemetryRouter(telemetryRouter)
 }

client/telemetry/PosthogClient.go

@@ -19,19 +19,23 @@ package telemetry
 
 import (
 	"github.com/caarlos0/env"
+	"github.com/patrickmn/go-cache"
 	"github.com/posthog/posthog-go"
 	"go.uber.org/zap"
+	"time"
 )
 
 type PosthogClient struct {
 	Client posthog.Client
+	cache  *cache.Cache
 }
 
 type PosthogConfig struct {
 	ApiKey            string `env:"API_KEY" envDefault:""`
 	PosthogEndpoint   string `env:"POSTHOG_ENDPOINT" envDefault:"https://app.posthog.com"`
 	SummaryInterval   int    `env:"SUMMARY_INTERVAL" envDefault:"24"`
 	HeartbeatInterval int    `env:"HEARTBEAT_INTERVAL" envDefault:"3"`
+	CacheExpiry       int    `env:"CACHE_EXPIRY" envDefault:"120"`
 }
 
 func GetPosthogConfig() (*PosthogConfig, error) {
@@ -52,8 +56,11 @@ func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) {
 	}
 	client, _ := posthog.NewWithConfig(cfg.ApiKey, posthog.Config{Endpoint: cfg.PosthogEndpoint})
 	//defer client.Close()
+	d := time.Duration(cfg.CacheExpiry)
+	c := cache.New(d*time.Minute, 240*time.Minute)
 	pgClient := &PosthogClient{
 		Client: client,
+		cache:  c,
 	}
 	return pgClient, nil
 }

client/telemetry/TelemetryEventClient.go

@@ -3,14 +3,14 @@ package telemetry
 import (
 	"encoding/json"
 	"fmt"
-	client "github.com/devtron-labs/devtron/client/events"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	util2 "github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/cluster"
 	"github.com/devtron-labs/devtron/pkg/user"
 	"github.com/devtron-labs/devtron/util"
 	"github.com/go-pg/pg"
+	"github.com/patrickmn/go-cache"
 	"github.com/posthog/posthog-go"
 	"go.uber.org/zap"
 	"google.golang.org/grpc/codes"
@@ -29,7 +29,6 @@ type TelemetryEventClientImpl struct {
 	clusterService       cluster.ClusterService
 	K8sUtil              *util2.K8sUtil
 	aCDAuthConfig        *user.ACDAuthConfig
-	config               *client.EventClientConfig
 	environmentService   cluster.EnvironmentService
 	userService          user.UserService
 	appListingRepository repository.AppListingRepository
@@ -40,10 +39,11 @@ type TelemetryEventClientImpl struct {
 }
 
 type TelemetryEventClient interface {
+	GetClientPlatformIdAndTelemetryUrl() (*PosthogData, error)
 }
 
 func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService,
-	K8sUtil *util2.K8sUtil, aCDAuthConfig *user.ACDAuthConfig, config *client.EventClientConfig,
+	K8sUtil *util2.K8sUtil, aCDAuthConfig *user.ACDAuthConfig,
 	environmentService cluster.EnvironmentService, userService user.UserService,
 	appListingRepository repository.AppListingRepository, PosthogClient *PosthogClient,
 	ciPipelineRepository pipelineConfig.CiPipelineRepository, pipelineRepository pipelineConfig.PipelineRepository,
@@ -55,12 +55,13 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client,
 		cron:   cron,
 		logger: logger,
 		client: client, clusterService: clusterService,
-		K8sUtil: K8sUtil, aCDAuthConfig: aCDAuthConfig, config: config,
+		K8sUtil: K8sUtil, aCDAuthConfig: aCDAuthConfig,
 		environmentService: environmentService, userService: userService,
 		appListingRepository: appListingRepository, PosthogClient: PosthogClient,
 		ciPipelineRepository: ciPipelineRepository, pipelineRepository: pipelineRepository,
 		posthogConfig: posthogConfig,
 	}
+
 	watcher.HeartbeatEventForTelemetry()
 	_, err := cron.AddFunc(fmt.Sprintf("@every %dm", watcher.posthogConfig.SummaryInterval), watcher.SummaryEventForTelemetry)
 	if err != nil {
@@ -122,30 +123,11 @@ func (d TelemetryEventType) String() string {
 }
 
 func (impl *TelemetryEventClientImpl) SummaryEventForTelemetry() {
-	client, err := impl.K8sUtil.GetClientForInCluster()
+	ucid, err := impl.getUCID()
 	if err != nil {
 		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
 		return
 	}
-	cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client)
-	if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown {
-		// if not found, create new cm
-		cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}}
-		data := map[string]string{}
-		data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number
-		cm.Data = data
-		_, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client)
-		if err != nil {
-			return
-		}
-	}
-	if cm == nil {
-		impl.logger.Errorw("cm found nil inside telemetry summary event", "cm", cm)
-		return
-	}
-	dataMap := cm.Data
-	ucid := dataMap[DevtronUniqueClientIdConfigMapKey]
-
 	discoveryClient, err := impl.K8sUtil.GetK8sDiscoveryClientInCluster()
 	if err != nil {
 		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
@@ -229,31 +211,11 @@ func (impl *TelemetryEventClientImpl) SummaryEventForTelemetry() {
 }
 
 func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() {
-	client, err := impl.K8sUtil.GetClientForInCluster()
+	ucid, err := impl.getUCID()
 	if err != nil {
 		impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err)
 		return
 	}
-	cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client)
-	if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown {
-		// if not found, create new cm
-		cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}}
-		data := map[string]string{}
-		data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number
-		cm.Data = data
-		_, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client)
-		if err != nil {
-			impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err)
-			return
-		}
-	}
-	if cm == nil {
-		impl.logger.Errorw("configmap found nil for telemetry heartbeat event", "cm", cm)
-		return
-	}
-	dataMap := cm.Data
-	ucid := dataMap[DevtronUniqueClientIdConfigMapKey]
-
 	discoveryClient, err := impl.K8sUtil.GetK8sDiscoveryClientInCluster()
 	if err != nil {
 		impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err)
@@ -283,3 +245,56 @@ func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() {
 		Properties: prop,
 	})
 }
+
+func (impl *TelemetryEventClientImpl) GetClientPlatformIdAndTelemetryUrl() (*PosthogData, error) {
+	ucid, err := impl.getUCID()
+	if err != nil {
+		impl.logger.Errorw("exception while getting unique client id", "error", err)
+		return nil, err
+	}
+	data := &PosthogData{
+		Url:  impl.posthogConfig.PosthogEndpoint,
+		UCID: ucid,
+	}
+	return data, err
+}
+
+type PosthogData struct {
+	Url  string `json:"url,omitempty"`
+	UCID string `json:"ucid,omitempty"`
+}
+
+func (impl *TelemetryEventClientImpl) getUCID() (string, error) {
+	ucid, found := impl.PosthogClient.cache.Get(DevtronUniqueClientIdConfigMapKey)
+	if found {
+		return ucid.(string), nil
+	} else {
+		client, err := impl.K8sUtil.GetClientForInCluster()
+		if err != nil {
+			impl.logger.Errorw("exception while getting unique client id", "error", err)
+			return "", err
+		}
+
+		cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client)
+		if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown {
+			// if not found, create new cm
+			cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}}
+			data := map[string]string{}
+			data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number
+			cm.Data = data
+			_, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client)
+			if err != nil {
+				impl.logger.Errorw("exception while getting unique client id", "error", err)
+				return "", err
+			}
+		}
+		dataMap := cm.Data
+		ucid = dataMap[DevtronUniqueClientIdConfigMapKey]
+		impl.PosthogClient.cache.Set(DevtronUniqueClientIdConfigMapKey, ucid, cache.DefaultExpiration)
+		if cm == nil {
+			impl.logger.Errorw("configmap not found while getting unique client id", "cm", cm)
+			return ucid.(string), err
+		}
+	}
+	return ucid.(string), nil
+}

pkg/sso/SSOLoginService.go

@@ -23,6 +23,7 @@ import (
 	"github.com/argoproj/argo-cd/util/session"
 	"github.com/devtron-labs/devtron/api/bean"
 	session2 "github.com/devtron-labs/devtron/client/argocdServer/session"
+	"github.com/devtron-labs/devtron/client/telemetry"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
 	"github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/cluster"
@@ -53,13 +54,14 @@ type SSOLoginServiceImpl struct {
 	clusterService      cluster.ClusterService
 	envService          cluster.EnvironmentService
 	aCDAuthConfig       *user.ACDAuthConfig
+	posthogConfig       *telemetry.PosthogConfig
 }
 
 func NewSSOLoginServiceImpl(userAuthRepository repository.UserAuthRepository, sessionManager *session.SessionManager,
 	client session2.ServiceClient, logger *zap.SugaredLogger, userRepository repository.UserRepository,
 	userGroupRepository repository.RoleGroupRepository, ssoLoginRepository repository.SSOLoginRepository,
 	K8sUtil *util.K8sUtil, clusterService cluster.ClusterService, envService cluster.EnvironmentService,
-	aCDAuthConfig *user.ACDAuthConfig) *SSOLoginServiceImpl {
+	aCDAuthConfig *user.ACDAuthConfig, posthogConfig *telemetry.PosthogConfig) *SSOLoginServiceImpl {
 	serviceImpl := &SSOLoginServiceImpl{
 		userAuthRepository:  userAuthRepository,
 		sessionManager:      sessionManager,
@@ -72,6 +74,7 @@ func NewSSOLoginServiceImpl(userAuthRepository repository.UserAuthRepository, se
 		clusterService:      clusterService,
 		envService:          envService,
 		aCDAuthConfig:       aCDAuthConfig,
+		posthogConfig:       posthogConfig,
 	}
 	return serviceImpl
 }