Merge branch 'develop' into audit-trigger

# Conflicts:
#	env_gen.json

Author: prakash100198

client/argocdServer/ArgoClientWrapperService.go

@@ -51,9 +51,9 @@ import (
 )
 
 type ACDConfig struct {
-	ArgoCDAutoSyncEnabled     bool `env:"ARGO_AUTO_SYNC_ENABLED" envDefault:"true" description:"If enabled all argocd application will have auto sync enabled"` // will gradually switch this flag to false in enterprise
-	RegisterRepoMaxRetryCount int  `env:"ARGO_REPO_REGISTER_RETRY_COUNT" envDefault:"3" description:"Argo app registration in argo retries on deployment"`
-	RegisterRepoMaxRetryDelay int  `env:"ARGO_REPO_REGISTER_RETRY_DELAY" envDefault:"10" description:"Argo app registration in argo cd on deployment delay between retry"`
+	ArgoCDAutoSyncEnabled     bool `env:"ARGO_AUTO_SYNC_ENABLED" envDefault:"true" description:"If enabled all argocd application will have auto sync enabled" example:"true" deprecated:"false"` // will gradually switch this flag to false in enterprise
+	RegisterRepoMaxRetryCount int  `env:"ARGO_REPO_REGISTER_RETRY_COUNT" envDefault:"4" description:"Retry count for registering a GitOps repository to ArgoCD" example:"3" deprecated:"false"`
+	RegisterRepoMaxRetryDelay int  `env:"ARGO_REPO_REGISTER_RETRY_DELAY" envDefault:"5" description:"Delay (in Seconds) between the retries for registering a GitOps repository to ArgoCD" example:"5" deprecated:"false"`
 }
 
 func (config *ACDConfig) IsManualSyncEnabled() bool {
@@ -105,7 +105,7 @@ type ApplicationClientWrapper interface {
 	// IsArgoAppPatchRequired decides weather the v1alpha1.ApplicationSource requires to be updated
 	IsArgoAppPatchRequired(argoAppSpec *v1alpha1.ApplicationSource, currentGitRepoUrl, currentTargetRevision, currentChartPath string) bool
 
-	// GetGitOpsRepoName returns the GitOps repository name, configured for the argoCd app
+	// GetGitOpsRepoNameForApplication returns the GitOps repository name, configured for the argoCd app
 	GetGitOpsRepoNameForApplication(ctx context.Context, appName string) (gitOpsRepoName string, err error)
 
 	GetGitOpsRepoURLForApplication(ctx context.Context, appName string) (gitOpsRepoURL string, err error)
@@ -127,6 +127,7 @@ type RepoCredsClientWrapper interface {
 type CertificateClientWrapper interface {
 	CreateCertificate(ctx context.Context, query *certificate.RepositoryCertificateCreateRequest) (*v1alpha1.RepositoryCertificateList, error)
 	DeleteCertificate(ctx context.Context, query *certificate.RepositoryCertificateQuery, opts ...grpc.CallOption) (*v1alpha1.RepositoryCertificateList, error)
+	CertificateClientWrapperEnt
 }
 
 type ClusterClientWrapper interface {
@@ -147,7 +148,7 @@ type ArgoClientWrapperServiceImpl struct {
 	repositoryService       repository.ServiceClient
 	clusterClient           cluster.ServiceClient
 	repoCredsClient         repocreds2.ServiceClient
-	CertificateClient       certificate2.ServiceClient
+	certificateClient       certificate2.ServiceClient
 	logger                  *zap.SugaredLogger
 	ACDConfig               *ACDConfig
 	gitOpsConfigReadService config.GitOpsConfigReadService
@@ -176,7 +177,7 @@ func NewArgoClientWrapperServiceImpl(
 		repositoryService:              repositoryService,
 		clusterClient:                  clusterClient,
 		repoCredsClient:                repocredsClient,
-		CertificateClient:              CertificateClient,
+		certificateClient:              CertificateClient,
 		logger:                         logger,
 		ACDConfig:                      ACDConfig,
 		gitOpsConfigReadService:        gitOpsConfigReadService,
@@ -521,7 +522,7 @@ func (impl *ArgoClientWrapperServiceImpl) createRepoInArgoCd(ctx context.Context
 	}
 	repo, err := impl.repositoryService.Create(ctx, grpcConfig, &repository2.RepoCreateRequest{Repo: repo, Upsert: true})
 	if err != nil {
-		impl.logger.Errorw("error in creating argo Repository", "err", err)
+		impl.logger.Errorw("error in creating argo Repository", "url", gitOpsRepoUrl, "err", err)
 		return err
 	}
 	return nil
@@ -530,7 +531,8 @@ func (impl *ArgoClientWrapperServiceImpl) createRepoInArgoCd(ctx context.Context
 // isRetryableArgoRepoCreationError returns whether to retry or not, based on the error returned from callback func
 // In createRepoInArgoCd, we will retry only if the error matches to bean.ArgoRepoSyncDelayErr
 func (impl *ArgoClientWrapperServiceImpl) isRetryableArgoRepoCreationError(argoCdErr error) bool {
-	return strings.Contains(argoCdErr.Error(), bean.ArgoRepoSyncDelayErr)
+	return strings.Contains(argoCdErr.Error(), bean.ArgoRepoSyncDelayErr) ||
+		strings.Contains(argoCdErr.Error(), bean.ArgoRepoSyncDelayErrOld)
 }
 
 // handleArgoRepoCreationError - manages the error thrown while performing createRepoInArgoCd
@@ -543,15 +545,18 @@ func (impl *ArgoClientWrapperServiceImpl) handleArgoRepoCreationError(ctx contex
 		}
 	}
 	if isEmptyRepoError {
-		// - found empty repository, create some file in repository
+		impl.logger.Infow("handling for empty repo", "url", gitOpsRepoUrl)
+		// - found empty repository (there is no origin/HEAD)
+		// - create new commit on HEAD (default branch) with a README file
+		// - then register the repository in ArgoCD
 		gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(gitOpsRepoUrl)
-		err := impl.gitOperationService.CreateReadmeInGitRepo(ctx, gitOpsRepoName, targetRevision, userId)
+		err := impl.gitOperationService.CreateFirstCommitOnHead(ctx, gitOpsRepoName, userId)
 		if err != nil {
 			impl.logger.Errorw("error in creating file in git repo", "err", err)
 			return err
 		}
 	}
-	// try to register with after creating readme file
+	// try to register with after commiting a file to origin/HEAD
 	return impl.createRepoInArgoCd(ctx, grpcConfig, gitOpsRepoUrl)
 }
 
@@ -586,13 +591,13 @@ func (impl *ArgoClientWrapperServiceImpl) CreateCertificate(ctx context.Context,
 	if err != nil {
 		return nil, err
 	}
-	return impl.CertificateClient.CreateCertificate(ctx, grpcConfig, query)
+	return impl.certificateClient.CreateCertificate(ctx, grpcConfig, query)
 }
 
 func (impl *ArgoClientWrapperServiceImpl) DeleteCertificate(ctx context.Context, query *certificate.RepositoryCertificateQuery, opts ...grpc.CallOption) (*v1alpha1.RepositoryCertificateList, error) {
 	grpcConfig, err := impl.acdConfigGetter.GetGRPCConfig()
 	if err != nil {
 		return nil, err
 	}
-	return impl.CertificateClient.DeleteCertificate(ctx, grpcConfig, query, opts...)
+	return impl.certificateClient.DeleteCertificate(ctx, grpcConfig, query, opts...)
 }

client/argocdServer/ArgoClientWrapperService_ent.go

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package argocdServer
+
+type CertificateClientWrapperEnt interface {
+}

client/argocdServer/bean/bean.go

@@ -46,7 +46,9 @@ const RegisterRepoMaxRetryCount = 3
 var EmptyRepoErrorList = []string{"failed to get index: 404 Not Found", "remote repository is empty"}
 
 // ArgoRepoSyncDelayErr - This error occurs inconsistently; ArgoCD requires 80-120s after last commit for create repository operation
-const ArgoRepoSyncDelayErr = "Unable to resolve 'HEAD' to a commit SHA"
+// Error message reference: https://github.com/argoproj/argo-cd/blob/master/util/git/client.go#L718
+const ArgoRepoSyncDelayErr = "unable to resolve 'HEAD' to a commit SHA"    // argocd version <= v2.13.0
+const ArgoRepoSyncDelayErrOld = "Unable to resolve 'HEAD' to a commit SHA" // argocd version > v2.13.0
 
 const (
 	Degraded    = "Degraded"

client/argocdServer/k8sClient.go

@@ -227,7 +227,6 @@ func (impl ArgoK8sClientImpl) DeleteArgoApplication(ctx context.Context, k8sConf
 	patchType := types.MergePatchType
 	patchJSON := ""
 
-	//TODO: ayush test cascade delete
 	if cascadeDelete {
 		patchJSON = `{"metadata": {"finalizers": ["resources-finalizer.argocd.argoproj.io"]}}`
 	} else {

client/telemetry/TelemetryEventClient.go

@@ -24,12 +24,8 @@ import (
 	"fmt"
 	cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier"
 	posthogTelemetry "github.com/devtron-labs/common-lib/telemetry"
-	"github.com/devtron-labs/common-lib/utils/k8s/commonBean"
 	"github.com/devtron-labs/devtron/api/helm-app/gRPC"
 	installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read"
-	bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean"
-	"github.com/devtron-labs/devtron/pkg/auth/user/bean"
-	bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean"
 	module2 "github.com/devtron-labs/devtron/pkg/module/bean"
 	ucidService "github.com/devtron-labs/devtron/pkg/ucid"
 	cron3 "github.com/devtron-labs/devtron/util/cron"
@@ -49,9 +45,7 @@ import (
 	"github.com/go-pg/pg"
 	"github.com/posthog/posthog-go"
 	"github.com/robfig/cron/v3"
-	"github.com/tidwall/gjson"
 	"go.uber.org/zap"
-	"k8s.io/apimachinery/pkg/version"
 )
 
 type TelemetryEventClientImpl struct {
@@ -154,115 +148,6 @@ func (impl *TelemetryEventClientImpl) StopCron() {
 	impl.cron.Stop()
 }
 
-func (impl *TelemetryEventClientImpl) SummaryDetailsForTelemetry() (cluster []bean3.ClusterBean, user []bean.UserInfo,
-	k8sServerVersion *version.Info, hostURL bool, ssoSetup bool, HelmAppAccessCount string, ChartStoreVisitCount string,
-	SkippedOnboarding bool, HelmAppUpdateCounter string, helmChartSuccessfulDeploymentCount int, ExternalHelmAppClusterCount map[int32]int) {
-
-	discoveryClient, err := impl.K8sUtil.GetK8sDiscoveryClientInCluster()
-	if err != nil {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return
-	}
-	k8sServerVersion, err = discoveryClient.ServerVersion()
-	if err != nil {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return
-	}
-
-	users, err := impl.userService.GetAll()
-	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("exception caught inside telemetry summery event", "err", err)
-		return
-	}
-
-	clusters, err := impl.clusterService.FindAllActive()
-
-	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return
-	}
-
-	hostURL = false
-
-	attribute, err := impl.attributeRepo.FindByKey(bean2.HostUrlKey)
-	if err == nil && attribute.Id > 0 {
-		hostURL = true
-	}
-
-	attribute, err = impl.attributeRepo.FindByKey("HelmAppAccessCounter")
-
-	if err == nil {
-		HelmAppAccessCount = attribute.Value
-	}
-
-	attribute, err = impl.attributeRepo.FindByKey("ChartStoreVisitCount")
-
-	if err == nil {
-		ChartStoreVisitCount = attribute.Value
-	}
-
-	attribute, err = impl.attributeRepo.FindByKey("HelmAppUpdateCounter")
-
-	if err == nil {
-		HelmAppUpdateCounter = attribute.Value
-	}
-
-	helmChartSuccessfulDeploymentCount, err = impl.installedAppReadService.GetDeploymentSuccessfulStatusCountForTelemetry()
-
-	//externalHelmCount := make(map[int32]int)
-	ExternalHelmAppClusterCount = make(map[int32]int)
-
-	for _, clusterDetail := range clusters {
-		req := &gRPC.AppListRequest{}
-		config := &gRPC.ClusterConfig{
-			ApiServerUrl:          clusterDetail.ServerUrl,
-			Token:                 clusterDetail.Config[commonBean.BearerToken],
-			ClusterId:             int32(clusterDetail.Id),
-			ClusterName:           clusterDetail.ClusterName,
-			InsecureSkipTLSVerify: clusterDetail.InsecureSkipTLSVerify,
-		}
-
-		if clusterDetail.InsecureSkipTLSVerify == false {
-			config.KeyData = clusterDetail.Config[commonBean.TlsKey]
-			config.CertData = clusterDetail.Config[commonBean.CertData]
-			config.CaData = clusterDetail.Config[commonBean.CertificateAuthorityData]
-		}
-		req.Clusters = append(req.Clusters, config)
-		applicationStream, err := impl.helmAppClient.ListApplication(context.Background(), req)
-		if err == nil {
-			clusterList, err1 := applicationStream.Recv()
-			if err1 != nil {
-				impl.logger.Errorw("error in list helm applications streams recv", "err", err)
-			}
-			if err1 != nil && clusterList != nil && !clusterList.Errored {
-				ExternalHelmAppClusterCount[clusterList.ClusterId] = len(clusterList.DeployedAppDetail)
-			}
-		} else {
-			impl.logger.Errorw("error while fetching list application from kubelink", "err", err)
-		}
-	}
-
-	//getting userData from emailId
-	userData, err := impl.userAttributesRepository.GetUserDataByEmailId(AdminEmailIdConst)
-
-	SkippedOnboardingValue := gjson.Get(userData, SkippedOnboardingConst).Str
-
-	if SkippedOnboardingValue == "true" {
-		SkippedOnboarding = true
-	} else {
-		SkippedOnboarding = false
-	}
-
-	ssoSetup = false
-
-	ssoConfig, err := impl.ssoLoginService.GetAll()
-	if err == nil && len(ssoConfig) > 0 {
-		ssoSetup = true
-	}
-
-	return clusters, users, k8sServerVersion, hostURL, ssoSetup, HelmAppAccessCount, ChartStoreVisitCount, SkippedOnboarding, HelmAppUpdateCounter, helmChartSuccessfulDeploymentCount, ExternalHelmAppClusterCount
-}
-
 // New methods for collecting additional telemetry metrics
 
 func (impl *TelemetryEventClientImpl) SummaryEventForTelemetryEA() {
@@ -291,7 +176,7 @@ func (impl *TelemetryEventClientImpl) SendSummaryEvent(eventType string) error {
 		return err
 	}
 
-	clusters, users, k8sServerVersion, hostURL, ssoSetup, HelmAppAccessCount, ChartStoreVisitCount, SkippedOnboarding, HelmAppUpdateCounter, helmChartSuccessfulDeploymentCount, ExternalHelmAppClusterCount := impl.SummaryDetailsForTelemetry()
+	clusters, users, k8sServerVersion, hostURL, ssoSetup, HelmAppAccessCount, ChartStoreVisitCount, SkippedOnboarding, HelmAppUpdateCounter, helmChartSuccessfulDeploymentCount, ExternalHelmAppClusterCount := impl.GetSummaryDetailsForTelemetry()
 
 	payload := &TelemetryEventEA{UCID: ucid, Timestamp: time.Now(), EventType: TelemetryEventType(eventType), DevtronVersion: "v1"}
 	payload.ServerVersion = k8sServerVersion.String()

client/telemetry/TelemetryEventClientExtended.go

@@ -182,7 +182,7 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string)
 		return err
 	}
 
-	clusters, users, k8sServerVersion, hostURL, ssoSetup, HelmAppAccessCount, ChartStoreVisitCount, SkippedOnboarding, HelmAppUpdateCounter, HelmChartSuccessfulDeploymentCount, ExternalHelmAppClusterCount := impl.SummaryDetailsForTelemetry()
+	clusters, users, k8sServerVersion, hostURL, ssoSetup, HelmAppAccessCount, ChartStoreVisitCount, SkippedOnboarding, HelmAppUpdateCounter, HelmChartSuccessfulDeploymentCount, ExternalHelmAppClusterCount := impl.GetSummaryDetailsForTelemetry()
 	payload := &TelemetryEventDto{UCID: ucid, Timestamp: time.Now(), EventType: TelemetryEventType(eventType), DevtronVersion: "v1"}
 	payload.ServerVersion = k8sServerVersion.String()
 
@@ -345,12 +345,39 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string)
 	payload.JobPipelineTriggeredLast24h = impl.getJobPipelineTriggeredLast24h()
 	payload.JobPipelineSucceededLast24h = impl.getJobPipelineSucceededLast24h()
 	payload.UserCreatedPluginCount = impl.getUserCreatedPluginCount()
-	payload.PolicyCount = impl.getPolicyCount()
+	payload.DeploymentWindowPolicyCount = impl.getDeploymentWindowPolicyCount()
+	payload.ApprovalPolicyCount = impl.getApprovalPolicyCount()
+	payload.PluginPolicyCount = impl.getPluginPolicyCount()
+	payload.TagsPolicyCount = impl.getTagsPolicyCount()
+	payload.FilterConditionPolicyCount = impl.getFilterConditionPolicyCount()
+	payload.LockDeploymentConfigurationPolicyCount = impl.getLockDeploymentConfigurationPolicyCount()
 	payload.AppliedPolicyRowCount = impl.getAppliedPolicyRowCount()
 	payload.PhysicalClusterCount, payload.IsolatedClusterCount = impl.getClusterCounts()
 	payload.ActiveUsersLast30Days = impl.getActiveUsersLast30Days()
 	payload.GitOpsPipelineCount = impl.getGitOpsPipelineCount()
 	payload.HelmPipelineCount = impl.helmPipelineCount()
+	payload.ProjectsWithZeroAppsCount = impl.getProjectsWithZeroAppsCount()
+	payload.AppsWithPropagationTagsCount = impl.getAppsWithPropagationTagsCount()
+	payload.AppsWithNonPropagationTagsCount = impl.getAppsWithNonPropagationTagsCount()
+	payload.AppsWithDescriptionCount = impl.getAppsWithDescriptionCount()
+	payload.AppsWithCatalogDataCount = impl.getAppsWithCatalogDataCount()
+	payload.AppsWithReadmeDataCount = impl.getAppsWithReadmeDataCount()
+	payload.HighestEnvironmentCountInApp = impl.getHighestEnvironmentCountInApp()
+	payload.HighestAppCountInEnvironment = impl.getHighestAppCountInEnvironment()
+	payload.HighestWorkflowCountInApp = impl.getHighestWorkflowCountInApp()
+	payload.HighestEnvironmentCountInWorkflow = impl.getHighestEnvironmentCountInWorkflow()
+	payload.HighestGitRepoCountInApp = impl.getHighestGitRepoCountInApp()
+	payload.AppsWithIncludeExcludeFilesCount = impl.getAppsWithIncludeExcludeFilesCount()
+	payload.DockerfileLanguagesList = impl.getDockerfileLanguagesList()
+	payload.BuildpackLanguagesList = impl.getBuildpackLanguagesList()
+	payload.AppsWithDeploymentChartCount = impl.getAppsWithDeploymentChartCount()
+	payload.AppsWithRolloutChartCount = impl.getAppsWithRolloutChartCount()
+	payload.AppsWithStatefulsetCount = impl.getAppsWithStatefulsetCount()
+	payload.AppsWithJobsCronjobsCount = impl.getAppsWithJobsCronjobsCount()
+	payload.EnvironmentsWithPatchStrategyCount = impl.getEnvironmentsWithPatchStrategyCount()
+	payload.EnvironmentsWithReplaceStrategyCount = impl.getEnvironmentsWithReplaceStrategyCount()
+	payload.ExternalConfigMapCount = impl.getExternalConfigMapCount()
+	payload.InternalConfigMapCount = impl.getInternalConfigMapCount()
 
 	payload.ClusterProvider, err = impl.GetCloudProvider()
 	if err != nil {