fix: argocd sync start/ finish validation

Author: Ash-exp

pkg/app/AppService.go

@@ -74,14 +74,14 @@ import (
 type AppServiceConfig struct {
 	CdPipelineStatusCronTime                   string `env:"CD_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description:"Cron time for CD pipeline status"`
 	CdHelmPipelineStatusCronTime               string `env:"CD_HELM_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description:"Cron time to check the pipeline status "`
-	CdPipelineStatusTimeoutDuration            string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20" description:"Timeout for CD pipeline to get healthy" `              // in minutes
-	PipelineDegradedTime                       string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10" description:"Time to mark a pipeline degraded if not healthy in defined time"`                                // in minutes
-	GetPipelineDeployedWithinHours             int    `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12" description:"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses."` // in hours
-	HelmPipelineStatusCheckEligibleTime        string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120" description:"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle."`             // in seconds
+	CdPipelineStatusTimeoutDuration            string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20" description:"Timeout for CD pipeline to get healthy" `                                                                                                                                                                                                               // in minutes
+	PipelineDegradedTime                       string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10" description:"Time to mark a pipeline degraded if not healthy in defined time"`                                                                                                                                                                                                    // in minutes
+	GetPipelineDeployedWithinHours             int    `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12" description:"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses."`              // in hours
+	HelmPipelineStatusCheckEligibleTime        string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120" description:"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle."` // in seconds
 	ExposeCDMetrics                            bool   `env:"EXPOSE_CD_METRICS" envDefault:"false"`
-	DevtronChartHelmInstallRequestTimeout      int    `env:"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT" envDefault:"6" description:"Context timeout for no gitops concurrent async deployments"`         // in minutes
-	DevtronChartArgoCdInstallRequestTimeout    int    `env:"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT" envDefault:"1" description:"Context timeout for gitops concurrent async deployments"` // in minutes
-	ArgoCdManualSyncCronPipelineDeployedBefore int    `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3" description:"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)"`                     // in minutes
+	DevtronChartHelmInstallRequestTimeout      int    `env:"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT" envDefault:"6" description:"Context timeout for no gitops concurrent async deployments"`                                                         // in minutes
+	DevtronChartArgoCdInstallRequestTimeout    int    `env:"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT" envDefault:"1" description:"Context timeout for gitops concurrent async deployments"`                                                    // in minutes
+	ArgoCdManualSyncCronPipelineDeployedBefore int    `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3" description:"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)"` // in minutes
 }
 
 func GetAppServiceConfig() (*AppServiceConfig, error) {
@@ -555,10 +555,15 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap
 		if err != nil {
 			impl.logger.Errorw("error in save/update pipeline status fetch detail", "err", err, "cdWfrId", runnerHistoryId)
 		}
+		syncStartTime, found := helper.GetSyncStartTime(app)
+		if !found {
+			impl.logger.Warnw("sync operation not started yet", "app", app)
+			return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, fmt.Errorf("sync operation not started yet")
+		}
 		// creating cd pipeline status timeline
 		timeline := &pipelineConfig.PipelineStatusTimeline{
 			CdWorkflowRunnerId: runnerHistoryId,
-			StatusTime:         helper.GetSyncStartTime(app, statusTime),
+			StatusTime:         syncStartTime,
 			AuditLog: sql.AuditLog{
 				CreatedBy: 1,
 				CreatedOn: time.Now(),
@@ -591,7 +596,12 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap
 			timeline.Id = 0
 			timeline.Status = timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED
 			timeline.StatusDetail = app.Status.OperationState.Message
-			timeline.StatusTime = helper.GetSyncFinishTime(app, statusTime)
+			syncFinishTime, found := helper.GetSyncFinishTime(app)
+			if !found {
+				impl.logger.Warnw("sync operation not found for the deployment", "app", app)
+				return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, fmt.Errorf("sync operation not found for the deployment")
+			}
+			timeline.StatusTime = syncFinishTime
 			// checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times
 			err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil)
 			if err != nil {
@@ -669,10 +679,15 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap
 		if err != nil {
 			impl.logger.Errorw("error in save/update pipeline status fetch detail", "err", err, "installedAppVersionHistoryId", runnerHistoryId)
 		}
+		syncStartTime, found := helper.GetSyncStartTime(app)
+		if !found {
+			impl.logger.Warnw("sync operation not started yet", "app", app)
+			return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, fmt.Errorf("sync operation not started yet")
+		}
 		// creating installedAppVersionHistory status timeline
 		timeline := &pipelineConfig.PipelineStatusTimeline{
 			InstalledAppVersionHistoryId: runnerHistoryId,
-			StatusTime:                   helper.GetSyncStartTime(app, statusTime),
+			StatusTime:                   syncStartTime,
 			AuditLog: sql.AuditLog{
 				CreatedBy: 1,
 				CreatedOn: time.Now(),
@@ -705,7 +720,12 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap
 			timeline.Id = 0
 			timeline.Status = timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED
 			timeline.StatusDetail = app.Status.OperationState.Message
-			timeline.StatusTime = helper.GetSyncFinishTime(app, statusTime)
+			syncFinishTime, found := helper.GetSyncFinishTime(app)
+			if !found {
+				impl.logger.Warnw("sync operation not found for the deployment", "app", app)
+				return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, fmt.Errorf("sync operation not found for the deployment")
+			}
+			timeline.StatusTime = syncFinishTime
 			// checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times
 			err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil)
 			if err != nil {

pkg/argoApplication/helper/deploymentStatusHelper.go

@@ -2,42 +2,41 @@ package helper
 
 import (
 	"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"time"
 )
 
 // GetSyncStartTime assumes that it is always called for calculating start time of latest git hash
-func GetSyncStartTime(app *v1alpha1.Application, defaultStartTime time.Time) time.Time {
-	startTime := metav1.NewTime(defaultStartTime)
+func GetSyncStartTime(app *v1alpha1.Application) (time.Time, bool) {
 	gitHash := app.Status.Sync.Revision
 	if app.Status.OperationState != nil &&
 		app.Status.OperationState.Operation.Sync != nil &&
 		app.Status.OperationState.Operation.Sync.Revision == gitHash {
-		startTime = app.Status.OperationState.StartedAt
+		return app.Status.OperationState.StartedAt.Time, true
 	} else if len(app.Status.History) != 0 {
 		if app.Status.History.LastRevisionHistory().Revision == gitHash &&
 			app.Status.History.LastRevisionHistory().DeployStartedAt != nil {
-			startTime = *app.Status.History.LastRevisionHistory().DeployStartedAt
+			startTime := *app.Status.History.LastRevisionHistory().DeployStartedAt
+			return startTime.Time, true
 		}
 	}
-	return startTime.Time
+	return time.Time{}, false
 }
 
 // GetSyncFinishTime assumes that it is always called for calculating finish time of latest git hash
-func GetSyncFinishTime(app *v1alpha1.Application, defaultEndTime time.Time) time.Time {
-	finishTime := metav1.NewTime(defaultEndTime)
+func GetSyncFinishTime(app *v1alpha1.Application) (time.Time, bool) {
 	gitHash := app.Status.Sync.Revision
 	if app.Status.OperationState != nil &&
 		app.Status.OperationState.Operation.Sync != nil &&
 		app.Status.OperationState.Operation.Sync.Revision == gitHash &&
 		app.Status.OperationState.FinishedAt != nil {
-		finishTime = *app.Status.OperationState.FinishedAt
-	} else if app.Status.History != nil {
-		for _, history := range app.Status.History {
-			if history.Revision == gitHash {
-				finishTime = history.DeployedAt
-			}
+		finishTime := *app.Status.OperationState.FinishedAt
+		return finishTime.Time, true
+	} else if len(app.Status.History) != 0 {
+		if app.Status.History.LastRevisionHistory().Revision == gitHash &&
+			app.Status.History.LastRevisionHistory().DeployStartedAt != nil {
+			finishTime := *app.Status.History.LastRevisionHistory().DeployStartedAt
+			return finishTime.Time, true
 		}
 	}
-	return finishTime.Time
+	return time.Time{}, false
 }

pkg/workflow/status/WorkflowStatusService.go

@@ -455,7 +455,8 @@ func (impl *WorkflowStatusServiceImpl) CheckAndSendArgoPipelineStatusSyncEventIf
 	}
 
 	// pipelineId can be cdPipelineId or installedAppVersionId, using isAppStoreApplication flag to identify between them
-	if lastSyncTime.IsZero() || (!lastSyncTime.IsZero() && time.Since(lastSyncTime) > 5*time.Second) { // create new nats event
+	if lastSyncTime.IsZero() || (!lastSyncTime.IsZero() && time.Since(lastSyncTime) > 5*time.Second) {
+		// create new nats event
 		err = impl.cdPipelineEventPublishService.PublishArgoTypePipelineSyncEvent(pipelineId, installedAppVersionId, userId, isAppStoreApplication)
 		if err != nil {
 			impl.logger.Errorw("error, PublishArgoTypePipelineSyncEvent", "err", err)