Skip to content

Commit 87f37cb

Browse files
Ash-expAsh-exp
authored
Merge pull request #6569 from devtron-labs/fix-argocd-update-event
fix: anomalies in deployment status timeline
2 parents b9e94fd + 2f9f5f5 commit 87f37cb

File tree

4 files changed

+66
-35
lines changed

4 files changed

+66
-35
lines changed

‎pkg/app/AppService.go‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -74,14 +74,14 @@ import (
7474
type AppServiceConfig struct {
7575
CdPipelineStatusCronTime string `env:"CD_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description:"Cron time for CD pipeline status"`
7676
CdHelmPipelineStatusCronTime string `env:"CD_HELM_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description:"Cron time to check the pipeline status "`
77-
CdPipelineStatusTimeoutDuration string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20" description:"Timeout for CD pipeline to get healthy" ` // in minutes
78-
PipelineDegradedTime string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10" description:"Time to mark a pipeline degraded if not healthy in defined time"` // in minutes
79-
GetPipelineDeployedWithinHours int `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12" description:"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses."` // in hours
80-
HelmPipelineStatusCheckEligibleTime string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120" description:"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle."` // in seconds
77+
CdPipelineStatusTimeoutDuration string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20" description:"Timeout for CD pipeline to get healthy" ` // in minutes
78+
PipelineDegradedTime string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10" description:"Time to mark a pipeline degraded if not healthy in defined time"` // in minutes
79+
GetPipelineDeployedWithinHours int `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12" description:"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses."` // in hours
80+
HelmPipelineStatusCheckEligibleTime string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120" description:"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle."` // in seconds
8181
ExposeCDMetrics bool `env:"EXPOSE_CD_METRICS" envDefault:"false"`
82-
DevtronChartHelmInstallRequestTimeout int `env:"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT" envDefault:"6" description:"Context timeout for no gitops concurrent async deployments"` // in minutes
83-
DevtronChartArgoCdInstallRequestTimeout int `env:"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT" envDefault:"1" description:"Context timeout for gitops concurrent async deployments"` // in minutes
84-
ArgoCdManualSyncCronPipelineDeployedBefore int `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3" description:"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)"` // in minutes
82+
DevtronChartHelmInstallRequestTimeout int `env:"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT" envDefault:"6" description:"Context timeout for no gitops concurrent async deployments"` // in minutes
83+
DevtronChartArgoCdInstallRequestTimeout int `env:"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT" envDefault:"1" description:"Context timeout for gitops concurrent async deployments"` // in minutes
84+
ArgoCdManualSyncCronPipelineDeployedBefore int `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3" description:"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)"` // in minutes
8585
}
8686

8787
func GetAppServiceConfig() (*AppServiceConfig, error) {
@@ -555,10 +555,15 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap
555555
if err != nil {
556556
impl.logger.Errorw("error in save/update pipeline status fetch detail", "err", err, "cdWfrId", runnerHistoryId)
557557
}
558+
syncStartTime, found := helper.GetSyncStartTime(app)
559+
if !found {
560+
impl.logger.Warnw("sync operation not started yet", "app", app)
561+
return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, fmt.Errorf("sync operation not started yet")
562+
}
558563
// creating cd pipeline status timeline
559564
timeline := &pipelineConfig.PipelineStatusTimeline{
560565
CdWorkflowRunnerId: runnerHistoryId,
561-
StatusTime: helper.GetSyncStartTime(app, statusTime),
566+
StatusTime: syncStartTime,
562567
AuditLog: sql.AuditLog{
563568
CreatedBy: 1,
564569
CreatedOn: time.Now(),
@@ -591,7 +596,12 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap
591596
timeline.Id = 0
592597
timeline.Status = timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED
593598
timeline.StatusDetail = app.Status.OperationState.Message
594-
timeline.StatusTime = helper.GetSyncFinishTime(app, statusTime)
599+
syncFinishTime, found := helper.GetSyncFinishTime(app)
600+
if !found {
601+
impl.logger.Warnw("sync operation not found for the deployment", "app", app)
602+
return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, fmt.Errorf("sync operation not found for the deployment")
603+
}
604+
timeline.StatusTime = syncFinishTime
595605
// checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times
596606
err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil)
597607
if err != nil {
@@ -610,6 +620,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap
610620
haveNewTimeline = true
611621
timeline.Status = timelineStatus.TIMELINE_STATUS_APP_HEALTHY
612622
timeline.StatusDetail = "App status is Healthy."
623+
timeline.StatusTime = statusTime
613624
}
614625
if haveNewTimeline {
615626
// not checking if this status is already present or not because already checked for terminal status existence earlier
@@ -668,10 +679,15 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap
668679
if err != nil {
669680
impl.logger.Errorw("error in save/update pipeline status fetch detail", "err", err, "installedAppVersionHistoryId", runnerHistoryId)
670681
}
682+
syncStartTime, found := helper.GetSyncStartTime(app)
683+
if !found {
684+
impl.logger.Warnw("sync operation not started yet", "app", app)
685+
return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, fmt.Errorf("sync operation not started yet")
686+
}
671687
// creating installedAppVersionHistory status timeline
672688
timeline := &pipelineConfig.PipelineStatusTimeline{
673689
InstalledAppVersionHistoryId: runnerHistoryId,
674-
StatusTime: helper.GetSyncStartTime(app, statusTime),
690+
StatusTime: syncStartTime,
675691
AuditLog: sql.AuditLog{
676692
CreatedBy: 1,
677693
CreatedOn: time.Now(),
@@ -704,7 +720,12 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap
704720
timeline.Id = 0
705721
timeline.Status = timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED
706722
timeline.StatusDetail = app.Status.OperationState.Message
707-
timeline.StatusTime = helper.GetSyncFinishTime(app, statusTime)
723+
syncFinishTime, found := helper.GetSyncFinishTime(app)
724+
if !found {
725+
impl.logger.Warnw("sync operation not found for the deployment", "app", app)
726+
return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, fmt.Errorf("sync operation not found for the deployment")
727+
}
728+
timeline.StatusTime = syncFinishTime
708729
// checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times
709730
err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil)
710731
if err != nil {

‎pkg/app/status/PipelineStatusTimelineResourcesService.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,18 +92,24 @@ func (impl *PipelineStatusTimelineResourcesServiceImpl) SaveOrUpdatePipelineTime
9292
if application != nil && application.Status.OperationState != nil && application.Status.OperationState.SyncResult != nil {
9393
for _, resource := range application.Status.OperationState.SyncResult.Resources {
9494
if resource != nil {
95+
resourceStatus := string(resource.HookPhase)
96+
if len(resourceStatus) == 0 {
97+
resourceStatus = string(resource.Status)
98+
}
99+
//if resource is already present in the timelineResources, then update it
95100
if index, ok := oldTimelineResourceMap[resource.Name]; ok {
96-
timelineResources[index].ResourceStatus = string(resource.HookPhase)
101+
timelineResources[index].ResourceStatus = resourceStatus
97102
timelineResources[index].StatusMessage = resource.Message
98103
timelineResources[index].UpdatedBy = userId
99104
timelineResources[index].UpdatedOn = time.Now()
100105
timelineResourcesToBeUpdated = append(timelineResourcesToBeUpdated, timelineResources[index])
101106
} else {
107+
//if resource is not present in the timelineResources, then create a new one
102108
newTimelineResource := &pipelineConfig.PipelineStatusTimelineResources{
103109
ResourceName: resource.Name,
104110
ResourceKind: resource.Kind,
105111
ResourceGroup: resource.Group,
106-
ResourceStatus: string(resource.HookPhase),
112+
ResourceStatus: resourceStatus,
107113
StatusMessage: resource.Message,
108114
AuditLog: sql.AuditLog{
109115
CreatedBy: userId,

‎pkg/argoApplication/helper/deploymentStatusHelper.go‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,38 +2,41 @@ package helper
22

33
import (
44
"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
5-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
65
"time"
76
)
87

98
// GetSyncStartTime assumes that it is always called for calculating start time of latest git hash
10-
func GetSyncStartTime(app *v1alpha1.Application, defaultStartTime time.Time) time.Time {
11-
startTime := metav1.NewTime(defaultStartTime)
9+
func GetSyncStartTime(app *v1alpha1.Application) (time.Time, bool) {
1210
gitHash := app.Status.Sync.Revision
13-
if app.Status.OperationState != nil {
14-
startTime = app.Status.OperationState.StartedAt
15-
} else if app.Status.History != nil {
16-
for _, history := range app.Status.History {
17-
if history.Revision == gitHash {
18-
startTime = *history.DeployStartedAt
19-
}
11+
if app.Status.OperationState != nil &&
12+
app.Status.OperationState.Operation.Sync != nil &&
13+
app.Status.OperationState.Operation.Sync.Revision == gitHash {
14+
return app.Status.OperationState.StartedAt.Time, true
15+
} else if len(app.Status.History) != 0 {
16+
if app.Status.History.LastRevisionHistory().Revision == gitHash &&
17+
app.Status.History.LastRevisionHistory().DeployStartedAt != nil {
18+
startTime := *app.Status.History.LastRevisionHistory().DeployStartedAt
19+
return startTime.Time, true
2020
}
2121
}
22-
return startTime.Time
22+
return time.Time{}, false
2323
}
2424

2525
// GetSyncFinishTime assumes that it is always called for calculating finish time of latest git hash
26-
func GetSyncFinishTime(app *v1alpha1.Application, defaultEndTime time.Time) time.Time {
27-
finishTime := metav1.NewTime(defaultEndTime)
26+
func GetSyncFinishTime(app *v1alpha1.Application) (time.Time, bool) {
2827
gitHash := app.Status.Sync.Revision
29-
if app.Status.OperationState != nil && app.Status.OperationState.FinishedAt != nil {
30-
finishTime = *app.Status.OperationState.FinishedAt
31-
} else if app.Status.History != nil {
32-
for _, history := range app.Status.History {
33-
if history.Revision == gitHash {
34-
finishTime = history.DeployedAt
35-
}
28+
if app.Status.OperationState != nil &&
29+
app.Status.OperationState.Operation.Sync != nil &&
30+
app.Status.OperationState.Operation.Sync.Revision == gitHash &&
31+
app.Status.OperationState.FinishedAt != nil {
32+
finishTime := *app.Status.OperationState.FinishedAt
33+
return finishTime.Time, true
34+
} else if len(app.Status.History) != 0 {
35+
if app.Status.History.LastRevisionHistory().Revision == gitHash &&
36+
app.Status.History.LastRevisionHistory().DeployStartedAt != nil {
37+
finishTime := *app.Status.History.LastRevisionHistory().DeployStartedAt
38+
return finishTime.Time, true
3639
}
3740
}
38-
return finishTime.Time
41+
return time.Time{}, false
3942
}

‎pkg/workflow/status/WorkflowStatusService.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,8 @@ func (impl *WorkflowStatusServiceImpl) CheckAndSendArgoPipelineStatusSyncEventIf
455455
}
456456

457457
// pipelineId can be cdPipelineId or installedAppVersionId, using isAppStoreApplication flag to identify between them
458-
if lastSyncTime.IsZero() || (!lastSyncTime.IsZero() && time.Since(lastSyncTime) > 5*time.Second) { // create new nats event
458+
if lastSyncTime.IsZero() || (!lastSyncTime.IsZero() && time.Since(lastSyncTime) > 5*time.Second) {
459+
// create new nats event
459460
err = impl.cdPipelineEventPublishService.PublishArgoTypePipelineSyncEvent(pipelineId, installedAppVersionId, userId, isAppStoreApplication)
460461
if err != nil {
461462
impl.logger.Errorw("error, PublishArgoTypePipelineSyncEvent", "err", err)

0 commit comments

Comments
 (0)