feat(issue-94): allow fetching all builds and other than completed (#102)

* feat(issue-94): allow fetching all builds and other than completed

* add handling for all case

* explicit handling of all case

* filter for fetching all builds

* readme

* Allow uuid and names

* filter builds by mintime in the case of  statusFilter=all

* must be startTime because builds could be running and not be finished

---------

Co-authored-by: Loeffler, Julian <[email protected]>
Co-authored-by: Markus Blaschke <[email protected]>

Author: 3 people

.dockerignore

@@ -1,2 +1,3 @@
 /azure-devops-exporter
 /release-assets
+.env

README.md

@@ -42,28 +42,24 @@ Application Options:
       --azuredevops.agentpool=                Enable scrape metrics for agent pool (IDs) [$AZURE_DEVOPS_AGENTPOOL]
       --whitelist.project=                    Filter projects (UUIDs) [$AZURE_DEVOPS_FILTER_PROJECT]
       --blacklist.project=                    Filter projects (UUIDs) [$AZURE_DEVOPS_BLACKLIST_PROJECT]
+      --timeline.state=                       Filter timeline states (completed, inProgress, pending) (default: completed) [$AZURE_DEVOPS_FILTER_TIMELINE_STATE]
+      --builds.all.project=                   Fetch all builds (even if they are not finished) [$AZURE_DEVOPS_FETCH_ALL_BUILDS_FILTER_PROJECT]
       --list.query=                           Pairs of query and project UUIDs in the form: '<queryId>@<projectId>' [$AZURE_DEVOPS_QUERIES]
-      --tags.schema=                          Tags to be extracted from builds in the format 'tagName:type' with following types: number,
-                                              info, bool [$AZURE_DEVOPS_TAG_SCHEMA]
+      --tags.schema=                          Tags to be extracted from builds in the format 'tagName:type' with following types: number, info, bool [$AZURE_DEVOPS_TAG_SCHEMA]
       --tags.build.definition=                Build definition ids to query tags (IDs) [$AZURE_DEVOPS_TAG_BUILD_DEFINITION]
-      --cache.path=                           Cache path (to folder, file://path... or
-                                              azblob://storageaccount.blob.core.windows.net/containername or
-                                              k8scm://{namespace}/{configmap}}) [$CACHE_PATH]
+      --cache.path=                           Cache path (to folder, file://path... or azblob://storageaccount.blob.core.windows.net/containername or k8scm://{namespace}/{configmap}}) [$CACHE_PATH]
       --request.concurrency=                  Number of concurrent requests against dev.azure.com (default: 10) [$REQUEST_CONCURRENCY]
       --request.retries=                      Number of retried requests against dev.azure.com (default: 3) [$REQUEST_RETRIES]
-      --servicediscovery.refresh=             Refresh duration for servicediscovery (time.duration) (default: 30m)
-                                              [$SERVICEDISCOVERY_REFRESH]
+      --servicediscovery.refresh=             Refresh duration for servicediscovery (time.duration) (default: 30m) [$SERVICEDISCOVERY_REFRESH]
       --limit.project=                        Limit number of projects (default: 100) [$LIMIT_PROJECT]
       --limit.builds-per-project=             Limit builds per project (default: 100) [$LIMIT_BUILDS_PER_PROJECT]
       --limit.builds-per-definition=          Limit builds per definition (default: 10) [$LIMIT_BUILDS_PER_DEFINITION]
       --limit.releases-per-project=           Limit releases per project (default: 100) [$LIMIT_RELEASES_PER_PROJECT]
       --limit.releases-per-definition=        Limit releases per definition (default: 100) [$LIMIT_RELEASES_PER_DEFINITION]
       --limit.deployments-per-definition=     Limit deployments per definition (default: 100) [$LIMIT_DEPLOYMENTS_PER_DEFINITION]
       --limit.releasedefinitions-per-project= Limit builds per definition (default: 100) [$LIMIT_RELEASEDEFINITION_PER_PROJECT]
-      --limit.build-history-duration=         Time (time.Duration) how long the exporter should look back for builds (default: 48h)
-                                              [$LIMIT_BUILD_HISTORY_DURATION]
-      --limit.release-history-duration=       Time (time.Duration) how long the exporter should look back for releases (default: 48h)
-                                              [$LIMIT_RELEASE_HISTORY_DURATION]
+      --limit.build-history-duration=         Time (time.Duration) how long the exporter should look back for builds (default: 48h) [$LIMIT_BUILD_HISTORY_DURATION]
+      --limit.release-history-duration=       Time (time.Duration) how long the exporter should look back for releases (default: 48h) [$LIMIT_RELEASE_HISTORY_DURATION]
       --server.bind=                          Server address (default: :8080) [$SERVER_BIND]
       --server.timeout.read=                  Server read timeout (default: 5s) [$SERVER_TIMEOUT_READ]
       --server.timeout.write=                 Server write timeout (default: 10s) [$SERVER_TIMEOUT_WRITE]

azure-devops-client/build.go

@@ -53,6 +53,7 @@ type TimelineRecord struct {
 	Result       string  `json:"result"`
 	WorkerName   string  `json:"workerName"`
 	Identifier   string  `json:"identifier"`
+	State        string  `json:"state"`
 	StartTime    time.Time
 	FinishTime   time.Time
 }
@@ -195,14 +196,26 @@ func (c *AzureDevopsClient) ListBuildHistoryWithStatus(project string, minTime t
 	defer c.concurrencyUnlock()
 	c.concurrencyLock()
 
-	url := fmt.Sprintf(
-		"%v/_apis/build/builds?api-version=%v&minTime=%s&statusFilter=%v",
-		url.QueryEscape(project),
-		url.QueryEscape(c.ApiVersion),
-		url.QueryEscape(minTime.Format(time.RFC3339)),
-		url.QueryEscape(statusFilter),
-	)
-	response, err := c.rest().R().Get(url)
+	requestUrl := ""
+
+	if statusFilter == "all" {
+		requestUrl = fmt.Sprintf(
+			"%v/_apis/build/builds?api-version=%v&statusFilter=%v",
+			url.QueryEscape(project),
+			url.QueryEscape(c.ApiVersion),
+			url.QueryEscape(statusFilter),
+		)
+	} else {
+		requestUrl = fmt.Sprintf(
+			"%v/_apis/build/builds?api-version=%v&minTime=%s&statusFilter=%v",
+			url.QueryEscape(project),
+			url.QueryEscape(c.ApiVersion),
+			url.QueryEscape(minTime.Format(time.RFC3339)),
+			url.QueryEscape(statusFilter),
+		)
+	}
+
+	response, err := c.rest().R().Get(requestUrl)
 	if err := c.checkResponse(response, err); err != nil {
 		error = err
 		return
@@ -214,6 +227,18 @@ func (c *AzureDevopsClient) ListBuildHistoryWithStatus(project string, minTime t
 		return
 	}
 
+	// if the status filter is "all", we need to filter the builds by minTime manually because Azure DevOps API does not support it
+	if statusFilter == "all" {
+		var filteredList BuildList
+		for _, build := range list.List {
+			if build.StartTime.After(minTime) {
+				filteredList.List = append(filteredList.List, build)
+			}
+		}
+		filteredList.Count = len(filteredList.List)
+		list = filteredList
+	}
+
 	return
 }
 

compose.yaml

@@ -0,0 +1,16 @@
+services:
+  exporter:
+    build: .
+    ports:
+      - "8000:8000"
+    environment:
+      SERVER_BIND: ${SERVER_BIND}
+      AZURE_DEVOPS_URL: ${AZURE_DEVOPS_URL}
+      AZURE_DEVOPS_ORGANISATION: ${AZURE_DEVOPS_ORGANISATION}
+      AZURE_DEVOPS_ACCESS_TOKEN: ${AZURE_DEVOPS_ACCESS_TOKEN}
+      AZURE_DEVOPS_FILTER_PROJECT: 72690669-de93-4a98-84a9-8300ce32a2f2
+      LIMIT_BUILDS_PER_PROJECT: 500
+      LIMIT_BUILDS_PER_DEFINITION: 100
+      SERVER_TIMEOUT_READ: 15s
+      AZURE_DEVOPS_FETCH_ALL_BUILDS: "true"
+      #AZURE_DEVOPS_FILTER_TIMELINE_STATE: "completed inProgress pending"

config/opts.go

@@ -58,6 +58,9 @@ type (
 			FilterProjects    []string `long:"whitelist.project"    env:"AZURE_DEVOPS_FILTER_PROJECT"    env-delim:" "   description:"Filter projects (UUIDs)"`
 			BlacklistProjects []string `long:"blacklist.project"    env:"AZURE_DEVOPS_BLACKLIST_PROJECT" env-delim:" "   description:"Filter projects (UUIDs)"`
 
+			FilterTimelineState  []string `long:"timeline.state"    env:"AZURE_DEVOPS_FILTER_TIMELINE_STATE"    env-delim:" "   description:"Filter timeline states (completed, inProgress, pending)" default:"completed"`
+			FetchAllBuildsFilter []string `long:"builds.all.project"   env:"AZURE_DEVOPS_FETCH_ALL_BUILDS_FILTER_PROJECT"  env-delim:" "  description:"Fetch all builds from projects (UUIDs or names)"`
+
 			// query settings
 			QueriesWithProjects []string `long:"list.query"    env:"AZURE_DEVOPS_QUERIES"    env-delim:" "   description:"Pairs of query and project UUIDs in the form: '<queryId>@<projectId>'"`
 

metrics_build.go

@@ -303,8 +303,20 @@ func (m *MetricsCollectorBuild) collectBuilds(ctx context.Context, logger *zap.S
 }
 
 func (m *MetricsCollectorBuild) collectBuildsTimeline(ctx context.Context, logger *zap.SugaredLogger, callback chan<- func(), project devopsClient.Project) {
+<<<<<<< main
+	minTime := time.Now().Add(-opts.Limit.BuildHistoryDuration)
+
+	statusFilter := "completed"
+	if arrayStringContains(opts.AzureDevops.FetchAllBuildsFilter, project.Name) || arrayStringContains(opts.AzureDevops.FetchAllBuildsFilter, project.Id) {
+		logger.Info("fetching all builds for project " + project.Name)
+		statusFilter = "all"
+	}
+
+	list, err := AzureDevopsClient.ListBuildHistoryWithStatus(project.Id, minTime, statusFilter)
+=======
 	minTime := time.Now().Add(-Opts.Limit.BuildHistoryDuration)
 	list, err := AzureDevopsClient.ListBuildHistoryWithStatus(project.Id, minTime, "completed")
+>>>>>>> main
 	if err != nil {
 		logger.Error(err)
 		return
@@ -316,8 +328,18 @@ func (m *MetricsCollectorBuild) collectBuildsTimeline(ctx context.Context, logge
 	buildTaskMetric := m.Collector.GetMetricList("buildTask")
 
 	for _, build := range list.List {
+
 		timelineRecordList, _ := AzureDevopsClient.ListBuildTimeline(project.Id, int64ToString(build.Id))
 		for _, timelineRecord := range timelineRecordList.List {
+
+			if opts.AzureDevops.FilterTimelineState != nil && !arrayStringContains(opts.AzureDevops.FilterTimelineState, timelineRecord.State) {
+				continue
+			}
+
+			if timelineRecord.Result == "" {
+				timelineRecord.Result = "unknown"
+			}
+
 			recordType := timelineRecord.RecordType
 			switch strings.ToLower(recordType) {
 			case "stage":
@@ -635,8 +657,20 @@ func (m *MetricsCollectorBuild) collectBuildsTimeline(ctx context.Context, logge
 }
 
 func (m *MetricsCollectorBuild) collectBuildsTags(ctx context.Context, logger *zap.SugaredLogger, callback chan<- func(), project devopsClient.Project) {
+<<<<<<< main
+	minTime := time.Now().Add(-opts.Limit.BuildHistoryDuration)
+
+	statusFilter := "completed"
+	if arrayStringContains(opts.AzureDevops.FetchAllBuildsFilter, project.Name) || arrayStringContains(opts.AzureDevops.FetchAllBuildsFilter, project.Id) {
+		logger.Info("fetching all builds for project " + project.Name)
+		statusFilter = "all"
+	}
+
+	list, err := AzureDevopsClient.ListBuildHistoryWithStatus(project.Id, minTime, statusFilter)
+=======
 	minTime := time.Now().Add(-Opts.Limit.BuildHistoryDuration)
 	list, err := AzureDevopsClient.ListBuildHistoryWithStatus(project.Id, minTime, "completed")
+>>>>>>> main
 	if err != nil {
 		logger.Error(err)
 		return