Merge pull request #479 from ClusterCockpit/dev

Dev

Author: moebiusband73

internal/graph/schema.resolvers.go

@@ -57,13 +57,13 @@ func (r *queryResolver) rooflineHeatmap(
 
 		jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
 		if err != nil {
-			cclog.Errorf("Error while loading roofline metrics for job %d", job.ID)
+			cclog.Warnf("Error while loading roofline metrics for job %d", job.ID)
 			return nil, err
 		}
 
 		flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"]
 		if flops_ == nil && membw_ == nil {
-			cclog.Infof("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", job.ID)
+			cclog.Warnf("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", job.ID)
 			continue
 			// return nil, fmt.Errorf("GRAPH/UTIL > 'flops_any' or 'mem_bw' missing for job %d", job.ID)
 		}

internal/graph/util.go

@@ -97,8 +97,8 @@ func LoadData(job *schema.Job,
 
 			ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
 			if err != nil {
-				cclog.Errorf("failed to load job data from metric store for job %d (user: %s, project: %s): %s",
-					job.JobID, job.User, job.Project, err.Error())
+				cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
+					job.Cluster, job.SubCluster, err.Error())
 				return err, 0, 0
 			}
 
@@ -116,11 +116,11 @@ func LoadData(job *schema.Job,
 			jd, err = ms.LoadData(job, metrics, scopes, ctx, resolution)
 			if err != nil {
 				if len(jd) != 0 {
-					cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s): %s",
-						job.JobID, job.User, job.Project, err.Error())
+					cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
+						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
 				} else {
-					cclog.Errorf("failed to load job data from metric store for job %d (user: %s, project: %s): %s",
-						job.JobID, job.User, job.Project, err.Error())
+					cclog.Warnf("failed to load job data from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
+						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
 					return err, 0, 0
 				}
 			}
@@ -129,8 +129,8 @@ func LoadData(job *schema.Job,
 			var jdTemp schema.JobData
 			jdTemp, err = archive.GetHandle().LoadJobData(job)
 			if err != nil {
-				cclog.Errorf("failed to load job data from archive for job %d (user: %s, project: %s): %s",
-					job.JobID, job.User, job.Project, err.Error())
+				cclog.Warnf("failed to load job data from archive for job %d (user: %s, project: %s, cluster: %s-%s): %s",
+					job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
 				return err, 0, 0
 			}
 
@@ -244,15 +244,15 @@ func LoadAverages(
 
 	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
 	if err != nil {
-		cclog.Errorf("failed to load job data from metric store for job %d (user: %s, project: %s): %s",
-			job.JobID, job.User, job.Project, err.Error())
+		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
+			job.Cluster, job.SubCluster, err.Error())
 		return err
 	}
 
 	stats, err := ms.LoadStats(job, metrics, ctx)
 	if err != nil {
-		cclog.Errorf("failed to load statistics from metric store for job %d (user: %s, project: %s): %s",
-			job.JobID, job.User, job.Project, err.Error())
+		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
+			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
 		return err
 	}
 
@@ -288,15 +288,15 @@ func LoadScopedJobStats(
 
 	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
 	if err != nil {
-		cclog.Errorf("failed to load job data from metric store for job %d (user: %s, project: %s): %s",
-			job.JobID, job.User, job.Project, err.Error())
+		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
+			job.Cluster, job.SubCluster, err.Error())
 		return nil, err
 	}
 
 	scopedStats, err := ms.LoadScopedStats(job, metrics, scopes, ctx)
 	if err != nil {
-		cclog.Errorf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s): %s",
-			job.JobID, job.User, job.Project, err.Error())
+		cclog.Warnf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
+			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
 		return nil, err
 	}
 
@@ -320,17 +320,17 @@ func LoadJobStats(
 
 	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
 	if err != nil {
-		cclog.Errorf("failed to load job data from metric store for job %d (user: %s, project: %s): %s",
-			job.JobID, job.User, job.Project, err.Error())
+		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
+			job.Cluster, job.SubCluster, err.Error())
 		return nil, err
 	}
 
 	data := make(map[string]schema.MetricStatistics, len(metrics))
 
 	stats, err := ms.LoadStats(job, metrics, ctx)
 	if err != nil {
-		cclog.Errorf("failed to load statistics from metric store for job %d (user: %s, project: %s): %s",
-			job.JobID, job.User, job.Project, err.Error())
+		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
+			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
 		return data, err
 	}
 
@@ -379,8 +379,8 @@ func LoadNodeData(
 
 	ms, err := GetMetricDataRepo(cluster, "")
 	if err != nil {
-		cclog.Errorf("failed to load node data from metric store: %s",
-			err.Error())
+		cclog.Errorf("failed to access metricDataRepo for cluster %s: %s",
+			cluster, err.Error())
 		return nil, err
 	}
 
@@ -389,7 +389,7 @@ func LoadNodeData(
 		if len(data) != 0 {
 			cclog.Warnf("partial error loading node data from metric store for cluster %s: %s", cluster, err.Error())
 		} else {
-			cclog.Errorf("failed to load node data from metric store for cluster %s: %s", cluster, err.Error())
+			cclog.Warnf("failed to load node data from metric store for cluster %s: %s", cluster, err.Error())
 			return nil, err
 		}
 	}
@@ -423,8 +423,8 @@ func LoadNodeListData(
 
 	ms, err := GetMetricDataRepo(cluster, subCluster)
 	if err != nil {
-		cclog.Errorf("failed to load node data from metric store: %s",
-			err.Error())
+		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
+			cluster, subCluster, err.Error())
 		return nil, err
 	}
 
@@ -434,7 +434,7 @@ func LoadNodeListData(
 			cclog.Warnf("partial error loading node list data from metric store for cluster %s, subcluster %s: %s",
 				cluster, subCluster, err.Error())
 		} else {
-			cclog.Errorf("failed to load node list data from metric store for cluster %s, subcluster %s: %s",
+			cclog.Warnf("failed to load node list data from metric store for cluster %s, subcluster %s: %s",
 				cluster, subCluster, err.Error())
 			return nil, err
 		}

internal/metricdispatch/dataLoader.go

@@ -329,7 +329,7 @@ func (ccms *CCMetricStore) LoadStats(
 		metric := query.Metric
 		data := res[0]
 		if data.Error != nil {
-			cclog.Errorf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
+			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
 			continue
 		}
 
@@ -556,7 +556,7 @@ func (ccms *CCMetricStore) LoadNodeListData(
 ) (map[string]schema.JobData, error) {
 	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
 	if err != nil {
-		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
+		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
 		return nil, err
 	}
 

internal/metricstoreclient/cc-metric-store.go

@@ -38,7 +38,7 @@
 // - All queries use prepared statements via stmtCache
 // - Complex aggregations use SQL for efficiency
 // - Histogram pre-initialization ensures consistent bin ranges
-// - Metric histogram queries limited to 500 jobs for running job analysis
+// - Metric histogram queries limited to 5000 jobs for running job analysis
 
 package repository
 
@@ -686,7 +686,7 @@ func (r *JobRepository) AddHistograms(
 //   - Pre-initialized with zeros for consistent visualization
 //
 // Limitations:
-//   - Running jobs: Limited to 500 jobs for performance
+//   - Running jobs: Limited to 5000 jobs for performance
 //   - Requires valid cluster configuration with metric peak values
 //   - Uses footprint statistic (avg/max/min) configured per metric
 func (r *JobRepository) AddMetricHistograms(
@@ -995,12 +995,12 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 // Returns slice of MetricHistoPoints, one per metric.
 //
 // Limitations:
-//   - Maximum 500 jobs (returns nil if more jobs match)
+//   - Maximum 5000 jobs (returns nil if more jobs match)
 //   - Requires metric backend availability
 //   - Bins based on metric peak values from cluster configuration
 //
 // Algorithm:
-//  1. Query first 501 jobs to check count limit
+//  1. Query first 5001 jobs to check count limit
 //  2. Load metric averages for all jobs via metricdispatch
 //  3. For each metric, create bins based on peak value
 //  4. Iterate averages and count jobs per bin
@@ -1011,13 +1011,13 @@ func (r *JobRepository) runningJobsMetricStatisticsHistogram(
 	bins *int,
 ) []*model.MetricHistoPoints {
 	// Get Jobs
-	jobs, err := r.QueryJobs(ctx, filters, &model.PageRequest{Page: 1, ItemsPerPage: 500 + 1}, nil)
+	jobs, err := r.QueryJobs(ctx, filters, &model.PageRequest{Page: 1, ItemsPerPage: 5000 + 1}, nil)
 	if err != nil {
 		cclog.Errorf("Error while querying jobs for footprint: %s", err)
 		return nil
 	}
-	if len(jobs) > 500 {
-		cclog.Errorf("too many jobs matched (max: %d)", 500)
+	if len(jobs) > 5000 {
+		cclog.Errorf("too many jobs matched (max: %d)", 5000)
 		return nil
 	}
 

internal/repository/stats.go

@@ -68,8 +68,8 @@ func RegisterFootprintWorker() {
 
 						ms, err := metricdispatch.GetMetricDataRepo(job.Cluster, job.SubCluster)
 						if err != nil {
-							cclog.Errorf("failed to load job data from metric store for job %d (user: %s, project: %s): %s",
-								job.JobID, job.User, job.Project, err.Error())
+							cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
+								job.Cluster, job.SubCluster, err.Error())
 							continue
 						}
 

internal/taskmanager/updateFootprintService.go

@@ -13,13 +13,14 @@ import (
 	"fmt"
 	"math"
 
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 	"github.com/ClusterCockpit/cc-lib/v2/schema"
 	"github.com/ClusterCockpit/cc-lib/v2/util"
 )
 
 var (
 	// ErrNoHostOrMetric is returned when the metric store does not find the host or the metric
-	ErrNoHostOrMetric error = errors.New("[METRICSTORE]> [METRICSTORE]> metric or host not found")
+	ErrNoHostOrMetric error = errors.New("[METRICSTORE]> metric or host not found")
 	// ErrInvalidTimeRange is returned when a query has 'from' >= 'to'
 	ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'")
 	// ErrEmptyCluster is returned when a query with ForAllNodes has no cluster specified
@@ -280,20 +281,16 @@ func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
 
 			data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
 			if err != nil {
-				// Check a special case where only the metric or host.
-				// Dont send errors, instead just send empty array
-				// where frontend already renders error for empty array.
-				if err == ErrNoHostOrMetric {
-					data.Data = make([]schema.Float, 0)
-					data.From = req.From
-					data.To = req.To
-					data.Resolution = query.Resolution
-				} else {
+				// Skip Error If Just Missing Host or Metric, Continue
+				// Empty Return For Metric Handled Gracefully By Frontend
+				if err != ErrNoHostOrMetric {
 					msg := err.Error()
 					data.Error = &msg
 					res = append(res, data)
-					continue
+				} else {
+					cclog.Warnf("failed to fetch '%s' from host '%s' (cluster: %s): %s", query.Metric, query.Hostname, req.Cluster, err.Error())
 				}
+				continue
 			}
 
 			if req.WithStats {

pkg/metricstore/api.go

@@ -104,6 +104,11 @@ func (ccms *InternalMetricStore) LoadData(
 	var errors []string
 	jobData := make(schema.JobData)
 	for i, row := range resBody.Results {
+		if len(row) == 0 {
+			// No Data Found For Metric, Logged in FetchData to Warn
+			continue
+		}
+
 		query := req.Queries[i]
 		metric := query.Metric
 		scope := assignedScope[i]
@@ -229,7 +234,7 @@ func buildQueries(
 	for _, metric := range metrics {
 		mc := archive.GetMetricConfig(job.Cluster, metric)
 		if mc == nil {
-			cclog.Infof("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
+			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
 			continue
 		}
 
@@ -535,11 +540,15 @@ func (ccms *InternalMetricStore) LoadStats(
 
 	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
 	for i, res := range resBody.Results {
+		if len(res) == 0 {
+			// No Data Found For Metric, Logged in FetchData to Warn
+			continue
+		}
 		query := req.Queries[i]
 		metric := query.Metric
 		data := res[0]
 		if data.Error != nil {
-			cclog.Errorf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
+			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
 			continue
 		}
 
@@ -609,6 +618,10 @@ func (ccms *InternalMetricStore) LoadScopedStats(
 	scopedJobStats := make(schema.ScopedJobStats)
 
 	for i, row := range resBody.Results {
+		if len(row) == 0 {
+			// No Data Found For Metric, Logged in FetchData to Warn
+			continue
+		}
 		query := req.Queries[i]
 		metric := query.Metric
 		scope := assignedScope[i]
@@ -717,6 +730,11 @@ func (ccms *InternalMetricStore) LoadNodeData(
 	var errors []string
 	data := make(map[string]map[string][]*schema.JobMetric)
 	for i, res := range resBody.Results {
+		if len(res) == 0 {
+			// No Data Found For Metric, Logged in FetchData to Warn
+			continue
+		}
+
 		var query APIQuery
 		if resBody.Queries != nil {
 			query = resBody.Queries[i]
@@ -816,6 +834,10 @@ func (ccms *InternalMetricStore) LoadNodeListData(
 	var errors []string
 	data := make(map[string]schema.JobData)
 	for i, row := range resBody.Results {
+		if len(row) == 0 {
+			// No Data Found For Metric, Logged in FetchData to Warn
+			continue
+		}
 		var query APIQuery
 		if resBody.Queries != nil {
 			query = resBody.Queries[i]