opt metric collection

Author: mfhonley

internal/commands/daemon.go

@@ -146,7 +146,7 @@ func runDaemon() {
 	// OTel SDK calls callbacks at CollectionInterval, we collect slightly faster
 	metricsInterval := time.Duration(cfg.CollectionInterval) * time.Second
 	if metricsInterval == 0 {
-		metricsInterval = 15 * time.Second
+		metricsInterval = 30 * time.Second
 	}
 	metricsTicker := time.NewTicker(metricsInterval)
 	defer metricsTicker.Stop()
@@ -202,7 +202,7 @@ func runDaemon() {
 func startMetricsCollection(cfg *config.Config, hostname string) bool {
 	interval := time.Duration(cfg.CollectionInterval) * time.Second
 	if interval == 0 {
-		interval = 15 * time.Second
+		interval = 30 * time.Second
 	}
 
 	otelCfg := &metrics.OTelConfig{

internal/metrics/collector.go

@@ -393,7 +393,7 @@ func StartOTelCollector(cfg *OTelConfig) error {
 
 	interval := cfg.CollectionInterval
 	if interval == 0 {
-		interval = 15 * time.Second
+		interval = 30 * time.Second
 	}
 
 	meterProvider = sdkmetric.NewMeterProvider(
@@ -1776,10 +1776,12 @@ func collectProcesses(limit int) ([]ProcessInfo, error) {
 			continue
 		}
 
-		cpuPercent, _ := p.CPUPercent()
+		// Only use MemoryPercent for filtering (non-blocking)
+		// Skip CPUPercent - it blocks for 100ms per process!
 		memPercent, _ := p.MemoryPercent()
 
-		if cpuPercent < 0.1 && memPercent < 0.1 {
+		// Filter by memory only (processes with < 0.1% memory are not interesting)
+		if memPercent < 0.1 {
 			continue
 		}
 
@@ -1788,82 +1790,32 @@ func collectProcesses(limit int) ([]ProcessInfo, error) {
 			Name: name,
 		}
 
-		if ppid, err := p.Ppid(); err == nil {
-			pi.PPID = int(ppid)
-		}
-
-		pi.CPUPercent = cpuPercent
 		pi.MemoryPercent = float64(memPercent)
 
+		// Minimal syscalls: only cmdline and memory info
 		if cmdline, err := p.Cmdline(); err == nil {
 			pi.Command = truncateString(cmdline, 200)
 		} else {
 			pi.Command = name
 		}
 
-		if exe, err := p.Exe(); err == nil {
-			pi.Exe = exe
-		}
-
-		if uids, err := p.Uids(); err == nil && len(uids) > 0 {
-			pi.UID = uids[0]
-			pi.User = fmt.Sprintf("%d", uids[0])
-		}
-
-		if gids, err := p.Gids(); err == nil && len(gids) > 0 {
-			pi.GID = gids[0]
-		}
-
 		if memInfo, err := p.MemoryInfo(); err == nil && memInfo != nil {
 			pi.MemoryRSS = memInfo.RSS
-			pi.MemoryVMS = memInfo.VMS
 		}
 
 		if status, err := p.Status(); err == nil && len(status) > 0 {
 			pi.Status = string(status[0])
 		}
 
-		if threads, err := p.NumThreads(); err == nil {
-			pi.NumThreads = uint16(threads)
-		}
-
-		if fds, err := p.NumFDs(); err == nil {
-			pi.NumFDs = uint32(fds)
-		}
-
-		if ioCounters, err := p.IOCounters(); err == nil && ioCounters != nil {
-			pi.IOReadBytes = ioCounters.ReadBytes
-			pi.IOWriteBytes = ioCounters.WriteBytes
-		}
-
-		if createTime, err := p.CreateTime(); err == nil {
-			pi.CreateTime = createTime / 1000
-		}
-
-		if times, err := p.Times(); err == nil && times != nil {
-			pi.CPUTimeUser = times.User
-			pi.CPUTimeSystem = times.System
-		}
-
-		if nice, err := p.Nice(); err == nil {
-			pi.Nice = int8(nice)
-		}
-
-		if terminal, err := p.Terminal(); err == nil {
-			pi.TTY = terminal
-		}
-
-		pi.CPUUsage = pi.CPUPercent
 		pi.MemoryUsage = pi.MemoryPercent
 		pi.MemoryKB = int64(pi.MemoryRSS / 1024)
-		pi.Threads = int(pi.NumThreads)
 
 		processes = append(processes, pi)
 	}
 
-	// Sort by CPU usage
+	// Sort by memory usage (since we don't have CPU anymore)
 	sort.Slice(processes, func(i, j int) bool {
-		return processes[i].CPUPercent > processes[j].CPUPercent
+		return processes[i].MemoryPercent > processes[j].MemoryPercent
 	})
 
 	if len(processes) > limit {
@@ -1890,8 +1842,9 @@ func collectContainers() ([]ContainerMetrics, error) {
 }
 
 func collectDockerContainers() ([]ContainerMetrics, error) {
-	// Check if docker is available
-	cmd := exec.Command("docker", "ps", "-q")
+	// Single call to docker stats - gets all running containers at once
+	// Skip "docker ps" check - if no containers, stats returns empty
+	cmd := exec.Command("docker", "stats", "--no-stream", "--format", "{{json .}}")
 	output, err := cmd.Output()
 	if err != nil {
 		return nil, err
@@ -1901,13 +1854,6 @@ func collectDockerContainers() ([]ContainerMetrics, error) {
 		return nil, nil
 	}
 
-	// Get container stats
-	cmd = exec.Command("docker", "stats", "--no-stream", "--format", "{{json .}}")
-	output, err = cmd.Output()
-	if err != nil {
-		return nil, err
-	}
-
 	var containers []ContainerMetrics
 
 	lines := strings.Split(strings.TrimSpace(string(output)), "\n")
@@ -1917,14 +1863,11 @@ func collectDockerContainers() ([]ContainerMetrics, error) {
 		}
 
 		var stats struct {
-			ID        string `json:"ID"`
-			Name      string `json:"Name"`
-			CPUPerc   string `json:"CPUPerc"`
-			MemUsage  string `json:"MemUsage"`
-			MemPerc   string `json:"MemPerc"`
-			NetIO     string `json:"NetIO"`
-			BlockIO   string `json:"BlockIO"`
-			PIDs      string `json:"PIDs"`
+			ID       string `json:"ID"`
+			Name     string `json:"Name"`
+			CPUPerc  string `json:"CPUPerc"`
+			MemUsage string `json:"MemUsage"`
+			MemPerc  string `json:"MemPerc"`
 		}
 
 		if err := json.Unmarshal([]byte(line), &stats); err != nil {
@@ -1953,38 +1896,8 @@ func collectDockerContainers() ([]ContainerMetrics, error) {
 		containers = append(containers, c)
 	}
 
-	// Get more details with docker inspect
-	for i := range containers {
-		cmd := exec.Command("docker", "inspect", "--format", "{{json .}}", containers[i].ContainerID)
-		output, err := cmd.Output()
-		if err != nil {
-			continue
-		}
-
-		var inspect struct {
-			Config struct {
-				Image  string            `json:"Image"`
-				Labels map[string]string `json:"Labels"`
-			} `json:"Config"`
-			State struct {
-				Status     string `json:"Status"`
-				Health     *struct{ Status string } `json:"Health"`
-				StartedAt  string `json:"StartedAt"`
-				ExitCode   int16  `json:"ExitCode"`
-			} `json:"State"`
-		}
-
-		if err := json.Unmarshal(output, &inspect); err == nil {
-			containers[i].ImageName = inspect.Config.Image
-			containers[i].Status = inspect.State.Status
-			if inspect.State.Health != nil {
-				containers[i].Health = inspect.State.Health.Status
-			}
-			if labels, err := json.Marshal(inspect.Config.Labels); err == nil {
-				containers[i].Labels = string(labels)
-			}
-		}
-	}
+	// Skip docker inspect for each container - too expensive (N syscalls)
+	// Basic stats from "docker stats" are enough for monitoring
 
 	return containers, nil
 }

internal/metrics/cpu.go

@@ -21,14 +21,13 @@ type CPUMetrics struct {
 
 // CPU state cache for delta-based calculation
 var (
-	lastCpuTimes         cpu.TimesStat
-	lastPerCoreCpuTimes  []cpu.TimesStat
-	lastCpuMetrics       CPUMetrics
-	cpuCacheMu           sync.RWMutex
-	cpuCacheInitialized  bool
-	perCoreCacheInit     bool
-	lastCpuSampleTime    time.Time
-	minCPUSampleInterval = 100 * time.Millisecond // Minimum time between samples for accurate delta
+	lastCpuTimes        cpu.TimesStat
+	lastPerCoreCpuTimes []cpu.TimesStat
+	lastCpuMetrics      CPUMetrics
+	cpuCacheMu          sync.RWMutex
+	cpuCacheInitialized bool
+	perCoreCacheInit    bool
+	lastCpuSampleTime   time.Time
 )
 
 // init initializes CPU monitoring by storing initial CPU times
@@ -47,81 +46,55 @@ func init() {
 	}
 }
 
-// GetCPUMetrics calculates detailed CPU usage metrics.
-// Uses gopsutil's built-in cpu.Percent for accurate cross-platform CPU measurement.
-// On first call or when cache is stale, performs a blocking measurement (100ms).
-// Subsequent calls within the sample interval return cached values instantly.
+// GetCPUMetrics calculates detailed CPU usage metrics using delta-based calculation.
+// Uses cpu.Times() which is non-blocking (reads /proc/stat on Linux, sysctl on macOS).
+// Calculates CPU usage as delta between current and previous sample.
+// First call returns zero values (baseline), subsequent calls return accurate metrics.
 func GetCPUMetrics() (CPUMetrics, error) {
 	cpuCacheMu.Lock()
 	defer cpuCacheMu.Unlock()
 
-	// Check if we have a recent enough measurement WITH valid data
-	// lastCpuMetrics.Total > 0 ensures we don't return empty cache from init()
-	if cpuCacheInitialized && lastCpuMetrics.Total > 0 && time.Since(lastCpuSampleTime) < minCPUSampleInterval {
-		// Return last calculated metrics (avoid too frequent measurements)
-		return lastCpuMetrics, nil
-	}
-
-	// Use gopsutil's built-in Percent function which handles all platform differences
-	// This is the most accurate way to measure CPU on macOS/Linux/Windows
-	percentages, err := cpu.Percent(100*time.Millisecond, false)
-	if err != nil || len(percentages) == 0 {
+	// Get current CPU times (non-blocking)
+	times, err := cpu.Times(false)
+	if err != nil || len(times) == 0 {
 		return CPUMetrics{}, err
 	}
 
-	totalCPU := percentages[0]
-
-	// Get CPU times for breakdown (user/system/idle/iowait)
-	times, err := cpu.Times(false)
-	if err != nil || len(times) == 0 {
-		// If times fail, at least return total CPU
-		metrics := CPUMetrics{
-			Total: totalCPU,
-			Idle:  100 - totalCPU,
-		}
-		lastCpuMetrics = metrics
+	// First call - just store baseline and return zeros
+	if !cpuCacheInitialized {
+		lastCpuTimes = times[0]
 		lastCpuSampleTime = time.Now()
 		cpuCacheInitialized = true
-		return metrics, nil
+		return CPUMetrics{}, nil
 	}
 
-	// Calculate breakdown if we have previous sample
-	var metrics CPUMetrics
-	if cpuCacheInitialized {
-		t1 := lastCpuTimes
-		t2 := times[0]
+	// Calculate delta from previous sample
+	t1 := lastCpuTimes
+	t2 := times[0]
 
-		t1All, _ := getAllBusy(t1)
-		t2All, _ := getAllBusy(t2)
-		totalDelta := t2All - t1All
+	t1All, t1Busy := getAllBusy(t1)
+	t2All, t2Busy := getAllBusy(t2)
+	totalDelta := t2All - t1All
+
+	var metrics CPUMetrics
+	if totalDelta > 0 {
+		// Calculate total CPU usage
+		totalCPU := clampPercent((t2Busy - t1Busy) / totalDelta * 100)
 
-		if totalDelta > 0 {
-			metrics = CPUMetrics{
-				Total:  totalCPU, // Use gopsutil's accurate total
-				User:   clampPercent((t2.User - t1.User) / totalDelta * 100),
-				System: clampPercent((t2.System - t1.System) / totalDelta * 100),
-				Iowait: clampPercent((t2.Iowait - t1.Iowait) / totalDelta * 100),
-				Steal:  clampPercent((t2.Steal - t1.Steal) / totalDelta * 100),
-				Idle:   clampPercent((t2.Idle - t1.Idle) / totalDelta * 100),
-			}
-		} else {
-			metrics = CPUMetrics{
-				Total: totalCPU,
-				Idle:  100 - totalCPU,
-			}
-		}
-	} else {
 		metrics = CPUMetrics{
-			Total: totalCPU,
-			Idle:  100 - totalCPU,
+			Total:  totalCPU,
+			User:   clampPercent((t2.User - t1.User) / totalDelta * 100),
+			System: clampPercent((t2.System - t1.System) / totalDelta * 100),
+			Iowait: clampPercent((t2.Iowait - t1.Iowait) / totalDelta * 100),
+			Steal:  clampPercent((t2.Steal - t1.Steal) / totalDelta * 100),
+			Idle:   clampPercent((t2.Idle - t1.Idle) / totalDelta * 100),
 		}
 	}
 
 	// Update cache
 	lastCpuTimes = times[0]
 	lastCpuMetrics = metrics
 	lastCpuSampleTime = time.Now()
-	cpuCacheInitialized = true
 
 	return metrics, nil
 }

internal/otelcol/manager.go

@@ -199,7 +199,7 @@ func (m *Manager) extractBinary(r io.Reader) error {
 func (m *Manager) GenerateConfig(cfg *Config) error {
 	interval := cfg.CollectionInterval
 	if interval == 0 {
-		interval = 15
+		interval = 30
 	}
 
 	config := fmt.Sprintf(`# CatOps OTel Collector Configuration