resources: store sys cpu usage per step

This can be used to convert step usage to relative units.

Signed-off-by: Tonis Tiigi <[email protected]>

Author: tonistiigi

executor/resources/monitor.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/moby/buildkit/executor/resources/types"
 	"github.com/moby/buildkit/util/network"
+	"github.com/prometheus/procfs"
 	"github.com/sirupsen/logrus"
 )
 
@@ -27,14 +28,16 @@ var initOnce sync.Once
 var isCgroupV2 bool
 
 type cgroupRecord struct {
-	once       sync.Once
-	ns         string
-	sampler    *Sub[*types.Sample]
-	samples    []*types.Sample
-	err        error
-	done       chan struct{}
-	monitor    *Monitor
-	netSampler NetworkSampler
+	once         sync.Once
+	ns           string
+	sampler      *Sub[*types.Sample]
+	samples      []*types.Sample
+	err          error
+	done         chan struct{}
+	monitor      *Monitor
+	netSampler   NetworkSampler
+	startCPUStat *procfs.CPUStat
+	sysCPUStat   *types.SysCPUStat
 }
 
 func (r *cgroupRecord) Wait() error {
@@ -44,6 +47,9 @@ func (r *cgroupRecord) Wait() error {
 }
 
 func (r *cgroupRecord) Start() {
+	if stat, err := r.monitor.proc.Stat(); err == nil {
+		r.startCPUStat = &stat.CPUTotal
+	}
 	s := NewSampler(2*time.Second, r.sample)
 	r.sampler = s.Record()
 }
@@ -73,6 +79,26 @@ func (r *cgroupRecord) close() {
 		} else {
 			r.samples = s
 		}
+
+		if r.startCPUStat != nil {
+			stat, err := r.monitor.proc.Stat()
+			if err == nil {
+				cpu := &types.SysCPUStat{
+					User:      stat.CPUTotal.User - r.startCPUStat.User,
+					Nice:      stat.CPUTotal.Nice - r.startCPUStat.Nice,
+					System:    stat.CPUTotal.System - r.startCPUStat.System,
+					Idle:      stat.CPUTotal.Idle - r.startCPUStat.Idle,
+					Iowait:    stat.CPUTotal.Iowait - r.startCPUStat.Iowait,
+					IRQ:       stat.CPUTotal.IRQ - r.startCPUStat.IRQ,
+					SoftIRQ:   stat.CPUTotal.SoftIRQ - r.startCPUStat.SoftIRQ,
+					Steal:     stat.CPUTotal.Steal - r.startCPUStat.Steal,
+					Guest:     stat.CPUTotal.Guest - r.startCPUStat.Guest,
+					GuestNice: stat.CPUTotal.GuestNice - r.startCPUStat.GuestNice,
+				}
+				r.sysCPUStat = cpu
+			}
+		}
+
 	})
 }
 
@@ -110,12 +136,15 @@ func (r *cgroupRecord) sample(tm time.Time) (*types.Sample, error) {
 	return sample, nil
 }
 
-func (r *cgroupRecord) Samples() ([]*types.Sample, error) {
+func (r *cgroupRecord) Samples() (*types.Samples, error) {
 	<-r.done
 	if r.err != nil {
 		return nil, r.err
 	}
-	return r.samples, nil
+	return &types.Samples{
+		Samples:    r.samples,
+		SysCPUStat: r.sysCPUStat,
+	}, nil
 }
 
 type nopRecord struct {
@@ -125,7 +154,7 @@ func (r *nopRecord) Wait() error {
 	return nil
 }
 
-func (r *nopRecord) Samples() ([]*types.Sample, error) {
+func (r *nopRecord) Samples() (*types.Samples, error) {
 	return nil, nil
 }
 
@@ -140,6 +169,7 @@ type Monitor struct {
 	mu      sync.Mutex
 	closed  chan struct{}
 	records map[string]*cgroupRecord
+	proc    procfs.FS
 }
 
 type NetworkSampler interface {
@@ -194,9 +224,15 @@ func NewMonitor() (*Monitor, error) {
 		}
 	})
 
+	fs, err := procfs.NewDefaultFS()
+	if err != nil {
+		return nil, err
+	}
+
 	return &Monitor{
 		closed:  make(chan struct{}),
 		records: make(map[string]*cgroupRecord),
+		proc:    fs,
 	}, nil
 }
 

executor/resources/types/types.go

@@ -11,7 +11,25 @@ type Recorder interface {
 	Start()
 	CloseAsync(func(context.Context) error) error
 	Wait() error
-	Samples() ([]*Sample, error)
+	Samples() (*Samples, error)
+}
+
+type SysCPUStat struct {
+	User      float64 `json:"user"`
+	Nice      float64 `json:"nice"`
+	System    float64 `json:"system"`
+	Idle      float64 `json:"idle"`
+	Iowait    float64 `json:"iowait"`
+	IRQ       float64 `json:"irq"`
+	SoftIRQ   float64 `json:"softirq"`
+	Steal     float64 `json:"steal"`
+	Guest     float64 `json:"guest"`
+	GuestNice float64 `json:"guestNice"`
+}
+
+type Samples struct {
+	Samples    []*Sample   `json:"samples,omitempty"`
+	SysCPUStat *SysCPUStat `json:"sysCPUStat,omitempty"`
 }
 
 // Sample represents a wrapper for sampled data of cgroupv2 controllers

solver/llbsolver/ops/exec.go

@@ -456,7 +456,7 @@ func (e *ExecOp) loadSecretEnv(ctx context.Context, g session.Group) ([]string,
 func (e *ExecOp) IsProvenanceProvider() {
 }
 
-func (e *ExecOp) Samples() ([]*resourcestypes.Sample, error) {
+func (e *ExecOp) Samples() (*resourcestypes.Samples, error) {
 	if e.rec == nil {
 		return nil, nil
 	}

solver/llbsolver/provenance.go

@@ -361,7 +361,7 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc
 			if err != nil {
 				return err
 			}
-			if len(samples) > 0 {
+			if samples != nil {
 				c.AddSamples(op.Digest(), samples)
 			}
 		case *ops.BuildOp:

solver/llbsolver/provenance/buildconfig.go

@@ -12,10 +12,10 @@ type BuildConfig struct {
 }
 
 type BuildStep struct {
-	ID            string                   `json:"id,omitempty"`
-	Op            interface{}              `json:"op,omitempty"`
-	Inputs        []string                 `json:"inputs,omitempty"`
-	ResourceUsage []*resourcestypes.Sample `json:"resourceUsage,omitempty"`
+	ID            string                  `json:"id,omitempty"`
+	Op            interface{}             `json:"op,omitempty"`
+	Inputs        []string                `json:"inputs,omitempty"`
+	ResourceUsage *resourcestypes.Samples `json:"resourceUsage,omitempty"`
 }
 
 type Source struct {

solver/llbsolver/provenance/capture.go

@@ -59,7 +59,7 @@ type Capture struct {
 	SSH                 []SSH
 	NetworkAccess       bool
 	IncompleteMaterials bool
-	Samples             map[digest.Digest][]*resourcestypes.Sample
+	Samples             map[digest.Digest]*resourcestypes.Samples
 }
 
 func (c *Capture) Merge(c2 *Capture) error {
@@ -217,9 +217,9 @@ func (c *Capture) AddSSH(s SSH) {
 	c.SSH = append(c.SSH, s)
 }
 
-func (c *Capture) AddSamples(dgst digest.Digest, samples []*resourcestypes.Sample) {
+func (c *Capture) AddSamples(dgst digest.Digest, samples *resourcestypes.Samples) {
 	if c.Samples == nil {
-		c.Samples = map[digest.Digest][]*resourcestypes.Sample{}
+		c.Samples = map[digest.Digest]*resourcestypes.Samples{}
 	}
 	c.Samples[dgst] = samples
 }