Merge pull request #351 from areed/concurrent-iops

Benchmark write latency with background IOPS

Author: areed

examples/preflight/host/filesystem-performance.yaml

@@ -10,22 +10,25 @@ spec:
         fileSize: 22Mi
         operationSizeBytes: 2300
         datasync: true
+        enableBackgroundIOPS: true
+        backgroundIOPSWarmupSeconds: 60
+        backgroundWriteIOPS: 300
+        backgroundWriteIOPSJobs: 6
+        backgroundReadIOPS: 50
+        backgroundReadIOPSJobs: 1
   analyzers:
     - filesystemPerformance:
         collectorName: etcd-perf
         outcomes:
-          - fail:
-              when: "iops < 50"
-              message: Insufficient random read IOPS
           - pass:
               when: "p99 < 3ms"
-              message: Write latency is great!
+              message: "Write latency is great! (p99: {{ .P99 }})"
           - pass:
               when: "p99 < 5ms"
-              message: Write latency is ok
+              message: "Write latency is ok (p99: {{ .P99 }})"
           - warn:
               when: "p99 < 8ms"
-              message: Write latency is high
+              message: "Write latency is high {{ .String }}"
           - fail:
               when: "p99 >= 8ms"
-              message: Write latency is too high
+              message: "Write latency is too high {{ .String }}"

pkg/analyze/host_filesystem_performance.go

@@ -1,10 +1,12 @@
 package analyzer
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"html/template"
+	"log"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
 
@@ -51,7 +53,7 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 		if outcome.Fail != nil {
 			if outcome.Fail.When == "" {
 				result.IsFail = true
-				result.Message = outcome.Fail.Message
+				result.Message = renderFSPerfOutcome(outcome.Fail.Message, fsPerf)
 				result.URI = outcome.Fail.URI
 
 				return &result, nil
@@ -64,15 +66,15 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 
 			if isMatch {
 				result.IsFail = true
-				result.Message = outcome.Fail.Message
+				result.Message = renderFSPerfOutcome(outcome.Fail.Message, fsPerf)
 				result.URI = outcome.Fail.URI
 
 				return &result, nil
 			}
 		} else if outcome.Warn != nil {
 			if outcome.Warn.When == "" {
 				result.IsWarn = true
-				result.Message = outcome.Warn.Message
+				result.Message = renderFSPerfOutcome(outcome.Warn.Message, fsPerf)
 				result.URI = outcome.Warn.URI
 
 				return &result, nil
@@ -85,15 +87,15 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 
 			if isMatch {
 				result.IsWarn = true
-				result.Message = outcome.Warn.Message
+				result.Message = renderFSPerfOutcome(outcome.Warn.Message, fsPerf)
 				result.URI = outcome.Warn.URI
 
 				return &result, nil
 			}
 		} else if outcome.Pass != nil {
 			if outcome.Pass.When == "" {
 				result.IsPass = true
-				result.Message = outcome.Pass.Message
+				result.Message = renderFSPerfOutcome(outcome.Pass.Message, fsPerf)
 				result.URI = outcome.Pass.URI
 
 				return &result, nil
@@ -106,7 +108,7 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 
 			if isMatch {
 				result.IsPass = true
-				result.Message = outcome.Pass.Message
+				result.Message = renderFSPerfOutcome(outcome.Pass.Message, fsPerf)
 				result.URI = outcome.Pass.URI
 
 				return &result, nil
@@ -125,14 +127,6 @@ func compareHostFilesystemPerformanceConditionalToActual(conditional string, fsP
 	keyword := strings.ToLower(parts[0])
 	comparator := parts[1]
 
-	if keyword == "iops" {
-		desiredInt, err := strconv.ParseInt(parts[2], 10, 64)
-		if err != nil {
-			return false, errors.Wrapf(err, "failed to parse desired iops %q as integer", parts[2])
-		}
-		return doCompareHostFilesystemIOPS(comparator, fsPerf.IOPS, int(desiredInt))
-	}
-
 	desiredDuration, err := time.ParseDuration(parts[2])
 	if err != nil {
 		return false, errors.Wrapf(err, "failed to parse duration %q", parts[2])
@@ -201,19 +195,17 @@ func doCompareHostFilesystemPerformance(operator string, actual time.Duration, d
 	return false, fmt.Errorf("Unknown filesystem performance operator %q", operator)
 }
 
-func doCompareHostFilesystemIOPS(operator string, actual int, desired int) (bool, error) {
-	switch operator {
-	case "<":
-		return actual < desired, nil
-	case "<=":
-		return actual <= desired, nil
-	case ">":
-		return actual > desired, nil
-	case ">=":
-		return actual >= desired, nil
-	case "=", "==", "===":
-		return actual == desired, nil
+func renderFSPerfOutcome(outcome string, fsPerf collect.FSPerfResults) string {
+	t, err := template.New("").Parse(outcome)
+	if err != nil {
+		log.Printf("Failed to parse filesystem performance outcome: %v", err)
+		return outcome
 	}
-
-	return false, fmt.Errorf("Unknown filesystem IOPS operator %q", operator)
+	var buf bytes.Buffer
+	err = t.Execute(&buf, fsPerf)
+	if err != nil {
+		log.Printf("Failed to render filesystem performance outcome: %v", err)
+		return outcome
+	}
+	return buf.String()
 }

pkg/analyze/host_filesystem_performance_test.go

@@ -19,51 +19,6 @@ func TestAnalyzeHostFilesystemPerformance(t *testing.T) {
 		result       *AnalyzeResult
 		expectErr    bool
 	}{
-		{
-			name: "IOPS",
-			fsPerf: &collect.FSPerfResults{
-				IOPS: 50,
-			},
-			hostAnalyzer: &troubleshootv1beta2.FilesystemPerformanceAnalyze{
-				Outcomes: []*troubleshootv1beta2.Outcome{
-					{
-						Fail: &troubleshootv1beta2.SingleOutcome{
-							When:    "iops < 20",
-							Message: "IOPS < 20",
-						},
-					},
-					{
-						Fail: &troubleshootv1beta2.SingleOutcome{
-							When:    "iops <= 20",
-							Message: "IOPS <= 30",
-						},
-					},
-					{
-						Fail: &troubleshootv1beta2.SingleOutcome{
-							When:    "iops > 70",
-							Message: "IOPS > 70",
-						},
-					},
-					{
-						Fail: &troubleshootv1beta2.SingleOutcome{
-							When:    "iops >= 100",
-							Message: "IOPS >= 100",
-						},
-					},
-					{
-						Pass: &troubleshootv1beta2.SingleOutcome{
-							When:    "iops == 50",
-							Message: "IOPS == 50",
-						},
-					},
-				},
-			},
-			result: &AnalyzeResult{
-				Title:   "Filesystem Performance",
-				IsPass:  true,
-				Message: "IOPS == 50",
-			},
-		},
 		{
 			name: "Cover",
 			fsPerf: &collect.FSPerfResults{

pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go

@@ -73,13 +73,43 @@ type TCPConnect struct {
 	Timeout           string `json:"timeout,omitempty"`
 }
 
+// FilesystemPerformance benchmarks sequential write latency on a single file.
+// The optional background IOPS feature attempts to mimic real-world conditions by running read and
+// write workloads prior to and during benchmark execution.
 type FilesystemPerformance struct {
-	HostCollectorMeta  `json:",inline" yaml:",inline"`
+	HostCollectorMeta `json:",inline" yaml:",inline"`
+	// The size of each write operation performed while benchmarking. This does not apply to the
+	// background IOPS feature if enabled, since those must be fixed at 4096.
 	OperationSizeBytes uint64 `json:"operationSize,omitempty"`
-	Directory          string `json:"directory,omitempty"`
-	FileSize           string `json:"fileSize,omitempty"`
-	Sync               bool   `json:"sync,omitempty"`
-	Datasync           bool   `json:"datasync,omitempty"`
+	// The directory where the benchmark will create files.
+	Directory string `json:"directory,omitempty"`
+	// The size of the file used in the benchmark. The number of IO operations for the benchmark
+	// will be FileSize / OperationSizeBytes. Accepts valid Kubernetes resource units such as Mi.
+	FileSize string `json:"fileSize,omitempty"`
+	// Whether to call sync on the file after each write. Does not apply to background IOPS task.
+	Sync bool `json:"sync,omitempty"`
+	// Whether to call datasync on the file after each write. Skipped if Sync is also true. Does not
+	// apply to background IOPS task.
+	Datasync bool `json:"datasync,omitempty"`
+
+	// Enable the background IOPS feature.
+	EnableBackgroundIOPS bool `json:"enableBackgroundIOPS"`
+	// How long to run the background IOPS read and write workloads prior to starting the benchmarks.
+	BackgroundIOPSWarmupSeconds int `json:"backgroundIOPSWarmupSeconds"`
+	// The target write IOPS to run while benchmarking. This is a limit and there is no guarantee
+	// it will be reached. This is the total IOPS for all background write jobs.
+	BackgroundWriteIOPS int `json:"backgroundWriteIOPS"`
+	// The target read IOPS to run while benchmarking. This is a limit and there is no guarantee
+	// it will be reached. This is the total IOPS for all background read jobs.
+	BackgroundReadIOPS int `json:"backgroundReadIOPS"`
+	// Number of threads to use for background write IOPS. This should be set high enough to reach
+	// the target specified in BackgroundWriteIOPS.
+	// Example: If BackgroundWriteIOPS is 100 and write latency is 10ms then a single job would
+	// barely be able to reach 100 IOPS so this should be at least 2.
+	BackgroundWriteIOPSJobs int `json:"backgroundWriteIOPSJobs"`
+	// Number of threads to use for background read IOPS. This should be set high enough to reach
+	// the target specified in BackgrounReadIOPS.
+	BackgroundReadIOPSJobs int `json:"backgroundReadIOPSJobs"`
 }
 
 type Certificate struct {

pkg/collect/host_filesystem_performance.go

@@ -1,8 +1,10 @@
 package collect
 
 import (
+	"bytes"
 	"math"
 	"math/rand"
+	"text/template"
 	"time"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
@@ -49,7 +51,6 @@ type FSPerfResults struct {
 	P999    time.Duration
 	P9995   time.Duration
 	P9999   time.Duration
-	IOPS    int
 }
 
 func getPercentileIndex(p float64, items int) int {
@@ -58,3 +59,33 @@ func getPercentileIndex(p float64, items int) int {
 	}
 	return int(math.Ceil(p*float64(items))) - 1
 }
+
+var fsPerfTmpl = template.Must(template.New("").Parse(`
+   Min: {{ .Min }}
+   Max: {{ .Max }}
+   Avg: {{ .Average }}
+    p1: {{ .P1 }}
+    p5: {{ .P5 }}
+   p10: {{ .P10 }}
+   p20: {{ .P20 }}
+   p30: {{ .P30 }}
+   p40: {{ .P40 }}
+   p50: {{ .P50 }}
+   p60: {{ .P60 }}
+   p70: {{ .P70 }}
+   p80: {{ .P80 }}
+   p90: {{ .P90 }}
+   p95: {{ .P95 }}
+   p99: {{ .P99 }}
+ p99.5: {{ .P995 }}
+ p99.9: {{ .P999 }}
+p99.95: {{ .P9995 }}
+p99.99: {{ .P9999 }}`))
+
+func (f FSPerfResults) String() string {
+	var buf bytes.Buffer
+
+	fsPerfTmpl.Execute(&buf, f)
+
+	return buf.String()
+}