Merge #150496 #150512

150496: asim: plot initial values only if nonzero r=tbg a=tbg

Epic: CRDB-25222


150512: teamcity: add `s390x-test-failure` label for relevant unit test failures r=rail a=rickystewart

Epic: CRDB-21133
Release note: None

Co-authored-by: Tobias Grieger <[email protected]>
Co-authored-by: Ricky Stewart <[email protected]>

Author: 3 people

build/teamcity/cockroach/ci/tests-ibm-cloud-linux-s390x/unit_tests_impl.sh

@@ -20,6 +20,7 @@ TESTS=$(bazel query 'kind(go_test, pkg/...) except attr("tags", "[\[ ]integratio
 
 set -x
 
-$(bazel info bazel-bin)/pkg/cmd/bazci/bazci_/bazci -- test --config=ci --config=dev \
-    $TESTS \
-    --profile=/artifacts/profile.gz
+$(bazel info bazel-bin)/pkg/cmd/bazci/bazci_/bazci --extralabels=s390x-test-failure -- \
+		       test --config=ci --config=dev \
+		       $TESTS \
+		       --profile=/artifacts/profile.gz

pkg/cmd/bazci/bazci.go

@@ -61,6 +61,7 @@ var (
 	port                    int
 	artifactsDir            string
 	githubPostFormatterName string
+	extraLabels             []string
 
 	rootCmd = &cobra.Command{
 		Use:   "bazci",
@@ -286,6 +287,12 @@ func init() {
 		8998,
 		"port to run the bazci server on",
 	)
+	rootCmd.Flags().StringSliceVar(
+		&extraLabels,
+		"extralabels",
+		[]string{},
+		"comma-separated list of extra labels to add to any filed GitHub issues",
+	)
 }
 
 func getRunEnvForBeaverHub() string {
@@ -488,7 +495,7 @@ func processTestXmls(testXmls []string) error {
 				postErrors = append(postErrors, fmt.Sprintf("Failed to parse test.xml file with the following error: %+v", err))
 				continue
 			}
-			if err := githubpost.PostFromTestXMLWithFormatterName(githubPostFormatterName, testSuites); err != nil {
+			if err := githubpost.PostFromTestXMLWithFormatterName(githubPostFormatterName, testSuites, extraLabels); err != nil {
 				postErrors = append(postErrors, fmt.Sprintf("Failed to process %s with the following error: %+v", testXml, err))
 				continue
 			}

pkg/cmd/bazci/githubpost/githubpost.go

@@ -111,14 +111,22 @@ func DefaultIssueFilerFromFormatter(
 
 }
 
-func getFailurePosterFromFormatterName(formatterName string) FailurePoster {
+func getFailurePosterFromFormatterName(formatterName string, extraLabels []string) FailurePoster {
 	var reqFromFailure Formatter
 	switch formatterName {
 	case "pebble-metamorphic":
 		reqFromFailure = formatPebbleMetamorphicIssue
 	default:
 		reqFromFailure = DefaultFormatter
 	}
+	if len(extraLabels) > 0 {
+		reqFromFailure2 := func(ctx context.Context, f Failure) (issues.IssueFormatter, issues.PostRequest) {
+			i, r := reqFromFailure(ctx, f)
+			r.Labels = append(r.Labels, extraLabels...)
+			return i, r
+		}
+		reqFromFailure = reqFromFailure2
+	}
 	return DefaultIssueFilerFromFormatter(reqFromFailure)
 }
 
@@ -129,7 +137,7 @@ func getFailurePosterFromFormatterName(formatterName string) FailurePoster {
 // GitHub.
 func PostFromJSON(formatterName string, in io.Reader) {
 	ctx := context.Background()
-	fileIssue := getFailurePosterFromFormatterName(formatterName)
+	fileIssue := getFailurePosterFromFormatterName(formatterName, []string{})
 	if err := listFailuresFromJSON(ctx, in, fileIssue); err != nil {
 		log.Println(err) // keep going
 	}
@@ -138,10 +146,13 @@ func PostFromJSON(formatterName string, in io.Reader) {
 // PostFromTestXMLWithFormatterName consumes a Bazel-style `test.xml` stream
 // and posts issues for any failed tests to GitHub. If there are no failed
 // tests, it does nothing. Unlike PostFromTestXMLWithFailurePoster, it takes a
-// formatter name.
-func PostFromTestXMLWithFormatterName(formatterName string, testXml buildutil.TestSuites) error {
+// formatter name. extraLabels is a list of extra labels to apply to the
+// created GitHub issue.
+func PostFromTestXMLWithFormatterName(
+	formatterName string, testXml buildutil.TestSuites, extraLabels []string,
+) error {
 	ctx := context.Background()
-	fileIssue := getFailurePosterFromFormatterName(formatterName)
+	fileIssue := getFailurePosterFromFormatterName(formatterName, extraLabels)
 	return PostFromTestXMLWithFailurePoster(ctx, fileIssue, testXml)
 }
 
@@ -738,7 +749,7 @@ func formatPebbleMetamorphicIssue(
 // insight into what caused the build failure and can't properly assign owners,
 // so a general issue is filed against test-eng in this case.
 func PostGeneralFailure(formatterName, logs string) {
-	fileIssue := getFailurePosterFromFormatterName(formatterName)
+	fileIssue := getFailurePosterFromFormatterName(formatterName, []string{})
 	postGeneralFailureImpl(logs, fileIssue)
 }
 

pkg/kv/kvserver/asim/history/history.go

@@ -32,32 +32,52 @@ func (h *History) Listen(ctx context.Context, sms []metrics.StoreMetrics) {
 	h.Recorded = append(h.Recorded, sms)
 }
 
-func (h *History) ShowRecordedValueAt(idx int, stat string) string {
-	var buf strings.Builder
-
+// PerStoreValuesAt returns, for tick `idx` and metric `stat`, the per-store
+// measurements at that tick, in the History's store order.
+func (h *History) PerStoreValuesAt(idx int, stat string) []float64 {
 	storeMetricsAtTick := h.Recorded[idx]
 	values := make([]float64, 0, len(storeMetricsAtTick))
 
+	// Extract values for each store. Note that h.Recorded[idx] is already sorted
+	// by store ID when appending to h.Recorded.
+	for _, sm := range storeMetricsAtTick {
+		value := sm.GetMetricValue(stat)
+		values = append(values, value)
+	}
+	return values
+}
+
+// ShowRecordedValueAt returns a string representation of the recorded values.
+// The returned boolean is false if (and only if) the recorded values were all
+// zero.
+func (h *History) ShowRecordedValueAt(idx int, stat string) (string, bool) {
+	var buf strings.Builder
+
+	values := h.PerStoreValuesAt(idx, stat)
+
 	_, _ = fmt.Fprintf(&buf, "[")
 
 	// Extract values for each store. Note that h.Recorded[idx] is already sorted
 	// by store ID when appending to h.Recorded.
-	for i, sm := range storeMetricsAtTick {
+	for i, v := range values {
 		if i > 0 {
 			_, _ = fmt.Fprintf(&buf, ", ")
 		}
-		value := sm.GetMetricValue(stat)
+		storeID := h.Recorded[idx][i].StoreID
+
 		if stat == "disk_fraction_used" {
-			_, _ = fmt.Fprintf(&buf, "s%v=%.2f", sm.StoreID, value)
+			_, _ = fmt.Fprintf(&buf, "s%v=%.2f", storeID, v)
 		} else {
-			_, _ = fmt.Fprintf(&buf, "s%v=%.0f", sm.StoreID, value)
+			_, _ = fmt.Fprintf(&buf, "s%v=%.0f", storeID, v)
 		}
-		values = append(values, value)
 	}
 	_, _ = fmt.Fprintf(&buf, "]")
 	stddev, _ := stats.StandardDeviation(values)
 	mean, _ := stats.Mean(values)
 	sum, _ := stats.Sum(values)
 	_, _ = fmt.Fprintf(&buf, " (stddev=%.2f, mean=%.2f, sum=%.0f)", stddev, mean, sum)
-	return buf.String()
+	// If the stddev is zero, all values are the same. If additionally the mean
+	// is zero, all values were zero.
+	nonzero := stddev > 0 || mean != 0
+	return buf.String(), nonzero
 }

pkg/kv/kvserver/asim/tests/datadriven_simulation_test.go

@@ -563,11 +563,17 @@ func TestDataDriven(t *testing.T) {
 					asciigraph.Height(height),
 					asciigraph.Width(width),
 				))
-
-				buf.WriteString("\ninitial store values: ")
-				buf.WriteString(history.ShowRecordedValueAt(0, stat))
-				buf.WriteString("\nlast store values: ")
-				buf.WriteString(history.ShowRecordedValueAt(len(history.Recorded)-1, stat))
+				buf.WriteString("\n")
+				at0, ok0 := history.ShowRecordedValueAt(0, stat)
+				if ok0 {
+					buf.WriteString("initial store values: ")
+					buf.WriteString(at0)
+					buf.WriteString("\n")
+				}
+				s, _ := history.ShowRecordedValueAt(len(history.Recorded)-1, stat)
+				buf.WriteString("last store values: ")
+				buf.WriteString(s)
+				buf.WriteString("\n")
 
 				return buf.String()
 			default:

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing.txt

@@ -62,7 +62,6 @@ plot stat=qps sample=2
   395 ┤ ││ │                       ││                         │
     0 ┼─────────────────────────────╯─────────────────────────╯
                                              qps
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=996, s2=998, s3=996, s4=1000, s5=1000, s6=1012, s7=1001] (stddev=5.07, mean=1000.43, sum=7003)
 
 plot stat=replica_moves sample=2
@@ -84,7 +83,6 @@ plot stat=replica_moves sample=2
  0.33 ┤│ │
  0.00 ┼───────────────────────────────────────────────────────────────────────────────
                                         replica_moves
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=5, s2=0, s3=1, s4=0, s5=0, s6=0, s7=0] (stddev=1.73, mean=0.86, sum=6)
 
 # The cluster settings and simulation variables can be modified to examine how
@@ -133,7 +131,6 @@ plot stat=qps sample=4
   467 ┤ │    │││      │││    │ │  ││   ││   │  │      │       │      ╭│       │
     0 ┼────────────────────────────╯──────────────────────────────────────────────────
                                              qps
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=999, s2=0, s3=2002, s4=1998, s5=0, s6=1003, s7=999] (stddev=755.93, mean=1000.14, sum=7001)
 
 # Plot the replica movements for the same sample, we should see a steadily
@@ -158,7 +155,6 @@ plot stat=replica_moves sample=4
   1.60 ┤     │╭╭─────╭──╭──────╯───────╭───────╯
   0.00 ┼───────────────────────────────╯
                                          replica_moves
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=24, s2=11, s3=16, s4=11, s5=5, s6=10, s7=6] (stddev=5.99, mean=11.86, sum=83)
 
 # vim:ft=sh

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_skewed_cpu_even_ranges_mma

@@ -83,7 +83,6 @@ plot stat=cpu sample=1
   335000000 ┤│  ╭──────────╯   ╭───────────╯─────────────╯───╯ ╭─╯
           0 ┼──────────────────╯───╯─────────────────╯─────────╯
                                                    cpu
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=569996683, s2=572459858, s3=577041089, s4=557385814, s5=555426485, s6=556145510, s7=570251533, s8=555498030, s9=557066732] (stddev=8256286.12, mean=563474637.11, sum=5071271734)
 
 plot stat=leases sample=1
@@ -149,5 +148,4 @@ plot stat=write_bytes_per_second
   335 ┤│  ╭╮             ╭───────────╯─────────────╯───╯ ╭──╯
     0 ┼───╯╰─────────────╯───╯─────────────────╯─────────╯
                                     write_bytes_per_second
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=3757, s2=3916, s3=4016, s4=544, s5=554, s6=559, s7=569, s8=559, s9=546] (stddev=1576.26, mean=1668.89, sum=15020)

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_skewed_cpu_even_ranges_mma_and_queues

@@ -83,7 +83,6 @@ plot stat=cpu sample=1
   159192871 ┤│ │╭─╭─╯─╭───╯───────────────────────────╯
           0 ┼─────╯───╯
                                                    cpu
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=571144223, s2=571097655, s3=570432769, s4=556522751, s5=555345020, s6=560111935, s7=557209028, s8=559901378, s9=571601108] (stddev=6738397.52, mean=563707318.56, sum=5073365867)
 
 plot stat=leases sample=1
@@ -149,5 +148,4 @@ plot stat=write_bytes_per_second
   335 ┤│ ╭────╭────────────────────────────╯╯   ╭──╯
     0 ┼───────╯─────╯───────────────────────────╯
                                     write_bytes_per_second
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=4023, s2=3324, s3=3483, s4=832, s5=563, s6=698, s7=819, s8=693, s9=564] (stddev=1387.79, mean=1666.56, sum=14999)

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_skewed_cpu_even_ranges_sma

@@ -83,7 +83,6 @@ plot stat=cpu sample=1
   335000000 ┤│  │   ╭╭│╭╯╯──╯╯
           0 ┼──────────╯╯
                                                    cpu
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=562526385, s2=573916363, s3=562940607, s4=574474977, s5=560862237, s6=561701547, s7=558426354, s8=558535231, s9=559898216] (stddev=5806267.41, mean=563697990.78, sum=5073281917)
 
 plot stat=leases sample=1
@@ -149,5 +148,4 @@ plot stat=write_bytes_per_second
   333 ┤│    ╭──╯╭╯
     0 ┼─────────╯
                                     write_bytes_per_second
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=2358, s2=4029, s3=2063, s4=1391, s5=1245, s6=1398, s7=851, s8=829, s9=834] (stddev=977.64, mean=1666.44, sum=14998)

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_skewed_cpu_even_ranges_sma_and_queues

@@ -83,7 +83,6 @@ plot stat=cpu sample=1
   167928572 ┤│   │╭╭╯─╯
           0 ┼────╯─╯╯
                                                    cpu
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=561422336, s2=564403791, s3=564929833, s4=562766740, s5=564001678, s6=576295997, s7=559127795, s8=561707672, s9=560029049] (stddev=4772070.32, mean=563853876.78, sum=5074684891)
 
 plot stat=leases sample=1
@@ -149,5 +148,4 @@ plot stat=write_bytes_per_second
   334 ┤│  ╭─╯╯╯─╯
     0 ┼───╯─╯╯╯
                                     write_bytes_per_second
-initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=0.00, mean=0.00, sum=0)
 last store values: [s1=1098, s2=1667, s3=2366, s4=1531, s5=1948, s6=1536, s7=1386, s8=1669, s9=1799] (stddev=337.21, mean=1666.67, sum=15000)