Add bulk indexing failure log entry to ignores (#17774)

Author: ericywl

integrationservertest/internal/asserts/apm_logs.go

@@ -32,8 +32,12 @@ import (
 // for displaying information.
 type APMLogEntry struct {
 	Timestamp time.Time `json:"@timestamp"`
-	Message   string
-	LogLogger string `json:"log.logger"`
+	Message   string    `json:"message"`
+	LogLogger string    `json:"log.logger"`
+	Service   struct {
+		// Version shows which version this log entry came from.
+		Version string `json:"version"`
+	} `json:"service"`
 }
 
 func ZeroAPMLogs(t *testing.T, resp search.Response) {

integrationservertest/logs_filters.go

@@ -36,6 +36,15 @@ func (e apmErrorLogs) ToQueries() []types.Query {
 }
 
 var (
+	// This error seems to happen occasionally in-between upgrades.
+	// On upgrade, the APM server gets restarted while metrics are not fully flushed,
+	// causing this error to occur.
+	bulkIndexingFailed = apmErrorLog(types.Query{
+		MatchPhrase: map[string]types.MatchPhraseQuery{
+			"message": {Query: "bulk indexing request failed"},
+		},
+	})
+
 	tlsHandshakeError = apmErrorLog(types.Query{
 		MatchPhrasePrefix: map[string]types.MatchPhrasePrefixQuery{
 			"message": {Query: "http: TLS handshake error from 127.0.0.1:"},

integrationservertest/standalone_test.go

@@ -154,6 +154,7 @@ func managed7Runner(fromVersion7, toVersion8, toVersion9 ech.Version, config upg
 					addIndexTemplateTracesError,
 				},
 				APMErrorLogsIgnored: apmErrorLogs{
+					bulkIndexingFailed,
 					tlsHandshakeError,
 					esReturnedUnknown503,
 					refreshCache403,
@@ -215,6 +216,7 @@ func managed8Runner(fromVersion7, toVersion8, toVersion9 ech.Version, config upg
 					eventLoopShutdown,
 				},
 				APMErrorLogsIgnored: apmErrorLogs{
+					bulkIndexingFailed,
 					tlsHandshakeError,
 					esReturnedUnknown503,
 					refreshCache403,
@@ -261,6 +263,7 @@ func managed9Runner(fromVersion7, toVersion8, toVersion9 ech.Version, config upg
 					eventLoopShutdown,
 				},
 				APMErrorLogsIgnored: apmErrorLogs{
+					bulkIndexingFailed,
 					tlsHandshakeError,
 					esReturnedUnknown503,
 					refreshCache503,

integrationservertest/upgrade_test.go

@@ -162,6 +162,7 @@ func buildTestSteps(t *testing.T, versions ech.Versions, config upgradeTestConfi
 	// unrelated to our test.
 	steps = append(steps, checkErrorLogsStep{
 		APMErrorLogsIgnored: apmErrorLogs{
+			bulkIndexingFailed,
 			tlsHandshakeError,
 			esReturnedUnknown503,
 			refreshCache403,