refactor: disable telemetry error sender by default (#3395)

We already emit metrics and those metrics are used to alert
internally/customers. There is no need to send alerts to incident.io via
webhook by default

Author: iamKunalGupta

flow/alerting/alerting.go

@@ -498,8 +498,8 @@ func (a *Alerter) logFlowErrorInternal(
 	errorClass, errInfo := GetErrorClass(ctx, inErr)
 	tags = append(tags, "errorClass:"+errorClass.String(), "errorAction:"+errorClass.ErrorAction().String())
 
-	if !internal.PeerDBTelemetryErrorActionBasedAlertingEnabled() || errorClass.ErrorAction() == NotifyTelemetry {
-		// Warnings alert us just like errors until there's a customer warning system
+	// Only send alerts to telemetry sender (incident.io) if the env is enabled
+	if internal.PeerDBTelemetrySenderSendErrorAlertsEnabled() {
 		a.sendTelemetryMessage(ctx, logger, flowName, inErrWithStack, telemetry.ERROR, tags...)
 	}
 	loggerFunc(fmt.Sprintf("Emitting error/warning metric: '%s'", inErr.Error()),

flow/internal/config.go

@@ -193,10 +193,11 @@ func PeerDBMaintenanceModeWaitAlertSeconds() int {
 	return getEnvConvert("PEERDB_MAINTENANCE_MODE_WAIT_ALERT_SECONDS", 600, strconv.Atoi)
 }
 
-func PeerDBTelemetryErrorActionBasedAlertingEnabled() bool {
-	enabled, err := strconv.ParseBool(GetEnvString("PEERDB_TELEMETRY_ERROR_ACTION_BASED_ALERTING_ENABLED", "false"))
+// PEERDB_TELEMETRY_SENDER_SEND_ERROR_ALERTS_ENABLED is whether to send error alerts to the telemetry sender
+func PeerDBTelemetrySenderSendErrorAlertsEnabled() bool {
+	enabled, err := strconv.ParseBool(GetEnvString("PEERDB_TELEMETRY_SENDER_SEND_ERROR_ALERTS_ENABLED", "false"))
 	if err != nil {
-		slog.Error("failed to parse PEERDB_TELEMETRY_ERROR_ACTION_BASED_ALERTING_ENABLED to bool", "error", err)
+		slog.Error("failed to parse PEERDB_TELEMETRY_SENDER_SEND_ERROR_ALERTS_ENABLED to bool", "error", err)
 		return false
 	}
 	return enabled