Experimental: Tackle tempo max trace reached

Author: mrnicegyu11

services/monitoring/tempo_config.yaml.j2

@@ -9,7 +9,9 @@ distributor:
           endpoint: "0.0.0.0:4318"  # Explicitly bind to all interfaces https://github.com/grafana/tempo/issues/4465
         grpc:
           endpoint: "0.0.0.0:4317"  # Explicitly bind to all interfaces https://github.com/grafana/tempo/issues/4465
-
+  log_discarded_spans:
+    enabled: true
+    include_all_attributes: false
 #ingester:
 #  max_block_duration: 5m               # cut the headblock when this much time passes. this should probably be left alone normally
 
@@ -62,3 +64,9 @@ overrides:
   defaults:
     metrics_generator:
       processors: ['service-graphs', 'local-blocks']
+    global:
+      max_bytes_per_trace: 25000000  # 25MB
+    ingestion:
+      rate_limit_bytes: 30000000
+      burst_size_bytes: 40000000
+      max_traces_per_user: 10000