[monitoring] Remove HostReportEvent from Kafka/Elasticsearch pipeline

Host reports are too large to store in Kafka/Elasticsearch due to their high frequency (~60s intervals) and data volume. Host metrics should use Prometheus instead.

Changes:
- Remove HostReportEvent and RunningFrameSummary from monitoring.proto
- Remove publishHostReportEvent from HostReportHandler, KafkaEventPublisher
- Remove host report indexing from ElasticsearchClient, KafkaEventConsumer
- Remove buildHostReportEvent from MonitoringEventBuilder
- Update documentation to note host metrics use Prometheus
- Keep HostEvent for state change audit trail (up/down/locked)

Author: ramonfigueiredo

cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java

@@ -72,7 +72,6 @@
 import com.imageworks.spcue.monitoring.MonitoringEventBuilder;
 import com.imageworks.spcue.grpc.monitoring.EventType;
 import com.imageworks.spcue.grpc.monitoring.HostEvent;
-import com.imageworks.spcue.grpc.monitoring.HostReportEvent;
 
 public class HostReportHandler {
 
@@ -170,9 +169,6 @@ public void queueHostReport(HostReport report) {
     public void handleHostReport(HostReport report, boolean isBoot) {
         long startTime = System.currentTimeMillis();
         try {
-            // Publish host report event to Kafka for monitoring
-            publishHostReportEvent(report, isBoot);
-
             // Record Prometheus metric for host report
             if (prometheusMetrics != null) {
                 String facility = report.getHost().getFacility();
@@ -1125,23 +1121,6 @@ public void setMonitoringEventBuilder(MonitoringEventBuilder monitoringEventBuil
         this.monitoringEventBuilder = monitoringEventBuilder;
     }
 
-    /**
-     * Publishes a host report event to Kafka for monitoring purposes. This method is called
-     * asynchronously to avoid blocking the report queue.
-     */
-    private void publishHostReportEvent(HostReport report, boolean isBoot) {
-        if (kafkaEventPublisher == null || !kafkaEventPublisher.isEnabled()) {
-            return;
-        }
-
-        try {
-            HostReportEvent event = monitoringEventBuilder.buildHostReportEvent(report, isBoot);
-            kafkaEventPublisher.publishHostReportEvent(event);
-        } catch (Exception e) {
-            logger.trace("Failed to publish host report event: {}", e.getMessage());
-        }
-    }
-
     /**
      * Publishes a host state change event to Kafka for monitoring purposes.
      */

cuebot/src/main/java/com/imageworks/spcue/monitoring/ElasticsearchClient.java

@@ -75,7 +75,6 @@ public class ElasticsearchClient {
     private static final String INDEX_LAYER_EVENTS = "opencue-layer-events";
     private static final String INDEX_FRAME_EVENTS = "opencue-frame-events";
     private static final String INDEX_HOST_EVENTS = "opencue-host-events";
-    private static final String INDEX_HOST_REPORTS = "opencue-host-reports";
     private static final String INDEX_PROC_EVENTS = "opencue-proc-events";
 
     @Autowired
@@ -125,7 +124,7 @@ private void initializeThreadPool() {
      */
     private void createIndexTemplates() {
         String[] indexPrefixes = {INDEX_JOB_EVENTS, INDEX_LAYER_EVENTS, INDEX_FRAME_EVENTS,
-                INDEX_HOST_EVENTS, INDEX_HOST_REPORTS, INDEX_PROC_EVENTS};
+                INDEX_HOST_EVENTS, INDEX_PROC_EVENTS};
 
         for (String prefix : indexPrefixes) {
             try {
@@ -241,15 +240,6 @@ public void indexHostEvent(String eventId, String jsonDocument) {
         indexDocument(INDEX_HOST_EVENTS, eventId, jsonDocument);
     }
 
-    /**
-     * Indexes a host report document.
-     */
-    public void indexHostReport(String eventId, String jsonDocument) {
-        if (!enabled)
-            return;
-        indexDocument(INDEX_HOST_REPORTS, eventId, jsonDocument);
-    }
-
     /**
      * Indexes a proc event document.
      */

cuebot/src/main/java/com/imageworks/spcue/monitoring/KafkaEventConsumer.java

@@ -47,7 +47,6 @@ public class KafkaEventConsumer {
     private static final String TOPIC_LAYER_EVENTS = "opencue.layer.events";
     private static final String TOPIC_FRAME_EVENTS = "opencue.frame.events";
     private static final String TOPIC_HOST_EVENTS = "opencue.host.events";
-    private static final String TOPIC_HOST_REPORTS = "opencue.host.reports";
     private static final String TOPIC_PROC_EVENTS = "opencue.proc.events";
 
     @Autowired
@@ -108,7 +107,7 @@ private void initializeConsumer() {
 
         // Subscribe to all event topics
         consumer.subscribe(Arrays.asList(TOPIC_JOB_EVENTS, TOPIC_LAYER_EVENTS, TOPIC_FRAME_EVENTS,
-                TOPIC_HOST_EVENTS, TOPIC_HOST_REPORTS, TOPIC_PROC_EVENTS));
+                TOPIC_HOST_EVENTS, TOPIC_PROC_EVENTS));
     }
 
     private void startConsumerThread() {
@@ -165,9 +164,6 @@ private void processRecord(ConsumerRecord<String, String> record) {
             case TOPIC_HOST_EVENTS:
                 elasticsearchClient.indexHostEvent(eventId, value);
                 break;
-            case TOPIC_HOST_REPORTS:
-                elasticsearchClient.indexHostReport(eventId, value);
-                break;
             case TOPIC_PROC_EVENTS:
                 elasticsearchClient.indexProcEvent(eventId, value);
                 break;

cuebot/src/main/java/com/imageworks/spcue/monitoring/KafkaEventPublisher.java

@@ -41,7 +41,6 @@
 import com.imageworks.spcue.grpc.monitoring.EventType;
 import com.imageworks.spcue.grpc.monitoring.FrameEvent;
 import com.imageworks.spcue.grpc.monitoring.HostEvent;
-import com.imageworks.spcue.grpc.monitoring.HostReportEvent;
 import com.imageworks.spcue.grpc.monitoring.JobEvent;
 import com.imageworks.spcue.grpc.monitoring.LayerEvent;
 import com.imageworks.spcue.grpc.monitoring.ProcEvent;
@@ -67,7 +66,6 @@ public class KafkaEventPublisher extends ThreadPoolExecutor {
     private static final String TOPIC_LAYER_EVENTS = "opencue.layer.events";
     private static final String TOPIC_FRAME_EVENTS = "opencue.frame.events";
     private static final String TOPIC_HOST_EVENTS = "opencue.host.events";
-    private static final String TOPIC_HOST_REPORTS = "opencue.host.reports";
     private static final String TOPIC_PROC_EVENTS = "opencue.proc.events";
 
     @Autowired
@@ -205,16 +203,6 @@ public void publishHostEvent(HostEvent event) {
                 event.getHeader().getEventType().name());
     }
 
-    /**
-     * Publishes a host report event to Kafka.
-     */
-    public void publishHostReportEvent(HostReportEvent event) {
-        if (!enabled)
-            return;
-        publishEvent(TOPIC_HOST_REPORTS, event.getHostName(), event,
-                event.getHeader().getEventType().name());
-    }
-
     /**
      * Publishes a proc event to Kafka.
      */

cuebot/src/main/java/com/imageworks/spcue/monitoring/MonitoringEventBuilder.java

@@ -36,15 +36,10 @@
 import com.imageworks.spcue.grpc.monitoring.EventType;
 import com.imageworks.spcue.grpc.monitoring.FrameEvent;
 import com.imageworks.spcue.grpc.monitoring.HostEvent;
-import com.imageworks.spcue.grpc.monitoring.HostReportEvent;
 import com.imageworks.spcue.grpc.monitoring.JobEvent;
 import com.imageworks.spcue.grpc.monitoring.LayerEvent;
 import com.imageworks.spcue.grpc.monitoring.ProcEvent;
-import com.imageworks.spcue.grpc.monitoring.RunningFrameSummary;
 import com.imageworks.spcue.grpc.report.FrameCompleteReport;
-import com.imageworks.spcue.grpc.report.HostReport;
-import com.imageworks.spcue.grpc.report.RenderHost;
-import com.imageworks.spcue.grpc.report.RunningFrameInfo;
 
 import java.util.Arrays;
 import java.util.List;
@@ -300,42 +295,6 @@ public HostEvent buildHostEvent(EventType eventType, DispatchHost host,
         return builder.build();
     }
 
-    /**
-     * Builds a HostReportEvent from a host report.
-     */
-    public HostReportEvent buildHostReportEvent(HostReport report, boolean isBoot) {
-        EventType eventType = isBoot ? EventType.HOST_BOOT : EventType.HOST_REPORT;
-        EventHeader header = publisher.createEventHeader(eventType).build();
-
-        RenderHost rhost = report.getHost();
-
-        HostReportEvent.Builder builder = HostReportEvent.newBuilder().setHeader(header)
-                .setHostName(rhost.getName()).setFacility(rhost.getFacility()).setHostData(rhost)
-                .setCoreInfo(report.getCoreInfo()).setIsBootReport(isBoot);
-
-        // Add running frame summaries
-        List<RunningFrameSummary> frameSummaries = report.getFramesList().stream()
-                .map(this::buildRunningFrameSummary).collect(Collectors.toList());
-        builder.addAllRunningFrames(frameSummaries);
-
-        return builder.build();
-    }
-
-    /**
-     * Builds a running frame summary from RunningFrameInfo.
-     */
-    private RunningFrameSummary buildRunningFrameSummary(RunningFrameInfo frame) {
-        return RunningFrameSummary.newBuilder().setFrameId(frame.getFrameId())
-                .setFrameName(frame.getFrameName()).setJobId(frame.getJobId())
-                .setJobName(frame.getJobName()).setLayerId(frame.getLayerId())
-                .setNumCores(frame.getNumCores()).setStartTime(frame.getStartTime())
-                .setRss(frame.getRss()).setMaxRss(frame.getMaxRss()).setVsize(frame.getVsize())
-                .setMaxVsize(frame.getMaxVsize()).setUsedGpuMemory(frame.getUsedGpuMemory())
-                .setMaxUsedGpuMemory(frame.getMaxUsedGpuMemory())
-                .setUsedSwapMemory(frame.getUsedSwapMemory()).setLluTime(frame.getLluTime())
-                .build();
-    }
-
     /**
      * Builds a ProcEvent for a proc booking/unbooking.
      */

docs/_docs/concepts/render-farm-monitoring.md

@@ -41,7 +41,6 @@ Cuebot publishes events to Apache Kafka topics when significant state changes oc
 | `opencue.layer.events` | Layer state changes |
 | `opencue.frame.events` | Frame execution events (started, completed, failed, retried) |
 | `opencue.host.events` | Host state changes (up, down, locked, nimby) |
-| `opencue.host.reports` | Periodic host status reports with resource utilization |
 | `opencue.proc.events` | Process allocation and deallocation events |
 
 Events are published asynchronously to avoid impacting render farm performance. A bounded queue ensures the system remains responsive even under high load.

docs/_docs/reference/monitoring-reference.md

@@ -49,7 +49,6 @@ This reference provides comprehensive documentation for all monitoring system co
 | `opencue.layer.events` | `layerId` | Layer state changes |
 | `opencue.frame.events` | `frameId` | Frame execution events |
 | `opencue.host.events` | `hostId` | Host state changes |
-| `opencue.host.reports` | `hostId` | Periodic host status reports |
 | `opencue.proc.events` | `procId` | Process allocation events |
 
 ![Kafka UI for Apache Kafka](/assets/images/opencue_monitoring/opencue_monitoring_ui_for_apache_kafka.png)
@@ -208,38 +207,6 @@ All events include a `header` field with common metadata, plus event-specific fi
 }
 ```
 
-### Host report payload
-
-```json
-{
-  "header": {
-    "event_id": "550e8400-e29b-41d4-a716-446655440004",
-    "event_type": "HOST_REPORT",
-    "timestamp": "1732446600000",
-    "source_cuebot": "cuebot-01",
-    "correlation_id": ""
-  },
-  "host_id": "550e8400-e29b-41d4-a716-446655440004",
-  "host_name": "render-node-01",
-  "facility": "cloud",
-  "alloc_name": "render.general",
-  "state": "UP",
-  "lock_state": "OPEN",
-  "nimby_enabled": false,
-  "total_cores": 64,
-  "idle_cores": 32,
-  "total_memory": "137438953472",
-  "free_memory": "68719476736",
-  "total_gpu_memory": "25769803776",
-  "free_gpu_memory": "25769803776",
-  "load": 1250,
-  "ping_time": "1732443000000",
-  "boot_time": "1732300000000",
-  "os": "Linux",
-  "running_procs": 4
-}
-```
-
 ## Prometheus metrics
 
 ### Job and frame metrics
@@ -357,7 +324,7 @@ metrics.prometheus.frame.memory.buckets=1073741824,2147483648,4294967296,8589934
 Examples:
 - `opencue-job-events-2024.11.24`
 - `opencue-frame-events-2024.11.24`
-- `opencue-host-reports-2024.11.24`
+- `opencue-host-events-2024.11.24`
 
 ### Index mappings
 

proto/src/monitoring.proto

@@ -44,7 +44,6 @@ enum EventType {
     FRAME_CHECKPOINT = 27;
 
     // Host events
-    HOST_REPORT = 30;
     HOST_BOOT = 31;
     HOST_STATE_CHANGED = 32;
     HOST_LOCKED = 33;
@@ -158,44 +157,6 @@ message HostEvent {
     string reason = 7;
 }
 
-// -------- Host Report Event --------
-
-message HostReportEvent {
-    EventHeader header = 1;
-
-    // Host information
-    string host_name = 2;
-    string facility = 3;
-
-    // Full host report data
-    report.RenderHost host_data = 4;
-    report.CoreDetail core_info = 5;
-
-    // Running frame summaries
-    repeated RunningFrameSummary running_frames = 6;
-
-    // Whether this is a boot report
-    bool is_boot_report = 7;
-}
-
-message RunningFrameSummary {
-    string frame_id = 1;
-    string frame_name = 2;
-    string job_id = 3;
-    string job_name = 4;
-    string layer_id = 5;
-    int32 num_cores = 6;
-    int64 start_time = 7;
-    int64 rss = 8;
-    int64 max_rss = 9;
-    int64 vsize = 10;
-    int64 max_vsize = 11;
-    int64 used_gpu_memory = 12;
-    int64 max_used_gpu_memory = 13;
-    int64 used_swap_memory = 14;
-    int64 llu_time = 15;
-}
-
 // -------- Proc Events --------
 
 message ProcEvent {

sandbox/kibana-queries.md

@@ -16,7 +16,6 @@ GET /opencue-job-events-*/_count
 GET /opencue-layer-events-*/_count
 GET /opencue-proc-events-*/_count
 GET /opencue-host-events-*/_count
-GET /opencue-host-reports-*/_count
 ```
 
 ## Pickup Time Tracking
@@ -478,7 +477,7 @@ GET /opencue-layer-events-*/_search
 }
 ```
 
-## Host Events and Reports
+## Host Events
 
 ```json
 # Recent host events
@@ -492,18 +491,6 @@ GET /opencue-host-events-*/_search
   ],
   "size": 10
 }
-
-# Recent host reports
-GET /opencue-host-reports-*/_search
-{
-  "query": {
-    "match_all": {}
-  },
-  "sort": [
-    { "header.timestamp": { "order": "desc" } }
-  ],
-  "size": 10
-}
 ```
 
 ## Time-Based Analytics