feat(): gather zone level statistics

Author: j-white

README.md

@@ -13,7 +13,7 @@ By running it as a worker and pushing metrics, we avoid the need to deploy a ded
 - [x] D1
 - [x] Durable Objects
 - [x] Queues
-- [ ] Zones
+- [x] Zones
 
 ## Usage
 

features/data/zone_http_requests_response.json

@@ -0,0 +1,75 @@
+{
+  "data": {
+    "viewer": {
+      "zones": [
+        {
+          "zoneTag": "zone123abc",
+          "httpRequestsAdaptiveGroups": [
+            {
+              "count": 150,
+              "dimensions": {
+                "edgeResponseStatus": 200,
+                "clientRequestHTTPHost": "example.com"
+              }
+            },
+            {
+              "count": 25,
+              "dimensions": {
+                "edgeResponseStatus": 304,
+                "clientRequestHTTPHost": "example.com"
+              }
+            },
+            {
+              "count": 10,
+              "dimensions": {
+                "edgeResponseStatus": 404,
+                "clientRequestHTTPHost": "example.com"
+              }
+            },
+            {
+              "count": 5,
+              "dimensions": {
+                "edgeResponseStatus": 500,
+                "clientRequestHTTPHost": "example.com"
+              }
+            },
+            {
+              "count": 80,
+              "dimensions": {
+                "edgeResponseStatus": 200,
+                "clientRequestHTTPHost": "api.example.com"
+              }
+            },
+            {
+              "count": 3,
+              "dimensions": {
+                "edgeResponseStatus": 401,
+                "clientRequestHTTPHost": "api.example.com"
+              }
+            }
+          ]
+        },
+        {
+          "zoneTag": "zone456def",
+          "httpRequestsAdaptiveGroups": [
+            {
+              "count": 200,
+              "dimensions": {
+                "edgeResponseStatus": 200,
+                "clientRequestHTTPHost": "myapp.io"
+              }
+            },
+            {
+              "count": 15,
+              "dimensions": {
+                "edgeResponseStatus": 503,
+                "clientRequestHTTPHost": "myapp.io"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  },
+  "errors": null
+}

gql/zone_http_requests_query.graphql

@@ -0,0 +1,18 @@
+query GetZoneHttpRequestsQuery($zoneIDs: [String!], $datetimeStart: Time!, $datetimeEnd: Time!, $limit: Int!) {
+  viewer {
+    zones(filter: { zoneTag_in: $zoneIDs }) {
+      zoneTag
+      httpRequestsAdaptiveGroups(limit: $limit, filter: {
+        datetime_geq: $datetimeStart,
+        datetime_lt: $datetimeEnd,
+        requestSource_in: ["eyeball"]
+      }) {
+        count
+        dimensions {
+          edgeResponseStatus
+          clientRequestHTTPHost
+        }
+      }
+    }
+  }
+}

src/gql.rs

@@ -44,6 +44,13 @@ pub struct GetQueueBacklogAnalyticsQuery;
 )]
 pub struct GetQueueOperationsAnalyticsQuery;
 
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "gql/schema.graphql",
+    query_path = "gql/zone_http_requests_query.graphql"
+)]
+pub struct GetZoneHttpRequestsQuery;
+
 #[allow(non_camel_case_types)]
 type float32 = f32;
 
@@ -62,6 +69,9 @@ type uint32 = u32;
 #[allow(non_camel_case_types)]
 type float64 = f64;
 
+#[allow(non_camel_case_types)]
+type uint16 = u16;
+
 pub async fn do_get_workers_analytics_query(cloudflare_api_url: &String, cloudflare_api_key: &String, variables: get_workers_analytics_query::Variables) -> Result<Vec<Metric>, Box<dyn Error>> {
     let request_body = GetWorkersAnalyticsQuery::build_query(variables);
     //console_log!("request_body: {:?}", request_body);
@@ -411,6 +421,53 @@ pub async fn do_get_queue_operations_analytics_query(cloudflare_api_url: &String
     Ok(prometheus_registry_to_opentelemetry_metrics(registry, timestamp_nanos))
 }
 
+pub async fn do_get_zone_http_requests_query(cloudflare_api_url: &String, cloudflare_api_key: &String, variables: get_zone_http_requests_query::Variables) -> Result<Vec<Metric>, Box<dyn Error>> {
+    let request_body = GetZoneHttpRequestsQuery::build_query(variables);
+    let client = reqwest::Client::new();
+    let res = client.post(cloudflare_api_url)
+        .bearer_auth(cloudflare_api_key)
+        .json(&request_body).send().await?;
+
+    if !res.status().is_success() {
+        console_log!("GraphQL query failed: {:?}", res.status());
+        return Err(Box::new(res.error_for_status().unwrap_err()));
+    }
+
+    let response_body: Response<get_zone_http_requests_query::ResponseData> = res.json().await?;
+    if response_body.errors.is_some() {
+        console_log!("GraphQL query failed: {:?}", response_body.errors);
+        return Err(Box::new(worker::Error::JsError("graphql".parse().unwrap())));
+    }
+    let response_data: get_zone_http_requests_query::ResponseData = response_body.data.expect("missing response data");
+
+    let registry = Registry::new();
+    let zone_requests_status_host_opts = Opts::new("cloudflare_zone_requests_status_host", "Count of requests per edge HTTP status per host");
+    let zone_requests_status_host = CounterVec::new(zone_requests_status_host_opts, &["zone", "status", "host"]).unwrap();
+    registry.register(Box::new(zone_requests_status_host.clone())).unwrap();
+
+    let last_datetime: Option<Time> = None;
+    for zone in response_data.viewer.unwrap().zones.iter() {
+        let zone_tag = zone.zone_tag.clone();
+        for group in zone.http_requests_adaptive_groups.iter() {
+            let dimensions = group.dimensions.as_ref().unwrap();
+            let status = dimensions.edge_response_status.to_string();
+            let host = dimensions.client_request_http_host.clone();
+            let count = group.count;
+
+            zone_requests_status_host.with_label_values(&[zone_tag.as_str(), status.as_str(), host.as_str()]).inc_by(count as f64);
+        }
+    }
+
+    let timestamp_nanos: u64 = last_datetime.map(|datetime| {
+        let datetime: NaiveDateTime = NaiveDateTime::parse_from_str(&datetime, "%+").unwrap();
+        datetime.and_utc().timestamp_nanos_opt().unwrap_or(0) as u64
+    }).unwrap_or_else(|| {
+        systemtime_to_nanos(SystemTime::now())
+    });
+
+    Ok(prometheus_registry_to_opentelemetry_metrics(registry, timestamp_nanos))
+}
+
 fn systemtime_to_nanos(time: web_time::SystemTime) -> u64 {
     let duration = time.duration_since(web_time::SystemTime::UNIX_EPOCH).unwrap();
     duration.as_nanos() as u64

src/lib.rs

@@ -7,7 +7,7 @@ use prost::Message;
 use worker::*;
 use worker::js_sys::Uint8Array;
 use worker::wasm_bindgen::JsValue;
-use crate::gql::{get_workers_analytics_query, do_get_workers_analytics_query, do_get_d1_analytics_query, get_d1_analytics_query, do_get_durableobjects_analytics_query, get_durable_objects_analytics_query, do_get_queue_backlog_analytics_query, get_queue_backlog_analytics_query, do_get_queue_operations_analytics_query, get_queue_operations_analytics_query};
+use crate::gql::{get_workers_analytics_query, do_get_workers_analytics_query, do_get_d1_analytics_query, get_d1_analytics_query, do_get_durableobjects_analytics_query, get_durable_objects_analytics_query, do_get_queue_backlog_analytics_query, get_queue_backlog_analytics_query, do_get_queue_operations_analytics_query, get_queue_operations_analytics_query, do_get_zone_http_requests_query, get_zone_http_requests_query};
 
 mod gql;
 mod metrics;
@@ -156,6 +156,36 @@ async fn do_trigger(env: Env) -> Result<()> {
             return Err(Error::JsError(e.to_string()));
         }
     };
+
+    // Zone HTTP requests metrics (optional - only if CLOUDFLARE_ZONE_IDS is set)
+    if let Ok(zone_ids_var) = env.var("CLOUDFLARE_ZONE_IDS") {
+        let zone_ids: Vec<String> = zone_ids_var.to_string()
+            .split(',')
+            .map(|s| s.trim().to_string())
+            .filter(|s| !s.is_empty())
+            .collect();
+
+        if !zone_ids.is_empty() {
+            let result = do_get_zone_http_requests_query(&cloudflare_api_url, &cloudflare_api_key, get_zone_http_requests_query::Variables {
+                zone_i_ds: Some(zone_ids),
+                datetime_start: start.to_rfc3339(),
+                datetime_end: end.to_rfc3339(),
+                limit: 9999,
+            }).await;
+            match result {
+                Ok(metrics) => {
+                    for metric in metrics {
+                        all_metrics.push(metric);
+                    }
+                },
+                Err(e) => {
+                    console_log!("Querying Cloudflare API for zone HTTP requests failed: {:?}", e);
+                    return Err(Error::JsError(e.to_string()));
+                }
+            };
+        }
+    }
+
     console_log!("Done fetching!");
 
     do_push_metrics(env, all_metrics).await