From 3613ba2fa7d20a431727ed3ac375e35a29fdd509 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 19 Aug 2025 13:03:03 +0100 Subject: [PATCH 1/8] Create api endpoint for new status page --- database/src/lib.rs | 16 ++++----- site/src/api.rs | 15 +++++++++ site/src/request_handlers.rs | 2 ++ site/src/request_handlers/status_page_new.rs | 35 ++++++++++++++++++++ site/src/server.rs | 32 ++++++++++++++++-- 5 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 site/src/request_handlers/status_page_new.rs diff --git a/database/src/lib.rs b/database/src/lib.rs index e6f918845..f91a60582 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -807,7 +807,7 @@ pub struct ArtifactCollection { pub end_time: DateTime, } -#[derive(Debug, Copy, Clone, PartialEq)] +#[derive(Debug, Copy, Clone, PartialEq, serde::Deserialize, serde::Serialize)] pub enum BenchmarkRequestStatus { WaitingForArtifacts, ArtifactsReady, @@ -865,7 +865,7 @@ const BENCHMARK_REQUEST_TRY_STR: &str = "try"; const BENCHMARK_REQUEST_MASTER_STR: &str = "master"; const BENCHMARK_REQUEST_RELEASE_STR: &str = "release"; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] pub enum BenchmarkRequestType { /// A Try commit Try { @@ -893,7 +893,7 @@ impl fmt::Display for BenchmarkRequestType { } } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] pub struct BenchmarkRequest { commit_type: BenchmarkRequestType, // When was the compiler artifact created @@ -1058,7 +1058,7 @@ impl BenchmarkRequestIndex { } } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] pub enum BenchmarkJobStatus { Queued, InProgress { @@ -1100,7 +1100,7 @@ impl fmt::Display for BenchmarkJobStatus { } } -#[derive(Debug, Copy, Clone, PartialEq)] +#[derive(Debug, Copy, Clone, PartialEq, serde::Deserialize, serde::Serialize)] pub struct BenchmarkSet(u32); impl BenchmarkSet { @@ -1115,7 +1115,7 @@ impl BenchmarkSet { /// Each request is split into several `BenchmarkJob`s. Collectors poll the /// queue and claim a job only when its `benchmark_set` matches one of the sets /// they are responsible for. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] pub struct BenchmarkJob { id: u32, target: Target, @@ -1188,7 +1188,7 @@ impl BenchmarkJobConclusion { } /// The configuration for a collector -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, serde::Deserialize, serde::Serialize)] pub struct CollectorConfig { name: String, target: Target, @@ -1226,7 +1226,7 @@ impl CollectorConfig { /// The data that can be retrived from the database directly to populate the /// status page -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, serde::Deserialize, serde::Serialize)] pub struct PartialStatusPageData { /// A Vector of; completed requests with any associated errors pub completed_requests: Vec<(BenchmarkRequest, Vec)>, diff --git a/site/src/api.rs b/site/src/api.rs index 3b78d729d..256d068af 100644 --- a/site/src/api.rs +++ b/site/src/api.rs @@ -391,6 +391,21 @@ pub mod status { } } +pub mod status_new { + use database::{BenchmarkJob, BenchmarkRequest, CollectorConfig}; + use serde::Serialize; + + #[derive(Serialize, Debug)] + pub struct Response { + /// Completed requests alongside any errors + pub completed: Vec<(BenchmarkRequest, Vec)>, + /// In progress requests alongside the jobs associated with the request + pub in_progress: Vec<(BenchmarkRequest, Vec)>, + /// Configuration for all collectors including ones that are inactive + pub collector_configs: Vec, + } +} + pub mod self_profile_raw { use serde::{Deserialize, Serialize}; diff --git a/site/src/request_handlers.rs b/site/src/request_handlers.rs index d73e3cbc1..3efea3894 100644 --- a/site/src/request_handlers.rs +++ b/site/src/request_handlers.rs @@ -5,6 +5,7 @@ mod graph; mod next_artifact; mod self_profile; mod status_page; +mod status_page_new; pub use bootstrap::handle_bootstrap; pub use dashboard::handle_dashboard; @@ -19,6 +20,7 @@ pub use self_profile::{ handle_self_profile_raw_download, }; pub use status_page::handle_status_page; +pub use status_page_new::handle_status_page_new; use crate::api::{info, ServerResult}; use crate::load::SiteCtxt; diff --git a/site/src/request_handlers/status_page_new.rs b/site/src/request_handlers/status_page_new.rs new file mode 100644 index 000000000..6afc3b53b --- /dev/null +++ b/site/src/request_handlers/status_page_new.rs @@ -0,0 +1,35 @@ +use std::sync::Arc; + +use crate::api::{status_new, ServerResult}; +use crate::load::SiteCtxt; + +pub async fn handle_status_page_new(ctxt: Arc) -> ServerResult { + let conn = ctxt.conn().await; + + let collector_configs = conn + .get_collector_configs() + .await + .map_err(|e| e.to_string())?; + // The query gives us `max_completed_requests` number of completed requests + // and all inprogress requests without us needing to specify + // + // TODO; for `in_progress` requests we could look at the the completed + // `requests`, then use the `duration_ms` to display an estimated job + // finish time. Could also do that on the frontend but probably makes + // sense to do in SQL. + let partial_data = conn + .get_status_page_data() + .await + .map_err(|e| e.to_string())?; + + Ok(status_new::Response { + completed: partial_data + .completed_requests + .iter() + // @TODO Remove this + .map(|it| (it.0.clone(), it.2.clone())) + .collect(), + in_progress: partial_data.in_progress, + collector_configs, + }) +} diff --git a/site/src/server.rs b/site/src/server.rs index 1452cafc6..2d8ce0f3f 100644 --- a/site/src/server.rs +++ b/site/src/server.rs @@ -382,6 +382,22 @@ async fn serve_req(server: Server, req: Request) -> Result { + let ctxt: Arc = server.ctxt.read().as_ref().unwrap().clone(); + let result = request_handlers::handle_status_page_new(ctxt).await; + return match result { + Ok(result) => Ok(http::Response::builder() + .header_typed(ContentType::json()) + .body(hyper::Body::from(serde_json::to_string(&result).unwrap())) + .unwrap()), + Err(err) => Ok(http::Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .header_typed(ContentType::text_utf8()) + .header_typed(CacheControl::new().with_no_cache().with_no_store()) + .body(hyper::Body::from(err)) + .unwrap()), + }; + } "/perf/next_artifact" => { return server .handle_get_async(&req, request_handlers::handle_next_artifact) @@ -640,6 +656,7 @@ async fn handle_fs_path( | "/dashboard.html" | "/detailed-query.html" | "/help.html" + | "/status_new.html" | "/status.html" => resolve_template(relative_path).await, _ => match TEMPLATES.get_static_asset(relative_path, use_compression)? { Payload::Compressed(data) => { @@ -714,14 +731,18 @@ fn verify_gh_sig(cfg: &Config, header: &str, body: &[u8]) -> Option { Some(false) } -fn to_response(result: ServerResult, compression: &Option) -> Response +fn to_response_with_content_type( + result: ServerResult, + content_type: ContentType, + compression: &Option, +) -> Response where S: Serialize, { match result { Ok(result) => { let response = http::Response::builder() - .header_typed(ContentType::octet_stream()) + .header_typed(content_type) .header_typed(CacheControl::new().with_no_cache().with_no_store()); let body = rmp_serde::to_vec_named(&result).unwrap(); maybe_compressed_response(response, body, compression) @@ -735,6 +756,13 @@ where } } +fn to_response(result: ServerResult, compression: &Option) -> Response +where + S: Serialize, +{ + to_response_with_content_type(result, ContentType::octet_stream(), compression) +} + pub fn maybe_compressed_response( response: http::response::Builder, body: Vec, From ab38465d7e1ac3f8e32063eaae9eace3ed6b5b17 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 19 Aug 2025 13:03:29 +0100 Subject: [PATCH 2/8] create minimal ui for new status page --- site/frontend/package.json | 4 + site/frontend/src/pages/status_new.ts | 8 + site/frontend/src/pages/status_new/data.ts | 130 ++++++++++++ .../src/pages/status_new/expansion.ts | 19 ++ site/frontend/src/pages/status_new/page.vue | 193 ++++++++++++++++++ site/frontend/src/urls.ts | 1 + site/frontend/src/utils/getType.ts | 8 + site/frontend/templates/pages/status_new.html | 10 + 8 files changed, 373 insertions(+) create mode 100644 site/frontend/src/pages/status_new.ts create mode 100644 site/frontend/src/pages/status_new/data.ts create mode 100644 site/frontend/src/pages/status_new/expansion.ts create mode 100644 site/frontend/src/pages/status_new/page.vue create mode 100644 site/frontend/src/utils/getType.ts create mode 100644 site/frontend/templates/pages/status_new.html diff --git a/site/frontend/package.json b/site/frontend/package.json index 2e47e95f0..452fe6332 100644 --- a/site/frontend/package.json +++ b/site/frontend/package.json @@ -38,6 +38,10 @@ "source": "src/pages/status.ts", "distDir": "dist/scripts" }, + "status_new": { + "source": "src/pages/status_new.ts", + "distDir": "dist/scripts" + }, "bootstrap": { "source": "src/pages/bootstrap.ts", "distDir": "dist/scripts" diff --git a/site/frontend/src/pages/status_new.ts b/site/frontend/src/pages/status_new.ts new file mode 100644 index 000000000..1af392fa8 --- /dev/null +++ b/site/frontend/src/pages/status_new.ts @@ -0,0 +1,8 @@ +import Status from "./status_new/page.vue"; +import {createApp} from "vue"; +import WithSuspense from "../components/with-suspense.vue"; + +const app = createApp(WithSuspense, { + component: Status, +}); +app.mount("#app"); diff --git a/site/frontend/src/pages/status_new/data.ts b/site/frontend/src/pages/status_new/data.ts new file mode 100644 index 000000000..ce4c653e9 --- /dev/null +++ b/site/frontend/src/pages/status_new/data.ts @@ -0,0 +1,130 @@ +import {getTypeString} from "../../utils/getType"; + +type CommitTypeMaster = { + sha: string; + parent_sha: string; + pr: number; +}; + +type CommitTypeRelease = { + tag: string; +}; + +type CommitTypeTry = { + sha: string; + parent_sha: string | null; + pr: number; +}; + +export type CommitType = CommitTypeRelease | CommitTypeMaster | CommitTypeTry; +export type CommitTypeString = "Master" | "Try" | "Release"; + +export type BenchmarkRequestComplete = { + commit_type: { + [K in CommitTypeString]: CommitType; + }; + commit_date: string | null; + created_at: string | null; + status: { + Completed: { + completed_at: string; + duration_ms: number; + }; + }; + backends: string; + profile: string; +}; + +export type BenchmarkRequestInProgress = { + commit_type: { + [K in CommitTypeString]: CommitType; + }; + commit_date: string | null; + created_at: string | null; + status: "InProgress"; + backends: string; + profiles: string; +}; + +export function isMasterBenchmarkRequest( + commitType: Object +): commitType is {["Master"]: CommitTypeMaster} { + return "Master" in commitType; +} + +export function isReleaseBenchmarkRequest( + commitType: Object +): commitType is {["Release"]: CommitTypeRelease} { + return "Release" in commitType; +} + +export function isTryBenchmarkRequest( + commitType: Object +): commitType is {["Try"]: CommitTypeTry} { + return "Try" in commitType; +} + +export type BenchmarkJobStatusInProgress = { + started_at: string; + collector_name: string; +}; + +export type BenchmarkJobStatusCompleted = { + started_at: string; + completed_at: string; + collector_name: string; + success: boolean; +}; + +export type BenchmarkJobStatusString = "InProgress" | "Completed"; +export type BenchmarkJobStatusQueued = "Queued"; + +export type BenchmarkJob = { + id: number; + target: string; + backend: string; + request_tag: string; + benchmark_set: number; + created_at: string; + status: + | BenchmarkJobStatusQueued + | { + [K in BenchmarkJobStatusQueued]: + | BenchmarkJobStatusInProgress + | BenchmarkJobStatusCompleted; + }; + deque_counter: number; +}; + +export function isQueuedBenchmarkJob( + status: unknown +): status is BenchmarkJobStatusQueued { + return getTypeString(status) === "string"; +} + +export function isInProgressBenchmarkJob( + status: unknown +): status is {["InProgress"]: BenchmarkJobStatusInProgress} { + return getTypeString(status) === "object" && "InProgress" in status; +} + +export function isCompletedBenchmarkJob( + status: unknown +): status is {["Completed"]: BenchmarkJobStatusCompleted} { + return getTypeString(status) === "object" && "Completed" in status; +} + +export type CollectorConfig = { + name: string; + target: string; + benchmark_set: number; + is_active: boolean; + last_heartbeat_at: string; + date_added: string; +}; + +export type StatusResponse = { + completed: [BenchmarkRequestComplete, string[]][]; + in_progress: [BenchmarkRequestInProgress, BenchmarkJob[]][]; + collector_configs: CollectorConfig[]; +}; diff --git a/site/frontend/src/pages/status_new/expansion.ts b/site/frontend/src/pages/status_new/expansion.ts new file mode 100644 index 000000000..c5347b40b --- /dev/null +++ b/site/frontend/src/pages/status_new/expansion.ts @@ -0,0 +1,19 @@ +import {ref} from "vue"; + +export function useExpandedStore() { + const expanded = ref(new Set()); + + function isExpanded(sha: string) { + return expanded.value.has(sha); + } + + function toggleExpanded(sha: string) { + if (isExpanded(sha)) { + expanded.value.delete(sha); + } else { + expanded.value.add(sha); + } + } + + return {toggleExpanded, isExpanded}; +} diff --git a/site/frontend/src/pages/status_new/page.vue b/site/frontend/src/pages/status_new/page.vue new file mode 100644 index 000000000..405d1a693 --- /dev/null +++ b/site/frontend/src/pages/status_new/page.vue @@ -0,0 +1,193 @@ + + + + + diff --git a/site/frontend/src/urls.ts b/site/frontend/src/urls.ts index f2f629270..37c97cb54 100644 --- a/site/frontend/src/urls.ts +++ b/site/frontend/src/urls.ts @@ -4,6 +4,7 @@ export const INFO_URL = `${BASE_URL}/info`; export const DASHBOARD_DATA_URL = `${BASE_URL}/dashboard`; export const STATUS_DATA_URL = `${BASE_URL}/status_page`; +export const STATUS_DATA_NEW_URL = `${BASE_URL}/status_page_new`; export const BOOTSTRAP_DATA_URL = `${BASE_URL}/bootstrap`; export const GRAPH_DATA_URL = `${BASE_URL}/graphs`; export const COMPARE_DATA_URL = `${BASE_URL}/get`; diff --git a/site/frontend/src/utils/getType.ts b/site/frontend/src/utils/getType.ts new file mode 100644 index 000000000..7754ec289 --- /dev/null +++ b/site/frontend/src/utils/getType.ts @@ -0,0 +1,8 @@ +export function getTypeString(obj: any): string { + return Object.prototype.toString + .call(obj) + .toLowerCase() + .replace("[", "") + .replace("]", "") + .split(" ")[1]; +} diff --git a/site/frontend/templates/pages/status_new.html b/site/frontend/templates/pages/status_new.html new file mode 100644 index 000000000..fb95a81fb --- /dev/null +++ b/site/frontend/templates/pages/status_new.html @@ -0,0 +1,10 @@ +{% extends "layout.html" %} +{% block head %} + +{% endblock %} +{% block content %} +
+{% endblock %} +{% block script %} + +{% endblock %} From 5ee49dc808e438f089215509aea1816a1c71619d Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 19 Aug 2025 13:10:51 +0100 Subject: [PATCH 3/8] remove un-used file --- .../src/pages/status_new/expansion.ts | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 site/frontend/src/pages/status_new/expansion.ts diff --git a/site/frontend/src/pages/status_new/expansion.ts b/site/frontend/src/pages/status_new/expansion.ts deleted file mode 100644 index c5347b40b..000000000 --- a/site/frontend/src/pages/status_new/expansion.ts +++ /dev/null @@ -1,19 +0,0 @@ -import {ref} from "vue"; - -export function useExpandedStore() { - const expanded = ref(new Set()); - - function isExpanded(sha: string) { - return expanded.value.has(sha); - } - - function toggleExpanded(sha: string) { - if (isExpanded(sha)) { - expanded.value.delete(sha); - } else { - expanded.value.add(sha); - } - } - - return {toggleExpanded, isExpanded}; -} From 92f116b3e5922fab49f9797e86549b6039c50696 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 19 Aug 2025 13:12:41 +0100 Subject: [PATCH 4/8] revert change to `to_response` --- site/src/server.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/site/src/server.rs b/site/src/server.rs index 2d8ce0f3f..cc507957f 100644 --- a/site/src/server.rs +++ b/site/src/server.rs @@ -731,18 +731,14 @@ fn verify_gh_sig(cfg: &Config, header: &str, body: &[u8]) -> Option { Some(false) } -fn to_response_with_content_type( - result: ServerResult, - content_type: ContentType, - compression: &Option, -) -> Response +fn to_response(result: ServerResult, compression: &Option) -> Response where S: Serialize, { match result { Ok(result) => { let response = http::Response::builder() - .header_typed(content_type) + .header_typed(ContentType::octet_stream()) .header_typed(CacheControl::new().with_no_cache().with_no_store()); let body = rmp_serde::to_vec_named(&result).unwrap(); maybe_compressed_response(response, body, compression) @@ -756,13 +752,6 @@ where } } -fn to_response(result: ServerResult, compression: &Option) -> Response -where - S: Serialize, -{ - to_response_with_content_type(result, ContentType::octet_stream(), compression) -} - pub fn maybe_compressed_response( response: http::response::Builder, body: Vec, From 12d6c1f1509b449e3ce7c4bd09c4eeb9cfac439a Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 19 Aug 2025 13:51:11 +0100 Subject: [PATCH 5/8] fix typechecking --- site/frontend/src/pages/status_new/data.ts | 8 ++++---- site/frontend/src/pages/status_new/page.vue | 10 ++-------- site/frontend/src/utils/getType.ts | 10 +++++++++- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/site/frontend/src/pages/status_new/data.ts b/site/frontend/src/pages/status_new/data.ts index ce4c653e9..d44598212 100644 --- a/site/frontend/src/pages/status_new/data.ts +++ b/site/frontend/src/pages/status_new/data.ts @@ -1,4 +1,4 @@ -import {getTypeString} from "../../utils/getType"; +import {isObject, isString} from "../../utils/getType"; type CommitTypeMaster = { sha: string; @@ -99,19 +99,19 @@ export type BenchmarkJob = { export function isQueuedBenchmarkJob( status: unknown ): status is BenchmarkJobStatusQueued { - return getTypeString(status) === "string"; + return isString(status) && status === "Queued"; } export function isInProgressBenchmarkJob( status: unknown ): status is {["InProgress"]: BenchmarkJobStatusInProgress} { - return getTypeString(status) === "object" && "InProgress" in status; + return isObject(status) && "InProgress" in status; } export function isCompletedBenchmarkJob( status: unknown ): status is {["Completed"]: BenchmarkJobStatusCompleted} { - return getTypeString(status) === "object" && "Completed" in status; + return isObject(status) && "Completed" in status; } export type CollectorConfig = { diff --git a/site/frontend/src/pages/status_new/page.vue b/site/frontend/src/pages/status_new/page.vue index 405d1a693..ddd4c2113 100644 --- a/site/frontend/src/pages/status_new/page.vue +++ b/site/frontend/src/pages/status_new/page.vue @@ -3,8 +3,7 @@ import {getJson} from "../../utils/requests"; import {STATUS_DATA_NEW_URL} from "../../urls"; import {withLoading} from "../../utils/loading"; import {ref, Ref} from "vue"; -import {StatusResponseNew} from "./data"; -import {useExpandedStore} from "./expansion"; +import {StatusResponse, CollectorConfig} from "./data"; async function loadStatusNew(loading: Ref) { dataNew.value = await withLoading(loading, () => @@ -15,12 +14,7 @@ async function loadStatusNew(loading: Ref) { const loading = ref(true); const dataNew: Ref = ref(null); -function statusLabel(c) { - if (!c.is_active) return "Inactive"; - return this.isStale(c) ? "Stale" : "Active"; -} - -function statusClass(c) { +function statusClass(c: CollectorConfig): string { return c.is_active ? "active" : "inactive"; } diff --git a/site/frontend/src/utils/getType.ts b/site/frontend/src/utils/getType.ts index 7754ec289..9a77c9891 100644 --- a/site/frontend/src/utils/getType.ts +++ b/site/frontend/src/utils/getType.ts @@ -1,4 +1,4 @@ -export function getTypeString(obj: any): string { +function getTypeString(obj: any): string { return Object.prototype.toString .call(obj) .toLowerCase() @@ -6,3 +6,11 @@ export function getTypeString(obj: any): string { .replace("]", "") .split(" ")[1]; } + +export function isObject(maybeObject: any): maybeObject is object { + return getTypeString(maybeObject) === "object"; +} + +export function isString(maybeString: any): maybeString is string { + return getTypeString(maybeString) === "string"; +} From 01f69bc77289de154d57db8bc9d8a8b49aa146ac Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 19 Aug 2025 14:52:05 +0100 Subject: [PATCH 6/8] Add the queue to the api response --- site/frontend/src/pages/status_new/data.ts | 35 ++++++++++++++++++++ site/src/api.rs | 2 ++ site/src/request_handlers/status_page_new.rs | 18 +++++++--- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/site/frontend/src/pages/status_new/data.ts b/site/frontend/src/pages/status_new/data.ts index d44598212..0661f1e85 100644 --- a/site/frontend/src/pages/status_new/data.ts +++ b/site/frontend/src/pages/status_new/data.ts @@ -46,6 +46,22 @@ export type BenchmarkRequestInProgress = { profiles: string; }; +export type BenchmarkRequestArtifactsReady = { + commit_type: { + [K in CommitTypeString]: CommitType; + }; + commit_date: string | null; + created_at: string | null; + status: "ArtifactsReady"; + backends: string; + profiles: string; +}; + +type BenchmarkRequest = + | BenchmarkRequestComplete + | BenchmarkRequestInProgress + | BenchmarkRequestArtifactsReady; + export function isMasterBenchmarkRequest( commitType: Object ): commitType is {["Master"]: CommitTypeMaster} { @@ -64,6 +80,24 @@ export function isTryBenchmarkRequest( return "Try" in commitType; } +export function isArtifactsReadyBenchmarkRequest( + req: BenchmarkRequest +): req is BenchmarkRequestArtifactsReady { + return isString(req.status) && req.status === "ArtifactsReady"; +} + +export function isInProgressBenchmarkRequest( + req: BenchmarkRequest +): req is BenchmarkRequestInProgress { + return isString(req.status) && req.status === "InProgress"; +} + +export function isCompleteBenchmarkRequest( + req: BenchmarkRequest +): req is BenchmarkRequestComplete { + return isObject(req.status) && "Completed" in req.status; +} + export type BenchmarkJobStatusInProgress = { started_at: string; collector_name: string; @@ -127,4 +161,5 @@ export type StatusResponse = { completed: [BenchmarkRequestComplete, string[]][]; in_progress: [BenchmarkRequestInProgress, BenchmarkJob[]][]; collector_configs: CollectorConfig[]; + queue: BenchmarkRequest[]; }; diff --git a/site/src/api.rs b/site/src/api.rs index 256d068af..32772053f 100644 --- a/site/src/api.rs +++ b/site/src/api.rs @@ -403,6 +403,8 @@ pub mod status_new { pub in_progress: Vec<(BenchmarkRequest, Vec)>, /// Configuration for all collectors including ones that are inactive pub collector_configs: Vec, + /// The current queue + pub queue: Vec, } } diff --git a/site/src/request_handlers/status_page_new.rs b/site/src/request_handlers/status_page_new.rs index 6afc3b53b..120921d80 100644 --- a/site/src/request_handlers/status_page_new.rs +++ b/site/src/request_handlers/status_page_new.rs @@ -1,15 +1,18 @@ use std::sync::Arc; use crate::api::{status_new, ServerResult}; +use crate::job_queue::build_queue; use crate::load::SiteCtxt; pub async fn handle_status_page_new(ctxt: Arc) -> ServerResult { let conn = ctxt.conn().await; + let error_to_string = |e: anyhow::Error| e.to_string(); + let collector_configs = conn .get_collector_configs() .await - .map_err(|e| e.to_string())?; + .map_err(error_to_string)?; // The query gives us `max_completed_requests` number of completed requests // and all inprogress requests without us needing to specify // @@ -17,10 +20,16 @@ pub async fn handle_status_page_new(ctxt: Arc) -> ServerResult) -> ServerResult Date: Wed, 20 Aug 2025 13:01:43 +0100 Subject: [PATCH 7/8] update endpoints to return something more ui friendly --- database/src/lib.rs | 36 ++--- site/src/api.rs | 80 ++++++++++- site/src/request_handlers/status_page_new.rs | 135 +++++++++++++++++-- 3 files changed, 220 insertions(+), 31 deletions(-) diff --git a/database/src/lib.rs b/database/src/lib.rs index f91a60582..6d112e6c4 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -199,7 +199,7 @@ impl Ord for Commit { /// The compilation profile (i.e., how the crate was built) #[derive( - Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, + Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Deserialize, serde::Serialize, )] pub enum Profile { /// A checked build (i.e., no codegen) @@ -356,9 +356,7 @@ impl PartialOrd for Scenario { /// https://doc.rust-lang.org/nightly/rustc/platform-support.html /// /// Presently we only support x86_64 -#[derive( - Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, -)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum Target { /// `x86_64-unknown-linux-gnu` X86_64UnknownLinuxGnu, @@ -393,9 +391,7 @@ impl fmt::Display for Target { } /// The codegen backend used for compilation. -#[derive( - Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, -)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum CodegenBackend { /// The default LLVM backend Llvm, @@ -807,7 +803,7 @@ pub struct ArtifactCollection { pub end_time: DateTime, } -#[derive(Debug, Copy, Clone, PartialEq, serde::Deserialize, serde::Serialize)] +#[derive(Debug, Copy, Clone, PartialEq)] pub enum BenchmarkRequestStatus { WaitingForArtifacts, ArtifactsReady, @@ -824,7 +820,7 @@ const BENCHMARK_REQUEST_STATUS_IN_PROGRESS_STR: &str = "in_progress"; const BENCHMARK_REQUEST_STATUS_COMPLETED_STR: &str = "completed"; impl BenchmarkRequestStatus { - pub(crate) fn as_str(&self) -> &str { + pub fn as_str(&self) -> &str { match self { Self::WaitingForArtifacts => BENCHMARK_REQUEST_STATUS_WAITING_FOR_ARTIFACTS_STR, Self::ArtifactsReady => BENCHMARK_REQUEST_STATUS_ARTIFACTS_READY_STR, @@ -865,7 +861,7 @@ const BENCHMARK_REQUEST_TRY_STR: &str = "try"; const BENCHMARK_REQUEST_MASTER_STR: &str = "master"; const BENCHMARK_REQUEST_RELEASE_STR: &str = "release"; -#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] +#[derive(Debug, Clone, PartialEq)] pub enum BenchmarkRequestType { /// A Try commit Try { @@ -893,7 +889,7 @@ impl fmt::Display for BenchmarkRequestType { } } -#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] +#[derive(Debug, Clone, PartialEq)] pub struct BenchmarkRequest { commit_type: BenchmarkRequestType, // When was the compiler artifact created @@ -986,6 +982,10 @@ impl BenchmarkRequest { self.created_at } + pub fn commit_date(&self) -> Option> { + self.commit_date + } + pub fn is_master(&self) -> bool { matches!(self.commit_type, BenchmarkRequestType::Master { .. }) } @@ -1058,7 +1058,7 @@ impl BenchmarkRequestIndex { } } -#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] +#[derive(Debug, Clone, PartialEq)] pub enum BenchmarkJobStatus { Queued, InProgress { @@ -1101,7 +1101,7 @@ impl fmt::Display for BenchmarkJobStatus { } #[derive(Debug, Copy, Clone, PartialEq, serde::Deserialize, serde::Serialize)] -pub struct BenchmarkSet(u32); +pub struct BenchmarkSet(pub u32); impl BenchmarkSet { pub fn new(id: u32) -> Self { @@ -1115,7 +1115,7 @@ impl BenchmarkSet { /// Each request is split into several `BenchmarkJob`s. Collectors poll the /// queue and claim a job only when its `benchmark_set` matches one of the sets /// they are responsible for. -#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] +#[derive(Debug, Clone, PartialEq)] pub struct BenchmarkJob { id: u32, target: Target, @@ -1169,6 +1169,10 @@ impl BenchmarkJob { pub fn status(&self) -> &BenchmarkJobStatus { &self.status } + + pub fn created_at(&self) -> DateTime { + self.created_at + } } /// Describes the final state of a job @@ -1188,7 +1192,7 @@ impl BenchmarkJobConclusion { } /// The configuration for a collector -#[derive(Debug, PartialEq, serde::Deserialize, serde::Serialize)] +#[derive(Debug, PartialEq)] pub struct CollectorConfig { name: String, target: Target, @@ -1226,7 +1230,7 @@ impl CollectorConfig { /// The data that can be retrived from the database directly to populate the /// status page -#[derive(Debug, PartialEq, serde::Deserialize, serde::Serialize)] +#[derive(Debug, PartialEq)] pub struct PartialStatusPageData { /// A Vector of; completed requests with any associated errors pub completed_requests: Vec<(BenchmarkRequest, Vec)>, diff --git a/site/src/api.rs b/site/src/api.rs index 32772053f..7d1ed2fcc 100644 --- a/site/src/api.rs +++ b/site/src/api.rs @@ -392,19 +392,89 @@ pub mod status { } pub mod status_new { - use database::{BenchmarkJob, BenchmarkRequest, CollectorConfig}; + use chrono::{DateTime, Utc}; + use database::BenchmarkSet; use serde::Serialize; + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct BenchmarkRequestStatusUi { + pub state: String, + pub completed_at: Option>, + pub duration: Option, + } + + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct BenchmarkRequestTypeUi { + pub r#type: String, + pub tag: Option, + pub parent_sha: Option, + pub pr: Option, + } + + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct BenchmarkRequestUi { + pub status: BenchmarkRequestStatusUi, + pub request_type: BenchmarkRequestTypeUi, + pub commit_date: Option>, + pub created_at: DateTime, + pub backends: Vec, + pub profiles: String, + pub errors: Vec, + } + + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct BenchmarkJobStatusUi { + pub state: String, + pub started_at: Option>, + pub completed_at: Option>, + pub collector_name: Option, + } + + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct BenchmarkJobUi { + pub target: String, + pub backend: String, + pub profile: String, + pub request_tag: String, + pub benchmark_set: BenchmarkSet, + pub created_at: DateTime, + pub status: BenchmarkJobStatusUi, + pub deque_counter: u32, + } + + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct BenchmarkInProgressUi { + pub request: BenchmarkRequestUi, + pub jobs: Vec, + } + + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct CollectorConfigUi { + pub name: String, + pub target: String, + pub benchmark_set: BenchmarkSet, + pub is_active: bool, + pub last_heartbeat_at: DateTime, + pub date_added: DateTime, + } + #[derive(Serialize, Debug)] pub struct Response { /// Completed requests alongside any errors - pub completed: Vec<(BenchmarkRequest, Vec)>, + pub completed: Vec, /// In progress requests alongside the jobs associated with the request - pub in_progress: Vec<(BenchmarkRequest, Vec)>, + pub in_progress: Vec, /// Configuration for all collectors including ones that are inactive - pub collector_configs: Vec, + pub collector_configs: Vec, /// The current queue - pub queue: Vec, + pub queue: Vec, } } diff --git a/site/src/request_handlers/status_page_new.rs b/site/src/request_handlers/status_page_new.rs index 120921d80..13cb92440 100644 --- a/site/src/request_handlers/status_page_new.rs +++ b/site/src/request_handlers/status_page_new.rs @@ -1,8 +1,107 @@ use std::sync::Arc; +use crate::api::status_new::{ + BenchmarkInProgressUi, BenchmarkJobStatusUi, BenchmarkJobUi, BenchmarkRequestStatusUi, + BenchmarkRequestTypeUi, BenchmarkRequestUi, CollectorConfigUi, +}; use crate::api::{status_new, ServerResult}; use crate::job_queue::build_queue; use crate::load::SiteCtxt; +use database::{ + BenchmarkJob, BenchmarkJobStatus, BenchmarkRequest, BenchmarkRequestStatus, CollectorConfig, +}; + +fn benchmark_request_status_to_ui(status: BenchmarkRequestStatus) -> BenchmarkRequestStatusUi { + let (completed_at, duration) = match status { + BenchmarkRequestStatus::Completed { + duration, + completed_at, + } => (Some(completed_at), u32::try_from(duration.as_millis()).ok()), + _ => (None, None), + }; + + BenchmarkRequestStatusUi { + state: status.as_str().to_owned(), + completed_at, + duration, + } +} + +fn benchmark_request_type_to_ui(req: &BenchmarkRequest) -> BenchmarkRequestTypeUi { + BenchmarkRequestTypeUi { + r#type: match (req.is_release(), req.is_master()) { + (true, _) => "Release", + (_, true) => "Master", + _ => "Try", + } + .to_owned(), + tag: req.tag().map(|it| it.to_owned()), + parent_sha: req.parent_sha().map(|it| it.to_owned()), + pr: req.pr().copied(), + } +} + +fn benchmark_request_to_ui( + req: &BenchmarkRequest, + errors: Vec, +) -> anyhow::Result { + Ok(BenchmarkRequestUi { + status: benchmark_request_status_to_ui(req.status()), + request_type: benchmark_request_type_to_ui(req), + commit_date: req.commit_date(), + created_at: req.created_at(), + backends: req.backends()?.iter().map(|it| it.to_string()).collect(), + profiles: req.profiles()?.iter().map(|it| it.to_string()).collect(), + errors, + }) +} + +fn benchmark_job_status_to_ui(status: &BenchmarkJobStatus) -> BenchmarkJobStatusUi { + let (started_at, completed_at, collector_name_ref) = match status { + BenchmarkJobStatus::Queued => (None, None, None), + BenchmarkJobStatus::InProgress { + started_at, + collector_name, + } => (Some(*started_at), None, Some(collector_name)), + BenchmarkJobStatus::Completed { + started_at, + completed_at, + collector_name, + .. + } => (Some(*started_at), Some(*completed_at), Some(collector_name)), + }; + + BenchmarkJobStatusUi { + state: status.as_str().to_owned(), + started_at, + completed_at, + collector_name: collector_name_ref.cloned(), + } +} + +fn benchmark_job_to_ui(job: &BenchmarkJob) -> BenchmarkJobUi { + BenchmarkJobUi { + target: job.target().as_str().to_owned(), + backend: job.backend().as_str().to_owned(), + profile: job.profile().as_str().to_owned(), + request_tag: job.request_tag().to_owned(), + benchmark_set: job.benchmark_set(), + created_at: job.created_at(), + status: benchmark_job_status_to_ui(job.status()), + deque_counter: job.deque_count(), + } +} + +fn collector_config_to_ui(config: &CollectorConfig) -> CollectorConfigUi { + CollectorConfigUi { + name: config.name().to_owned(), + target: config.target().as_str().to_owned(), + benchmark_set: config.benchmark_set(), + is_active: config.is_active(), + last_heartbeat_at: config.last_heartbeat_at(), + date_added: config.date_added(), + } +} pub async fn handle_status_page_new(ctxt: Arc) -> ServerResult { let conn = ctxt.conn().await; @@ -12,11 +111,14 @@ pub async fn handle_status_page_new(ctxt: Arc) -> ServerResult) -> ServerResult = vec![]; + for it in partial_data.completed_requests { + completed.push(benchmark_request_to_ui(&it.0, it.1).map_err(error_to_string)?); + } + + let mut in_progress: Vec = vec![]; + for it in partial_data.in_progress { + in_progress.push(BenchmarkInProgressUi { + request: benchmark_request_to_ui(&it.0, vec![]).map_err(error_to_string)?, + jobs: it.1.iter().map(benchmark_job_to_ui).collect(), + }); + } + + let mut queue_ui: Vec = vec![]; + for it in queue { + queue_ui.push(benchmark_request_to_ui(&it, vec![]).map_err(error_to_string)?); + } + Ok(status_new::Response { - completed: partial_data - .completed_requests - .iter() - // @TODO Remove this - .map(|it| (it.0.clone(), it.2.clone())) - .collect(), - in_progress: partial_data.in_progress, + completed, + in_progress, collector_configs, - queue, + queue: queue_ui, }) } From 5d0a107a06784096f2c331d15b0469c4066da36d Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Wed, 20 Aug 2025 15:40:06 +0100 Subject: [PATCH 8/8] Use camelcase for all of the objects in the JSON blob, update typescript types --- site/frontend/src/pages/status_new/data.ts | 216 +++++++++----------- site/frontend/src/pages/status_new/page.vue | 26 +-- site/src/api.rs | 1 + 3 files changed, 110 insertions(+), 133 deletions(-) diff --git a/site/frontend/src/pages/status_new/data.ts b/site/frontend/src/pages/status_new/data.ts index 0661f1e85..6347e72c6 100644 --- a/site/frontend/src/pages/status_new/data.ts +++ b/site/frontend/src/pages/status_new/data.ts @@ -1,165 +1,139 @@ -import {isObject, isString} from "../../utils/getType"; +type BenchmarkRequestStatusComplete = { + state: "completed"; + completedAt: string; + duration: number; // time in milliseconds +}; -type CommitTypeMaster = { - sha: string; - parent_sha: string; - pr: number; +type BenchmarkRequestStatusInProgress = { + state: "in_progress"; }; -type CommitTypeRelease = { - tag: string; +type BenchmarkRequestStatusArtifactsReady = { + state: "artifacts_ready"; }; -type CommitTypeTry = { - sha: string; +export type BenchmarkRequestStatus = + | BenchmarkRequestStatusComplete + | BenchmarkRequestStatusInProgress + | BenchmarkRequestStatusArtifactsReady; + +type BenchmarkRequestTypeTry = { + type: "Try"; + tag: string | null; parent_sha: string | null; pr: number; }; -export type CommitType = CommitTypeRelease | CommitTypeMaster | CommitTypeTry; -export type CommitTypeString = "Master" | "Try" | "Release"; +type BenchmarkRequestTypeMaster = { + type: "Master"; + tag: string; + parent_sha: string; + pr: number; +}; + +type BenchmarkRequestTypeRelease = { + type: "Try"; + tag: string; +}; + +type BenchmarkRequestType = + | BenchmarkRequestTypeTry + | BenchmarkRequestTypeMaster + | BenchmarkRequestTypeRelease; export type BenchmarkRequestComplete = { - commit_type: { - [K in CommitTypeString]: CommitType; - }; - commit_date: string | null; - created_at: string | null; - status: { - Completed: { - completed_at: string; - duration_ms: number; - }; - }; - backends: string; - profile: string; + status: BenchmarkRequestStatusComplete; + requestType: BenchmarkRequestType; + commitDate: string | null; + createdAt: string | null; + backends: string[]; + profiles: string; + errors: string[]; }; export type BenchmarkRequestInProgress = { - commit_type: { - [K in CommitTypeString]: CommitType; - }; - commit_date: string | null; - created_at: string | null; - status: "InProgress"; - backends: string; + status: BenchmarkRequestStatusInProgress; + requestType: BenchmarkRequestType; + commitDate: string | null; + createdAt: string | null; + backends: string[]; profiles: string; + errors: string[]; }; export type BenchmarkRequestArtifactsReady = { - commit_type: { - [K in CommitTypeString]: CommitType; - }; - commit_date: string | null; - created_at: string | null; - status: "ArtifactsReady"; - backends: string; + status: BenchmarkRequestStatusArtifactsReady; + requestType: BenchmarkRequestType; + commitDate: string | null; + createdAt: string | null; + backends: string[]; profiles: string; + errors: string[]; }; -type BenchmarkRequest = +export type BenchmarkRequest = | BenchmarkRequestComplete | BenchmarkRequestInProgress | BenchmarkRequestArtifactsReady; -export function isMasterBenchmarkRequest( - commitType: Object -): commitType is {["Master"]: CommitTypeMaster} { - return "Master" in commitType; -} - -export function isReleaseBenchmarkRequest( - commitType: Object -): commitType is {["Release"]: CommitTypeRelease} { - return "Release" in commitType; -} - -export function isTryBenchmarkRequest( - commitType: Object -): commitType is {["Try"]: CommitTypeTry} { - return "Try" in commitType; -} - -export function isArtifactsReadyBenchmarkRequest( - req: BenchmarkRequest -): req is BenchmarkRequestArtifactsReady { - return isString(req.status) && req.status === "ArtifactsReady"; -} - -export function isInProgressBenchmarkRequest( - req: BenchmarkRequest -): req is BenchmarkRequestInProgress { - return isString(req.status) && req.status === "InProgress"; -} - -export function isCompleteBenchmarkRequest( - req: BenchmarkRequest -): req is BenchmarkRequestComplete { - return isObject(req.status) && "Completed" in req.status; -} +export type BenchmarkJobStatusQueued = { + state: "queued"; +}; export type BenchmarkJobStatusInProgress = { - started_at: string; - collector_name: string; + state: "in_progress"; + startedAt: string; + collectorName: string; }; -export type BenchmarkJobStatusCompleted = { - started_at: string; - completed_at: string; - collector_name: string; - success: boolean; +export type BenchmarkJobStatusFailed = { + state: "failed"; + startedAt: string; + completedAt: string; + collectorName: string; }; -export type BenchmarkJobStatusString = "InProgress" | "Completed"; -export type BenchmarkJobStatusQueued = "Queued"; +export type BenchmarkJobStatusSuccess = { + state: "success"; + startedAt: string; + completedAt: string; + collectorName: string; +}; + +export type BenchmarkJobStatus = + | BenchmarkJobStatusSuccess + | BenchmarkJobStatusFailed + | BenchmarkJobStatusInProgress + | BenchmarkJobStatusQueued; export type BenchmarkJob = { - id: number; target: string; backend: string; - request_tag: string; - benchmark_set: number; - created_at: string; - status: - | BenchmarkJobStatusQueued - | { - [K in BenchmarkJobStatusQueued]: - | BenchmarkJobStatusInProgress - | BenchmarkJobStatusCompleted; - }; - deque_counter: number; -}; - -export function isQueuedBenchmarkJob( - status: unknown -): status is BenchmarkJobStatusQueued { - return isString(status) && status === "Queued"; -} - -export function isInProgressBenchmarkJob( - status: unknown -): status is {["InProgress"]: BenchmarkJobStatusInProgress} { - return isObject(status) && "InProgress" in status; -} - -export function isCompletedBenchmarkJob( - status: unknown -): status is {["Completed"]: BenchmarkJobStatusCompleted} { - return isObject(status) && "Completed" in status; -} + profile: string; + requestTag: string; + benchmarkSet: number; + createdAt: string; + status: BenchmarkJobStatus; + dequeCounter: number; +}; export type CollectorConfig = { name: string; target: string; - benchmark_set: number; - is_active: boolean; - last_heartbeat_at: string; - date_added: string; + benchmarkSet: number; + isActive: boolean; + lastHeartbeatAt: string; + dateAdded: string; +}; + +export type StatusResponseInProgress = { + request: BenchmarkRequestInProgress; + jobs: BenchmarkJob[]; }; export type StatusResponse = { - completed: [BenchmarkRequestComplete, string[]][]; - in_progress: [BenchmarkRequestInProgress, BenchmarkJob[]][]; - collector_configs: CollectorConfig[]; + completed: BenchmarkRequestComplete[]; + inProgress: StatusResponseInProgress[]; + collectorConfigs: CollectorConfig[]; queue: BenchmarkRequest[]; }; diff --git a/site/frontend/src/pages/status_new/page.vue b/site/frontend/src/pages/status_new/page.vue index ddd4c2113..6c3a46aab 100644 --- a/site/frontend/src/pages/status_new/page.vue +++ b/site/frontend/src/pages/status_new/page.vue @@ -15,7 +15,7 @@ const loading = ref(true); const dataNew: Ref = ref(null); function statusClass(c: CollectorConfig): string { - return c.is_active ? "active" : "inactive"; + return c.isActive ? "active" : "inactive"; } loadStatusNew(loading); @@ -34,27 +34,32 @@ loadStatusNew(loading);
-
{{ c.name }}:
-
- {{ c.is_active ? "Active" : "Inactive" }} +
+ {{ c.name }} +
+
+ Status: + + {{ c.isActive ? "Active" : "Inactive" }} +
Target: {{ c.target }}
-
Benchmark Set: #{{ c.benchmark_set }}
+
Benchmark Set: #{{ c.benchmarkSet }}
Last Heartbeat: - {{ c.last_heartbeat_at }} + {{ c.lastHeartbeatAt }}
Date Added: - {{ c.date_added }} + {{ c.dateAdded }}
@@ -103,16 +108,13 @@ loadStatusNew(loading); } .header { - display: flex; - align-items: center; } .status { - padding: 2px 8px; + padding: 2px 8px 2px 0px; border-radius: 20px; font-size: 0.75rem; width: 50px; - font-weight: bold; } .status.active { diff --git a/site/src/api.rs b/site/src/api.rs index 7d1ed2fcc..3002a2fa0 100644 --- a/site/src/api.rs +++ b/site/src/api.rs @@ -466,6 +466,7 @@ pub mod status_new { } #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] pub struct Response { /// Completed requests alongside any errors pub completed: Vec,