Skip to content

Commit 29737b0

Browse files
EdwardKaravakisEdwardKaravakis
authored
Merge pull request #117 from PanDAWMS/v0.0.31
Changes for 0.0.31 for Oracle
2 parents 08cb0cf + e5313ca commit 29737b0

File tree

7 files changed

+203
-3
lines changed

7 files changed

+203
-3
lines changed

‎schema/oracle/ATLAS_PANDA.sql‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
-- IMPORTANT: Please always update to up2date version
3535
--------------------------------------------------------
3636

37-
INSERT INTO "ATLAS_PANDA"."PANDADB_VERSION" VALUES ('PanDA', 0, 0, 30);
37+
INSERT INTO "ATLAS_PANDA"."PANDADB_VERSION" VALUES ('PanDA', 0, 0, 31);
3838
--------------------------------------------------------
3939
-- DDL for Sequence FILESTABLE4_ROW_ID_SEQ
4040
--------------------------------------------------------
@@ -3023,6 +3023,18 @@ COMMENT ON COLUMN "ATLAS_PANDA"."WORKER_NODE_GPUS"."DRIVER_VERSION" IS 'Version
30233023
COMMENT ON COLUMN "ATLAS_PANDA"."WORKER_NODE_GPUS"."LAST_SEEN" IS 'Timestamp of the last time the worker node was active.';
30243024

30253025

3026+
--------------------------------------------------------
3027+
-- DDL for Table WORKER_NODE_QUEUE
3028+
--------------------------------------------------------
3029+
3030+
CREATE TABLE "ATLAS_PANDA"."WORKER_NODE_QUEUE" (
3031+
site VARCHAR2(128),
3032+
host_name VARCHAR2(128),
3033+
panda_queue VARCHAR2(128),
3034+
last_seen DATE,
3035+
CONSTRAINT PK_WORKER_NODE_QUEUE PRIMARY KEY (site, host_name, panda_queue)
3036+
);
3037+
30263038
--------------------------------------------------------
30273039
-- DDL for Table CPU_BENCHMARKS
30283040
--------------------------------------------------------

‎schema/oracle/ATLAS_PANDA_VIEW.sql‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
-- CPU summary by site / queue / CPU arch
2+
CREATE MATERIALIZED VIEW "ATLAS_PANDA"."MV_WORKER_NODE_SUMMARY"
3+
BUILD IMMEDIATE
4+
REFRESH COMPLETE ON DEMAND
5+
START WITH SYSDATE
6+
NEXT SYSDATE + 1/24
7+
AS
8+
SELECT
9+
wn.site,
10+
wnq.panda_queue,
11+
wn.cpu_architecture_level,
12+
SUM(wn.n_logical_cpus) AS total_logical_cpus,
13+
ROUND(
14+
100 * SUM(wn.n_logical_cpus)
15+
/ SUM(SUM(wn.n_logical_cpus)) OVER (PARTITION BY wnq.panda_queue),
16+
2
17+
) AS pct_within_pq
18+
FROM atlas_panda.worker_node wn, atlas_panda.worker_node_queue wnq
19+
WHERE wnq.last_seen > sysdate - interval '1' month
20+
AND wn.site = wnq.site AND wn.host_name = wnq.host_name
21+
GROUP BY wn.site, wnq.panda_queue, wn.cpu_architecture_level;
22+
23+
-- GPU inventory summary (per host configuration) by site/queue
24+
CREATE MATERIALIZED VIEW "ATLAS_PANDA"."MV_WORKER_NODE_GPU_SUMMARY"
25+
BUILD IMMEDIATE
26+
REFRESH COMPLETE ON DEMAND
27+
START WITH SYSDATE
28+
NEXT SYSDATE + 1/24
29+
AS
30+
SELECT
31+
wng.site,
32+
wnq.panda_queue,
33+
wng.vendor,
34+
wng.model,
35+
wng.vram,
36+
wng.framework,
37+
wng.framework_version,
38+
wng.count AS gpus_per_host,
39+
COUNT(*) AS host_count
40+
FROM ATLAS_PANDA.worker_node_gpus wng,
41+
ATLAS_PANDA.worker_node_queue wnq
42+
WHERE wnq.last_seen > SYSDATE - INTERVAL '1' MONTH
43+
AND wng.site = wnq.site
44+
AND wng.host_name = wnq.host_name
45+
GROUP BY
46+
wng.site,
47+
wnq.panda_queue,
48+
wng.vendor,
49+
wng.model,
50+
wng.vram,
51+
wng.framework,
52+
wng.framework_version,
53+
wng.count;
54+
55+
GRANT SELECT ON ATLAS_PANDA.MV_WORKER_NODE_SUMMARY     TO ATLAS_PANDA_READER;
56+
GRANT SELECT ON ATLAS_PANDA.MV_WORKER_NODE_GPU_SUMMARY TO ATLAS_PANDA_READER;

‎schema/postgres/sqls/patches/0.0.31.README.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# PanDA Database Schema Patch 0.0.31
2+
3+
**Version:** 0.0.31
4+
**Upgrade from:** 0.0.30
5+
**Date:** November 2025
6+
7+
## Overview
8+
9+
This patch adds worker-node queue tracking and two reporting materialized views in the `doma_panda` schema:
10+
11+
- `doma_panda.worker_node_queue` (new table)
12+
- `doma_panda.mv_worker_node_summary` (CPU summary per panda_queue)
13+
- `doma_panda.mv_worker_node_gpu_summary` (GPU inventory summary)
14+
15+
The materialized views compute per-queue percentages and are intended to refresh hourly.
16+
Per 0.0.30, versioning continues to use a single **PanDA** component in `pandadb_version`. (See 0.0.30 docs for the versioning shift.).
17+
18+
## Files in this Patch
19+
20+
1. **0.0.31.patch.sql** — Main DB updates (run against `panda_db`)
21+
2. **0.0.31.cron.patch.sql** — Cron job updates (run where `pg_cron` is installed)
22+
3. **0.0.31.README.md** — This file
23+
24+
## Order of Operations
25+
26+
1. Apply `0.0.31.patch.sql` to `panda_db`
27+
2. Apply `0.0.31.cron.patch.sql` to the DB that hosts `pg_cron`

‎schema/postgres/sqls/patches/0.0.31.cron.patch.sql‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-- Cron job updates for patch 0.0.31
2+
-- This file contains cron-specific updates and must be run against the database where pg_cron is installed.
3+
-- IMPORTANT: pg_cron may be installed in 'postgres' or in 'panda_db'.
4+
-- To check: SELECT extname FROM pg_extension WHERE extname = 'pg_cron';
5+
6+
-- Examples:
7+
-- psql -U postgres -d postgres -f 0.0.31.cron.patch.sql -- if pg_cron in 'postgres'
8+
-- psql -U postgres -d panda_db -f 0.0.31.cron.patch.sql -- if pg_cron in 'panda_db'
9+
10+
-- =========================
11+
-- Hourly MV refresh jobs
12+
-- =========================
13+
-- Create top-of-hour schedules for MV refreshes
14+
SELECT cron.schedule ('0 * * * *', 'REFRESH MATERIALIZED VIEW CONCURRENTLY doma_panda.mv_worker_node_summary;');
15+
SELECT cron.schedule ('0 * * * *', 'REFRESH MATERIALIZED VIEW CONCURRENTLY doma_panda.mv_worker_node_gpu_summary;');
16+
17+
-- Point the jobs at the correct database/user/node
18+
UPDATE cron.job SET database = 'panda_db', nodename = '' WHERE command LIKE '%doma_panda.mv_worker_node_summary%';
19+
UPDATE cron.job SET database = 'panda_db', nodename = '' WHERE command LIKE '%doma_panda.mv_worker_node_gpu_summary%';

‎schema/postgres/sqls/patches/0.0.31.patch.sql‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
-- patch to be used to upgrade from version 0.0.30
2+
-- =========================
3+
-- Table: WORKER_NODE_QUEUE
4+
-- =========================
5+
CREATE TABLE IF NOT EXISTS doma_panda.worker_node_queue (
6+
site VARCHAR(128),
7+
host_name VARCHAR(128),
8+
panda_queue VARCHAR(128),
9+
last_seen timestamp,
10+
CONSTRAINT pk_worker_node_queue PRIMARY KEY (site, host_name, panda_queue)
11+
);
12+
13+
-- ===========================================
14+
-- MV: MV_WORKER_NODE_SUMMARY (partition = pq)
15+
-- ===========================================
16+
CREATE MATERIALIZED VIEW IF NOT EXISTS doma_panda.mv_worker_node_summary AS
17+
SELECT
18+
wn.site,
19+
wnq.panda_queue,
20+
wn.cpu_architecture_level,
21+
SUM(wn.n_logical_cpus) AS total_logical_cpus,
22+
ROUND(
23+
(SUM(wn.n_logical_cpus)::numeric * 100)
24+
/ NULLIF(SUM(SUM(wn.n_logical_cpus)) OVER (PARTITION BY wnq.panda_queue), 0)::numeric
25+
, 2) AS pct_within_pq
26+
FROM doma_panda.worker_node AS wn,
27+
doma_panda.worker_node_queue AS wnq
28+
WHERE wnq.last_seen > (CURRENT_TIMESTAMP - INTERVAL '1 month')
29+
AND wn.site = wnq.site
30+
AND wn.host_name = wnq.host_name
31+
GROUP BY wn.site, wnq.panda_queue, wn.cpu_architecture_level
32+
WITH NO DATA;
33+
34+
CREATE UNIQUE INDEX IF NOT EXISTS ux_mv_wn_summary
35+
ON doma_panda.mv_worker_node_summary (site, panda_queue, cpu_architecture_level);
36+
37+
-- ===============================================
38+
-- MV: MV_WORKER_NODE_GPU_SUMMARY (per-host config)
39+
-- ===============================================
40+
CREATE MATERIALIZED VIEW IF NOT EXISTS doma_panda.mv_worker_node_gpu_summary AS
41+
SELECT
42+
wng.site,
43+
wnq.panda_queue,
44+
wng.vendor,
45+
wng.model,
46+
wng.vram,
47+
wng.framework,
48+
wng.framework_version,
49+
wng.count AS gpus_per_host,
50+
COUNT(*) AS host_count
51+
FROM doma_panda.worker_node_gpus AS wng,
52+
doma_panda.worker_node_queue AS wnq
53+
WHERE wnq.last_seen > (CURRENT_TIMESTAMP - INTERVAL '1 month')
54+
AND wng.site = wnq.site
55+
AND wng.host_name = wnq.host_name
56+
GROUP BY
57+
wng.site,
58+
wnq.panda_queue,
59+
wng.vendor,
60+
wng.model,
61+
wng.vram,
62+
wng.framework,
63+
wng.framework_version,
64+
wng.count
65+
WITH NO DATA;
66+
67+
CREATE UNIQUE INDEX IF NOT EXISTS ux_mv_wn_gpu_summary
68+
ON doma_panda.mv_worker_node_gpu_summary
69+
(site, panda_queue, vendor, model, vram, framework, framework_version, gpus_per_host);
70+
71+
-- ===================
72+
-- Initial population (must be non-concurrent the first time)
73+
-- ===================
74+
REFRESH MATERIALIZED VIEW doma_panda.mv_worker_node_summary;
75+
REFRESH MATERIALIZED VIEW doma_panda.mv_worker_node_gpu_summary;
76+
77+
-- Update schema version
78+
UPDATE doma_panda.pandadb_version
79+
SET major = 0, minor = 0, patch = 31
80+
WHERE component = 'PanDA';
81+
commit;

‎schema/postgres/sqls/post_step_cron.sql‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,10 @@ SELECT cron.schedule ('@daily', $$DELETE FROM cron.job_run_details WHERE end_tim
1010
SELECT cron.schedule ('@daily', 'call partman.run_maintenance_proc()');
1111
UPDATE cron.job SET database='panda_db',nodename='' WHERE command like '%partman.run_maintenance_proc%';
1212
SELECT cron.schedule ('0 8 * * *', 'CALL doma_panda.update_worker_node_metrics()');
13-
UPDATE cron.job SET database = 'panda_db',nodename = '' WHERE command LIKE '%update_worker_node_map%' OR command LIKE '%update_worker_node_metrics%';
13+
SELECT cron.schedule ('0 * * * *', 'REFRESH MATERIALIZED VIEW CONCURRENTLY doma_panda.mv_worker_node_summary;');
14+
SELECT cron.schedule ('0 * * * *', 'REFRESH MATERIALIZED VIEW CONCURRENTLY doma_panda.mv_worker_node_gpu_summary;');
15+
16+
UPDATE cron.job SET database = 'panda_db',nodename = '' WHERE command LIKE '%update_worker_node_metrics%';
17+
UPDATE cron.job SET database = 'panda_db', nodename = '' WHERE command LIKE '%mv_worker_node_summary%';
18+
UPDATE cron.job SET database = 'panda_db', nodename = '' WHERE command LIKE '%mv_worker_node_gpu_summary%';
1419

‎schema/postgres/version‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.0.30
1+
0.0.31

0 commit comments

Comments
 (0)