Skip to content

Commit 4796068

Browse files
atalmanatalman
authored
Nightlies page use clickhouse, deprecate rockset (#6090)
Rewrite Nightlies page using clickhouse
1 parent 5c8b064 commit 4796068

File tree

8 files changed

+100
-314
lines changed

8 files changed

+100
-314
lines changed

‎torchci/clickhouse_queries/nightly_jobs_red/params.json

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,5 @@
22
"granularity": "String",
33
"repo": "String",
44
"startTime": "DateTime64(3)",
5-
"stopTime": "DateTime64(3)",
6-
"timezone": "String"
7-
}
5+
"stopTime": "DateTime64(3)"
6+
}

‎torchci/clickhouse_queries/nightly_jobs_red/query.sql

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,37 @@
1-
-- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted
21
SELECT
3-
FORMAT_ISO8601(
4-
DATE_TRUNC(:granularity, time)
2+
DATE_TRUNC(
3+
{granularity: String },
4+
time
55
) AS granularity_bucket,
66
AVG(
77
CASE
88
when conclusion = 'failure' THEN 1
99
when conclusion = 'timed_out' THEN 1
1010
when conclusion = 'cancelled' THEN 1
11-
when conclusion = 'skipped' THEN 1
11+
when conclusion = 'skipped' THEN 1
1212
ELSE 0
1313
END
14-
) as red,
14+
) as red
1515
FROM
16-
(
17-
SELECT
18-
job._event_time AT TIME ZONE :timezone as time,
19-
job.conclusion as conclusion,
20-
FROM
21-
commons.workflow_job job
22-
JOIN commons.workflow_run workflow on workflow.id = job.run_id
23-
JOIN push on workflow.head_commit.id = push.head_commit.id
24-
WHERE
25-
job.name NOT LIKE '%generate-matrix%'
26-
AND job.name NOT LIKE '%unittests%'
27-
AND workflow.name NOT IN ('cron', 'Bandit', 'tests')
28-
AND push.ref = 'refs/heads/nightly'
29-
AND push.repository.owner.name = 'pytorch'
30-
AND push.repository.name = :repo
31-
AND job._event_time >= PARSE_DATETIME_ISO8601(:startTime)
32-
AND job._event_time < PARSE_DATETIME_ISO8601(:stopTime)
33-
) as all_job
16+
(
17+
SELECT
18+
job.created_at AS time,
19+
job.conclusion AS conclusion
20+
FROM
21+
workflow_job job
22+
JOIN workflow_run workflow ON workflow.id = job.run_id
23+
JOIN push ON workflow.head_commit.'id' = push.head_commit.'id'
24+
WHERE
25+
job.name NOT LIKE '%generate-matrix%'
26+
AND job.name NOT LIKE '%unittests%'
27+
AND workflow.name NOT IN ('cron', 'Bandit', 'tests')
28+
AND push.ref = 'refs/heads/nightly'
29+
AND push.repository.'owner'.'name' = 'pytorch'
30+
AND push.repository.'name' = {repo: String }
31+
AND job.created_at >= {startTime: DateTime64(3)}
32+
AND job.created_at < {stopTime: DateTime64(3)}
33+
)
3434
GROUP BY
35-
DATE_TRUNC(:granularity, time)
35+
granularity_bucket
3636
ORDER BY
37-
DATE_TRUNC(:granularity, time) ASC
37+
granularity_bucket ASC

‎torchci/clickhouse_queries/nightly_jobs_red_by_name/query.sql

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,19 @@
1-
-- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted
2-
SELECT
3-
COUNT(*) COUNT, workflow.name
1+
SELECT
2+
COUNT(*) AS COUNT,
3+
workflow.name as name
44
FROM
5-
commons.workflow_job job
6-
JOIN commons.workflow_run workflow on workflow.id = job.run_id
7-
JOIN push on workflow.head_commit.id = push.head_commit.id
5+
workflow_job job
6+
JOIN workflow_run workflow on workflow.id = job.run_id
7+
JOIN push ON workflow.head_commit.'id' = push.head_commit.'id'
88
WHERE
99
job.name NOT LIKE '%generate-matrix%'
1010
AND job.name NOT LIKE '%unittests%'
1111
AND workflow.name NOT IN ('cron', 'Bandit', 'tests', 'Lint')
1212
AND push.ref = 'refs/heads/nightly'
13-
AND push.repository.owner.name = 'pytorch'
14-
AND push.repository.name in ('pytorch', 'vision', 'audio', 'text')
15-
AND job._event_time >= PARSE_DATETIME_ISO8601(:startTime)
16-
AND job._event_time < PARSE_DATETIME_ISO8601(:stopTime)
17-
AND job.conclusion in ('failure', 'timed_out', 'cancelled')
13+
AND push.repository.'owner'.'name' = 'pytorch'
14+
AND push.repository.'name' IN ('pytorch', 'vision', 'audio')
15+
AND job.created_at >= {startTime: DateTime64(3)}
16+
AND job.created_at < {stopTime: DateTime64(3)}
17+
AND job.conclusion IN ('failure', 'timed_out', 'cancelled')
1818
GROUP BY
1919
workflow.name
20-
ORDER BY COUNT DESC

‎torchci/clickhouse_queries/nightly_jobs_red_by_platform/params.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
22
"startTime": "DateTime64(3)",
3-
"stopTime": "DateTime64(3)"
4-
}
3+
"stopTime": "DateTime64(3)",
4+
"repo": "String"
5+
}

‎torchci/clickhouse_queries/nightly_jobs_red_by_platform/query.sql

Lines changed: 13 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,30 @@
1-
-- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted
21
WITH all_failed_jobs AS (
3-
SELECT
4-
COUNT(*) COUNT, workflow.path
2+
SELECT
3+
COUNT(*) AS COUNT, workflow.path
54
FROM
6-
commons.workflow_job job
7-
JOIN commons.workflow_run workflow on workflow.id = job.run_id
8-
JOIN push on workflow.head_commit.id = push.head_commit.id
5+
workflow_job job
6+
JOIN workflow_run workflow on workflow.id = job.run_id
7+
JOIN push ON workflow.head_commit.'id' = push.head_commit.'id'
98
WHERE
109
job.name NOT LIKE '%generate-matrix%'
1110
AND job.name NOT LIKE '%unittests%'
1211
AND workflow.name NOT IN ('cron', 'Bandit', 'tests', 'Lint')
1312
AND push.ref = 'refs/heads/nightly'
14-
AND push.repository.owner.name = 'pytorch'
15-
AND push.repository.name in ('pytorch', 'vision', 'audio', 'text')
16-
AND job._event_time >= PARSE_DATETIME_ISO8601(:startTime)
17-
AND job._event_time < PARSE_DATETIME_ISO8601(:stopTime)
18-
AND job.conclusion in ('failure', 'timed_out', 'cancelled')
13+
AND push.repository.'owner'.'name' = 'pytorch'
14+
AND push.repository.'name' = {repo: String }
15+
AND job.created_at >= {startTime: DateTime64(3)}
16+
AND job.created_at < {stopTime: DateTime64(3)}
17+
AND job.conclusion IN ('failure', 'timed_out', 'cancelled')
1918
GROUP BY
2019
workflow.path )
21-
SELECT
22-
SUM(COUNT) as Count, 'Conda' as Platform
23-
FROM
24-
all_failed_jobs
25-
where path like '%conda%'
26-
UNION
2720
SELECT
2821
SUM(COUNT) as Count, 'Wheel' as Platform
2922
FROM
3023
all_failed_jobs
31-
where path like '%wheel%'
32-
UNION
24+
where workflow.path like '%wheel%'
25+
UNION ALL
3326
SELECT
3427
SUM(COUNT) as Count, 'Libtorch' as Platform
3528
FROM
3629
all_failed_jobs
37-
where path like '%libtorch%'
38-
30+
where workflow.path like '%libtorch%'

‎torchci/clickhouse_queries/nightly_jobs_red_past_day/query.sql

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,18 @@
1-
-- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted
21
SELECT
3-
COUNT(*) COUNT,
4-
job.name
2+
COUNT(*) AS COUNT,
3+
job.name as name
54
FROM
6-
commons.workflow_job job
7-
JOIN commons.workflow_run workflow on workflow.id = job.run_id
8-
JOIN push on workflow.head_commit.id = push.head_commit.id
5+
workflow_job job
6+
JOIN workflow_run workflow ON workflow.id = job.run_id
7+
join push on push.head_commit.'id' = workflow.head_commit.'id'
98
WHERE
109
job.name NOT LIKE '%generate-matrix%'
1110
AND job.name NOT LIKE '%unittests%'
1211
AND workflow.name NOT IN ('cron', 'Bandit', 'tests')
1312
AND push.ref = 'refs/heads/nightly'
14-
AND push.repository.owner.name = 'pytorch'
15-
AND push.repository.name = :repo
16-
AND job.conclusion in ('failure', 'timed_out', 'cancelled')
17-
AND job._event_time >= CURRENT_DATE() - INTERVAL 1 DAY
13+
AND push.repository.'owner'.'name' = 'pytorch'
14+
AND push.repository.'name' = {repo: String }
15+
AND job.conclusion IN ('failure', 'timed_out', 'cancelled')
16+
AND job.completed_at >= today() - 1
1817
GROUP BY job.name
19-
ORDER BY COUNT
20-
18+
ORDER BY COUNT;

‎torchci/clickhouse_queries/validation_jobs_red_past_day/query.sql

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,15 @@
1-
-- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted
21
SELECT
3-
COUNT(*) COUNT,
4-
job.name
2+
COUNT(*) AS COUNT,
3+
job.name as name
54
FROM
6-
commons.workflow_job job
7-
JOIN commons.workflow_run workflow on workflow.id = job.run_id
5+
workflow_job job
6+
JOIN workflow_run workflow ON workflow.id = job.run_id
87
WHERE
9-
job.head_branch = 'main'
10-
AND workflow.name = 'cron'
11-
AND workflow.event = 'schedule'
12-
AND job.conclusion in ('failure', 'timed_out', 'cancelled')
13-
AND job.name like CONCAT('%',:channel,'%')
14-
AND workflow.repository.full_name = 'pytorch/builder'
15-
AND job._event_time >= CURRENT_DATE() - INTERVAL 1 DAY
8+
job.head_branch = 'main'
9+
AND workflow.name like '%Binaries Validations%'
10+
AND workflow.event = 'schedule'
11+
AND job.name like concat('%', {channel: String}, '%')
12+
AND job.conclusion IN ('failure', 'timed_out', 'cancelled')
13+
AND job.completed_at >= today() - 1
1614
GROUP BY job.name
17-
ORDER BY COUNT DESC
15+
ORDER BY COUNT DESC;

0 commit comments

Comments
 (0)