Skip to content

Commit b2a9102

Browse files
authored
[ch] set up ingestion for metrics and stable pushes tables (#5738)
* Renamed metrics table to ossci_uploaded_metrics * Relevant PR for metrics table: pytorch/pytorch#136799 * Relevant info for stable pushes table: pytorch/pytorch#136470 --- Tested by running the lambda handler on some docs currently in those bucket/paths
1 parent 482a7a6 commit b2a9102

File tree

1 file changed

+36
-0
lines changed

1 file changed

+36
-0
lines changed

aws/lambda/clickhouse-replicator-s3/lambda_function.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,38 @@ def torchbench_userbenchmark_adapter(table, bucket, key):
388388
general_adapter(table, bucket, key, schema, ["none"], "JSONEachRow")
389389

390390

391+
def ossci_uploaded_metrics_adapter(table, bucket, key):
392+
schema = """
393+
`repo` String,
394+
`workflow` String,
395+
`build_environment` String,
396+
`job` String,
397+
`test_config` String,
398+
`pr_number` Int64,
399+
`run_id` Int64,
400+
`run_number` Int64,
401+
`run_attempt` Int64,
402+
`job_id` Int64,
403+
`job_name` String,
404+
`metric_name` String,
405+
`calling_file` String,
406+
`calling_module` String,
407+
`calling_function` String,
408+
`timestamp` DateTime64(9),
409+
`info` String
410+
"""
411+
general_adapter(table, bucket, key, schema, "gzip", "JSONEachRow")
412+
413+
414+
def stable_pushes_adapter(table, bucket, key):
415+
schema = """
416+
`sha` String,
417+
`repository` String,
418+
`timestamp` DateTime
419+
"""
420+
general_adapter(table, bucket, key, schema, "none", "JSONEachRow")
421+
422+
391423
SUPPORTED_PATHS = {
392424
"merges": "default.merges",
393425
"queue_times_historical": "default.queue_times_historical",
@@ -400,6 +432,8 @@ def torchbench_userbenchmark_adapter(table, bucket, key):
400432
"test_data_aggregates": "misc.aggregated_test_metrics",
401433
"torchbench-csv/torchao": "benchmark.inductor_torchao_perf_stats",
402434
"torchbench-userbenchmark": "benchmark.torchbench_userbenchmark",
435+
"ossci_uploaded_metrics": "misc.ossci_uploaded_metrics",
436+
"stable_pushes": "misc.stable_pushes",
403437
}
404438

405439
OBJECT_CONVERTER = {
@@ -414,6 +448,8 @@ def torchbench_userbenchmark_adapter(table, bucket, key):
414448
"misc.aggregated_test_metrics": external_aggregated_test_metrics_adapter,
415449
"benchmark.inductor_torchao_perf_stats": torchao_perf_stats_adapter,
416450
"benchmark.torchbench_userbenchmark": torchbench_userbenchmark_adapter,
451+
"misc.ossci_uploaded_metrics": ossci_uploaded_metrics_adapter,
452+
"misc.stable_pushes": stable_pushes_adapter,
417453
}
418454

419455

0 commit comments

Comments
 (0)