Skip to content

Commit 48e2958

Browse files
divanikdivanik
committed
Resolve conflicts
2 parents 5e35373 + 4d96a71 commit 48e2958

File tree

174 files changed

+4119
-1161
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

174 files changed

+4119
-1161
lines changed

‎.github/workflows/nightly_statistics.yml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# generated by praktika
2+
3+
name: NightlyStatistics
4+
on:
5+
schedule:
6+
- cron: 13 5 * * *
7+
workflow_dispatch:
8+
9+
concurrency:
10+
group: ${{ github.workflow }}
11+
12+
env:
13+
PYTHONUNBUFFERED: 1
14+
CHECKOUT_REF: ""
15+
16+
jobs:
17+
18+
config_workflow:
19+
runs-on: [self-hosted, style-checker-aarch64]
20+
needs: []
21+
name: "Config Workflow"
22+
outputs:
23+
data: ${{ steps.run.outputs.DATA }}
24+
steps:
25+
- name: Checkout code
26+
uses: actions/checkout@v4
27+
with:
28+
ref: ${{ env.CHECKOUT_REF }}
29+
30+
- name: Prepare env script
31+
run: |
32+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
33+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
34+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
35+
export PYTHONPATH=./ci:.:
36+
37+
cat > ./ci/tmp/workflow_status.json << 'EOF'
38+
${{ toJson(needs) }}
39+
EOF
40+
ENV_SETUP_SCRIPT_EOF
41+
42+
- name: Run
43+
id: run
44+
run: |
45+
. ./ci/tmp/praktika_setup_env.sh
46+
set -o pipefail
47+
if command -v ts &> /dev/null; then
48+
python3 -m praktika run 'Config Workflow' --workflow "NightlyStatistics" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
49+
else
50+
python3 -m praktika run 'Config Workflow' --workflow "NightlyStatistics" --ci |& tee ./ci/tmp/job.log
51+
fi
52+
53+
collect_statistics:
54+
runs-on: [self-hosted, style-checker-aarch64]
55+
needs: [config_workflow]
56+
name: "Collect Statistics"
57+
outputs:
58+
data: ${{ steps.run.outputs.DATA }}
59+
steps:
60+
- name: Checkout code
61+
uses: actions/checkout@v4
62+
with:
63+
ref: ${{ env.CHECKOUT_REF }}
64+
65+
- name: Prepare env script
66+
run: |
67+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
68+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
69+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
70+
export PYTHONPATH=./ci:.:
71+
72+
cat > ./ci/tmp/workflow_status.json << 'EOF'
73+
${{ toJson(needs) }}
74+
EOF
75+
ENV_SETUP_SCRIPT_EOF
76+
77+
- name: Run
78+
id: run
79+
run: |
80+
. ./ci/tmp/praktika_setup_env.sh
81+
set -o pipefail
82+
if command -v ts &> /dev/null; then
83+
python3 -m praktika run 'Collect Statistics' --workflow "NightlyStatistics" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
84+
else
85+
python3 -m praktika run 'Collect Statistics' --workflow "NightlyStatistics" --ci |& tee ./ci/tmp/job.log
86+
fi

‎base/base/BFloat16.h‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,13 @@ class BFloat16
4848
{
4949
}
5050

51+
static constexpr BFloat16 fromBits(UInt16 bits) noexcept
52+
{
53+
BFloat16 res;
54+
res.x = bits;
55+
return res;
56+
}
57+
5158
template <typename T>
5259
constexpr BFloat16 & operator=(const T & other)
5360
{
@@ -312,3 +319,16 @@ constexpr inline auto operator/(BFloat16 a, T b)
312319
{
313320
return Float32(a) / b;
314321
}
322+
323+
namespace std
324+
{
325+
template <>
326+
class numeric_limits<BFloat16>
327+
{
328+
public:
329+
static constexpr BFloat16 lowest() noexcept { return BFloat16::fromBits(0b1111111101111111); }
330+
static constexpr BFloat16 min() noexcept { return BFloat16::fromBits(0b0000000100000000); }
331+
static constexpr BFloat16 max() noexcept { return BFloat16::fromBits(0b0111111101111111); }
332+
static constexpr BFloat16 infinity() noexcept { return BFloat16::fromBits(0b0111111110000000); }
333+
};
334+
}

‎ci/jobs/collect_statistics.py‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
import json
2+
import sys
3+
from datetime import datetime
4+
5+
from ci.praktika import Secret
6+
from ci.praktika.cidb import CIDB
7+
from ci.praktika.result import Result
8+
from ci.praktika.s3 import S3
9+
from ci.praktika.utils import Shell
10+
from ci.settings.settings import S3_REPORT_BUCKET_NAME
11+
12+
# Job collects overall CI statistics per each job
13+
14+
# TODO: Should work for generic CI and become native praktika job
15+
16+
17+
QUANTILES = [
18+
0,
19+
5,
20+
10,
21+
15,
22+
20,
23+
25,
24+
35,
25+
40,
26+
45,
27+
50,
28+
55,
29+
60,
30+
65,
31+
70,
32+
75,
33+
80,
34+
85,
35+
90,
36+
95,
37+
100,
38+
]
39+
40+
DAYS = [1, 3]
41+
42+
JOB_STATISTICS_QUERY = """
43+
SELECT
44+
count(),
45+
{QUANTILES_SECTION},
46+
FROM default.checks
47+
WHERE
48+
check_start_time >= now() - INTERVAL {DAYS} DAY
49+
AND check_name = '{JOB_NAME}'
50+
AND check_status = 'success'
51+
AND base_ref = '{BASE_REF}'
52+
"""
53+
54+
JOB_NAMES_QUERY = """
55+
SELECT DISTINCT check_name AS JOB_NAME
56+
FROM default.checks
57+
WHERE
58+
check_start_time >= NOW() - INTERVAL {DAYS} DAY
59+
AND check_status = 'success'
60+
AND base_ref = '{BASE_REF}'
61+
AND head_ref not LIKE '2%'
62+
AND head_ref not LIKE 'release/2%'
63+
"""
64+
65+
66+
# Format-friendly helper
67+
def format_quantiles(q_list, column="check_duration_ms"):
68+
return ",\n ".join(f"quantile({q / 100:.2f})({column})" for q in q_list)
69+
70+
71+
def format_quantiles_names(q_list):
72+
return [f"{q}" for q in q_list]
73+
74+
75+
RESULTS = ["runs"] + format_quantiles_names(QUANTILES)
76+
77+
78+
def get_job_stat_for_interval(name, interval_days, overall_statistics):
79+
job_stats = {"quantiles": {}}
80+
81+
query = JOB_STATISTICS_QUERY.format(
82+
JOB_NAME=name,
83+
QUANTILES_SECTION=format_quantiles(QUANTILES),
84+
DAYS=interval_days,
85+
BASE_REF=BASE_REF,
86+
)
87+
88+
output = cidb.query(query)
89+
values = output.split()
90+
91+
key, val = None, None
92+
try:
93+
assert len(values) == len(RESULTS), f"Mismatch: {len(values)} vs {len(RESULTS)}"
94+
for key, val in zip(RESULTS, values):
95+
parsed_val = int(float(val))
96+
if key == "runs":
97+
job_stats["runs"] = parsed_val
98+
else:
99+
assert int(key) >= 0, f"Invalid quantile key: {key}"
100+
job_stats["quantiles"][key] = parsed_val
101+
102+
overall_statistics[name][f"{interval_days}d"] = job_stats
103+
except Exception as e:
104+
print(
105+
f" ERROR: Failed to parse stats — key [{key}], value [{val}], error: {e}"
106+
)
107+
return False
108+
return True
109+
110+
111+
if __name__ == "__main__":
112+
113+
cidb = CIDB(
114+
url=Secret.Config(
115+
name="clickhouse-test-stat-url",
116+
type=Secret.Type.AWS_SSM_VAR,
117+
).get_value(),
118+
user=Secret.Config(
119+
name="clickhouse-test-stat-login",
120+
type=Secret.Type.AWS_SSM_VAR,
121+
).get_value(),
122+
passwd=Secret.Config(
123+
name="clickhouse-test-stat-password",
124+
type=Secret.Type.AWS_SSM_VAR,
125+
).get_value(),
126+
)
127+
128+
BASE_REF = "master"
129+
130+
names = None
131+
results = []
132+
133+
print(f"--- Get Job Names ---")
134+
135+
def get_all_job_names():
136+
query = JOB_NAMES_QUERY.format(DAYS=3, BASE_REF=BASE_REF)
137+
global names
138+
names = cidb.query(query).splitlines()
139+
return True
140+
141+
results.append(
142+
Result.from_commands_run(
143+
name="Get all job names", command=get_all_job_names, with_info=True
144+
)
145+
)
146+
if not results[-1].is_ok():
147+
sys.exit()
148+
149+
print(f"--- Get statistics for each job ---")
150+
overall_statistics = {}
151+
is_collected = False
152+
results_stat = []
153+
for job_name in names:
154+
overall_statistics[job_name] = {}
155+
156+
def do():
157+
res = False
158+
for days in DAYS:
159+
res = (
160+
get_job_stat_for_interval(
161+
name=job_name,
162+
interval_days=days,
163+
overall_statistics=overall_statistics,
164+
)
165+
or res
166+
)
167+
return res
168+
169+
results_stat.append(Result.from_commands_run(name=job_name, command=do))
170+
171+
if results_stat[-1].is_ok():
172+
is_collected = True
173+
results.append(
174+
Result(
175+
name="Fetch statistics",
176+
status=Result.Status.SUCCESS if is_collected else Result.Status.FAILED,
177+
results=results_stat,
178+
)
179+
)
180+
181+
print(f"--- Upload statistics ---")
182+
statistics_link = None
183+
if is_collected:
184+
185+
def do():
186+
global statistics_link
187+
file_name = "./ci/tmp/statistics.json"
188+
archive_name = "./ci/tmp/statistics.json.gz"
189+
archive_name_with_date = (
190+
f"./ci/tmp/statistics_{datetime.now().strftime('%d_%m_%Y')}.json.gz"
191+
)
192+
193+
with open(file_name, "w") as f:
194+
json.dump(overall_statistics, f, indent=2)
195+
196+
Shell.check(
197+
f"rm -f {archive_name} {archive_name_with_date} && gzip -k {file_name} && cp {archive_name} {archive_name_with_date}"
198+
)
199+
_ = S3.copy_file_to_s3(
200+
local_path=archive_name,
201+
s3_path=f"{S3_REPORT_BUCKET_NAME}/statistics",
202+
content_type="application/json",
203+
content_encoding="gzip",
204+
)
205+
statistics_link = S3.copy_file_to_s3(
206+
local_path=archive_name_with_date,
207+
s3_path=f"{S3_REPORT_BUCKET_NAME}/statistics",
208+
content_type="application/json",
209+
content_encoding="gzip",
210+
)
211+
212+
results.append(Result.from_commands_run("Upload", command=do))
213+
214+
Result.create_from(
215+
results=results, links=[statistics_link] if statistics_link else []
216+
).complete_job(with_job_summary_in_info=False)

0 commit comments

Comments
 (0)