Skip to content

Commit 1aa2c13

Browse files
bassosimonebassosimone
authored
refactor(data): use IQBPipeline to query (#38)
This diff changes `data/run_query.py` to use the `IQBPipeline` to query for data and saves v1 cached data as a side effect. Additionally, we continue to generate and update v0 data as part of running the pipeline. In fact, the rest of the pipeline has not been modified and only `data/run_query.py` internals have changed. This diff opens up the way for stopping to use v0 data and just keep it around as golden files for testing. On this note, add `.gitignore` rules to `./data/cache` so that only the files currently part of the hierarchy are kept around as reference and anything else is instead ignored.
1 parent 6af0111 commit 1aa2c13

File tree

257 files changed

+6908
-6607
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

257 files changed

+6908
-6607
lines changed

‎data/cache/.gitignore‎

Lines changed: 263 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,263 @@
1+
# Ignore everything by default
2+
*
3+
4+
# Keep this file
5+
/.gitignore
6+
7+
# Keep specific v0 files (golden/reference data)
8+
!/v0/ae_2024_10.json
9+
!/v0/ae_2025_10.json
10+
!/v0/af_2024_10.json
11+
!/v0/al_2024_10.json
12+
!/v0/al_2025_10.json
13+
!/v0/am_2024_10.json
14+
!/v0/am_2025_10.json
15+
!/v0/ar_2024_10.json
16+
!/v0/ar_2025_10.json
17+
!/v0/at_2024_10.json
18+
!/v0/at_2025_10.json
19+
!/v0/au_2024_10.json
20+
!/v0/au_2025_10.json
21+
!/v0/az_2024_10.json
22+
!/v0/az_2025_10.json
23+
!/v0/ba_2024_10.json
24+
!/v0/ba_2025_10.json
25+
!/v0/bd_2024_10.json
26+
!/v0/bd_2025_10.json
27+
!/v0/be_2024_10.json
28+
!/v0/be_2025_10.json
29+
!/v0/bg_2024_10.json
30+
!/v0/bg_2025_10.json
31+
!/v0/bh_2025_10.json
32+
!/v0/bm_2024_10.json
33+
!/v0/bm_2025_10.json
34+
!/v0/bo_2024_10.json
35+
!/v0/bo_2025_10.json
36+
!/v0/br_2024_10.json
37+
!/v0/br_2025_10.json
38+
!/v0/bw_2024_10.json
39+
!/v0/bw_2025_10.json
40+
!/v0/by_2024_10.json
41+
!/v0/by_2025_10.json
42+
!/v0/bz_2025_10.json
43+
!/v0/ca_2024_10.json
44+
!/v0/ca_2025_10.json
45+
!/v0/ch_2024_10.json
46+
!/v0/ch_2025_10.json
47+
!/v0/ci_2024_10.json
48+
!/v0/cl_2024_10.json
49+
!/v0/cl_2025_10.json
50+
!/v0/cn_2024_10.json
51+
!/v0/cn_2025_10.json
52+
!/v0/co_2024_10.json
53+
!/v0/co_2025_10.json
54+
!/v0/cr_2024_10.json
55+
!/v0/cr_2025_10.json
56+
!/v0/cw_2024_10.json
57+
!/v0/cy_2024_10.json
58+
!/v0/cy_2025_10.json
59+
!/v0/cz_2024_10.json
60+
!/v0/cz_2025_10.json
61+
!/v0/de_2024_10.json
62+
!/v0/de_2025_10.json
63+
!/v0/dk_2024_10.json
64+
!/v0/dk_2025_10.json
65+
!/v0/do_2024_10.json
66+
!/v0/do_2025_10.json
67+
!/v0/dz_2024_10.json
68+
!/v0/dz_2025_10.json
69+
!/v0/ec_2024_10.json
70+
!/v0/ec_2025_10.json
71+
!/v0/ee_2024_10.json
72+
!/v0/ee_2025_10.json
73+
!/v0/eg_2024_10.json
74+
!/v0/eg_2025_10.json
75+
!/v0/es_2024_10.json
76+
!/v0/es_2025_10.json
77+
!/v0/et_2024_10.json
78+
!/v0/et_2025_10.json
79+
!/v0/fi_2024_10.json
80+
!/v0/fi_2025_10.json
81+
!/v0/fj_2024_10.json
82+
!/v0/fj_2025_10.json
83+
!/v0/fr_2024_10.json
84+
!/v0/fr_2025_10.json
85+
!/v0/gb_2024_10.json
86+
!/v0/gb_2025_10.json
87+
!/v0/ge_2024_10.json
88+
!/v0/ge_2025_10.json
89+
!/v0/gh_2024_10.json
90+
!/v0/gh_2025_10.json
91+
!/v0/gp_2024_10.json
92+
!/v0/gp_2025_10.json
93+
!/v0/gr_2024_10.json
94+
!/v0/gr_2025_10.json
95+
!/v0/gt_2024_10.json
96+
!/v0/gt_2025_10.json
97+
!/v0/hk_2024_10.json
98+
!/v0/hk_2025_10.json
99+
!/v0/hn_2024_10.json
100+
!/v0/hn_2025_10.json
101+
!/v0/hr_2024_10.json
102+
!/v0/hr_2025_10.json
103+
!/v0/hu_2024_10.json
104+
!/v0/hu_2025_10.json
105+
!/v0/id_2024_10.json
106+
!/v0/id_2025_10.json
107+
!/v0/ie_2024_10.json
108+
!/v0/ie_2025_10.json
109+
!/v0/il_2024_10.json
110+
!/v0/il_2025_10.json
111+
!/v0/in_2024_10.json
112+
!/v0/in_2025_10.json
113+
!/v0/iq_2024_10.json
114+
!/v0/iq_2025_10.json
115+
!/v0/ir_2025_10.json
116+
!/v0/it_2024_10.json
117+
!/v0/it_2025_10.json
118+
!/v0/jm_2024_10.json
119+
!/v0/jm_2025_10.json
120+
!/v0/jo_2024_10.json
121+
!/v0/jo_2025_10.json
122+
!/v0/jp_2024_10.json
123+
!/v0/jp_2025_10.json
124+
!/v0/ke_2024_10.json
125+
!/v0/ke_2025_10.json
126+
!/v0/kg_2024_10.json
127+
!/v0/kh_2024_10.json
128+
!/v0/kh_2025_10.json
129+
!/v0/kr_2024_10.json
130+
!/v0/kr_2025_10.json
131+
!/v0/kw_2024_10.json
132+
!/v0/kw_2025_10.json
133+
!/v0/ky_2024_10.json
134+
!/v0/kz_2024_10.json
135+
!/v0/kz_2025_10.json
136+
!/v0/lb_2024_10.json
137+
!/v0/lb_2025_10.json
138+
!/v0/lk_2024_10.json
139+
!/v0/lk_2025_10.json
140+
!/v0/lt_2024_10.json
141+
!/v0/lt_2025_10.json
142+
!/v0/lu_2024_10.json
143+
!/v0/lu_2025_10.json
144+
!/v0/lv_2024_10.json
145+
!/v0/lv_2025_10.json
146+
!/v0/ly_2024_10.json
147+
!/v0/ly_2025_10.json
148+
!/v0/ma_2024_10.json
149+
!/v0/ma_2025_10.json
150+
!/v0/mg_2024_10.json
151+
!/v0/mg_2025_10.json
152+
!/v0/mm_2024_10.json
153+
!/v0/mm_2025_10.json
154+
!/v0/mn_2024_10.json
155+
!/v0/mn_2025_10.json
156+
!/v0/mq_2024_10.json
157+
!/v0/mq_2025_10.json
158+
!/v0/mt_2024_10.json
159+
!/v0/mx_2024_10.json
160+
!/v0/mx_2025_10.json
161+
!/v0/my_2024_10.json
162+
!/v0/my_2025_10.json
163+
!/v0/mz_2024_10.json
164+
!/v0/mz_2025_10.json
165+
!/v0/na_2025_10.json
166+
!/v0/ng_2024_10.json
167+
!/v0/ng_2025_10.json
168+
!/v0/ni_2024_10.json
169+
!/v0/ni_2025_10.json
170+
!/v0/nl_2024_10.json
171+
!/v0/nl_2025_10.json
172+
!/v0/no_2024_10.json
173+
!/v0/no_2025_10.json
174+
!/v0/np_2024_10.json
175+
!/v0/np_2025_10.json
176+
!/v0/nz_2024_10.json
177+
!/v0/nz_2025_10.json
178+
!/v0/om_2024_10.json
179+
!/v0/om_2025_10.json
180+
!/v0/pa_2024_10.json
181+
!/v0/pa_2025_10.json
182+
!/v0/pe_2024_10.json
183+
!/v0/pe_2025_10.json
184+
!/v0/pf_2024_10.json
185+
!/v0/pf_2025_10.json
186+
!/v0/ph_2024_10.json
187+
!/v0/ph_2025_10.json
188+
!/v0/pk_2024_10.json
189+
!/v0/pk_2025_10.json
190+
!/v0/pl_2024_10.json
191+
!/v0/pl_2025_10.json
192+
!/v0/pr_2024_10.json
193+
!/v0/pr_2025_10.json
194+
!/v0/ps_2024_10.json
195+
!/v0/ps_2025_10.json
196+
!/v0/pt_2024_10.json
197+
!/v0/pt_2025_10.json
198+
!/v0/py_2024_10.json
199+
!/v0/qa_2024_10.json
200+
!/v0/qa_2025_10.json
201+
!/v0/re_2024_10.json
202+
!/v0/re_2025_10.json
203+
!/v0/ro_2024_10.json
204+
!/v0/ro_2025_10.json
205+
!/v0/rs_2024_10.json
206+
!/v0/rs_2025_10.json
207+
!/v0/ru_2024_10.json
208+
!/v0/ru_2025_10.json
209+
!/v0/sa_2024_10.json
210+
!/v0/sa_2025_10.json
211+
!/v0/se_2024_10.json
212+
!/v0/se_2025_10.json
213+
!/v0/sg_2024_10.json
214+
!/v0/sg_2025_10.json
215+
!/v0/si_2024_10.json
216+
!/v0/si_2025_10.json
217+
!/v0/sk_2024_10.json
218+
!/v0/sk_2025_10.json
219+
!/v0/sn_2024_10.json
220+
!/v0/sv_2024_10.json
221+
!/v0/sv_2025_10.json
222+
!/v0/sy_2025_10.json
223+
!/v0/th_2024_10.json
224+
!/v0/th_2025_10.json
225+
!/v0/tm_2024_10.json
226+
!/v0/tn_2024_10.json
227+
!/v0/tn_2025_10.json
228+
!/v0/tr_2024_10.json
229+
!/v0/tr_2025_10.json
230+
!/v0/tt_2024_10.json
231+
!/v0/tw_2024_10.json
232+
!/v0/tw_2025_10.json
233+
!/v0/tz_2024_10.json
234+
!/v0/tz_2025_10.json
235+
!/v0/ua_2024_10.json
236+
!/v0/ua_2025_10.json
237+
!/v0/ug_2024_10.json
238+
!/v0/ug_2025_10.json
239+
!/v0/us_2024_10.json
240+
!/v0/us_2025_10.json
241+
!/v0/uy_2025_10.json
242+
!/v0/uz_2024_10.json
243+
!/v0/uz_2025_10.json
244+
!/v0/ve_2024_10.json
245+
!/v0/ve_2025_10.json
246+
!/v0/vn_2024_10.json
247+
!/v0/vn_2025_10.json
248+
!/v0/xk_2024_10.json
249+
!/v0/ye_2024_10.json
250+
!/v0/ye_2025_10.json
251+
!/v0/za_2024_10.json
252+
!/v0/za_2025_10.json
253+
!/v0/zm_2024_10.json
254+
!/v0/zw_2024_10.json
255+
!/v1/20241001T000000Z/20241101T000000Z/downloads_by_country/data.parquet
256+
!/v1/20241001T000000Z/20241101T000000Z/downloads_by_country/stats.json
257+
!/v1/20241001T000000Z/20241101T000000Z/uploads_by_country/data.parquet
258+
!/v1/20241001T000000Z/20241101T000000Z/uploads_by_country/stats.json
259+
!/v1/20251001T000000Z/20251101T000000Z/downloads_by_country/data.parquet
260+
!/v1/20251001T000000Z/20251101T000000Z/downloads_by_country/stats.json
261+
!/v1/20251001T000000Z/20251101T000000Z/uploads_by_country/data.parquet
262+
!/v1/20251001T000000Z/20251101T000000Z/uploads_by_country/stats.json
263+

‎data/cache/v0/ae_2024_10.json‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -5,44 +5,44 @@
55
},
66
"metrics": {
77
"download_throughput_mbps": {
8-
"p1": 0.3371936720531858,
9-
"p5": 1.9956779377443443,
10-
"p10": 4.852578761942473,
11-
"p25": 15.231393668639102,
12-
"p50": 50.024713581243944,
13-
"p75": 138.9549726831921,
14-
"p90": 363.55685621320964,
15-
"p95": 519.0213934174361,
16-
"p99": 751.4689773815049
8+
"p1": 0.34227098341874695,
9+
"p5": 2.0050602946104297,
10+
"p10": 4.829645130602473,
11+
"p25": 15.22434290003249,
12+
"p50": 49.98571146159887,
13+
"p75": 138.97628724187012,
14+
"p90": 363.1102566674616,
15+
"p95": 520.609253755341,
16+
"p99": 750.4640072574977
1717
},
1818
"upload_throughput_mbps": {
19-
"p1": 0.07496650709665575,
20-
"p5": 0.5403482374849342,
21-
"p10": 1.65150662938528,
22-
"p25": 7.0573572777807225,
23-
"p50": 22.8636517764728,
24-
"p75": 71.51900278880757,
25-
"p90": 121.92188386559485,
26-
"p95": 221.50857463311786,
27-
"p99": 305.3448650611846
19+
"p1": 0.07458375126168135,
20+
"p5": 0.5436737880479822,
21+
"p10": 1.652840778324904,
22+
"p25": 7.04807750189541,
23+
"p50": 22.881643674139134,
24+
"p75": 71.5588192971058,
25+
"p90": 121.8726513231747,
26+
"p95": 221.21403403021233,
27+
"p99": 305.76851703610623
2828
},
2929
"latency_ms": {
30-
"p1": 339.084,
31-
"p5": 199.102,
32-
"p10": 145.892,
33-
"p25": 41.169,
34-
"p50": 20.935,
35-
"p75": 18.194,
30+
"p1": 343.322,
31+
"p5": 198.446,
32+
"p10": 145.931,
33+
"p25": 41.213,
34+
"p50": 20.938,
35+
"p75": 18.191,
3636
"p90": 16.476,
3737
"p95": 15.0,
38-
"p99": 1.474
38+
"p99": 1.4
3939
},
4040
"packet_loss": {
41-
"p1": 0.29170729698462744,
42-
"p5": 0.16668561502288604,
43-
"p10": 0.11759452283540514,
44-
"p25": 0.05783743933641156,
45-
"p50": 0.00658763808078303,
41+
"p1": 0.2913825220205726,
42+
"p5": 0.1666607533558039,
43+
"p10": 0.11750268567179813,
44+
"p25": 0.057920036175228395,
45+
"p50": 0.006589320482259453,
4646
"p75": 0.0,
4747
"p90": 0.0,
4848
"p95": 0.0,

0 commit comments

Comments
 (0)