Skip to content
This repository was archived by the owner on Jan 6, 2026. It is now read-only.

Commit b6854fc

Browse files
juliangruberjuliangruber
committed
Merge branch 'main' into add/http_head
2 parents 879d761 + ee75ab1 commit b6854fc

File tree

8 files changed

+174
-16
lines changed

8 files changed

+174
-16
lines changed

‎observablehq.config.js‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
11
import { jsonFetcher } from './src/data/json-fetcher.js'
22
import { getDateXDaysAgo } from './src/utils/date-utils.js'
33

4-
const start = '2024-04-07'
4+
const startProviders = '2024-04-07'
5+
const startClients = '2025-02-25'
56
const end = getDateXDaysAgo(1)
67

7-
const result = await jsonFetcher(
8-
`https://stats.filspark.com/miners/retrieval-success-rate/summary?from=${start}&to=${end}`,
8+
const providersSummary = await jsonFetcher(
9+
`https://stats.filspark.com/miners/retrieval-success-rate/summary?from=${startProviders}&to=${end}`,
10+
)
11+
const clientsSummary = await jsonFetcher(
12+
`https://stats.filspark.com/clients/retrieval-success-rate/summary?from=${startClients}&to=${end}`,
13+
)
14+
const providerPaths = providersSummary.map(
15+
(provider) => `/provider/${provider.miner_id}`,
16+
)
17+
const clientPaths = clientsSummary.map(
18+
(client) => `/client/${client.client_id}`,
919
)
10-
const providerPaths = result.map((provider) => `/provider/${provider.miner_id}`)
11-
1220
// See https://observablehq.com/framework/config for documentation.
1321
export default {
1422
// The app’s title; used in the sidebar and webpage titles.
@@ -45,5 +53,5 @@ export default {
4553
// linkify: true, // convert URLs in Markdown to links
4654
// typographer: false, // smart quotes and other typographic improvements
4755
// cleanUrls: true, // drop .html from URLs
48-
dynamicPaths: providerPaths,
56+
dynamicPaths: [...providerPaths, ...clientPaths],
4957
}

‎src/client/[client].md‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
---
2+
toc: false
3+
title: Storage Client Summary
4+
---
5+
6+
```js
7+
import { LineGraph } from '../components/line-graph.js'
8+
import { getDateXDaysAgo } from '../utils/date-utils.js'
9+
10+
const data = FileAttachment(
11+
`../data/${observable.params.client}-spark-client-rsr-summary.json`,
12+
).json()
13+
```
14+
15+
<div class="hero">
16+
<body><a href="/"><img src="../media/spark-logomark-blue-with-bbox.png" alt="Spark Logo" width="300" /></a><body>
17+
<h2>Dashboard Beta</h2>
18+
<body><a href="https://filspark.com/dashboard" target="_blank" rel="noopener noreferrer">(Click here for Legacy Spark Grafana Dashboard)</a><body>
19+
</div>
20+
21+
<h4>Storage Client Spark RSR Summary</h4>
22+
<body>This section shows the storage client Spark Retrieval Success Rate Score summary. You can adjust the date range. Records start on the 25th February 2025.</body>
23+
24+
```js
25+
const startDate = getDateXDaysAgo(180)
26+
const minStartDate = '2025-02-25'
27+
const start = view(
28+
Inputs.date({
29+
label: 'Start',
30+
value:
31+
new Date(startDate) >= new Date(minStartDate) ? startDate : minStartDate,
32+
}),
33+
)
34+
const end = view(Inputs.date({ label: 'End', value: getDateXDaysAgo(1) }))
35+
```
36+
37+
<h3>Stats for ${observable.params.client}</h3>
38+
39+
<div class="grid grid-cols" style="grid-auto-rows: 500px;">
40+
<div class="card">${
41+
resize((width) => LineGraph(data, {width, title: "Retrieval Success Rate", start, end }))
42+
}</div>
43+
</div>
44+
45+
<style>
46+
47+
.hero {
48+
display: flex;
49+
flex-direction: column;
50+
align-items: center;
51+
font-family: var(--sans-serif);
52+
margin: 4rem 0 8rem;
53+
text-wrap: balance;
54+
text-align: center;
55+
}
56+
57+
.hero h1 {
58+
margin: 1rem 0;
59+
padding: 1rem 0;
60+
max-width: none;
61+
font-size: 14vw;
62+
font-weight: 900;
63+
line-height: 1;
64+
background: linear-gradient(30deg, var(--theme-foreground-focus), currentColor);
65+
-webkit-background-clip: text;
66+
-webkit-text-fill-color: transparent;
67+
background-clip: text;
68+
}
69+
70+
.hero h2 {
71+
margin: 0;
72+
max-width: 34em;
73+
font-size: 20px;
74+
font-style: initial;
75+
font-weight: 500;
76+
line-height: 1.5;
77+
color: var(--theme-foreground-muted);
78+
}
79+
80+
@media (min-width: 640px) {
81+
.hero h1 {
82+
font-size: 90px;
83+
}
84+
}
85+
86+
</style>

‎src/components/line-graph.js‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import * as Plot from 'npm:@observablehq/plot'
22

3-
export function LineGraph(events, { width, height, title, start, end } = {}) {
3+
export function LineGraph(
4+
events,
5+
{ width, height, title, start, end, fromZero } = {},
6+
) {
47
const startDate = new Date(start)
58
const endDate = new Date(end)
69
const filteredEvents = events.filter((event) => {
@@ -30,7 +33,13 @@ export function LineGraph(events, { width, height, title, start, end } = {}) {
3033
width,
3134
height,
3235
x: { type: 'utc', ticks: 'month', label: null },
33-
y: { grid: true, inset: 10, label: 'RSR (%)', percent: true },
36+
y: {
37+
grid: true,
38+
inset: 10,
39+
label: 'RSR (%)',
40+
percent: true,
41+
zero: fromZero,
42+
},
3443
color: { legend: true },
3544
marks: [
3645
Plot.lineY(combinedData, {

‎src/data/[client]-spark-client-rsr-summary.json.js‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import pRetry from 'p-retry'
2+
import { jsonFetcher } from './json-fetcher.js'
3+
import { getDateXDaysAgo } from '../utils/date-utils.js'
4+
import { parseArgs } from 'node:util'
5+
6+
const {
7+
values: { client },
8+
} = parseArgs({
9+
options: { client: { type: 'string' } },
10+
})
11+
const start = '2025-02-25'
12+
const end = getDateXDaysAgo(1)
13+
const summary = await pRetry(
14+
() =>
15+
jsonFetcher(
16+
`https://stats.filspark.com/client/${client}/retrieval-success-rate/summary?from=${start}&to=${end}`,
17+
),
18+
{ retries: 3 },
19+
)
20+
process.stdout.write(JSON.stringify(summary))

‎src/data/spark-clients-rsr.json.js‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { jsonFetcher } from './json-fetcher.js'
2+
import { getDateXDaysAgo } from '../utils/date-utils.js'
3+
4+
const from = getDateXDaysAgo(31)
5+
const to = getDateXDaysAgo(1)
6+
7+
const output = await jsonFetcher(
8+
`https://stats.filspark.com/clients/retrieval-success-rate/summary?from=${from}&to=${to}`,
9+
)
10+
process.stdout.write(JSON.stringify(output))

‎src/data/spark-miner-rsr-summaries.json.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ import { getDateXDaysAgo } from '../utils/date-utils.js'
33

44
const summaries = {}
55

6-
for (let i = 201; i >= 1; i--) {
6+
// Fetch the last 100 days of data
7+
// We want this number to be small enough so that the resulting json file that is generated is not too large (<25MB)
8+
for (let i = 100; i >= 1; i--) {
79
const dayString = getDateXDaysAgo(i)
810
summaries[dayString] = await jsonFetcher(
911
`https://stats.filspark.com/miners/retrieval-success-rate/summary?from=${dayString}&to=${dayString}`,

‎src/index.md‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ const SparkMinerRsrSummaries = FileAttachment(
2424
const SparkRetrievalTimes = FileAttachment(
2525
'./data/spark-retrieval-timings.json',
2626
).json()
27+
const SparkClientRates = FileAttachment('./data/spark-clients-rsr.json').json()
2728
```
2829

2930
```js
@@ -50,6 +51,13 @@ const tidySparkMinerRates = SparkMinerRates.sort(
5051
success_rate_http_head: `${(record.success_rate_http_head * 100).toFixed(2)}%`,
5152
}
5253
})
54+
const tidySparkClientRates = SparkClientRates.sort(
55+
(recordA, recordB) => recordB.success_rate - recordA.success_rate,
56+
).map((record) => ({
57+
...record,
58+
success_rate: `${(record.success_rate * 100).toFixed(2)}%`,
59+
success_rate_http: `${(record.success_rate_http * 100).toFixed(2)}%`,
60+
}))
5361
```
5462
5563
<div class="hero">
@@ -295,8 +303,7 @@ const tidy = clone(SparkRetrievalResultCodes).flatMap(({ day, rates }) => {
295303
<div class="card">
296304
${Plot.plot({
297305
title: 'Time to First Byte (ms)',
298-
// TODO: Change tick to month once we have more data
299-
x: { type: 'utc', ticks: 'day' },
306+
x: { type: 'utc', ticks: 'month' },
300307
y: { grid: true, zero: true},
301308
marks: [
302309
Plot.lineY(SparkRetrievalTimes, {
@@ -331,15 +338,32 @@ ${JSON.stringify(
331338
<body>The following table shows the Spark RSR and TTFB values calculated in aggregate for each Filecoin Storage Provider over the past 30 days. Click on a miner id to view stats about this storage provider.</body>
332339
333340
```js
334-
const search = view(
341+
const searchMinerStats = view(
335342
Inputs.search(tidySparkMinerRates, {
336343
placeholder: 'Search Storage Providers…',
337344
}),
338345
)
339346
```
340347
341348
<div class="card" style="padding: 0;">
342-
${Inputs.table(search, {rows: 16, format: {miner_id: id => htl.html`<a href=./provider/${id} target=_blank>${id}</a>`}})}
349+
${Inputs.table(searchMinerStats, {rows: 16, format: {miner_id: id => htl.html`<a href=./provider/${id} target=_blank>${id}</a>`}})}
350+
</div>
351+
352+
<div class="divider"></div>
353+
354+
<h4>Spark Client RSR Table</h4>
355+
<body>The following table shows the Spark RSR values calculated in aggregate for each Filecoin Storage Client over the past 30 days. Click on a client id to view stats about this storage client.</body>
356+
357+
```js
358+
const searchClientStats = view(
359+
Inputs.search(tidySparkClientRates, {
360+
placeholder: 'Search Storage Clients...',
361+
}),
362+
)
363+
```
364+
365+
<div class="card" style="padding: 0;">
366+
${Inputs.table(searchClientStats, {rows: 16, format: {client_id: id => htl.html`<a href=./client/${id} target=_blank>${id}</a>`}})}
343367
</div>
344368
345369
<style>

‎src/provider/[provider].md‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ const end = view(Inputs.date({ label: 'End', value: getDateXDaysAgo(1) }))
3333
<h4>Storage Provider Spark RSR Summary</h4>
3434
<body>This section shows the storage provider Spark Retrieval Success Rate Score summary.</body>
3535
<div class="card">${
36-
resize((width) => LineGraph(rsrData, {width, title: "Retrieval Success Rate", start, end }))
36+
resize((width) => LineGraph(rsrData, {width, title: "Retrieval Success Rate", start, end, fromZero: true }))
3737
}</div>
3838
</div>
3939
<div>
@@ -42,8 +42,7 @@ const end = view(Inputs.date({ label: 'End', value: getDateXDaysAgo(1) }))
4242
<div class="card">
4343
${Plot.plot({
4444
title: 'Time to First Byte (ms)',
45-
// TODO: Change tick to month once we have more data
46-
x: { type: 'utc', ticks: 'day' },
45+
x: { type: 'utc', ticks: 'month' },
4746
y: { grid: true, zero: true },
4847
marks: [
4948
Plot.lineY(ttfbData, {

0 commit comments

Comments
 (0)