Skip to content

data: add trace schema for haskell & rust #194

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Feb 25, 2025
Merged

data: add trace schema for haskell & rust #194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d83aab2
data: add trace schema for haskell & rust
will-break-it Feb 14, 2025
c5944d7
data: add example logs for all events for make validate
will-break-it Feb 14, 2025
eb8b987
data: simplify haskell trace schema
will-break-it Feb 14, 2025
1b86f95
data: remove verbose extract of message kind
will-break-it Feb 14, 2025
84d7bd1
data: updated rust schema
will-break-it Feb 14, 2025
aed2c2c
data: composed rust message & action types
will-break-it Feb 14, 2025
9a64c9c
Trace Schema Changes (#202)
will-break-it Feb 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 40 additions & 3 deletions data/simulation/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
all: config.schema.json topology.schema.json
all: config.schema.json topology.schema.json trace.haskell.schema.json trace.rust.schema.json

config.schema.json: config.d.ts
npx typescript-json-schema config.d.ts Config > config.schema.json
Expand All @@ -8,7 +8,44 @@ topology.schema.json: topology.d.ts
npx typescript-json-schema topology.d.ts Topology > topology.schema.json
npx prettier -w topology.schema.json

trace.haskell.schema.json: trace.haskell.d.ts
npx typescript-json-schema trace.haskell.d.ts HaskellTraceEvent > trace.haskell.schema.json
npx prettier -w trace.haskell.schema.json

trace.rust.schema.json: trace.rust.d.ts
npx typescript-json-schema trace.rust.d.ts RustTraceEvent > trace.rust.schema.json
npx prettier -w trace.rust.schema.json

.PHONY: validate
validate:
npx pajv -m schema.schema.json -s config.schema.json -d config.default.yaml
npx pajv -m schema.schema.json -s topology.schema.json -d topology-dense-52.yaml
@echo "Validating config..."
@npx pajv -m schema.schema.json -s config.schema.json -d config.default.yaml > /dev/null && echo "config.default.yaml valid" || exit 1
@echo "Validating topology..."
@npx pajv -m schema.schema.json -s topology.schema.json -d topology-dense-52.yaml > /dev/null && echo "topology-dense-52.yaml valid" || exit 1
@echo "Validating Haskell trace events..."
@line_num=0; \
while IFS= read -r line; do \
line_num=$$((line_num + 1)); \
echo "$$line" > temp.json; \
if ! npx ajv-cli validate -s trace.haskell.schema.json -d temp.json 2>/dev/null >/dev/null; then \
echo "Error in example.haskell.jsonl at line $$line_num:"; \
echo "$$line"; \
npx ajv-cli validate -s trace.haskell.schema.json -d temp.json; \
rm -f temp.json; \
exit 1; \
fi; \
done < example.haskell.jsonl && echo "example.haskell.jsonl valid"
@echo "Validating Rust trace events..."
@line_num=0; \
while IFS= read -r line; do \
line_num=$$((line_num + 1)); \
echo "$$line" > temp.json; \
if ! npx ajv-cli validate -s trace.rust.schema.json -d temp.json 2>/dev/null >/dev/null; then \
echo "Error in example.rust.jsonl at line $$line_num:"; \
echo "$$line"; \
npx ajv-cli validate -s trace.rust.schema.json -d temp.json; \
rm -f temp.json; \
exit 1; \
fi; \
done < example.rust.jsonl && echo "example.rust.jsonl valid"
@rm -f temp.json
22 changes: 22 additions & 0 deletions data/simulation/example.haskell.jsonl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{"time_s": 1.0, "event": {"tag": "Cpu", "node": 1, "node_name": "node1", "duration_s": 0.1, "task_label": "ValIB: 6-29"}}
{"time_s": 1.2, "event": {"tag": "Cpu", "node": 1, "node_name": "node1", "duration_s": 0.05, "task_label": "ValIH: 6253064077348247640"}}
{"time_s": 1.5, "event": {"tag": "Cpu", "node": 2, "node_name": "node2", "duration_s": 0.15, "task_label": "ValEB: 7-30"}}
{"time_s": 1.8, "event": {"tag": "Cpu", "node": 2, "node_name": "node2", "duration_s": 0.08, "task_label": "ValRH: 7253064077348247641"}}
{"time_s": 2.0, "event": {"tag": "Cpu", "node": 3, "node_name": "node3", "duration_s": 0.12, "task_label": "ValRB: 8-31"}}
{"time_s": 2.3, "event": {"tag": "Cpu", "node": 3, "node_name": "node3", "duration_s": 0.03, "task_label": "ValVote: vote123"}}
{"time_s": 3.0, "event": {"tag": "generated", "kind": "IB", "id": "block1", "node": 1, "node_name": "node1", "size_bytes": 1000, "slot": 1, "payload_bytes": 800, "rb_ref": "rb123"}}
{"time_s": 3.1, "event": {"tag": "Sent", "sender": 1, "receipient": 2, "msg_size_bytes": 1000, "sending_s": 0.01, "kind": "IB", "ids": ["block1"]}}
{"time_s": 3.2, "event": {"tag": "received", "kind": "IB", "id": "block1", "node": 2, "node_name": "node2"}}
{"time_s": 3.3, "event": {"tag": "enteredstate", "kind": "IB", "id": "block1", "node": 2, "node_name": "node2"}}
{"time_s": 4.0, "event": {"tag": "generated", "kind": "EB", "id": "eb1", "node": 2, "node_name": "node2", "size_bytes": 500, "input_blocks": ["block1"]}}
{"time_s": 4.1, "event": {"tag": "Sent", "sender": 2, "receipient": 3, "msg_size_bytes": 500, "sending_s": 0.01, "kind": "EB", "ids": ["eb1"]}}
{"time_s": 4.2, "event": {"tag": "received", "kind": "EB", "id": "eb1", "node": 3, "node_name": "node3"}}
{"time_s": 4.3, "event": {"tag": "enteredstate", "kind": "EB", "id": "eb1", "node": 3, "node_name": "node3"}}
{"time_s": 5.0, "event": {"tag": "generated", "kind": "RB", "id": "rb1", "node": 3, "node_name": "node3", "size_bytes": 300}}
{"time_s": 5.1, "event": {"tag": "Sent", "sender": 3, "receipient": 1, "msg_size_bytes": 300, "sending_s": 0.01, "kind": "RB", "ids": ["rb1"]}}
{"time_s": 5.2, "event": {"tag": "received", "kind": "RB", "id": "rb1", "node": 1, "node_name": "node1"}}
{"time_s": 5.3, "event": {"tag": "enteredstate", "kind": "RB", "id": "rb1", "node": 1, "node_name": "node1"}}
{"time_s": 6.0, "event": {"tag": "generated", "kind": "VT", "id": "vote1", "node": 1, "node_name": "node1", "size_bytes": 200, "votes": 1, "endorse_blocks": ["eb1"]}}
{"time_s": 6.1, "event": {"tag": "Sent", "sender": 1, "receipient": 2, "msg_size_bytes": 200, "sending_s": 0.01, "kind": "VT", "ids": ["vote1"]}}
{"time_s": 6.2, "event": {"tag": "received", "kind": "VT", "id": "vote1", "node": 2, "node_name": "node2"}}
{"time_s": 6.3, "event": {"tag": "enteredstate", "kind": "VT", "id": "vote1", "node": 2, "node_name": "node2"}}
32 changes: 32 additions & 0 deletions data/simulation/example.rust.jsonl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{"time": 1000, "message": {"type": "Slot", "number": 1}}
{"time": 2000, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 0}, "task_type": "PraosBlockValidated"}}
{"time": 2100, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 0}, "task_type": "PraosBlockValidated"}}
{"time": 2200, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 1}, "task_type": "EndorserBlockValidated"}}
{"time": 2300, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 1}, "task_type": "EndorserBlockValidated"}}
{"time": 2400, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 2}, "task_type": "VoteBundleValidated"}}
{"time": 2500, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 2}, "task_type": "VoteBundleValidated"}}
{"time": 2600, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 3}, "task_type": "InputBlockValidated"}}
{"time": 2700, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 3}, "task_type": "InputBlockValidated"}}
{"time": 2800, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 4}, "task_type": "TransactionValidated"}}
{"time": 2900, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 4}, "task_type": "TransactionValidated"}}
{"time": 3000, "message": {"type": "CpuSubtaskStarted", "task": {"node": "node1", "index": 5}, "task_type": "VoteBundleValidated", "subtask_id": 0, "subtasks": 2}}
{"time": 3100, "message": {"type": "CpuSubtaskFinished", "task": {"node": "node1", "index": 5}, "task_type": "VoteBundleValidated", "subtask_id": 0, "subtasks": 2}}
{"time": 3200, "message": {"type": "CpuSubtaskStarted", "task": {"node": "node1", "index": 5}, "task_type": "VoteBundleValidated", "subtask_id": 1, "subtasks": 2}}
{"time": 3300, "message": {"type": "CpuSubtaskFinished", "task": {"node": "node1", "index": 5}, "task_type": "VoteBundleValidated", "subtask_id": 1, "subtasks": 2}}
{"time": 4000, "message": {"type": "InputBlockGenerated", "slot": 1, "producer": "node1", "index": 0, "header_bytes": 100, "transactions": ["tx1", "tx2"]}}
{"time": 4100, "message": {"type": "InputBlockSent", "slot": 1, "producer": "node1", "sender": "node1", "recipient": "node2", "index": 0, "header_bytes": 100, "transactions": ["tx1", "tx2"]}}
{"time": 4200, "message": {"type": "InputBlockReceived", "slot": 1, "producer": "node1", "sender": "node1", "recipient": "node2", "index": 0, "header_bytes": 100, "transactions": ["tx1", "tx2"]}}
{"time": 4300, "message": {"type": "InputBlockLotteryWon", "slot": 1, "producer": "node1", "index": 0}}
{"time": 5000, "message": {"type": "EndorserBlockGenerated", "slot": 1, "producer": "node2"}}
{"time": 5100, "message": {"type": "EndorserBlockSent", "slot": 1, "producer": "node2", "sender": "node2", "recipient": "node3"}}
{"time": 5200, "message": {"type": "EndorserBlockReceived", "slot": 1, "producer": "node2", "sender": "node2", "recipient": "node3"}}
{"time": 5300, "message": {"type": "EndorserBlockLotteryWon", "slot": 1, "producer": "node2"}}
{"time": 6000, "message": {"type": "PraosBlockGenerated", "slot": 1, "producer": "node3", "header_bytes": 200, "transactions": ["tx1", "tx2"], "vrf": 584785879898000}}
{"time": 6100, "message": {"type": "PraosBlockSent", "slot": 1, "producer": "node3", "sender": "node3", "recipient": "node1", "header_bytes": 200, "transactions": ["tx1", "tx2"], "vrf": 584785879898836}}
{"time": 6200, "message": {"type": "PraosBlockReceived", "slot": 1, "producer": "node3", "sender": "node3", "recipient": "node1", "header_bytes": 200, "transactions": ["tx1", "tx2"], "vrf": 584785870008836}}
{"time": 6300, "message": {"type": "PraosBlockLotteryWon", "slot": 1, "producer": "node3", "vrf": 384785879898000}}
{"time": 7000, "message": {"type": "TransactionSent", "id": "tx1", "sender": "node1", "recipient": "node2", "publisher": "node1", "bytes": 50}}
{"time": 8000, "message": {"type": "VotesGenerated", "id": "vote1", "slot": 1, "producer": "node1", "votes": {"node1": 1}}}
{"time": 8100, "message": {"type": "VotesSent", "id": "vote1", "slot": 1, "producer": "node1", "sender": "node1", "recipient": "node2", "votes": {"node1": 1}}}
{"time": 8200, "message": {"type": "VotesReceived", "id": "vote1", "slot": 1, "producer": "node1", "sender": "node1", "recipient": "node2", "votes": {"node1": 1}}}
{"time": 8300, "message": {"type": "VoteLotteryWon", "id": "vote1", "slot": 1, "producer": "node1"}}
74 changes: 74 additions & 0 deletions data/simulation/trace.haskell.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/** Haskell simulation trace format */
export interface HaskellTraceEvent {
time_s: number;
event: HaskellEvent;
}

type BlockKind = "IB" | "EB" | "RB" | "VT";
type BlockAction = "Generated" | "EnteredState";

type HaskellEvent =
| HaskellCpuEvent
| HaskellBlockEvent // Unified block event type
| HaskellNetworkEvent; // Combine Sent/Received into network events

interface HaskellCpuEvent {
type: "Cpu";
node: string;
cpu_time_s: number;
// CPU task types: Block validation (ValIB, ValEB, ValRB), Header validation (ValIH, ValRH), Vote validation (ValVote). Format: "<task_type>: <id>"
task_label: string; // e.g., "ValIB: 6-29" or "ValRH: 6253064077348247640"
}

// Base block event interface with just identification info
interface BaseBlockEvent {
type: `${BlockKind}${BlockAction}`;
node: string;
id: string;
slot: number;
}

// Additional fields for Generated events
interface GeneratedBlockEvent extends BaseBlockEvent {
size_bytes: number;
}

interface GeneratedInputBlock extends GeneratedBlockEvent {
payload_bytes: number;
rb_ref: string;
}

interface GeneratedEndorserBlock extends GeneratedBlockEvent {
input_blocks: string[];
}

interface GeneratedRankingBlock extends GeneratedBlockEvent {
endorse_blocks: string[];
payload_bytes: number;
}

interface GeneratedVote extends GeneratedBlockEvent {
votes: number;
endorse_blocks: string[];
}

// EnteredState events only need the base identification info
type EnteredStateBlock = BaseBlockEvent;

type HaskellBlockEvent =
| GeneratedInputBlock
| GeneratedEndorserBlock
| GeneratedRankingBlock
| GeneratedVote
| EnteredStateBlock;

interface HaskellNetworkEvent {
type: "NetworkMessage";
action: "Sent" | "Received"; // Added to distinguish direction
sender: string;
recipient: string;
block_kind: BlockKind;
msg_size_bytes: number;
sending_s: number;
ids: string[];
}
Loading
Loading