From 74823b3df01553eb304111c3de19a5ff31b3c2a2 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Mon, 28 Apr 2025 13:21:30 +0200 Subject: [PATCH 1/6] Add notes about ledger state query API --- src/client/node-to-client/state-query/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/client/node-to-client/state-query/README.md b/src/client/node-to-client/state-query/README.md index 3d5f24e..f5c9485 100644 --- a/src/client/node-to-client/state-query/README.md +++ b/src/client/node-to-client/state-query/README.md @@ -79,6 +79,9 @@ _Since: v9_ Query the chain's start time as a `UTCTime`. +> [!CAUTION] +> Is this the Byron or Shelly start time? + ```cddl {{#include messages.cddl:api}} ``` @@ -99,6 +102,10 @@ Example response: {{#include examples/getSystemStart/result.cbor}} ``` +> [!WARNING] +> TODO: Create more realistic examples and use a tool to probabilistically verify more data. +> TODO: Also use CBOR diagnostic notation for examples + > [!CAUTION] > FIXME: While I experimented in using the network / consensus cddl parts above, `time` would be defined in the CDDL prelude (a number, assuming seconds since epoch), but is actually incorrect and the result is serialized using `ToCBOR UTCTime` following this cddl: > From b48d5aee9aa140fb63e925889763d037d3e836b8 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 8 May 2025 14:39:45 +0200 Subject: [PATCH 2/6] Use cddl socket in n2c api example cddl --- src/client/node-to-client/state-query/getSystemStart.cddl | 5 +++-- src/client/node-to-client/state-query/messages.cddl | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/client/node-to-client/state-query/getSystemStart.cddl b/src/client/node-to-client/state-query/getSystemStart.cddl index d0a6d8a..7802841 100644 --- a/src/client/node-to-client/state-query/getSystemStart.cddl +++ b/src/client/node-to-client/state-query/getSystemStart.cddl @@ -1,5 +1,6 @@ -query = 1 -result = [year, dayOfYear, timeOfDayPico] +$query /= 1 +$result /= [year, dayOfYear, timeOfDayPico] + year = bigint dayOfYear = int timeOfDayPico = bigint diff --git a/src/client/node-to-client/state-query/messages.cddl b/src/client/node-to-client/state-query/messages.cddl index f61210b..ba3ad6d 100644 --- a/src/client/node-to-client/state-query/messages.cddl +++ b/src/client/node-to-client/state-query/messages.cddl @@ -18,8 +18,8 @@ acquireFailurePointNotOnChain = 1 failure = acquireFailurePointTooOld / acquireFailurePointNotOnChain -query = any -result = any +$query = any +$result = any msgAcquire = [0, point] / [8] @@ -27,8 +27,8 @@ msgAcquire = [0, point] msgAcquired = [1] msgFailure = [2, failure] ; ANCHOR: api -msgQuery = [3, query] -msgResult = [4, result] +msgQuery = [3, $query] +msgResult = [4, $result] ; ANCHOR_END: api msgRelease = [5] msgReAcquire = [6, point] From e7e87dfbaf32f9f323f350fb00366673714ee28b Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 8 May 2025 16:14:36 +0200 Subject: [PATCH 3/6] Check api example cbor in CI --- .github/workflows/check-examples.sh | 27 +++++++++++++++++ .github/workflows/check-examples.yaml | 29 +++++++++++++++++++ flake.nix | 6 ++-- .../state-query/getSystemStart.cddl | 2 +- .../node-to-client/state-query/messages.cddl | 3 -- 5 files changed, 61 insertions(+), 6 deletions(-) create mode 100755 .github/workflows/check-examples.sh create mode 100644 .github/workflows/check-examples.yaml diff --git a/.github/workflows/check-examples.sh b/.github/workflows/check-examples.sh new file mode 100755 index 0000000..4cc7c91 --- /dev/null +++ b/.github/workflows/check-examples.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -uo pipefail + +# Check consistency of .cbor files against .cddl in the repository. + +# TODO: check whether 'cddl' is installed + +# Check that given cbor is valid against all given cddls files concatenated +function check_cbor() { + local cbor_file=$1 + shift + local cddl_files=("$@") + local cddl=$(cat "${cddl_files[@]}") + # NOTE: cddl spams stdout on invalid cbor, so we drop everything after '-- cannot complete' + out=$(cddl <(echo "${cddl}") validate <(xxd -r -p "${cbor_file}") 2> /dev/null | \ + awk -v RS='--' 'NR==1{print} NR==2{exit}') + if [ $? -ne 0 ]; then + echo "Invalid cbor file: ${cbor_file}" + echo "Error: ${out%--*}" + printf "CDDL: \n${cddl}" + exit 1 + fi +} + +cd src/api +check_cbor examples/getSystemStart/query.cbor cddl/local-state-query.cddl cddl/getSystemStart.cddl +check_cbor examples/getSystemStart/result.cbor cddl/local-state-query.cddl cddl/getSystemStart.cddl diff --git a/.github/workflows/check-examples.yaml b/.github/workflows/check-examples.yaml new file mode 100644 index 0000000..0fdc366 --- /dev/null +++ b/.github/workflows/check-examples.yaml @@ -0,0 +1,29 @@ +name: Check cbor examples + +on: + pull_request: + push: + branches: + - main + schedule: + # Everyday at 4:00 AM + - cron: "0 4 * * *" + +jobs: + check-cbor-examples: + name: Check cbor examples + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install nix + uses: cachix/install-nix-action@V27 + with: + extra_nix_config: | + accept-flake-config = true + log-lines = 1000 + + - name: Enter nix shell + uses: nicknovitski/nix-develop@v1 + + - run: ./.github/workflows/check-examples.sh diff --git a/flake.nix b/flake.nix index 3f81183..a75905b 100644 --- a/flake.nix +++ b/flake.nix @@ -23,7 +23,6 @@ rec { inherit inputs; legacyPackages = pkgs; - defaultPackage = packages.mdbook; packages.mdbook = pkgs.stdenv.mkDerivation { name = "cardano-blueprint-book"; @@ -43,7 +42,10 @@ devShells.default = pkgs.mkShell { inputsFrom = [ packages.mdbook ]; - buildInputs = formattingPkgs; + buildInputs = formattingPkgs ++ [ + pkgs.cddl + pkgs.xxd + ]; }; } ); diff --git a/src/client/node-to-client/state-query/getSystemStart.cddl b/src/client/node-to-client/state-query/getSystemStart.cddl index 7802841..1ca368f 100644 --- a/src/client/node-to-client/state-query/getSystemStart.cddl +++ b/src/client/node-to-client/state-query/getSystemStart.cddl @@ -1,4 +1,4 @@ -$query /= 1 +$query /= [1] $result /= [year, dayOfYear, timeOfDayPico] year = bigint diff --git a/src/client/node-to-client/state-query/messages.cddl b/src/client/node-to-client/state-query/messages.cddl index ba3ad6d..487bdbf 100644 --- a/src/client/node-to-client/state-query/messages.cddl +++ b/src/client/node-to-client/state-query/messages.cddl @@ -18,9 +18,6 @@ acquireFailurePointNotOnChain = 1 failure = acquireFailurePointTooOld / acquireFailurePointNotOnChain -$query = any -$result = any - msgAcquire = [0, point] / [8] / [10] From f04a24f7318073a6564a4a5e220b3416ec4e955d Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 8 May 2025 16:20:10 +0200 Subject: [PATCH 4/6] Require dayOfYear to be an unsigned int --- src/client/node-to-client/state-query/getSystemStart.cddl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/node-to-client/state-query/getSystemStart.cddl b/src/client/node-to-client/state-query/getSystemStart.cddl index 1ca368f..f482ecb 100644 --- a/src/client/node-to-client/state-query/getSystemStart.cddl +++ b/src/client/node-to-client/state-query/getSystemStart.cddl @@ -2,5 +2,5 @@ $query /= [1] $result /= [year, dayOfYear, timeOfDayPico] year = bigint -dayOfYear = int +dayOfYear = uint timeOfDayPico = bigint From bfbf62fe027605e75fd152740800cf7b73641b57 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Fri, 9 May 2025 17:08:54 +0200 Subject: [PATCH 5/6] Regenerate getSystemStart/result.cbor --- .../state-query/examples/getSystemStart/result.cbor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/node-to-client/state-query/examples/getSystemStart/result.cbor b/src/client/node-to-client/state-query/examples/getSystemStart/result.cbor index ea0efe6..6d9892e 100644 --- a/src/client/node-to-client/state-query/examples/getSystemStart/result.cbor +++ b/src/client/node-to-client/state-query/examples/getSystemStart/result.cbor @@ -1 +1 @@ -820483c2581e65fea62360470c59141d0ba6cc897f99e050184606937264a1f8c5026abc3b3a5d754770442481c3581e50670ee65e805e3cc5aadf6619e791db8b1c2237dd918ba3b6818e7c258a \ No newline at end of file +820483c2581ecca45aff5ff13d202641114479adc1b566b8a778514878b6b8c7410aa4a41a1fbe21aac2581e9de64341ff22211c81b5479ea4ad812682e3db52f36447dfc107a992e49d \ No newline at end of file From 890666aa354d63a998cae308dd30c773df697a6f Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Fri, 9 May 2025 17:36:14 +0200 Subject: [PATCH 6/6] Update logbook --- src/logbook.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/logbook.md b/src/logbook.md index 69cc991..966ac30 100644 --- a/src/logbook.md +++ b/src/logbook.md @@ -1,5 +1,52 @@ Logbook about `cardano-blueprint` that contains thinking, discussions, pains, joys, events, and experiences that happen on a daily basis. It is supposed to be a kind of [Stream of consciousness](https://en.wikipedia.org/wiki/Stream_of_consciousness) that can later be searched, linked to or reviewed. It may also be used as a very informal decision log. +## 2025-05-09 + +By @ch1bo on writing local state query serialization tests in `ouroboros-consensus` + +- If we want to use `cddlc`, we need to package it using `nix` to make it available to CI. +- Cuddle based tests in `cardano-ledger` are way faster than the ones running the ruby-based `cddl` tool. +- The LocalStateQuery API description and example cbor I envisioned to have in the cardano-blueprint would be *full messages* as sent over the wire. +- After moving the test suite into `ouroboros-consensus-cardano` tests I seem to have everything in scope, network decoders and `CardanoBlock` which selects all the right things. +- Making sure the right type class instances are in scope was a bit tricky (needed imports like `import Ouroboros.Consensus.Shelley.Ledger.SupportsProtocol ()`) +- Finally I have a working roundtrip test for queries and when doing it for results (and some `typed-protocols` annoyance) I came across this weird thing: + - After correctly decoding the `GetSystemStart` result from: + + 820483c2581e65fea62360470c59141d0ba6cc897f99e050184606937264a1f8c5026abc3b3a5d754770442481c3581e50670ee65e805e3cc5aadf6619e791db8b1c2237dd918ba3b6818e7c258a + + - Re-encoding it as `MsgResult` produces: + + 820483c2581e65fea62360470c59141d0ba6cc897f99e050184606937264a1f8c5026abc01c3581e50670ee65e805e3cc5aadf6619e791db8b1c2237dd918ba3b6818e7c258a + + - In diagnostic notation: + + [ + 4, + [ + 2(h'65fea62360470c59141d0ba6cc897f99e050184606937264a1f8c5026abc'), + -4205646576720553090_3, + 3(h'50670ee65e805e3cc5aadf6619e791db8b1c2237dd918ba3b6818e7c258a'), + ], + ] + + becomes + + [ + 4, + [ + 2(h'65fea62360470c59141d0ba6cc897f99e050184606937264a1f8c5026abc'), + 1, + 3(h'50670ee65e805e3cc5aadf6619e791db8b1c2237dd918ba3b6818e7c258a'), + ], + ] + + - Overflow on the second field of `UTCTime`!? + + - Turns out the `FromCBOR UTCTime` is using [`fromOrdinalDate`](https://hackage.haskell.org/package/time-1.14/docs/Data-Time-Calendar-OrdinalDate.html#v:fromOrdinalDate) which clips `dayOfYear` (the second field, an `Int`) to valid range `[1, 366]` +- So in fact `dayOfYear` in `cddl` should not be an `int` (with which the example was generated)! +- Made a CI job to check cbor against cddl in `cardano-blueprint`, switched `dayOfYear` to `uint` and regenerate example. Now it fails because the generated `532554154` is clamped to `366`. +- Are roundtrips even required? As long as we can decode what comes from the CDDL and what we encode is conformant that might be good enough? + ## 2025-03-25 By @ch1bo