Add NTN cddls

.github/workflows/ci.yml

@@ -55,6 +55,11 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Check CDDLs up-to-date with Blueprints
+      continue-on-error: true
+      run: |
+        ./scripts/ci/diff-cddls.sh
+
     - name: Install base libraries
       uses: input-output-hk/actions/base@latest
       with:
@@ -138,6 +143,17 @@ jobs:
     - name: Build projects [build]
       run: cabal build all -j
 
+    - name: Install Ruby for CDDL
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.4'
+        bundler-cache: true
+
+    - name: Install cddl tools
+      run: |
+        gem install cddlc
+        cabal install cuddle-0.5.0.0
+
     - name: Test
       if: matrix.test-set == 'all'
       run: cabal test all -j --test-show-details=streaming

nix/cddlc/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem 'cddlc'

nix/cddlc/Gemfile.lock

@@ -0,0 +1,19 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cddlc (0.4.2)
+      neatjson (~> 0.10)
+      treetop (~> 1)
+    neatjson (0.10.5)
+    polyglot (0.3.5)
+    treetop (1.6.14)
+      polyglot (~> 0.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cddlc
+
+BUNDLED WITH
+   2.6.2

nix/cddlc/gemset.nix

@@ -0,0 +1,47 @@
+{
+  cddlc = {
+    dependencies = [
+      "neatjson"
+      "treetop"
+    ];
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "1s3fbgd5yqgji162zsmlwnva1v1r3zc1qiyv6im7karv5f08r8m3";
+      type = "gem";
+    };
+    version = "0.4.2";
+  };
+  neatjson = {
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "0wm1lq8yl6rzysh3wg6fa55w5534k6ppiz0qb7jyvdy582mk5i0s";
+      type = "gem";
+    };
+    version = "0.10.5";
+  };
+  polyglot = {
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "1bqnxwyip623d8pr29rg6m8r0hdg08fpr2yb74f46rn1wgsnxmjr";
+      type = "gem";
+    };
+    version = "0.3.5";
+  };
+  treetop = {
+    dependencies = [ "polyglot" ];
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "1m5fqy7vq6y7bgxmw7jmk7y6pla83m16p7lb41lbqgg53j8x2cds";
+      type = "gem";
+    };
+    version = "1.6.14";
+  };
+}

nix/cddlc/package.nix

@@ -0,0 +1,23 @@
+{ lib
+, bundlerApp
+, bundlerUpdateScript
+}:
+
+bundlerApp {
+  pname = "cddlc";
+
+  gemdir = ./.;
+
+  exes = [ "cddlc" ];
+
+  passthru.updateScript = bundlerUpdateScript "cddlc";
+
+  meta = {
+    description = "CDDL conversion utilities";
+    homepage = "https://github.com/cabo/cddlc";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ amesgen ];
+    platforms = lib.platforms.unix;
+    mainProgram = "cddlc";
+  };
+}

nix/haskell.nix

@@ -38,6 +38,19 @@ let
       ({ pkgs, lib, ... }: lib.mkIf pkgs.stdenv.hostPlatform.isWindows {
         # https://github.com/input-output-hk/haskell.nix/issues/2332
         packages.basement.configureFlags = [ "--hsc2hs-option=--cflag=-Wno-int-conversion" ];
+        # We can't cross-compile the ruby gem `cddlc` so we decided to skip this
+        # test on Windows in Hydra.
+        packages.ouroboros-consensus-cardano.components.tests.cardano-test.preCheck = ''
+          export DISABLE_CDDLC=1
+        '';
+      })
+      ({ pkgs, ... }: lib.mkIf (!pkgs.stdenv.hostPlatform.isWindows) {
+        # Tools for CBOR/CDDL tests:
+        packages.ouroboros-consensus-cardano.components.tests.cardano-test = {
+          build-tools =
+            [ pkgs.cddlc pkgs.cuddle ];
+          extraSrcFiles = [ "cddl/**/*" ];
+        };
       })
     ];
     flake.variants = {

nix/shell.nix

@@ -16,6 +16,8 @@ hsPkgs.shellFor {
     pkgs.ghcid
     pkgs.xrefcheck
     pkgs.fourmolu
+    pkgs.cuddle
+    pkgs.cddlc
 
     # release management
     pkgs.scriv

nix/tools.nix

@@ -38,6 +38,18 @@ in
 
   fourmolu = tool "fourmolu" "0.18.0.0" { };
 
+  cuddle = tool "cuddle" "git" {
+    src = final.fetchFromGitHub {
+      owner = "input-output-hk";
+      repo = "cuddle";
+      rev = "cuddle-0.5.0.0";
+      hash = "sha256-06a9N1IAh0kKW/xPu1qiLK22HpXyARnipA1YJxY4jOQ=";
+    };
+  };
+
+  # remove once our nixpkgs contains https://github.com/NixOS/nixpkgs/pull/394873
+  cddlc = final.callPackage ./cddlc/package.nix { };
+
   haskellBuildUtils = prev.haskellBuildUtils.override {
     inherit (final.hsPkgs.args) compiler-nix-name;
     index-state = tool-index-state;

ouroboros-consensus-cardano/cddl/base.cddl

@@ -0,0 +1,53 @@
+telescope7<byron, shelley, allegra, mary, alonzo, babbage, conway>
+  = [pastEra, pastEra, pastEra, pastEra, pastEra, pastEra, currentEra<conway>] /
+    [pastEra, pastEra, pastEra, pastEra, pastEra, currentEra<babbage>] /
+    [pastEra, pastEra, pastEra, pastEra, currentEra<alonzo>] /
+    [pastEra, pastEra, pastEra, currentEra<mary>] /
+    [pastEra, pastEra, currentEra<allegra>] /
+    [pastEra, currentEra<shelley>] /
+    [currentEra<byron>]
+
+ns7<byron, shelley, allegra, mary, alonzo, babbage, conway>
+  = [6, conway] /
+    [5, babbage] /
+    [4, alonzo] /
+    [3, mary] /
+    [2, allegra] /
+    [1, shelley] /
+    [0, byron]
+
+;; Blockchain types
+pastEra = [bound, bound]
+currentEra<st> = [bound, st]
+bound = [relativeTime, slotno, epochno]
+eraIdx = word8
+individualPoolStake = [stake, hash]
+nonce = [0] / [1, hash]
+point = [] / [ slotno, hash ]
+poolDistr = map<keyhash, individualPoolStake>
+slotno = word64
+stake = rational
+
+withOrigin<v> = [] / [v]
+withOriginTH<v> = [0] / [1, v]
+
+;; Collections
+either<x, y> = [0, x] / [1, y]
+map<x, y> = { * x => y }
+maybe<x> = [] / [x]
+seq<x> = [*23 x] / [24* x] ; encoded with indefinite-length encoding
+set<x> = #6.258([* x])
+
+;; Types from other packages
+blockno = word64
+epochno = word64
+coin = word64
+rational = [int, int]
+keyhash = bstr .size 28
+hash = bstr .size 32
+relativeTime = int
+
+;; Base word types
+word8  = uint .size 1
+word32 = uint .size 4
+word64 = uint .size 8

ouroboros-consensus-cardano/cddl/disk/immutable/chunkFile.cddl

@@ -0,0 +1,3 @@
+chunkFile = bytes .cborseq block.cardanoBlock
+
+;# import block as block

ouroboros-consensus-cardano/cddl/disk/immutable/primary.ksy

@@ -0,0 +1,12 @@
+meta:
+  id: primary_file
+  endian: be
+
+seq:
+  - id: current_version
+    type: u1
+    valid: 0x01
+
+  - id: secondary_offsets
+    type: u4
+    repeat: eos

ouroboros-consensus-cardano/cddl/disk/immutable/secondary.ksy

@@ -0,0 +1,29 @@
+meta:
+  id: secondary_file
+  endian: be
+
+types:
+  entry:
+    seq:
+      - id: block_offset
+        type: u8
+
+      - id: header_offset
+        type: u2
+
+      - id: header_size
+        type: u2
+
+      - id: crc
+        type: u4
+
+      - id: header_hash
+        size: 32
+
+      - id: block_or_ebb
+        type: u8
+
+seq:
+  - id: secondary_offsets
+    type: entry
+    repeat: eos

ouroboros-consensus-cardano/cddl/disk/ledger/headerstate.cddl

@@ -0,0 +1,28 @@
+headerState =
+  [withOrigin<headerStateTip>, headerStateChainDep]
+
+headerStateTip =
+  ns7<byronAnnTip, annTip, annTip, annTip, annTip, annTip, annTip>
+
+byronAnnTip = [slotno,
+               hash,
+               blockno,
+               bool]
+
+annTip = [slotno,
+          hash,
+          blockno]
+
+headerStateChainDep =
+  telescope7<versionedPbftState,
+             versionedTPraosState,
+             versionedTPraosState,
+             versionedTPraosState,
+             versionedTPraosState,
+             versionedPraosState,
+             versionedPraosState>
+
+;# import withOrigin, ns7, telescope7, slotno, hash, blockno from base
+;# import versionedPbftState from pbft
+;# import versionedPraosState from praos
+;# import versionedTPraosState from tpraos

ouroboros-consensus-cardano/cddl/disk/ledger/ledgerstate.cddl

@@ -0,0 +1,30 @@
+ledgerState =
+  telescope7<byron.ledgerstate,
+             versionedShelleyLedgerState<shelley.ledgerstate>,
+             versionedShelleyLedgerState<allegra.ledgerstate>,
+             versionedShelleyLedgerState<mary.ledgerstate>,
+             versionedShelleyLedgerState<alonzo.ledgerstate>,
+             versionedShelleyLedgerState<babbage.ledgerstate>,
+             versionedShelleyLedgerState<conway.ledgerstate>>
+
+versionedShelleyLedgerState<eraSt1> = [ shelleyVersion2, shelleyLedgerState<eraSt1> ]
+
+shelleyVersion2 = 2
+
+shelleyLedgerState<eraSt2> = [ withOrigin<shelleyTip>, eraSt2, shelleyTransition ]
+
+shelleyTip = [slotno, blockno, hash]
+
+shelleyTransition = word32
+
+; TODO these should be imports from the ledger however they do not
+; provide these definitions yet.
+byron.ledgerstate = any
+shelley.ledgerstate = any
+allegra.ledgerstate = any
+mary.ledgerstate = any
+alonzo.ledgerstate = any
+babbage.ledgerstate = any
+conway.ledgerstate = any
+
+;# import telescope7, withOrigin, slotno, blockno, hash, word32 from base

ouroboros-consensus-cardano/cddl/disk/ledger/pbft.cddl

@@ -0,0 +1,5 @@
+versionedPbftState = [serializationFormat1, {* keyhash => [* slotno]}]
+
+serializationFormat1 = 1
+
+;# import keyhash, slotno from base

ouroboros-consensus-cardano/cddl/disk/ledger/praos.cddl

@@ -0,0 +1,13 @@
+versionedPraosState = [praosVersion, praosState]
+
+praosVersion = 0
+
+praosState = [withOrigin<slotno>,
+              {* keyhash => word64},
+              nonce,
+              nonce,
+              nonce,
+              nonce,
+              nonce]
+
+;# import withOrigin, slotno, keyhash, word64, nonce from base

ouroboros-consensus-cardano/cddl/disk/ledger/stateFile.cddl

@@ -0,0 +1,9 @@
+ledgerStateSnapshot =
+  [snapshotEncodingVersion1, extLedgerState]
+
+snapshotEncodingVersion1 = 1
+
+extLedgerState = [ledgerState, headerState]
+
+;# import headerState from headerstate
+;# import ledgerState from ledgerstate

ouroboros-consensus-cardano/cddl/disk/ledger/tpraos.cddl

@@ -0,0 +1,12 @@
+versionedTPraosState =
+  [serializationFormat1, [withOriginTH<slotno>, tpraosState]]
+
+tpraosState = [prtclState, ticknState, nonce]
+
+prtclState = [{* keyhash => word64}, nonce, nonce]
+
+ticknState = [nonce, nonce]
+
+serializationFormat1 = 1
+
+;# import withOriginTH, slotno, keyhash, word64, nonce from base

ouroboros-consensus-cardano/cddl/disk/volatile/blocksDatFile.cddl

@@ -0,0 +1,3 @@
+blockDatFile = bytes .cborseq block.cardanoBlock
+
+;# import block as block

ouroboros-consensus-cardano/cddl/node-to-node/blockfetch/block.cddl

@@ -0,0 +1,17 @@
+serialisedCardanoBlock = #6.24(bytes .cbor cardanoBlock)
+
+cardanoBlock = byron.block
+             / [2, shelley.block]
+             / [3, allegra.block]
+             / [4, mary.block]
+             / [5, alonzo.block]
+             / [6, babbage.block]
+             / [7, conway.block]
+
+;# import byron as byron
+;# import shelley as shelley
+;# import allegra as allegra
+;# import mary as mary
+;# import alonzo as alonzo
+;# import babbage as babbage
+;# import conway as conway