diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fb597460e..055235cf33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Check CDDLs up-to-date with Blueprints + continue-on-error: true + run: | + ./scripts/ci/diff-cddls.sh + - name: Install base libraries uses: input-output-hk/actions/base@latest with: @@ -138,6 +143,17 @@ jobs: - name: Build projects [build] run: cabal build all -j + - name: Install Ruby for CDDL + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.4' + bundler-cache: true + + - name: Install cddl tools + run: | + gem install cddlc + cabal install cuddle-0.5.0.0 + - name: Test if: matrix.test-set == 'all' run: cabal test all -j --test-show-details=streaming diff --git a/nix/cddlc/Gemfile b/nix/cddlc/Gemfile new file mode 100644 index 0000000000..3cc0760bb2 --- /dev/null +++ b/nix/cddlc/Gemfile @@ -0,0 +1,2 @@ +source 'https://rubygems.org' +gem 'cddlc' diff --git a/nix/cddlc/Gemfile.lock b/nix/cddlc/Gemfile.lock new file mode 100644 index 0000000000..da7f3d68cd --- /dev/null +++ b/nix/cddlc/Gemfile.lock @@ -0,0 +1,19 @@ +GEM + remote: https://rubygems.org/ + specs: + cddlc (0.4.2) + neatjson (~> 0.10) + treetop (~> 1) + neatjson (0.10.5) + polyglot (0.3.5) + treetop (1.6.14) + polyglot (~> 0.3) + +PLATFORMS + ruby + +DEPENDENCIES + cddlc + +BUNDLED WITH + 2.6.2 diff --git a/nix/cddlc/gemset.nix b/nix/cddlc/gemset.nix new file mode 100644 index 0000000000..997b4c7cab --- /dev/null +++ b/nix/cddlc/gemset.nix @@ -0,0 +1,47 @@ +{ + cddlc = { + dependencies = [ + "neatjson" + "treetop" + ]; + groups = [ "default" ]; + platforms = [ ]; + source = { + remotes = [ "https://rubygems.org" ]; + sha256 = "1s3fbgd5yqgji162zsmlwnva1v1r3zc1qiyv6im7karv5f08r8m3"; + type = "gem"; + }; + version = "0.4.2"; + }; + neatjson = { + groups = [ "default" ]; + platforms = [ ]; + source = { + remotes = [ "https://rubygems.org" ]; + sha256 = "0wm1lq8yl6rzysh3wg6fa55w5534k6ppiz0qb7jyvdy582mk5i0s"; + type = "gem"; + }; + version = "0.10.5"; + }; + polyglot = { + groups = [ "default" ]; + platforms = [ ]; + source = { + remotes = [ "https://rubygems.org" ]; + sha256 = "1bqnxwyip623d8pr29rg6m8r0hdg08fpr2yb74f46rn1wgsnxmjr"; + type = "gem"; + }; + version = "0.3.5"; + }; + treetop = { + dependencies = [ "polyglot" ]; + groups = [ "default" ]; + platforms = [ ]; + source = { + remotes = [ "https://rubygems.org" ]; + sha256 = "1m5fqy7vq6y7bgxmw7jmk7y6pla83m16p7lb41lbqgg53j8x2cds"; + type = "gem"; + }; + version = "1.6.14"; + }; +} diff --git a/nix/cddlc/package.nix b/nix/cddlc/package.nix new file mode 100644 index 0000000000..ad8c2eb888 --- /dev/null +++ b/nix/cddlc/package.nix @@ -0,0 +1,23 @@ +{ lib +, bundlerApp +, bundlerUpdateScript +}: + +bundlerApp { + pname = "cddlc"; + + gemdir = ./.; + + exes = [ "cddlc" ]; + + passthru.updateScript = bundlerUpdateScript "cddlc"; + + meta = { + description = "CDDL conversion utilities"; + homepage = "https://github.com/cabo/cddlc"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ amesgen ]; + platforms = lib.platforms.unix; + mainProgram = "cddlc"; + }; +} diff --git a/nix/haskell.nix b/nix/haskell.nix index 7f55ab425a..f099458567 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -38,6 +38,19 @@ let ({ pkgs, lib, ... }: lib.mkIf pkgs.stdenv.hostPlatform.isWindows { # https://github.com/input-output-hk/haskell.nix/issues/2332 packages.basement.configureFlags = [ "--hsc2hs-option=--cflag=-Wno-int-conversion" ]; + # We can't cross-compile the ruby gem `cddlc` so we decided to skip this + # test on Windows in Hydra. + packages.ouroboros-consensus-cardano.components.tests.cardano-test.preCheck = '' + export DISABLE_CDDLC=1 + ''; + }) + ({ pkgs, ... }: lib.mkIf (!pkgs.stdenv.hostPlatform.isWindows) { + # Tools for CBOR/CDDL tests: + packages.ouroboros-consensus-cardano.components.tests.cardano-test = { + build-tools = + [ pkgs.cddlc pkgs.cuddle ]; + extraSrcFiles = [ "cddl/**/*" ]; + }; }) ]; flake.variants = { diff --git a/nix/shell.nix b/nix/shell.nix index 4f346a5a40..ecbf552c93 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -16,6 +16,8 @@ hsPkgs.shellFor { pkgs.ghcid pkgs.xrefcheck pkgs.fourmolu + pkgs.cuddle + pkgs.cddlc # release management pkgs.scriv diff --git a/nix/tools.nix b/nix/tools.nix index cd272bbd54..2753d8bba4 100644 --- a/nix/tools.nix +++ b/nix/tools.nix @@ -38,6 +38,18 @@ in fourmolu = tool "fourmolu" "0.18.0.0" { }; + cuddle = tool "cuddle" "git" { + src = final.fetchFromGitHub { + owner = "input-output-hk"; + repo = "cuddle"; + rev = "cuddle-0.5.0.0"; + hash = "sha256-06a9N1IAh0kKW/xPu1qiLK22HpXyARnipA1YJxY4jOQ="; + }; + }; + + # remove once our nixpkgs contains https://github.com/NixOS/nixpkgs/pull/394873 + cddlc = final.callPackage ./cddlc/package.nix { }; + haskellBuildUtils = prev.haskellBuildUtils.override { inherit (final.hsPkgs.args) compiler-nix-name; index-state = tool-index-state; diff --git a/ouroboros-consensus-cardano/cddl/base.cddl b/ouroboros-consensus-cardano/cddl/base.cddl new file mode 100644 index 0000000000..c9dc8bb120 --- /dev/null +++ b/ouroboros-consensus-cardano/cddl/base.cddl @@ -0,0 +1,52 @@ +telescope7 + = [pastEra, pastEra, pastEra, pastEra, pastEra, pastEra, currentEra] / + [pastEra, pastEra, pastEra, pastEra, pastEra, currentEra] / + [pastEra, pastEra, pastEra, pastEra, currentEra] / + [pastEra, pastEra, pastEra, currentEra] / + [pastEra, pastEra, currentEra] / + [pastEra, currentEra] / + [currentEra] + +ns7 + = [6, conway] / + [5, babbage] / + [4, alonzo] / + [3, mary] / + [2, allegra] / + [1, shelley] / + [0, byron] + +;; Blockchain types +pastEra = [bound, bound] +currentEra = [bound, st] +bound = [relativeTime, slotno, epochno] +eraIdx = word8 +individualPoolStake = [stake, hash] +nonce = [0] / [1, hash] +point = [] / [ slotno, hash ] +poolDistr = map +slotno = word64 +stake = rational + +withOrigin = [] / [v] + +;; Collections +either = [0, x] / [1, y] +map = { * x => y } +maybe = [] / [x] +seq = [*23 x] / [24* x] ; encoded with indefinite-length encoding +set = #6.258([* x]) + +;; Types from other packages +blockno = word64 +epochno = word64 +coin = word64 +rational = [int, int] +keyhash = bstr .size 28 +hash = bstr .size 32 +relativeTime = int + +;; Base word types +word8 = uint .size 1 +word32 = uint .size 4 +word64 = uint .size 8 diff --git a/ouroboros-consensus-cardano/cddl/node-to-node/blockfetch/block.cddl b/ouroboros-consensus-cardano/cddl/node-to-node/blockfetch/block.cddl new file mode 100644 index 0000000000..f2b4d9847d --- /dev/null +++ b/ouroboros-consensus-cardano/cddl/node-to-node/blockfetch/block.cddl @@ -0,0 +1,17 @@ +serialisedCardanoBlock = #6.24(bytes .cbor cardanoBlock) + +cardanoBlock = byron.block + / [2, shelley.block] + / [3, allegra.block] + / [4, mary.block] + / [5, alonzo.block] + / [6, babbage.block] + / [7, conway.block] + +;# import byron as byron +;# import shelley as shelley +;# import allegra as allegra +;# import mary as mary +;# import alonzo as alonzo +;# import babbage as babbage +;# import conway as conway diff --git a/ouroboros-consensus-cardano/cddl/node-to-node/chainsync/header.cddl b/ouroboros-consensus-cardano/cddl/node-to-node/chainsync/header.cddl new file mode 100644 index 0000000000..a77d541447 --- /dev/null +++ b/ouroboros-consensus-cardano/cddl/node-to-node/chainsync/header.cddl @@ -0,0 +1,25 @@ +header + = base.ns7, + serialisedShelleyHeader, + serialisedShelleyHeader, + serialisedShelleyHeader, + serialisedShelleyHeader, + serialisedShelleyHeader> + +byronHeader = [byronRegularIdx, #6.24(bytes .cbor byron.blockhead)] + / [byronBoundaryIdx, #6.24(bytes .cbor byron.ebbhead)] + +byronBoundaryIdx = [0, base.word32] +byronRegularIdx = [1, base.word32] + +serialisedShelleyHeader = #6.24(bytes .cbor era) + +;# include byron as byron +;# include shelley as shelley +;# include allegra as allegra +;# include mary as mary +;# include alonzo as alonzo +;# include babbage as babbage +;# include conway as conway +;# import base as base diff --git a/ouroboros-consensus-cardano/cddl/node-to-node/txsubmission2/tx.cddl b/ouroboros-consensus-cardano/cddl/node-to-node/txsubmission2/tx.cddl new file mode 100644 index 0000000000..a4393619d0 --- /dev/null +++ b/ouroboros-consensus-cardano/cddl/node-to-node/txsubmission2/tx.cddl @@ -0,0 +1,23 @@ +tx = + base.ns7, + serialisedShelleyTx, + serialisedShelleyTx, + serialisedShelleyTx, + serialisedShelleyTx, + serialisedShelleyTx> + +serialisedShelleyTx = #6.24(bytes .cbor era) + +;# include byron as byron + +; See https://github.com/IntersectMBO/cardano-ledger/issues/5124 +byron.transaction = [0, [byron.tx, [+ byron.twit]]] / [1, any] / [2, any] / [3, any] + +;# include shelley as shelley +;# include allegra as allegra +;# include mary as mary +;# include alonzo as alonzo +;# include babbage as babbage +;# include conway as conway +;# import base as base diff --git a/ouroboros-consensus-cardano/cddl/node-to-node/txsubmission2/txId.cddl b/ouroboros-consensus-cardano/cddl/node-to-node/txsubmission2/txId.cddl new file mode 100644 index 0000000000..fb16c56a43 --- /dev/null +++ b/ouroboros-consensus-cardano/cddl/node-to-node/txsubmission2/txId.cddl @@ -0,0 +1,22 @@ +txId = + base.ns7 + +byronTxId = [0, byron.txid] + / [1, byron.certificateid] + / [2, byron.updid] + / [3, byron.voteid] + +;# include byron as byron +;# include shelley as shelley +;# include allegra as allegra +;# include mary as mary +;# include alonzo as alonzo +;# include babbage as babbage +;# include conway as conway +;# import base as base diff --git a/ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal b/ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal index 2615cd7183..41b714e2b4 100644 --- a/ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal +++ b/ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal @@ -23,6 +23,10 @@ extra-doc-files: CHANGELOG.md README.md +data-files: + cddl/**/*.cddl + cddl/base.cddl + source-repository head type: git location: https://github.com/IntersectMBO/ouroboros-consensus @@ -435,6 +439,7 @@ test-suite cardano-test main-is: Main.hs other-modules: Test.Consensus.Cardano.DiffusionPipelining + Test.Consensus.Cardano.GenCDDLs Test.Consensus.Cardano.Golden Test.Consensus.Cardano.MiniProtocol.LocalTxSubmission.ByteStringTxParser Test.Consensus.Cardano.MiniProtocol.LocalTxSubmission.Server @@ -448,20 +453,26 @@ test-suite cardano-test Test.ThreadNet.MaryAlonzo Test.ThreadNet.ShelleyAllegra + other-modules: Paths_ouroboros_consensus_cardano build-depends: QuickCheck, base, base16-bytestring, bytestring, + cardano-ledger-allegra:testlib, cardano-ledger-alonzo, + cardano-ledger-alonzo:testlib, cardano-ledger-alonzo-test, cardano-ledger-api, + cardano-ledger-babbage:testlib, cardano-ledger-babbage-test, cardano-ledger-binary:testlib, - cardano-ledger-byron, + cardano-ledger-byron:{cardano-ledger-byron, testlib}, cardano-ledger-conway:testlib, cardano-ledger-core:{cardano-ledger-core, testlib}, + cardano-ledger-mary:testlib, cardano-ledger-shelley, + cardano-ledger-shelley:testlib, cardano-ledger-shelley-test, cardano-protocol-tpraos, cardano-slotting, @@ -469,6 +480,7 @@ test-suite cardano-test constraints, containers, contra-tracer, + directory, filepath, microlens, ouroboros-consensus:{ouroboros-consensus, unstable-consensus-testlib, unstable-mempool-test-utils}, @@ -478,12 +490,14 @@ test-suite cardano-test ouroboros-network-api, ouroboros-network-protocols:{ouroboros-network-protocols, testlib}, pretty-simple, + process-extras, sop-core, sop-extras, strict-sop-core, tasty, tasty-hunit, tasty-quickcheck, + temporary, typed-protocols ^>=0.3, unstable-byron-testlib, unstable-cardano-testlib, diff --git a/ouroboros-consensus-cardano/src/unstable-cardano-testlib/Test/Consensus/Cardano/Generators.hs b/ouroboros-consensus-cardano/src/unstable-cardano-testlib/Test/Consensus/Cardano/Generators.hs index f2187a7d6d..2973a72b1a 100644 --- a/ouroboros-consensus-cardano/src/unstable-cardano-testlib/Test/Consensus/Cardano/Generators.hs +++ b/ouroboros-consensus-cardano/src/unstable-cardano-testlib/Test/Consensus/Cardano/Generators.hs @@ -55,7 +55,6 @@ import Test.Cardano.Ledger.Alonzo.Arbitrary () import Test.Cardano.Ledger.Alonzo.Serialisation.Generators () import Test.Cardano.Ledger.Conway.Arbitrary () import Test.Consensus.Byron.Generators () -import Test.Consensus.Cardano.MockCrypto import Test.Consensus.Protocol.Serialisation.Generators () import Test.Consensus.Shelley.Generators import Test.Consensus.Shelley.MockCrypto (CanMock) @@ -70,14 +69,14 @@ import Test.Util.Serialisation.Roundtrip Disk -------------------------------------------------------------------------------} -instance Arbitrary (CardanoBlock MockCryptoCompatByron) where +instance Arbitrary (CardanoBlock StandardCrypto) where arbitrary = oneof $ catMaybes $ hcollapse generators where generators :: NP - (K (Maybe (Gen (CardanoBlock MockCryptoCompatByron)))) - (CardanoEras MockCryptoCompatByron) + (K (Maybe (Gen (CardanoBlock StandardCrypto)))) + (CardanoEras StandardCrypto) generators = mk BlockByron :* mk BlockShelley @@ -91,18 +90,18 @@ instance Arbitrary (CardanoBlock MockCryptoCompatByron) where mk :: forall a x. Arbitrary a => - (a -> CardanoBlock MockCryptoCompatByron) -> - K (Maybe (Gen (CardanoBlock MockCryptoCompatByron))) x + (a -> CardanoBlock StandardCrypto) -> + K (Maybe (Gen (CardanoBlock StandardCrypto))) x mk f = K $ Just $ f <$> arbitrary -instance Arbitrary (Coherent (CardanoBlock MockCryptoCompatByron)) where +instance Arbitrary (Coherent (CardanoBlock StandardCrypto)) where arbitrary = fmap Coherent $ oneof $ catMaybes $ hcollapse generators where generators :: NP - (K (Maybe (Gen (CardanoBlock MockCryptoCompatByron)))) - (CardanoEras MockCryptoCompatByron) + (K (Maybe (Gen (CardanoBlock StandardCrypto)))) + (CardanoEras StandardCrypto) generators = mk BlockByron :* mk BlockShelley @@ -116,11 +115,11 @@ instance Arbitrary (Coherent (CardanoBlock MockCryptoCompatByron)) where mk :: forall a x. Arbitrary (Coherent a) => - (a -> CardanoBlock MockCryptoCompatByron) -> - K (Maybe (Gen (CardanoBlock MockCryptoCompatByron))) x + (a -> CardanoBlock StandardCrypto) -> + K (Maybe (Gen (CardanoBlock StandardCrypto))) x mk f = K $ Just $ f . getCoherent <$> arbitrary -instance Arbitrary (CardanoHeader MockCryptoCompatByron) where +instance Arbitrary (CardanoHeader StandardCrypto) where arbitrary = getHeader <$> arbitrary instance @@ -139,7 +138,7 @@ instance aux = K . OneEraHash . toShortRawHash (Proxy @blk) . unwrapHeaderHash instance - (c ~ MockCryptoCompatByron, ShelleyBasedEra ShelleyEra) => + (c ~ StandardCrypto, ShelleyBasedEra ShelleyEra) => Arbitrary (AnnTip (CardanoBlock c)) where arbitrary = @@ -344,7 +343,7 @@ arbitraryNodeToNode injByron injShelley injAllegra injMary injAlonzo injBabbage x instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToNodeVersion (CardanoEras c)) @@ -362,7 +361,7 @@ instance injConway = mapSomeNestedCtxt (NCS . NCS . NCS . NCS . NCS . NCS . NCZ) instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToNodeVersion (CardanoEras c)) @@ -380,7 +379,7 @@ instance BlockConway instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToNodeVersion (CardanoEras c)) @@ -398,7 +397,7 @@ instance HeaderConway instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToNodeVersion (CardanoEras c)) @@ -416,7 +415,7 @@ instance GenTxConway instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToNodeVersion (CardanoEras c)) @@ -695,7 +694,7 @@ arbitraryNodeToClient injByron injShelley injAllegra injMary injAlonzo injBabbag ] instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToClientVersion (CardanoEras c)) @@ -713,7 +712,7 @@ instance BlockConway instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToClientVersion (CardanoEras c)) @@ -731,7 +730,7 @@ instance GenTxConway instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToClientVersion (CardanoEras c)) @@ -761,8 +760,8 @@ instance shrink = traverse aux where aux :: - CardanoApplyTxErr MockCryptoCompatByron -> - [CardanoApplyTxErr MockCryptoCompatByron] + CardanoApplyTxErr StandardCrypto -> + [CardanoApplyTxErr StandardCrypto] aux (HardForkApplyTxErrFromEra (OneEraApplyTxErr x)) = HardForkApplyTxErrFromEra . OneEraApplyTxErr <$> shrink x aux (HardForkApplyTxErrWrongEra x) = @@ -796,7 +795,7 @@ instance ] instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToClientVersion (CardanoEras c)) @@ -916,7 +915,7 @@ instance Arbitrary (EraIndex (CardanoEras c)) where Just ns -> return $ eraIndexFromNS ns instance - c ~ MockCryptoCompatByron => + c ~ StandardCrypto => Arbitrary ( WithVersion (HardForkNodeToClientVersion (CardanoEras c)) diff --git a/ouroboros-consensus-cardano/test/byron-test/Test/Consensus/Byron/Golden.hs b/ouroboros-consensus-cardano/test/byron-test/Test/Consensus/Byron/Golden.hs index c704066c59..31bb6bc1b0 100644 --- a/ouroboros-consensus-cardano/test/byron-test/Test/Consensus/Byron/Golden.hs +++ b/ouroboros-consensus-cardano/test/byron-test/Test/Consensus/Byron/Golden.hs @@ -14,7 +14,7 @@ import Test.Util.Paths import Test.Util.Serialisation.Golden tests :: TestTree -tests = goldenTest_all codecConfig ($(getGoldenDir) "byron") examples +tests = goldenTest_all codecConfig ($(getGoldenDir) "byron") Nothing examples instance ToGoldenDirectory ByronNodeToNodeVersion diff --git a/ouroboros-consensus-cardano/test/byron-test/Test/Consensus/Byron/Serialisation.hs b/ouroboros-consensus-cardano/test/byron-test/Test/Consensus/Byron/Serialisation.hs index ddc746b449..fd5f292237 100644 --- a/ouroboros-consensus-cardano/test/byron-test/Test/Consensus/Byron/Serialisation.hs +++ b/ouroboros-consensus-cardano/test/byron-test/Test/Consensus/Byron/Serialisation.hs @@ -37,7 +37,7 @@ tests :: TestTree tests = testGroup "Byron" - [ roundtrip_all testCodecCfg dictNestedHdr + [ roundtrip_all testCodecCfg dictNestedHdr Nothing , testProperty "BinaryBlockInfo sanity check" prop_byronBinaryBlockInfo , testGroup "Integrity" diff --git a/ouroboros-consensus-cardano/test/cardano-test/Main.hs b/ouroboros-consensus-cardano/test/cardano-test/Main.hs index eefdc14769..0a78071a28 100644 --- a/ouroboros-consensus-cardano/test/cardano-test/Main.hs +++ b/ouroboros-consensus-cardano/test/cardano-test/Main.hs @@ -2,6 +2,7 @@ module Main (main) where import System.IO (BufferMode (LineBuffering), hSetBuffering, stdout) import qualified Test.Consensus.Cardano.DiffusionPipelining +import Test.Consensus.Cardano.GenCDDLs import qualified Test.Consensus.Cardano.Golden import qualified Test.Consensus.Cardano.MiniProtocol.LocalTxSubmission.Server import qualified Test.Consensus.Cardano.Serialisation (tests) @@ -29,8 +30,12 @@ tests = testGroup "cardano" [ Test.Consensus.Cardano.DiffusionPipelining.tests - , Test.Consensus.Cardano.Golden.tests - , Test.Consensus.Cardano.Serialisation.tests + , withCDDLs $ + testGroup + "Serialisation" + [ Test.Consensus.Cardano.Golden.tests + , Test.Consensus.Cardano.Serialisation.tests + ] , Test.Consensus.Cardano.SupportedNetworkProtocolVersion.tests , Test.Consensus.Cardano.SupportsSanityCheck.tests , Test.ThreadNet.AllegraMary.tests diff --git a/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/GenCDDLs.hs b/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/GenCDDLs.hs new file mode 100644 index 0000000000..693dcc1ffb --- /dev/null +++ b/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/GenCDDLs.hs @@ -0,0 +1,202 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE ViewPatterns #-} + +module Test.Consensus.Cardano.GenCDDLs (withCDDLs) where + +import qualified Control.Monad as Monad +import qualified Data.ByteString as BS +import qualified Data.ByteString.Char8 as BS8 +import qualified Data.ByteString.Lazy as BSL +import qualified Data.List as L +import Data.Maybe (isNothing) +import Paths_ouroboros_consensus_cardano +import qualified System.Directory as D +import qualified System.Environment as E +import System.Exit +import qualified System.FilePath as F +import System.IO +import System.IO.Temp +import qualified System.Process.ByteString.Lazy as P +import qualified Test.Cardano.Chain.Binary.Cddl as Byron +import qualified Test.Cardano.Ledger.Allegra.Binary.Cddl as Allegra +import qualified Test.Cardano.Ledger.Alonzo.Binary.Cddl as Alonzo +import qualified Test.Cardano.Ledger.Babbage.Binary.Cddl as Babbage +import qualified Test.Cardano.Ledger.Conway.Binary.Cddl as Conway +import qualified Test.Cardano.Ledger.Mary.Binary.Cddl as Mary +import qualified Test.Cardano.Ledger.Shelley.Binary.Cddl as Shelley +import Test.Tasty +import Test.Util.Serialisation.CDDL (isCDDLCDisabled) + +newtype CDDLSpec = CDDLSpec {cddlSpec :: BS.ByteString} deriving Show + +-- | This function will run the provided test-tree after generating the node to +-- node cddls for Blocks and Headers. As more CDDLs are stabilized they will +-- have to be added here. Eventually we can have a datatype with one field for +-- each CDDL so that we know always what is available. +withCDDLs :: TestTree -> TestTree +withCDDLs f = + if isCDDLCDisabled + then f + else + withResource + ( do + probeTools + setupCDDLCEnv + + ntnBlock <- cddlc "cddl/node-to-node/blockfetch/block.cddl" + ntnBlock' <- fixupBlockCDDL ntnBlock + BS.writeFile "ntnblock.cddl" . cddlSpec $ ntnBlock' + + ntnHeader <- cddlc "cddl/node-to-node/chainsync/header.cddl" + BS.writeFile "ntnheader.cddl" . cddlSpec $ ntnHeader + + ntnTx <- cddlc "cddl/node-to-node/txsubmission2/tx.cddl" + ntnTx' <- fixupBlockCDDL ntnTx + BS.writeFile "ntntx.cddl" . cddlSpec $ ntnTx' + + ntnTxId <- cddlc "cddl/node-to-node/txsubmission2/txId.cddl" + BS.writeFile "ntntxid.cddl" . cddlSpec $ ntnTxId + ) + ( \() -> do + D.removeFile "ntnblock.cddl" + D.removeFile "ntnheader.cddl" + D.removeFile "ntntx.cddl" + D.removeFile "ntntxid.cddl" + ) + (\_ -> f) + +-- | The Ledger CDDL specs are not _exactly_ correct. Here we do some dirty +-- sed-replace to make them able to validate blocks. See cardano-ledger#5054. +fixupBlockCDDL :: CDDLSpec -> IO CDDLSpec +fixupBlockCDDL spec = + withTempFile "." "block-temp.cddl" $ \fp h -> do + hClose h + BS.writeFile fp . cddlSpec $ spec + -- For plutus, the type is actually `bytes`, but the distinct construct is + -- for forcing generation of different values. See cardano-ledger#5054 + sed fp ["-i", "s/\\(conway\\.distinct_VBytes = \\)/\\1 bytes ;\\//g"] + -- These 3 below are hardcoded for generation. See cardano-ledger#5054 + sed fp ["-i", "s/\\([yaoye]\\.address = \\)/\\1 bytes ;/g"] + sed fp ["-i", "s/\\(reward_account = \\)/\\1 bytes ;/g"] + sed + fp + [ "-i" + , "-z" + , "s/unit_interval = #6\\.30(\\[\\n\\s*1,\\n\\s*2,\\n\\])/unit_interval = #6.30([uint, uint])/g" + ] + + sed fp ["-i", "s/\\(chain_code: bytes\\)/\\1, ;/g"] + CDDLSpec <$> BS.readFile fp + +-- | This sets the environment variables needed for `cddlc` to run properly. +setupCDDLCEnv :: IO () +setupCDDLCEnv = do + byron <- map takePath <$> Byron.readByronCddlFileNames + shelley <- map takePath <$> Shelley.readShelleyCddlFileNames + allegra <- map takePath <$> Allegra.readAllegraCddlFileNames + mary <- map takePath <$> Mary.readMaryCddlFileNames + alonzo <- map takePath <$> Alonzo.readAlonzoCddlFileNames + babbage <- map takePath <$> Babbage.readBabbageCddlFileNames + conway <- map takePath <$> Conway.readConwayCddlFileNames + + localDataDir <- windowsPathHack <$> getDataDir + let local_paths = + map + (localDataDir F.) + ["cddl"] -- Directories with other cddls that we import should go here + include_path = + mconcat $ + L.intersperse ":" $ + map (mconcat . L.intersperse ":") [byron, shelley, allegra, mary, alonzo, babbage, conway] + <> local_paths + + E.setEnv "CDDL_INCLUDE_PATH" (include_path <> ":") + +-- | Call @sed@ on the given file with the given args +sed :: FilePath -> [String] -> IO () +sed fp args = + Monad.void $ P.readProcessWithExitCode "sed" (args ++ [fp]) mempty + +{- FOURMOLU_DISABLE -} + +cddlc :: FilePath -> IO CDDLSpec +cddlc dataFile = do + putStrLn $ "Generating: " <> dataFile + path <- getDataFileName dataFile + (_, BSL.toStrict -> cddl, BSL.toStrict -> err) <- +#ifdef mingw32_HOST_OS + -- we cannot call @cddlc@ directly because it is not an executable in + -- Haskell eyes, but we can call @ruby@ and pass the @cddlc@ script path as + -- an argument + do + prefix <- E.getEnv "MSYSTEM_PREFIX" + P.readProcessWithExitCode "ruby" [prefix F. "bin/cddlc", "-u", "-2", "-t", "cddl", path] mempty +#else + P.readProcessWithExitCode "cddlc" ["-u", "-2", "-t", "cddl", path] mempty +#endif + Monad.unless (BS.null err) $ red $ BS8.unpack err + return $ CDDLSpec cddl + where + red s = putStrLn $ "\ESC[31m" <> s <> "\ESC[0m" + +-- | @cddlc@ is not capable of using backlashes +-- +-- @cddlc@ mixes @C:@ with the separator in @CDDL_INCLUDE_PATH@, and it +-- doesn't understand @;@ as a separator. It works if we remove @C:@ and we +-- are running in the same drive as the cddl files. +windowsPathHack :: FilePath -> FilePath +windowsPathHack x = +#ifdef mingw32_HOST_OS + let f = [ if c /= '\\' then c else '/' | c <- x ] + in if "C:" `L.isPrefixOf` f + then drop 2 f + else f +#else + x +#endif + +takePath :: FilePath -> FilePath +takePath = windowsPathHack . F.takeDirectory + +probeTools :: IO () +probeTools = do + putStrLn "Probing tools:" +#ifdef mingw32_HOST_OS + -- On Windows, the cddl and cddlc files are POSIX scripts and therefore not + -- recognized as executables by @findExecutable@, so we need to do some dirty + -- tricks here. We check that ruby executable exists and then that there are + -- cddl and cddlc files in the binary folder of the MSYS2 installation. + putStr "- ruby " + rubyExe <- D.findExecutable "ruby" + if (isNothing rubyExe) + then do + putStrLn "not found!\nPlease install ruby" + exitFailure + else + putStrLn "found" + + putStr "- cddlc " + cddlcExe <- D.doesFileExist . (F. "bin/cddlc") =<< E.getEnv "MSYSTEM_PREFIX" + if cddlcExe + then putStrLn "found" + else do + putStrLn "not found!\nPlease install the `cddlc` ruby gem" + exitFailure + pure () +#else + posixProbeTool "cddlc" "install the `cddlc` ruby gem" + where + posixProbeTool :: String -> String -> IO () + posixProbeTool tool suggestion = do + putStr $ "- " <> tool <> " " + exe <- D.findExecutable tool + if isNothing exe + then do + putStrLn "not found!" + putStrLn $ "Please " <> suggestion + exitFailure + else + putStrLn "found" +#endif + +{- FOURMOLU_ENABLE -} diff --git a/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/Golden.hs b/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/Golden.hs index 500b24a4ab..0d2267eb21 100644 --- a/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/Golden.hs +++ b/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/Golden.hs @@ -1,5 +1,6 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE TypeFamilies #-} {-# OPTIONS_GHC -Wno-orphans #-} @@ -16,10 +17,22 @@ import System.FilePath (()) import Test.Consensus.Cardano.Examples import Test.Tasty import Test.Util.Paths +import Test.Util.Serialisation.CDDL import Test.Util.Serialisation.Golden tests :: TestTree -tests = goldenTest_all codecConfig ($(getGoldenDir) "cardano") examples +tests = + goldenTest_all + codecConfig + ($(getGoldenDir) "cardano") + ( Just $ + CDDLsForNodeToNode + ("ntnblock.cddl", "serialisedCardanoBlock") + ("ntnheader.cddl", "header") + ("ntntx.cddl", "tx") + ("ntntxid.cddl", "txId") + ) + examples instance CardanoHardForkConstraints c => diff --git a/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/Serialisation.hs b/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/Serialisation.hs index fa104caa50..857b5cc9c6 100644 --- a/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/Serialisation.hs +++ b/ouroboros-consensus-cardano/test/cardano-test/Test/Consensus/Cardano/Serialisation.hs @@ -1,6 +1,7 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} module Test.Consensus.Cardano.Serialisation (tests) where @@ -21,7 +22,6 @@ import Ouroboros.Network.Block (Serialised (..)) import Test.Consensus.Byron.Generators (epochSlots) import qualified Test.Consensus.Cardano.Examples as Cardano.Examples import Test.Consensus.Cardano.Generators () -import Test.Consensus.Cardano.MockCrypto (MockCryptoCompatByron) import Test.Tasty import Test.Tasty.QuickCheck (Property, testProperty, (===)) import Test.Util.Orphans.Arbitrary () @@ -33,7 +33,21 @@ tests = "Cardano" [ testGroup "Examples roundtrip" $ examplesRoundtrip Cardano.Examples.codecConfig Cardano.Examples.examples - , roundtrip_all_skipping result testCodecCfg dictNestedHdr + , roundtrip_all_skipping + result + testCodecCfg + dictNestedHdr + -- We would want to use this instead, but the generated blocks + -- do not quite validate yet or sometimes they are not + -- entirely coherent, so for now this is commented out. + -- + -- It is also the case that some (conway in particular) blocks take a + -- very long time to validate or consume too much memory. + -- + -- ( Just $ + -- CDDLsForNodeToNode ("ntnblock.cddl", "serialisedCardanoBlock") ("ntnheader.cddl", "header") + -- ) + Nothing , testProperty "BinaryBlockInfo sanity check" prop_CardanoBinaryBlockInfo ] where @@ -41,7 +55,7 @@ tests = result "roundtrip Result" = DoNotCheckCBORValidity result _ = CheckCBORValidity -testCodecCfg :: CardanoCodecConfig MockCryptoCompatByron +testCodecCfg :: CardanoCodecConfig StandardCrypto testCodecCfg = CardanoCodecConfig (ByronCodecConfig epochSlots) @@ -54,7 +68,7 @@ testCodecCfg = dictNestedHdr :: forall a. - NestedCtxt_ (CardanoBlock MockCryptoCompatByron) Header a -> + NestedCtxt_ (CardanoBlock StandardCrypto) Header a -> Dict (Eq a, Show a) dictNestedHdr = \case NCZ (CtxtByronBoundary{}) -> Dict @@ -70,7 +84,7 @@ dictNestedHdr = \case BinaryBlockInfo -------------------------------------------------------------------------------} -prop_CardanoBinaryBlockInfo :: CardanoBlock MockCryptoCompatByron -> Property +prop_CardanoBinaryBlockInfo :: CardanoBlock StandardCrypto -> Property prop_CardanoBinaryBlockInfo blk = encodedNestedHeader === extractedHeader where diff --git a/ouroboros-consensus-cardano/test/shelley-test/Test/Consensus/Shelley/Golden.hs b/ouroboros-consensus-cardano/test/shelley-test/Test/Consensus/Shelley/Golden.hs index acfab846ec..f3fb94e6c3 100644 --- a/ouroboros-consensus-cardano/test/shelley-test/Test/Consensus/Shelley/Golden.hs +++ b/ouroboros-consensus-cardano/test/shelley-test/Test/Consensus/Shelley/Golden.hs @@ -17,7 +17,7 @@ import Test.Util.Paths import Test.Util.Serialisation.Golden tests :: TestTree -tests = goldenTest_all codecConfig ($(getGoldenDir) "shelley") examplesShelley +tests = goldenTest_all codecConfig ($(getGoldenDir) "shelley") Nothing examplesShelley instance ToGoldenDirectory ShelleyNodeToNodeVersion diff --git a/ouroboros-consensus-cardano/test/shelley-test/Test/Consensus/Shelley/Serialisation.hs b/ouroboros-consensus-cardano/test/shelley-test/Test/Consensus/Shelley/Serialisation.hs index 4761357c58..b0aa8a53eb 100644 --- a/ouroboros-consensus-cardano/test/shelley-test/Test/Consensus/Shelley/Serialisation.hs +++ b/ouroboros-consensus-cardano/test/shelley-test/Test/Consensus/Shelley/Serialisation.hs @@ -31,7 +31,7 @@ tests :: TestTree tests = testGroup "Shelley" - [ roundtrip_all testCodecCfg dictNestedHdr + [ roundtrip_all testCodecCfg dictNestedHdr Nothing , -- Test for real crypto too testProperty "hashSize real crypto" $ prop_hashSize pReal , testProperty "ConvertRawHash real crypto" $ roundtrip_ConvertRawHash pReal diff --git a/ouroboros-consensus-diffusion/test/mock-test/Test/ThreadNet/BFT.hs b/ouroboros-consensus-diffusion/test/mock-test/Test/ThreadNet/BFT.hs index 26a34a87a1..3f82ec83e3 100644 --- a/ouroboros-consensus-diffusion/test/mock-test/Test/ThreadNet/BFT.hs +++ b/ouroboros-consensus-diffusion/test/mock-test/Test/ThreadNet/BFT.hs @@ -55,7 +55,7 @@ instance Arbitrary TestSetup where tests :: TestTree tests = testGroup "BFT" $ - [ roundtrip_all SimpleCodecConfig dictNestedHdr + [ roundtrip_all SimpleCodecConfig dictNestedHdr Nothing , testProperty "simple convergence" $ \setup -> prop_simple_bft_convergence setup ] diff --git a/ouroboros-consensus/ouroboros-consensus.cabal b/ouroboros-consensus/ouroboros-consensus.cabal index e7486dd37c..cbfffc1026 100644 --- a/ouroboros-consensus/ouroboros-consensus.cabal +++ b/ouroboros-consensus/ouroboros-consensus.cabal @@ -407,6 +407,7 @@ library unstable-consensus-testlib Test.Util.SOP Test.Util.SanityCheck Test.Util.Schedule + Test.Util.Serialisation.CDDL Test.Util.Serialisation.Examples Test.Util.Serialisation.Golden Test.Util.Serialisation.Roundtrip @@ -458,6 +459,7 @@ library unstable-consensus-testlib ouroboros-network-api, ouroboros-network-mock, pretty-simple, + process, quickcheck-instances, quickcheck-state-machine:no-vendored-treediff ^>=0.10, quiet, @@ -476,6 +478,7 @@ library unstable-consensus-testlib tasty-quickcheck >=0.11, tasty-rerun, template-haskell, + temporary, text, time, transformers-base, diff --git a/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/CDDL.hs b/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/CDDL.hs new file mode 100644 index 0000000000..cf36cb5786 --- /dev/null +++ b/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/CDDL.hs @@ -0,0 +1,63 @@ +{-# LANGUAGE LambdaCase #-} + +module Test.Util.Serialisation.CDDL + ( cddlTestCase + , cddlTest + , isCDDLCDisabled + , CDDLsForNodeToNode (..) + ) where + +import qualified Data.ByteString as BS +import Data.Maybe (isJust) +import qualified Data.Text as T +import qualified System.Environment as E +import System.Exit +import System.IO +import System.IO.Temp +import System.IO.Unsafe (unsafePerformIO) +import System.Process +import Test.Tasty +import Test.Tasty.HUnit + +-- | Windows on Hydra cannot cross-compile CDDLC so we decided to skip the tests +-- there. +isCDDLCDisabled :: Bool +isCDDLCDisabled = isJust $ unsafePerformIO (E.lookupEnv "DISABLE_CDDLC") + +-- | A Tasty test case running the @cuddle@ +cddlTestCase :: IO BS.ByteString -> FilePath -> T.Text -> TestTree +cddlTestCase cborM cddl rule = + testCase "CDDL compliance" $ + if isCDDLCDisabled + then assertBool "Skipped" True + else + cddlTest cborM cddl rule >>= \case + Left err -> assertFailure err + Right _ -> pure () + +-- | Test the CDDL conformance of the given bytestring +cddlTest :: + IO BS.ByteString -> + String -> + T.Text -> + IO (Either String ()) +cddlTest cborM cddl rule = + withTempFile "." "testcase.cbor" $ \fp h -> do + bs <- cborM + BS.hPutStr h bs + hClose h + (code, _out, err) <- + readProcessWithExitCode "cuddle" ["validate-cbor", "-c", fp, "-r", T.unpack rule, cddl] mempty + case code of + ExitFailure _ -> do + BS.writeFile "failing.cbor" bs + pure (Left err) + ExitSuccess -> pure (Right ()) + +-- | A collection of CDDL spec and the relevant rule to use +data CDDLsForNodeToNode = CDDLsForNodeToNode + { blockCDDL :: (FilePath, T.Text) + , headerCDDL :: (FilePath, T.Text) + , txCDDL :: (FilePath, T.Text) + , txIdCDDL :: (FilePath, T.Text) + } diff --git a/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/Golden.hs b/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/Golden.hs index 80cdd39fe3..78e06a3eea 100644 --- a/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/Golden.hs +++ b/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/Golden.hs @@ -47,6 +47,7 @@ import qualified Data.ByteString.UTF8 as BS.UTF8 import Data.List (nub) import qualified Data.Map.Strict as Map import Data.Proxy (Proxy (..)) +import qualified Data.Text as T import Data.TreeDiff import GHC.Stack (HasCallStack) import Ouroboros.Consensus.Block (CodecConfig) @@ -81,6 +82,7 @@ import System.FilePath (takeDirectory, ()) import Test.Cardano.Binary.TreeDiff (CBORBytes (..)) import Test.Tasty import Test.Tasty.Golden.Advanced (goldenTest) +import Test.Util.Serialisation.CDDL import Test.Util.Serialisation.Examples (Examples (..), Labelled) import Test.Util.Serialisation.SomeResult (SomeResult (..)) @@ -99,14 +101,27 @@ goldenTestCBOR :: (a -> Encoding) -> -- | Path to the file containing the golden output FilePath -> + -- | Path to the CDDL file that defines this CBOR, and the rule name + Maybe (FilePath, T.Text) -> TestTree -goldenTestCBOR testName example enc goldenFile = - goldenTest - testName - (Strict.readFile goldenFile) - (either exceptionToByteString id <$> try (evaluate actualValue)) - diff - updateGoldenFile +goldenTestCBOR testName example enc goldenFile mCddlPath = + testGroup testName $ + [ goldenTest + "Golden == actual" + (Strict.readFile goldenFile) + (either exceptionToByteString id <$> try (evaluate actualValue)) + diff + updateGoldenFile + ] + ++ ( case mCddlPath of + Nothing -> [] + Just (cddlPath, rule) -> + [ cddlTestCase + (Strict.readFile goldenFile) + cddlPath + rule + ] + ) where -- Copied from tasty-golden because it isn't exported updateGoldenFile :: Strict.ByteString -> IO () @@ -188,18 +203,19 @@ goldenTests :: (a -> Encoding) -> -- | Folder containing the golden files FilePath -> + Maybe (FilePath, T.Text) -> TestTree -goldenTests testName examples enc goldenFolder +goldenTests testName examples enc goldenFolder mCDDL | nub labels /= labels = error $ "Examples with the same label for " <> testName | [(Nothing, example)] <- examples = -- If there's just a single unlabelled example, no need for grouping, -- which makes the output more verbose. - goldenTestCBOR testName example enc (goldenFolder testName) + goldenTestCBOR testName example enc (goldenFolder testName) mCDDL | otherwise = testGroup testName - [ goldenTestCBOR testName' example enc (goldenFolder testName') + [ goldenTestCBOR testName' example enc (goldenFolder testName') mCDDL | (mbLabel, example) <- examples , let testName' = case mbLabel of Nothing -> testName @@ -215,18 +231,19 @@ goldenTests' :: Labelled (a, a -> Encoding) -> -- | Folder containing the golden files FilePath -> + Maybe (FilePath, T.Text) -> TestTree -goldenTests' testName examples goldenFolder +goldenTests' testName examples goldenFolder mCDDL | nub labels /= labels = error $ "Examples with the same label for " <> testName | [(Nothing, (example, exampleEncoder))] <- examples = -- If there's just a single unlabelled example, no need for grouping, -- which makes the output more verbose. - goldenTestCBOR testName example exampleEncoder (goldenFolder testName) + goldenTestCBOR testName example exampleEncoder (goldenFolder testName) mCDDL | otherwise = testGroup testName - [ goldenTestCBOR testName' example exampleEncoder (goldenFolder testName') + [ goldenTestCBOR testName' example exampleEncoder (goldenFolder testName') mCDDL | (mbLabel, (example, exampleEncoder)) <- examples , let testName' = case mbLabel of Nothing -> testName @@ -276,13 +293,14 @@ goldenTest_all :: -- | Path relative to the root of the repository that contains the golden -- files FilePath -> + Maybe CDDLsForNodeToNode -> Examples blk -> TestTree -goldenTest_all codecConfig goldenDir examples = +goldenTest_all codecConfig goldenDir mCDDLs examples = testGroup "Golden tests" [ goldenTest_SerialiseDisk codecConfig goldenDir examples - , goldenTest_SerialiseNodeToNode codecConfig goldenDir examples + , goldenTest_SerialiseNodeToNode codecConfig goldenDir mCDDLs examples , goldenTest_SerialiseNodeToClient codecConfig goldenDir examples ] @@ -316,6 +334,7 @@ goldenTest_SerialiseDisk codecConfig goldenDir Examples{..} = exampleValues enc (goldenDir "disk") + Nothing testLedgerTables :: TestTree testLedgerTables = @@ -327,6 +346,7 @@ goldenTest_SerialiseDisk codecConfig goldenDir Examples{..} = exampleLedgerState ) (goldenDir "disk") + Nothing encodeExt = encodeDiskExtLedgerState codecConfig @@ -341,9 +361,10 @@ goldenTest_SerialiseNodeToNode :: ) => CodecConfig blk -> FilePath -> + Maybe CDDLsForNodeToNode -> Examples blk -> TestTree -goldenTest_SerialiseNodeToNode codecConfig goldenDir Examples{..} = +goldenTest_SerialiseNodeToNode codecConfig goldenDir mCDDLs Examples{..} = testGroup "SerialiseNodeToNode" [ testVersion version @@ -354,15 +375,15 @@ goldenTest_SerialiseNodeToNode codecConfig goldenDir Examples{..} = testVersion version = testGroup (toGoldenDirectory version) - [ test "Block" exampleBlock - , test "Header" exampleHeader - , test "SerialisedBlock" exampleSerialisedBlock - , test "SerialisedHeader" exampleSerialisedHeader - , test "GenTx" exampleGenTx - , test "GenTxId" exampleGenTxId + [ test "Block" exampleBlock $ fmap blockCDDL mCDDLs + , test "Header" exampleHeader $ fmap headerCDDL mCDDLs + , test "SerialisedBlock" exampleSerialisedBlock Nothing + , test "SerialisedHeader" exampleSerialisedHeader Nothing + , test "GenTx" exampleGenTx $ fmap txCDDL mCDDLs + , test "GenTxId" exampleGenTxId $ fmap txIdCDDL mCDDLs ] where - test :: SerialiseNodeToNode blk a => TestName -> Labelled a -> TestTree + test :: SerialiseNodeToNode blk a => TestName -> Labelled a -> Maybe (FilePath, T.Text) -> TestTree test testName exampleValues = goldenTests testName @@ -421,6 +442,7 @@ goldenTest_SerialiseNodeToClient codecConfig goldenDir Examples{..} = exampleValues enc (goldenDir toGoldenDirectory versions) + Nothing testQuery name values = test name (filter (\(_, SomeBlockQuery q) -> blockQueryIsSupportedOnVersion q blockVersion) values) diff --git a/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/Roundtrip.hs b/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/Roundtrip.hs index 6130e7f9c4..4aa0db4b38 100644 --- a/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/Roundtrip.hs +++ b/ouroboros-consensus/src/unstable-consensus-testlib/Test/Util/Serialisation/Roundtrip.hs @@ -6,6 +6,8 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuantifiedConstraints #-} {-# LANGUAGE RankNTypes #-} @@ -13,6 +15,7 @@ {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeOperators #-} {-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE ViewPatterns #-} module Test.Util.Serialisation.Roundtrip ( -- * Basic test helpers @@ -48,6 +51,7 @@ import Codec.CBOR.Write (toLazyByteString) import Codec.Serialise (decode, encode) import Control.Arrow (left) import Control.Monad (unless, when) +import qualified Data.ByteString as BS import qualified Data.ByteString.Base16.Lazy as Base16 import qualified Data.ByteString.Lazy as Lazy import qualified Data.ByteString.Lazy.Char8 as Char8 @@ -55,7 +59,8 @@ import qualified Data.ByteString.Short as Short import Data.Constraint import Data.Function (on) import Data.Maybe (fromMaybe) -import qualified Data.Text.Lazy as T +import qualified Data.Text as T +import qualified Data.Text.Lazy as TL import Data.Typeable import GHC.Generics (Generic) import Ouroboros.Consensus.Block @@ -90,9 +95,11 @@ import Ouroboros.Network.Block , mkSerialised ) import Quiet (Quiet (..)) +import Test.QuickCheck.Monadic import Test.Tasty import Test.Tasty.QuickCheck import Test.Util.Orphans.Arbitrary () +import Test.Util.Serialisation.CDDL import Test.Util.Serialisation.Examples (Examples (..), Labelled) import Test.Util.Serialisation.SomeResult (SomeResult (..)) import Test.Util.TestEnv (adjustQuickCheckTests) @@ -103,9 +110,10 @@ import Text.Pretty.Simple (pShow) ------------------------------------------------------------------------------} roundtrip :: - (Eq a, Show a) => + (Eq a, Show a, Show e) => (a -> Encoding) -> (forall s. Decoder s a) -> + (BS.ByteString -> IO (Either e ())) -> a -> Property roundtrip enc dec = roundtrip' enc (const <$> dec) @@ -114,11 +122,12 @@ roundtrip enc dec = roundtrip' enc (const <$> dec) -- -- See 'roundtripAnd' roundtrip' :: - forall a. - (Eq a, Show a) => + forall a e. + (Eq a, Show a, Show e) => -- | @enc@ (a -> Encoding) -> (forall s. Decoder s (Lazy.ByteString -> a)) -> + (BS.ByteString -> IO (Either e ())) -> a -> Property roundtrip' = roundtripAnd CheckCBORValidity @@ -141,29 +150,38 @@ data ShouldCheckCBORValidity = CheckCBORValidity | DoNotCheckCBORValidity -- might happen is if the annotation is not canonical CBOR, but @enc@ does -- produce canonical CBOR. roundtripAnd :: - forall a. - (Eq a, Show a) => + forall a e. + (Eq a, Show a, Show e) => ShouldCheckCBORValidity -> -- | @enc@ (a -> Encoding) -> (forall s. Decoder s (Lazy.ByteString -> a)) -> + (BS.ByteString -> IO (Either e ())) -> a -> Property -roundtripAnd check enc dec a = checkRoundtripResult $ do +roundtripAnd check enc dec checkCddlValid a = let enc_a = enc a bs = toLazyByteString enc_a - - when (check == CheckCBORValidity) $ - (validFlatTerm (toFlatTerm enc_a) ?! "Encoded flat term is not valid: " <> show enc_a) - (bsRem, a') <- deserialiseFromBytes dec bs `onError` showByteString bs - Lazy.null bsRem ?! "Left-over bytes: " <> toBase16 bsRem - a == a' bs ?! pShowNeq a (a' bs) + cborValid = + throwLeft $ + when (check == CheckCBORValidity) $ + validFlatTerm (toFlatTerm enc_a) ?! "Encoded flat term is not valid: " <> show enc_a + doesRoundtrip = throwLeft $ do + (bsRem, a') <- deserialiseFromBytes dec bs `onError` showByteString bs + Lazy.null bsRem ?! "Left-over bytes: " <> toBase16 bsRem + a == a' bs ?! pShowNeq a (a' bs) + cddlValid = + monadicIO $ + run (checkCddlValid $ Lazy.toStrict bs) >>= \case + Left err -> assertWith False (show err) + Right _ -> pure () + in cborValid .&&. doesRoundtrip .&&. cddlValid where (?!) :: Bool -> String -> Either String () cond ?! msg = unless cond $ Left msg infix 1 ?! - pShowNeq x y = T.unpack (pShow x) <> "\n \t/= \n" <> T.unpack (pShow y) + pShowNeq x y = TL.unpack (pShow x) <> "\n \t/= \n" <> TL.unpack (pShow y) onError :: Either DeserialiseFailure (Char8.ByteString, Char8.ByteString -> a) -> @@ -182,9 +200,9 @@ roundtripAnd check enc dec a = checkRoundtripResult $ do toBase16 :: Lazy.ByteString -> String toBase16 = Char8.unpack . Base16.encode - checkRoundtripResult :: Either String () -> Property - checkRoundtripResult (Left str) = counterexample str False - checkRoundtripResult (Right ()) = property () + throwLeft :: Either String () -> Property + throwLeft (Left str) = counterexample str False + throwLeft (Right ()) = property () roundtripComparingEncoding :: (a -> Encoding) -> @@ -241,7 +259,6 @@ roundtrip_all :: , Arbitrary' (LedgerState blk EmptyMK) , Arbitrary' (AnnTip blk) , Arbitrary' (ChainDepState (BlockProtocol blk)) - , ArbitraryWithVersion (BlockNodeToNodeVersion blk) blk , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (Coherent blk) , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (Header blk) , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (GenTx blk) @@ -259,6 +276,7 @@ roundtrip_all :: ) => CodecConfig blk -> (forall a. NestedCtxt_ blk Header a -> Dict (Eq a, Show a)) -> + Maybe CDDLsForNodeToNode -> TestTree roundtrip_all = roundtrip_all_skipping (const CheckCBORValidity) @@ -285,7 +303,6 @@ roundtrip_all_skipping :: , Arbitrary' (LedgerState blk EmptyMK) , Arbitrary' (AnnTip blk) , Arbitrary' (ChainDepState (BlockProtocol blk)) - , ArbitraryWithVersion (BlockNodeToNodeVersion blk) blk , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (Coherent blk) , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (Header blk) , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (GenTx blk) @@ -304,12 +321,13 @@ roundtrip_all_skipping :: (TestName -> ShouldCheckCBORValidity) -> CodecConfig blk -> (forall a. NestedCtxt_ blk Header a -> Dict (Eq a, Show a)) -> + Maybe CDDLsForNodeToNode -> TestTree -roundtrip_all_skipping shouldCheckCBORvalidity ccfg dictNestedHdr = +roundtrip_all_skipping shouldCheckCBORvalidity ccfg dictNestedHdr mCDDLs = testGroup "Roundtrip" [ testGroup "SerialiseDisk" $ roundtrip_SerialiseDisk ccfg dictNestedHdr - , testGroup "SerialiseNodeToNode" $ roundtrip_SerialiseNodeToNode ccfg + , testGroup "SerialiseNodeToNode" $ roundtrip_SerialiseNodeToNode ccfg mCDDLs , testGroup "SerialiseNodeToClient" $ roundtrip_SerialiseNodeToClient shouldCheckCBORvalidity @@ -336,7 +354,7 @@ roundtrip_SerialiseDisk :: [TestTree] roundtrip_SerialiseDisk ccfg dictNestedHdr = [ testProperty "roundtrip block" $ - roundtrip' @blk (encodeDisk ccfg) (decodeDisk ccfg) + roundtrip' @blk (encodeDisk ccfg) (decodeDisk ccfg) (const $ pure (Right () :: Either () ())) , testProperty "roundtrip Header" $ \hdr -> case unnest hdr of DepPair ctxt nestedHdr -> case dictNestedHdr (flipNestedCtxt ctxt) of @@ -344,6 +362,7 @@ roundtrip_SerialiseDisk ccfg dictNestedHdr = roundtrip' (encodeDiskDep ccfg ctxt) (decodeDiskDep ccfg ctxt) + (const $ pure (Right () :: Either () ())) nestedHdr , -- Since the 'LedgerState' is a large data structure, we lower the -- number of tests to avoid slowing down the testsuite too much @@ -362,6 +381,7 @@ roundtrip_SerialiseDisk ccfg dictNestedHdr = roundtrip @a (encodeDisk ccfg) (decodeDisk ccfg) + (const $ pure (Right () :: Either () ())) -- | Used to generate arbitrary values for the serialisation roundtrip tests. -- As the serialisation format can change with the version, not all arbitrary @@ -462,13 +482,17 @@ newtype Coherent a = Coherent {getCoherent :: a} deriving (Eq, Generic) deriving Show via (Quiet (Coherent a)) +instance SerialiseNodeToNode blk blk => SerialiseNodeToNode blk (Coherent blk) where + encodeNodeToNode ccfg v = encodeNodeToNode ccfg v . getCoherent + decodeNodeToNode ccfg v = Coherent <$> decodeNodeToNode ccfg v + -- TODO how can we ensure that we have a test for each constraint listed in -- 'SerialiseNodeToNodeConstraints'? roundtrip_SerialiseNodeToNode :: forall blk. ( SerialiseNodeToNodeConstraints blk , Show (BlockNodeToNodeVersion blk) - , ArbitraryWithVersion (BlockNodeToNodeVersion blk) blk + , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (Coherent blk) , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (Header blk) , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (GenTx blk) , ArbitraryWithVersion (BlockNodeToNodeVersion blk) (GenTxId blk) @@ -479,14 +503,17 @@ roundtrip_SerialiseNodeToNode :: HasNestedContent Header blk , EncodeDiskDep (NestedCtxt Header) blk , DecodeDiskDep (NestedCtxt Header) blk + , Eq blk + , Show blk ) => CodecConfig blk -> + Maybe CDDLsForNodeToNode -> [TestTree] -roundtrip_SerialiseNodeToNode ccfg = - [ rt (Proxy @blk) "blk" - , rt (Proxy @(Header blk)) "Header" - , rt (Proxy @(GenTx blk)) "GenTx" - , rt (Proxy @(GenTxId blk)) "GenTxId" +roundtrip_SerialiseNodeToNode ccfg mCDDLs = + [ rt (Proxy @(Coherent blk)) "blk" Nothing + , rt (Proxy @(Header blk)) "Header" Nothing + , rt (Proxy @(GenTx blk)) "GenTx" Nothing + , rt (Proxy @(GenTxId blk)) "GenTxId" Nothing , -- Roundtrip a @'Serialised' blk@ -- -- We generate a random @blk@, convert it to 'Serialised' (using @@ -494,10 +521,22 @@ roundtrip_SerialiseNodeToNode ccfg = -- CBOR-in-CBOR), decode that 'Serialised' and convert (using -- 'decodeNodeToNode') it to a @blk@ again. testProperty "roundtrip Serialised blk" $ - \(WithVersion version blk) -> + \(WithVersion version (getCoherent -> blk)) -> roundtrip @blk (encodeThroughSerialised (encodeDisk ccfg) (enc version)) (decodeThroughSerialised (decodeDisk ccfg) (dec version)) + ( case fmap blockCDDL mCDDLs of + Nothing -> (const $ pure (Right ())) + Just (cddl, rule) -> + ( \bs -> do + BS.writeFile "current.cbor" bs + fmap (const ()) + <$> cddlTest + (pure bs) + cddl + rule + ) + ) blk , -- Same as above but for 'Header' testProperty "roundtrip Serialised Header" $ @@ -505,22 +544,35 @@ roundtrip_SerialiseNodeToNode ccfg = roundtrip @(Header blk) (enc version . SerialisedHeaderFromDepPair . encodeDepPair ccfg . unnest) (nest <$> (decodeDepPair ccfg . serialisedHeaderToDepPair =<< dec version)) + ( case fmap headerCDDL mCDDLs of + Nothing -> (const $ pure (Right ())) + Just (cddl, rule) -> + ( \bs -> do + fmap (const ()) + <$> cddlTest + (pure bs) + cddl + rule + ) + ) hdr , -- Check the compatibility between 'encodeNodeToNode' for @'Serialised' -- blk@ and 'decodeNodeToNode' for @blk@. testProperty "roundtrip Serialised blk compat 1" $ - \(WithVersion version blk) -> + \(WithVersion version (getCoherent -> blk)) -> roundtrip @blk (encodeThroughSerialised (encodeDisk ccfg) (enc version)) (dec version) + (const $ pure (Right () :: Either () ())) blk , -- Check the compatibility between 'encodeNodeToNode' for @blk@ and -- 'decodeNodeToNode' for @'Serialised' blk@. testProperty "roundtrip Serialised blk compat 2" $ - \(WithVersion version blk) -> + \(WithVersion version (getCoherent -> blk)) -> roundtrip @blk (enc version) (decodeThroughSerialised (decodeDisk ccfg) (dec version)) + (const $ pure (Right () :: Either () ())) blk , -- Same as above but for 'Header' testProperty "roundtrip Serialised Header compat 1" $ @@ -528,12 +580,14 @@ roundtrip_SerialiseNodeToNode ccfg = roundtrip @(Header blk) (enc version . SerialisedHeaderFromDepPair . encodeDepPair ccfg . unnest) (dec version) + (const $ pure (Right () :: Either () ())) hdr , testProperty "roundtrip Serialised Header compat 2" $ \(WithVersion version hdr) -> roundtrip @(Header blk) (enc version) (nest <$> (decodeDepPair ccfg . serialisedHeaderToDepPair =<< dec version)) + (const $ pure (Right () :: Either () ())) hdr ] where @@ -554,10 +608,22 @@ roundtrip_SerialiseNodeToNode ccfg = , Show a , SerialiseNodeToNode blk a ) => - Proxy a -> String -> TestTree - rt _ name = - testProperty ("roundtrip " <> name) $ \(WithVersion version x) -> - roundtrip @a (enc version) (dec version) x + Proxy a -> String -> Maybe (FilePath, T.Text) -> TestTree + rt _ name mCDDL = + testProperty ("roundtrip " <> name) $ \(WithVersion version x) -> do + roundtrip @a + (enc version) + (dec version) + ( case mCDDL of + Nothing -> const $ pure $ Right () + Just (cddl, rule) -> \bs -> + fmap (const ()) + <$> cddlTest + (pure bs) + cddl + rule + ) + x -- TODO how can we ensure that we have a test for each constraint listed in -- 'SerialiseNodeToClientConstraints'? @@ -616,6 +682,7 @@ roundtrip_SerialiseNodeToClient shouldCheckCBORvalidity ccfg = (shouldCheckCBORvalidity testLabel) (encodeThroughSerialised (encodeDisk ccfg) (enc version)) (const <$> decodeThroughSerialised (decodeDisk ccfg) (dec version)) + (const $ pure (Right () :: Either () ())) blk , -- See roundtrip_SerialiseNodeToNode for more info let testLabel = "roundtrip Serialised blk compat" @@ -625,6 +692,7 @@ roundtrip_SerialiseNodeToClient shouldCheckCBORvalidity ccfg = (shouldCheckCBORvalidity testLabel) (encodeThroughSerialised (encodeDisk ccfg) (enc version)) (const <$> dec version) + (const $ pure (Right () :: Either () ())) blk , let testLabel = "roundtrip Result" in testProperty testLabel $ @@ -633,6 +701,7 @@ roundtrip_SerialiseNodeToClient shouldCheckCBORvalidity ccfg = (shouldCheckCBORvalidity testLabel) (encodeBlockQueryResult ccfg version query) (const <$> decodeBlockQueryResult ccfg version query) + (const $ pure (Right () :: Either () ())) result ] where @@ -674,6 +743,7 @@ roundtrip_SerialiseNodeToClient shouldCheckCBORvalidity ccfg = (shouldCheckCBORvalidity testLabel) (enc' version) (const <$> dec' version) + (const $ pure (Right () :: Either () ())) a where testLabel = "roundtrip " <> name @@ -697,6 +767,7 @@ roundtrip_envelopes ccfg (WithVersion v (SomeSecond ctxt)) = roundtrip (encodeNodeToNode ccfg v . unBase16) (Base16 <$> decodeNodeToNode ccfg v) + (const (pure (Right () :: Either () ()))) (Base16 serialisedHeader) where serialisedHeader :: SerialisedHeader blk @@ -873,4 +944,4 @@ examplesRoundtrip codecConfig examples = mkTest exampleName example = testProperty (fromMaybe "" exampleName) $ once $ - roundtrip' enc dec example + roundtrip' enc dec (const $ pure (Right () :: Either () ())) example diff --git a/scripts/cbor/unwrap24serialised.hs b/scripts/cbor/unwrap24serialised.hs new file mode 100644 index 0000000000..b6c62ff718 --- /dev/null +++ b/scripts/cbor/unwrap24serialised.hs @@ -0,0 +1,26 @@ +{- cabal: + build-depends: cborg, bytestring, base +-} + +-- | A simple script that unwraps a CBOR term serialized as +-- CBOR-in-CBOR. It gets input from stdin and emits on stdout. +-- +-- > cat pre.cbor | cabal run ./scripts/unwrap24serialised.hs > post.cbor +module Main where + +import Prelude hiding (interact) +import Data.ByteString +import Codec.CBOR.Term +import Codec.CBOR.Write +import Codec.CBOR.Read +import Data.ByteString.Lazy (fromStrict, toStrict) + +main = interact $ + toStrict + . toLazyByteString + . encodeTerm + . (\(Right (_, t)) -> t) + . deserialiseFromBytes decodeTerm + . (\(Right (_, TTagged _ (TBytes t))) -> fromStrict t) + . deserialiseFromBytes decodeTerm + . fromStrict diff --git a/scripts/ci/diff-cddls.sh b/scripts/ci/diff-cddls.sh new file mode 100755 index 0000000000..094e092baa --- /dev/null +++ b/scripts/ci/diff-cddls.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +git clone https://github.com/cardano-scaling/cardano-blueprint + +check_diff () { + diff "ouroboros-consensus-cardano/cddl/node-to-node/$1" "cardano-blueprint/src/network/node-to-node/$1" + if [ $? -ne 0 ] + then + echo "::warning ouroboros-consensus-cardano/cddl/node-to-node/$1 differs from cardano-blueprint/src/network/node-to-node/$1" + else + echo "$1 OK" + fi +} + +check_diff "blockfetch/block.cddl" +check_diff "chainsync/header.cddl" +check_diff "txsubmission2/txId.cddl" +check_diff "txsubmission2/tx.cddl"