Merge #532: Fix CI. Move bitcoind tests to inner crate

8a07904 Add int tests to workflow (sanket1729)
4311760 Move bitcoind tests to inner crate (sanket1729)

Pull request description:

  This will allow us to have separate test dependacies and test CI only on
  stable. Ideally, we would want to have to a conditional dev dependancy
  but that is not possible currently

  Everytime `bitcoind` breaks with MSRV and we don't really need int tests with MSRV.
  Another issue is that testing this MSRV(2 years old) becomes a part of CI. When backporting fixes, we need to take care of making it pass. We need to support MSRV of old release(which is 2 + 2 = 4 years old). While our main code passes, the CI fails because we cannot build old bitcoind and we waste time figuring old versions.

ACKs for top commit:
  apoelstra:
    ACK 8a07904

Tree-SHA512: 869c938600a283be6ba9d17f1244343a018f670b75eb7e581ec1f1ff96a22ea004a58d1332c3187c5622645a4576ec2fd37701660dab51028a838278fda27a19

Author: apoelstra

.github/workflows/rust.yml

@@ -47,6 +47,23 @@ jobs:
           DO_FMT: true
         run: ./contrib/test.sh
 
+  Int-tests:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Crate
+        uses: actions/checkout@v2
+      - name: Checkout Toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - name: Running integration tests
+        env:
+          DO_BITCOIND_TESTS: true
+        run: ./contrib/test.sh
+
   Tests:
     name: Tests
     runs-on: ubuntu-latest

Cargo.toml

@@ -30,10 +30,10 @@ hashbrown = { version = "0.11", optional = true }
 actual-serde = { package = "serde", version = "1.0", optional = true }
 
 [dev-dependencies]
-bitcoind = { version = "0.27.0", features=["23_0"] }
-actual-rand = { package = "rand", version = "0.8.4"}
-secp256k1 = {version = "0.24.0", features = ["rand-std"]}
 serde_test = "1.0.147"
+bitcoin = { version = "0.29.2", features = ["base64"] }
+secp256k1 = {version = "0.24.0", features = ["rand-std"]}
+actual-base64 = { package = "base64", version = "0.13.0" }
 
 [[example]]
 name = "htlc"
@@ -65,4 +65,4 @@ required-features = ["compiler","std"]
 
 [[example]]
 name = "psbt_sign_finalize"
-required-features = ["std"]
+required-features = ["std", "base64"]

bitcoind-tests/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "bitcoind-tests"
+version = "0.1.0"
+authors = ["sanket1729 <[email protected]>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+miniscript = {path = "../"}
+bitcoind = { version = "0.29.3", features=["23_0"] }
+actual-rand = { package = "rand", version = "0.8.4"}
+secp256k1 = {version = "0.24.0", features = ["rand-std"]}

bitcoind-tests/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

tests/data/random_ms.txt renamed to bitcoind-tests/tests/data/random_ms.txt

@@ -19,6 +19,7 @@
 
 use std::str::FromStr;
 
+use miniscript::bitcoin;
 use actual_rand as rand;
 use bitcoin::hashes::hex::ToHex;
 use bitcoin::hashes::{hash160, ripemd160, sha256, Hash};
@@ -29,6 +30,7 @@ use miniscript::{
     Translator,
 };
 use rand::RngCore;
+
 #[derive(Clone, Debug)]
 pub struct PubData {
     pub pks: Vec<bitcoin::PublicKey>,

tests/setup/mod.rs renamed to bitcoind-tests/tests/setup/mod.rs

@@ -13,8 +13,9 @@ use bitcoin::hashes::{sha256d, Hash};
 use bitcoin::secp256k1::{self, Secp256k1};
 use bitcoin::util::psbt;
 use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
-use bitcoin::{self, Amount, LockTime, OutPoint, Sequence, Transaction, TxIn, TxOut, Txid};
+use bitcoin::{Amount, LockTime, OutPoint, Sequence, Transaction, TxIn, TxOut, Txid};
 use bitcoind::bitcoincore_rpc::{json, Client, RpcApi};
+use miniscript::bitcoin;
 use miniscript::psbt::PsbtExt;
 use miniscript::Descriptor;
 

tests/setup/test_util.rs renamed to bitcoind-tests/tests/setup/test_util.rs

@@ -7,6 +7,7 @@
 use std::collections::BTreeMap;
 use std::{error, fmt};
 
+use miniscript::bitcoin;
 use actual_rand as rand;
 use bitcoin::blockdata::witness::Witness;
 use bitcoin::hashes::{sha256d, Hash};
@@ -15,7 +16,7 @@ use bitcoin::util::sighash::SighashCache;
 use bitcoin::util::taproot::{LeafVersion, TapLeafHash};
 use bitcoin::util::{psbt, sighash};
 use bitcoin::{
-    self, secp256k1, Amount, LockTime, OutPoint, SchnorrSig, Script, Sequence, Transaction, TxIn,
+    secp256k1, Amount, LockTime, OutPoint, SchnorrSig, Script, Sequence, Transaction, TxIn,
     TxOut, Txid,
 };
 use bitcoind::bitcoincore_rpc::{json, Client, RpcApi};

tests/test_cpp.rs renamed to bitcoind-tests/tests/test_cpp.rs

@@ -4,33 +4,26 @@ set -ex
 
 FEATURES="compiler serde rand base64"
 
-cargo update -p serde --precise 1.0.142
-cargo update -p serde_derive --precise 1.0.142
-
 cargo --version
 rustc --version
 
+# Format if told to
+if [ "$DO_FMT" = true ]
+then
+    rustup component add rustfmt
+    cargo fmt -- --check
+fi
+
 # Pin dependencies required to build with Rust 1.41.1
 if cargo --version | grep "1\.41\.0"; then
-    cargo update -p url --precise 2.2.2
-    cargo update -p form_urlencoded --precise 1.0.1
     cargo update -p once_cell --precise 1.13.1
-    cargo update -p bzip2 --precise 0.4.2
-    cargo update -p which --precise 4.3.0
 fi
 
 # Pin dependencies required to build with Rust 1.47.0
 if cargo --version | grep "1\.47\.0"; then
     cargo update -p once_cell --precise 1.13.1
 fi
 
-# Format if told to
-if [ "$DO_FMT" = true ]
-then
-    rustup component add rustfmt
-    cargo fmt -- --check
-fi
-
 # Fuzz if told to
 if [ "$DO_FUZZ" = true ]
 then
@@ -42,6 +35,15 @@ then
     exit 0
 fi
 
+# Test bitcoind integration tests if told to (this only works with the stable toolchain)
+if [ "$DO_BITCOIND_TESTS" = true ]; then
+    cd bitcoind-tests
+    cargo test --verbose
+
+    # Exit integration tests, do not run other tests.
+    exit 0
+fi
+
 # Defaults / sanity checks
 cargo test
 
@@ -65,7 +67,7 @@ then
     cargo run --example psbt
     cargo run --example xpub_descriptors
     cargo run --example taproot --features=compiler
-    cargo run --example psbt_sign_finalize
+    cargo run --example psbt_sign_finalize --features=base64
 fi
 
 if [ "$DO_NO_STD" = true ]