Merge branch 'master' into unused_format

Author: jovfer

Jenkinsfile.ci

@@ -234,6 +234,7 @@ def linuxTesting(file, env_name, network_name) {
 
         sh "cp libindy/target/debug/libindy.so wrappers/java/lib"
         sh "cp libindy/target/debug/libindy.so wrappers/python"
+        sh "cp libindy/target/debug/libindy.so wrappers/nodejs"
         sh "cp libindy/target/debug/libindy.so cli"
         sh "cp libindy/target/debug/libindy.so libnullpay"
 
@@ -250,6 +251,7 @@ def linuxTesting(file, env_name, network_name) {
 
         stash includes: 'wrappers/java/lib/libindy.so', name: "LibindyJavaSO${env_name}"
         stash includes: 'wrappers/python/libindy.so', name: "LibindyPythonSO${env_name}"
+        stash includes: 'wrappers/nodejs/libindy.so', name: "LibindyNodejsSO${env_name}"
         stash includes: 'cli/libindy.so', name: "LibindyCliSO${env_name}"
         stash includes: 'cli/libnullpay.so', name: "LibnullpayCliSO${env_name}"
 
@@ -282,6 +284,7 @@ def linuxTesting(file, env_name, network_name) {
                 },
                 "${env_name}-java-test"   : { linuxModuleTesting(file, env_name, network_name, this.&linuxJavaTesting) },
                 "${env_name}-python-test" : { linuxModuleTesting(file, env_name, network_name, this.&linuxPythonTesting) },
+                "${env_name}-nodejs-test" : { linuxModuleTesting(file, env_name, network_name, this.&linuxNodejsTesting) },
                 "${env_name}-cli-test"    : { linuxModuleTesting(file, env_name, network_name, this.&linuxCLITesting) }
         ])
     }
@@ -338,6 +341,20 @@ def linuxPythonTesting(env_name, network_name, testEnv) {
     }
 }
 
+def linuxNodejsTesting(env_name, network_name, testEnv) {
+    unstash name: "LibindyNodejsSO${env_name}"
+    testEnv.inside("--network=${network_name}") {
+        echo "${env_name} Libindy Test: Test nodejs wrapper"
+
+        sh '''
+                cd wrappers/nodejs
+                npm run prepare
+                npm install
+                LD_LIBRARY_PATH=./:${LD_LIBRARY_PATH} RUST_LOG=trace TEST_POOL_IP=10.0.0.2 npm test
+            '''
+    }
+}
+
 def linuxCLITesting(env_name, network_name, testEnv) {
     if (env_name == "RedHat"){ // TODO: Delete it IS-702
         return;

cli/README.md

@@ -32,7 +32,7 @@ by beginning the line with a `#`.
 
 ### Getting help
 The most simple way is just start cli by `indy-cli` command and put `help` command. Also you can look to
-[Indy CLI Design](../doc/cli-design.md) doc that contains the list of commands and architecture overview.
+[Indy CLI Design](../doc/design/001-cli) doc that contains the list of commands and architecture overview.
 
 ### Old python-based CLI migration
 It is possible to import did's stored in the wallet of deprecated python-based CLI tool.

cli/src/commands/wallet.rs

@@ -161,7 +161,7 @@ pub mod list_command {
                 print_list_table(&wallets,
                                  &vec![("name", "Name"),
                                        ("pool_name", "Associated pool name"),
-                                       ("xtype", "Type")],
+                                       ("type", "Type")],
                                  "There are no wallets");
 
                 if let Some((_, cur_wallet)) = get_opened_wallet(ctx) {
@@ -424,7 +424,7 @@ pub mod tests {
             {
                 let cmd = list_command::new();
                 let params = CommandParams::new();
-                cmd.execute(&ctx, &params).unwrap_err();
+                cmd.execute(&ctx, &params).unwrap();
             }
             TestUtils::cleanup_storage();
         }

doc/design/008-state-proof-pluggable-parsing/README.md

@@ -0,0 +1,133 @@
+## Legend
+There are some types of requests to Nodes in the Pool allowing to use StateProof optimization in Client-Node communication.
+Instead of sending requests to all nodes in the Pool, a client can send a request to a single Node and expect StateProof signed by BLS multi-signature.
+
+BLS multi-signature (BLS MS) guaranties that there was a consensus of Nodes which signed some State identified by State RootHash.
+StateProof (SP) is small amount of data which allows to verify particular values against RootHash.
+Combination of BLS MS and SP allows clients to be sure about response of single node is a part of State signed by enough Nodes. 
+
+## Goals
+Libindy allows to extend building of supported requests via plugged interface and send them.
+It is nice to have a way to support BLS MS and SP verification for these plugged transactions.
+
+Implementation of math for SP verification is a bit complicate for plugin logic.
+Therefore libindy should perform all math calculation inside.
+A plugin should provide handler to parse custom reply to fixed data structure.
+
+## API
+The signature of the handler is described below together with custom `free` call to deallocate result data.
+
+```rust
+extern fn CustomTransactionParser(reply_from_node: *const c_char, parsed_sp: *mut *const c_char) -> ErrorCode;
+extern fn CustomFree(data: *mut c_char) -> ErrorCode;
+``` 
+
+Libindy API will contain call to register handler for specific transaction type:
+```rust
+extern fn indy_register_transaction_parser_for_sp(command_handle: i32,
+                                                  pool_handle: i32,
+                                                  txn_type: *const c_char,
+                                                  parser: CustomTransactionParser,
+                                                  free: CustomFree,
+                                                  cb: extern fn(command_handle_: i32, err: ErrorCode)) -> ErrorCode;
+```
+
+### Parsed Data structure
+
+A plugin should parse `reply_from_node` and return back to libindy parsed data as JSON string.
+Actually this data is array of entities each of them is describe SP Trie and set of key-value pairs to verify against this trie.
+It can be represented as `Vec<ParsedSP>` serialized to JSON.
+
+
+```rust
+/**
+ Single item to verification:
+ - SP Trie with RootHash
+ - BLS MS
+ - set of key-value to verify
+*/
+struct ParsedSP {
+    /// encoded SP Trie transferred from Node to Client
+    proof_nodes: String,
+    /// RootHash of the Trie, start point for verification. Should be same with appropriate filed in BLS MS data
+    root_hash: String,
+    /// entities to verification against current SP Trie
+    kvs_to_verify: KeyValuesInSP,
+    /// BLS MS data for verification
+    multi_signature: serde_json::Value,
+}
+
+/**
+ Variants of representation for items to verify against SP Trie
+ Right now 2 options are specified:
+ - simple array of key-value pair
+ - whole subtrie
+*/
+enum KeyValuesInSP {
+    Simple(KeyValueSimpleData),
+    SubTrie(KeyValuesSubTrieData),
+}
+
+/**
+ Simple variant of `KeyValuesInSP`.
+
+ All required data already present in parent SP Trie (built from `proof_nodes`).
+ `kvs` can be verified directly in parent trie
+*/
+struct KeyValueSimpleData {
+    kvs: Vec<(String /* b64-encoded key */, Option<String /* val */>)>
+}
+
+/**
+ Subtrie variant of `KeyValuesInSP`.
+
+ In this case Client (libindy) should construct subtrie and append it into trie based on `proof_nodes`.
+ After this preparation each kv pair can be checked.
+*/
+struct KeyValuesSubTrieData {
+    /// base64-encoded common prefix of each pair in `kvs`. Should be used to correct merging initial trie and subtrie
+    sub_trie_prefix: Option<String>,
+    kvs: Vec<(String /* b64-encoded key_suffix */, Option<String /* val */>)>,
+}
+```
+
+Expected libindy and plugin workflow is the following:
+1. Libindy receive reply from a Node, perform initial processing and pass raw reply to plugin.
+1. Plugin parse reply from the Node and specify one or more SP Trie with metadata and items for verification.
+1. Each SP Trie described by plugin as `ParsedSP`:
+    1. Set of encoded nodes of the SP Trie, received from Node - `proof_nodes`. May be fetched from response "as is".
+    1. RootHash of this Trie. May be fetched from the response "as is" also.
+    1. BLS MS data. Again may be fetched from the response "as is".
+    1. Key-value items to verification. Here plugin should define correct keys (path in the trie) and corresponded values.
+1. Plugin return serialized as JSON array of `ParsedSP`
+1. For each `ParsedSP` libindy:
+    1. build base trie from `proof_nodes`
+    1. if items to verify is `SubTrie`, construct this subtrie from (key-suffix, value) pairs and merge it with trie from clause above
+    1. iterate other key-value pairs and verify that trie (with signed `root_hash`) contains `value` at specified `key`
+    1. verify multi-signature
+1. If any verification is failed, libindy ignore particular SP + BLS MS and try to request same data from another node,
+or collect consensus of same replies from enough count of Nodes.
+
+
+Below is JSON structure for `Simple` case.
+```json
+[
+ {
+   "proof_nodes": "string with serialized SP tree",
+   "root_hash": "string with root hash",
+   "kvs_to_verify": {
+     "type": "simple",
+     "kvs": [["key1", "value1"], ["key2", "value2"]]
+   },
+   "multi_signature": "JSON object from Node`s reply as is"
+ }
+]
+```
+
+### Simple and SubTrie verification
+
+Some use cases require verification multiply pairs of key-value in one Trie.
+Moreover there is possible situation when client would like to verify whole subtrie.
+In this case, the amount of data transferred from Node to Client can be significantly reduced.
+Instead of including all nodes for SP verification to `proof_nodes`, Node can include only prefix path down to subtrie.
+The entire subtrie to verification can be restored on Client side from key-value pairs and combined with prefix part.

libindy/ci/amazon.dockerfile

@@ -17,6 +17,10 @@ RUN \
            libsodium-devel \
            spectool
 
+# install nodejs and npm
+RUN curl --silent --location https://rpm.nodesource.com/setup_8.x | bash -
+RUN yum -y install nodejs
+
 RUN cd /tmp && \
    curl https://download.libsodium.org/libsodium/releases/libsodium-1.0.14.tar.gz | tar -xz && \
     cd /tmp/libsodium-1.0.14 && \

libindy/ci/ubuntu.dockerfile

@@ -23,6 +23,10 @@ RUN apt-get update && \
       libncursesw5-dev \
       libzmq3-dev
 
+# install nodejs and npm
+RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
+RUN apt-get install -y nodejs
+
 RUN pip3 install -U \
 	pip \
 	setuptools \
@@ -49,4 +53,4 @@ USER indy
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.26.0
 ENV PATH /home/indy/.cargo/bin:$PATH
 
-WORKDIR /home/indy
+WORKDIR /home/indy

libindy/src/domain/anoncreds/mod.rs

@@ -14,4 +14,4 @@ pub mod revocation_state;
 pub mod schema;
 pub mod master_secret;
 
-pub const DELIMITER: &'static str = ":";
+pub const DELIMITER: char = ':';

libindy/src/errors/pool.rs

@@ -16,7 +16,7 @@ pub enum PoolError {
     Terminate,
     Timeout,
     AlreadyExists(String),
-    CommonError(CommonError)
+    CommonError(CommonError),
 }
 
 impl fmt::Display for PoolError {
@@ -27,7 +27,7 @@ impl fmt::Display for PoolError {
             PoolError::Terminate => write!(f, "Pool work terminated"),
             PoolError::Timeout => write!(f, "Timeout"),
             PoolError::AlreadyExists(ref description) => write!(f, "Pool ledger config already exists {}", description),
-            PoolError::CommonError(ref err) => err.fmt(f)
+            PoolError::CommonError(ref err) => err.fmt(f),
         }
     }
 }
@@ -40,7 +40,7 @@ impl error::Error for PoolError {
             PoolError::Terminate => "Pool work terminated",
             PoolError::Timeout => "Timeout",
             PoolError::AlreadyExists(ref description) => description,
-            PoolError::CommonError(ref err) => err.description()
+            PoolError::CommonError(ref err) => err.description(),
         }
     }
 
@@ -49,7 +49,7 @@ impl error::Error for PoolError {
             PoolError::NotCreated(_) | PoolError::InvalidHandle(_) => None,
             PoolError::Terminate | PoolError::Timeout => None,
             PoolError::AlreadyExists(_) => None,
-            PoolError::CommonError(ref err) => Some(err)
+            PoolError::CommonError(ref err) => Some(err),
         }
     }
 }

libindy/src/services/crypto/mod.rs

@@ -142,8 +142,8 @@ impl CryptoService {
     pub fn sign(&self, my_key: &Key, doc: &[u8]) -> Result<Vec<u8>, CryptoError> {
         trace!("sign >>> my_key: {:?}, doc: {:?}", my_key, doc);
 
-        let crypto_type_name = if my_key.verkey.contains(":") {
-            let splits: Vec<&str> = my_key.verkey.split(":").collect();
+        let crypto_type_name = if my_key.verkey.contains(':') {
+            let splits: Vec<&str> = my_key.verkey.split(':').collect();
             splits[1]
         } else {
             DEFAULT_CRYPTO_TYPE
@@ -168,8 +168,8 @@ impl CryptoService {
     pub fn verify(&self, their_vk: &str, msg: &[u8], signature: &[u8]) -> Result<bool, CryptoError> {
         trace!("verify >>> their_vk: {:?}, msg: {:?}, signature: {:?}", their_vk, msg, signature);
 
-        let (their_vk, crypto_type_name) = if their_vk.contains(":") {
-            let splits: Vec<&str> = their_vk.split(":").collect();
+        let (their_vk, crypto_type_name) = if their_vk.contains(':') {
+            let splits: Vec<&str> = their_vk.split(':').collect();
             (splits[0], splits[1])
         } else {
             (their_vk, DEFAULT_CRYPTO_TYPE)
@@ -210,15 +210,15 @@ impl CryptoService {
     pub fn encrypt(&self, my_key: &Key, their_vk: &str, doc: &[u8]) -> Result<(Vec<u8>, Vec<u8>), CryptoError> {
         trace!("encrypt >>> my_key: {:?}, their_vk: {:?}, doc: {:?}", my_key, their_vk, doc);
 
-        let (_my_vk, crypto_type_name) = if my_key.verkey.contains(":") {
-            let splits: Vec<&str> = my_key.verkey.split(":").collect();
+        let (_my_vk, crypto_type_name) = if my_key.verkey.contains(':') {
+            let splits: Vec<&str> = my_key.verkey.split(':').collect();
             (splits[0], splits[1])
         } else {
             (my_key.verkey.as_str(), DEFAULT_CRYPTO_TYPE)
         };
 
-        let (their_vk, their_crypto_type_name) = if their_vk.contains(":") {
-            let splits: Vec<&str> = their_vk.split(":").collect();
+        let (their_vk, their_crypto_type_name) = if their_vk.contains(':') {
+            let splits: Vec<&str> = their_vk.split(':').collect();
             (splits[0], splits[1])
         } else {
             (their_vk, DEFAULT_CRYPTO_TYPE)
@@ -252,15 +252,15 @@ impl CryptoService {
     pub fn decrypt(&self, my_key: &Key, their_vk: &str, doc: &[u8], nonce: &[u8]) -> Result<Vec<u8>, CryptoError> {
         trace!("decrypt >>> my_key: {:?}, their_vk: {:?}, doc: {:?}, nonce: {:?}", my_key, their_vk, doc, nonce);
 
-        let (_my_vk, crypto_type_name) = if my_key.verkey.contains(":") {
-            let splits: Vec<&str> = my_key.verkey.split(":").collect();
+        let (_my_vk, crypto_type_name) = if my_key.verkey.contains(':') {
+            let splits: Vec<&str> = my_key.verkey.split(':').collect();
             (splits[0], splits[1])
         } else {
             (my_key.verkey.as_str(), DEFAULT_CRYPTO_TYPE)
         };
 
-        let (their_vk, their_crypto_type_name) = if their_vk.contains(":") {
-            let splits: Vec<&str> = their_vk.split(":").collect();
+        let (their_vk, their_crypto_type_name) = if their_vk.contains(':') {
+            let splits: Vec<&str> = their_vk.split(':').collect();
             (splits[0], splits[1])
         } else {
             (their_vk, DEFAULT_CRYPTO_TYPE)
@@ -294,8 +294,8 @@ impl CryptoService {
     pub fn encrypt_sealed(&self, their_vk: &str, doc: &[u8]) -> Result<Vec<u8>, CryptoError> {
         trace!("encrypt_sealed >>> their_vk: {:?}, doc: {:?}", their_vk, doc);
 
-        let (their_vk, crypto_type_name) = if their_vk.contains(":") {
-            let splits: Vec<&str> = their_vk.split(":").collect();
+        let (their_vk, crypto_type_name) = if their_vk.contains(':') {
+            let splits: Vec<&str> = their_vk.split(':').collect();
             (splits[0], splits[1])
         } else {
             (their_vk, DEFAULT_CRYPTO_TYPE)
@@ -319,8 +319,8 @@ impl CryptoService {
     pub fn decrypt_sealed(&self, my_key: &Key, doc: &[u8]) -> Result<Vec<u8>, CryptoError> {
         trace!("decrypt_sealed >>> my_key: {:?}, doc: {:?}", my_key, doc);
 
-        let (my_vk, crypto_type_name) = if my_key.verkey.contains(":") {
-            let splits: Vec<&str> = my_key.verkey.split(":").collect();
+        let (my_vk, crypto_type_name) = if my_key.verkey.contains(':') {
+            let splits: Vec<&str> = my_key.verkey.split(':').collect();
             (splits[0], splits[1])
         } else {
             (my_key.verkey.as_str(), DEFAULT_CRYPTO_TYPE)
@@ -348,7 +348,7 @@ impl CryptoService {
 
         let res = match seed {
             Some(ref seed) =>
-                if seed.ends_with("=") {
+                if seed.ends_with('=') {
                     Some(base64::decode(&seed)
                         .map_err(|err| CommonError::InvalidStructure(format!("Can't deserialize Seed from Base64 string: {:?}", err)))?)
                 } else {
@@ -365,8 +365,8 @@ impl CryptoService {
     pub fn validate_key(&self, vk: &str) -> Result<(), CryptoError> {
         trace!("validate_key >>> vk: {:?}", vk);
 
-        let (vk, crypto_type_name) = if vk.contains(":") {
-            let splits: Vec<&str> = vk.split(":").collect();
+        let (vk, crypto_type_name) = if vk.contains(':') {
+            let splits: Vec<&str> = vk.split(':').collect();
             (splits[0], splits[1])
         } else {
             (vk, DEFAULT_CRYPTO_TYPE)
@@ -378,7 +378,7 @@ impl CryptoService {
 
         let crypto_type = self.crypto_types.get(crypto_type_name).unwrap();
 
-        let vk = if vk.starts_with("~") { &vk[1..] } else { vk };
+        let vk = if vk.starts_with('~') { &vk[1..] } else { vk };
         let vk = Base58::decode(vk)?;
 
         let res =crypto_type.validate_key(&vk)?;