diff --git a/common/tools/armageddon/cryptogen.go b/common/tools/armageddon/cryptogen.go index 8275cc07..32da300b 100644 --- a/common/tools/armageddon/cryptogen.go +++ b/common/tools/armageddon/cryptogen.go @@ -552,7 +552,19 @@ func copyFile(src, dst string) error { return err } -func CreateNewCertificateFromCA(caCertPath string, caPrivateKeyPath string, pathToNewTLSCert string, pathToNewTLSKey string, nodesIPs []string) ([]byte, error) { +func CreateNewCertificateFromCA(caCertPath string, caPrivateKeyPath string, certType string, pathToNewCert string, pathToNewPrivateKey string, nodesIPs []string) ([]byte, error) { + var ku x509.KeyUsage + switch certType { + case "tls": + certType = "tls" + ku = x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature + case "sign": + certType = "sign" + ku = x509.KeyUsageDigitalSignature + default: + return nil, fmt.Errorf("unsupported cert type: %s", certType) + } + caCertBytes, err := utils.ReadPem(caCertPath) if err != nil { return nil, err @@ -595,7 +607,7 @@ func CreateNewCertificateFromCA(caCertPath string, caPrivateKeyPath string, path return nil, fmt.Errorf("failed marshaling private key, err: %s", err) } - _, err = ca.SignCertificate(pathToNewTLSCert, "tls", nil, nodesIPs, GetPublicKey(privateKey), x509.KeyUsageCertSign|x509.KeyUsageCRLSign, []x509.ExtKeyUsage{ + _, err = ca.SignCertificate(pathToNewCert, certType, nil, nodesIPs, GetPublicKey(privateKey), ku, []x509.ExtKeyUsage{ x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth, }) @@ -603,12 +615,12 @@ func CreateNewCertificateFromCA(caCertPath string, caPrivateKeyPath string, path return nil, err } - err = utils.WritePEMToFile(pathToNewTLSKey, "PRIVATE KEY", privateKeyBytes) + err = utils.WritePEMToFile(pathToNewPrivateKey, "PRIVATE KEY", privateKeyBytes) if err != nil { return nil, err } - newCertBytes, err := os.ReadFile(filepath.Join(pathToNewTLSCert, "tls-cert.pem")) + newCertBytes, err := os.ReadFile(filepath.Join(pathToNewCert, fmt.Sprintf("%s-cert.pem", certType))) if err != nil { return nil, err } diff --git a/common/utils/net.go b/common/utils/net.go index 63a4f1d9..0239af0d 100644 --- a/common/utils/net.go +++ b/common/utils/net.go @@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0 package utils import ( + "bytes" "context" "crypto/x509" "fmt" @@ -64,6 +65,20 @@ func CertificateBytesToString(cert []byte) (string, error) { return CertificateToString(x509Cert), nil } +func AreCertificatesEqual(cert1, cert2 []byte) (bool, error) { + x509Cert1, err := Parsex509Cert(cert1) + if err != nil { + return false, err + } + x509Cert2, err := Parsex509Cert(cert2) + if err != nil { + return false, err + } + + // Compare RawTBSCertificate fields + return bytes.Equal(x509Cert1.RawTBSCertificate, x509Cert2.RawTBSCertificate), nil +} + func CertificateToString(cert *x509.Certificate) string { var sb strings.Builder fmt.Fprintf(&sb, "Certificate:\n") diff --git a/config/config.go b/config/config.go index f0375ac9..79cc09af 100644 --- a/config/config.go +++ b/config/config.go @@ -698,7 +698,11 @@ func (config *Configuration) CheckIfBatcherNodeExistsInSharedConfig(localSignCer return fmt.Errorf("batcher in shard%d does not exist for party%d in the shared config", localShardID, localPartyID) } - if !bytes.Equal(localTLSCert, sharedBatcherConfig.TlsCert) { + equal, err := utils.AreCertificatesEqual(localTLSCert, sharedBatcherConfig.TlsCert) + if err != nil { + return err + } + if !equal { localTLSCertString, err := utils.CertificateBytesToString(localTLSCert) if err != nil { return err @@ -710,7 +714,11 @@ func (config *Configuration) CheckIfBatcherNodeExistsInSharedConfig(localSignCer return fmt.Errorf("certificate mismatch: the batcher of party %d shard %d is attempting to load with TLS certificate: %v that differs from the shared configuration TLS certificate: %v", localPartyID, localShardID, localTLSCertString, sharedTLSCertString) } - if !bytes.Equal(localSignCert, sharedBatcherConfig.SignCert) { + equal, err = utils.AreCertificatesEqual(localSignCert, sharedBatcherConfig.SignCert) + if err != nil { + return err + } + if !equal { localSignCertString, err := utils.CertificateBytesToString(localSignCert) if err != nil { return err @@ -738,7 +746,11 @@ func (config *Configuration) CheckIfConsenterNodeExistsInSharedConfig(localSignC return fmt.Errorf("consenter configuration of partyID %d is missing from the shared configuration: %+v", localPartyID, sharedPartyConfig) } - if !bytes.Equal(localSignCert, sharedPartyConfig.ConsenterConfig.SignCert) { + equal, err := utils.AreCertificatesEqual(localSignCert, sharedPartyConfig.ConsenterConfig.SignCert) + if err != nil { + return err + } + if !equal { localSignCertString, err := utils.CertificateBytesToString(localSignCert) if err != nil { return err @@ -750,7 +762,11 @@ func (config *Configuration) CheckIfConsenterNodeExistsInSharedConfig(localSignC return fmt.Errorf("sign certificate mismatch: Consenter%d is attempting to load with sign certificate: %v that differs from the shared configuration sign certificate: %v", localPartyID, localSignCertString, sharedSignCertString) } - if !bytes.Equal(localTLSCert, sharedPartyConfig.ConsenterConfig.TlsCert) { + equal, err = utils.AreCertificatesEqual(localTLSCert, sharedPartyConfig.ConsenterConfig.TlsCert) + if err != nil { + return err + } + if !equal { localTLSCertString, err := utils.CertificateBytesToString(localTLSCert) if err != nil { return err @@ -776,7 +792,11 @@ func (config *Configuration) CheckIfAssemblerNodeExistsInSharedConfig() error { if sharedPartyConfig.AssemblerConfig == nil { return fmt.Errorf("assembler configuration of partyID %d is missing from the shared configuration: %+v", localPartyID, sharedPartyConfig) } - if !bytes.Equal(localTLSCert, sharedPartyConfig.AssemblerConfig.TlsCert) { + equal, err := utils.AreCertificatesEqual(localTLSCert, sharedPartyConfig.AssemblerConfig.TlsCert) + if err != nil { + return err + } + if !equal { localTLSCertString, err := utils.CertificateBytesToString(localTLSCert) if err != nil { return err diff --git a/node/consensus/consensus_real_reconfig_test.go b/node/consensus/consensus_real_reconfig_test.go index c54f64ba..ef2cef75 100644 --- a/node/consensus/consensus_real_reconfig_test.go +++ b/node/consensus/consensus_real_reconfig_test.go @@ -165,7 +165,7 @@ func TestConsensusWithRealConfigUpdate(t *testing.T) { caPrivKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", fmt.Sprintf("org%d", consenterToUpdate), "tlsca", "priv_sk") newCertPath := filepath.Join(dir, "crypto", "ordererOrganizations", fmt.Sprintf("org%d", consenterToUpdate), "orderers", fmt.Sprintf("party%d", consenterToUpdate), "consenter", "tls") newKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", fmt.Sprintf("org%d", consenterToUpdate), "orderers", fmt.Sprintf("party%d", consenterToUpdate), "consenter", "tls", "key.pem") - newCert, err := armageddon.CreateNewCertificateFromCA(caCertPath, caPrivKeyPath, newCertPath, newKeyPath, nodesIPs) + newCert, err := armageddon.CreateNewCertificateFromCA(caCertPath, caPrivKeyPath, "tls", newCertPath, newKeyPath, nodesIPs) require.NoError(t, err) configUpdatePbData := configUpdateBuilder.UpdateConsensusTLSCert(t, consenterToUpdate, newCert) env := configutil.CreateConfigTX(t, dir, parties, 1, configUpdatePbData) diff --git a/test/send_config_update_test.go b/test/send_config_update_test.go index cdf95fce..a2de9d9a 100644 --- a/test/send_config_update_test.go +++ b/test/send_config_update_test.go @@ -645,7 +645,7 @@ func TestRemoveParty(t *testing.T) { // TestAddNewParty verifies that adding a party via a config update succeeds, // that the new party's config is included in the updated shared config, -// and that the new party can be join (start) and process transactions after the config update. +// and that the new party can join (start) and process transactions after the config update. func TestAddNewParty(t *testing.T) { // Prepare Arma config and crypto and get the genesis block dir, err := os.MkdirTemp("", t.Name()) @@ -907,3 +907,242 @@ func TestAddNewParty(t *testing.T) { Signer: pullRequestSigner, }) } + +// TestChangePartyCertificates verifies that updating a party's certificates via a config update succeeds, +// and that the party can continue processing transactions after the config update with the new certificates. +func TestChangePartyCertificates(t *testing.T) { + // Prepare Arma config and crypto and get the genesis block + dir, err := os.MkdirTemp("", t.Name()) + require.NoError(t, err) + defer os.RemoveAll(dir) + + configPath := filepath.Join(dir, "config.yaml") + numOfParties := 3 + numOfShards := 2 + submittingParty := types.PartyID(2) + submittingOrg := fmt.Sprintf("org%d", submittingParty) + partyToUpdate := types.PartyID(1) + updateOrg := fmt.Sprintf("org%d", partyToUpdate) + + netInfo := testutil.CreateNetwork(t, configPath, numOfParties, numOfShards, "mTLS", "mTLS") + require.NotNil(t, netInfo) + require.NoError(t, err) + + armageddon.NewCLI().Run([]string{"generate", "--config", configPath, "--output", dir}) + + configFilePath := filepath.Join(dir, fmt.Sprintf("config/party%d/local_config_router.yaml", types.PartyID(submittingParty))) + conf, _, err := config.LoadLocalConfig(configFilePath) + require.NoError(t, err) + + // Modify the router configuration to require client signature verification. + conf.NodeLocalConfig.GeneralConfig.ClientSignatureVerificationRequired = true + utils.WriteToYAML(conf.NodeLocalConfig, configFilePath) + + armaBinaryPath, err := gexec.BuildWithEnvironment("github.com/hyperledger/fabric-x-orderer/cmd/arma", []string{"GOPRIVATE=" + os.Getenv("GOPRIVATE")}) + defer gexec.CleanupBuildArtifacts() + require.NoError(t, err) + require.NotNil(t, armaBinaryPath) + + // Start Arma nodes + numOfArmaNodes := len(netInfo) + readyChan := make(chan string, numOfArmaNodes) + armaNetwork := testutil.RunArmaNodes(t, dir, armaBinaryPath, readyChan, netInfo) + + testutil.WaitReady(t, readyChan, numOfArmaNodes, 10) + + parties := make([]types.PartyID, 0, numOfParties) + for i := 1; i <= numOfParties; i++ { + parties = append(parties, types.PartyID(i)) + } + + uc, err := testutil.GetUserConfig(dir, submittingParty) + require.NoError(t, err) + + totalTxNumber := 10 + // Send transactions to all parties to ensure network is operational before config update + signer, certBytes, err := testutil.LoadCryptoMaterialsFromDir(t, uc.MSPDir) + require.NoError(t, err) + broadcastClient := client.NewBroadcastTxClient(uc, 10*time.Second) + + for i := range totalTxNumber { + txContent := tx.PrepareTxWithTimestamp(i+totalTxNumber, 64, []byte("sessionNumber")) + env := tx.CreateSignedStructuredEnvelope(txContent, signer, certBytes, submittingOrg) + err = broadcastClient.SendTx(env) + require.NoError(t, err) + } + pullRequestSigner := signutil.CreateTestSigner(t, submittingOrg, dir) + statusUnknown := common.Status_UNKNOWN + // Pull blocks to verify all transactions are included + PullFromAssemblers(t, &BlockPullerOptions{ + UserConfig: uc, + Parties: parties, + Transactions: totalTxNumber, + ErrString: "cancelled pull from assembler: %d; pull ended: failed to receive a deliver response: rpc error: code = Canceled desc = grpc: the client connection is closing", + Timeout: 60, + Status: &statusUnknown, + Signer: pullRequestSigner, + }) + + // Create config update to change a party's certificates + configUpdateBuilder, _ := configutil.NewConfigUpdateBuilder(t, dir, filepath.Join(dir, "bootstrap", "bootstrap.block")) + + nodesIPs := testutil.GetNodesIPsFromNetInfo(netInfo) + require.NotNil(t, nodesIPs) + + tlsCACertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "tlsca", "tlsca-cert.pem") + tlsCAPrivKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "tlsca", "priv_sk") + + signCACertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "ca", "ca-cert.pem") + signCAPrivKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "ca", "priv_sk") + + // Update the router TLS certs in the config + newRouterTlsCertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "router", "tls") + newRouterTlsKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "router", "tls", "key.pem") + newRouterTlsCertBytes, err := armageddon.CreateNewCertificateFromCA(tlsCACertPath, tlsCAPrivKeyPath, "tls", newRouterTlsCertPath, newRouterTlsKeyPath, nodesIPs) + require.NoError(t, err) + configUpdateBuilder.UpdateRouterTLSCert(t, partyToUpdate, newRouterTlsCertBytes) + + // Update the batchers TLS certs and signing certs in the config + for shardToUpdate := types.ShardID(1); int(shardToUpdate) <= numOfShards; shardToUpdate++ { + newBatcherTlsCertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), fmt.Sprintf("batcher%d", shardToUpdate), "tls") + newBatcherTlsKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), fmt.Sprintf("batcher%d", shardToUpdate), "tls", "key.pem") + newBatcherTlsCertBytes, err := armageddon.CreateNewCertificateFromCA(tlsCACertPath, tlsCAPrivKeyPath, "tls", newBatcherTlsCertPath, newBatcherTlsKeyPath, nodesIPs) + require.NoError(t, err) + configUpdateBuilder.UpdateBatcherTLSCert(t, partyToUpdate, shardToUpdate, newBatcherTlsCertBytes) + + newBatcherSignCertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), fmt.Sprintf("batcher%d", shardToUpdate), "msp", "signcerts") + newBatcherSignKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), fmt.Sprintf("batcher%d", shardToUpdate), "msp", "keystore", "priv_sk") + newBatcherSignCertBytes, err := armageddon.CreateNewCertificateFromCA(signCACertPath, signCAPrivKeyPath, "sign", newBatcherSignCertPath, newBatcherSignKeyPath, nodesIPs) + require.NoError(t, err) + configUpdateBuilder.UpdateBatcherSignCert(t, partyToUpdate, shardToUpdate, newBatcherSignCertBytes) + } + + // Update the assembler TLS certs in the config + newAssemblerTlsCertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "assembler", "tls") + newAssemblerTlsKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "assembler", "tls", "key.pem") + newAssemblerTlsCertBytes, err := armageddon.CreateNewCertificateFromCA(tlsCACertPath, tlsCAPrivKeyPath, "tls", newAssemblerTlsCertPath, newAssemblerTlsKeyPath, nodesIPs) + require.NoError(t, err) + configUpdateBuilder.UpdateAssemblerTLSCert(t, partyToUpdate, newAssemblerTlsCertBytes) + + // Update the consenter TLS certs in the config + newConsenterTlsCertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "consenter", "tls") + newConsenterTlsKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "consenter", "tls", "key.pem") + newConsenterTlsCertBytes, err := armageddon.CreateNewCertificateFromCA(tlsCACertPath, tlsCAPrivKeyPath, "tls", newConsenterTlsCertPath, newConsenterTlsKeyPath, nodesIPs) + require.NoError(t, err) + configUpdateBuilder.UpdateConsensusTLSCert(t, partyToUpdate, newConsenterTlsCertBytes) + + // Update the consenter signing certs in the config + newConsenterSignCertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "consenter", "msp", "signcerts") + newConsenterSignKeyPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "consenter", "msp", "keystore", "priv_sk") + newConsenterSignCertBytes, err := armageddon.CreateNewCertificateFromCA(signCACertPath, signCAPrivKeyPath, "sign", newConsenterSignCertPath, newConsenterSignKeyPath, nodesIPs) + require.NoError(t, err) + configUpdateBuilder.UpdateConsenterSignCert(t, partyToUpdate, newConsenterSignCertBytes) + + // Submit config update + env := configutil.CreateConfigTX(t, dir, parties, int(submittingParty), configUpdateBuilder.ConfigUpdatePBData(t)) + require.NotNil(t, env) + + // Send the config tx + err = broadcastClient.SendTxTo(env, submittingParty) + require.NoError(t, err) + + broadcastClient.Stop() + + // Wait for Arma nodes to stop + testutil.WaitSoftStopped(t, netInfo) + + // Stop Arma nodes + armaNetwork.Stop() + + // Verify that the party's certificates are updated by checking the router's shared config + assemblerNodeConfigPath := filepath.Join(dir, "config", fmt.Sprintf("party%d", partyToUpdate), "local_config_assembler.yaml") + assemblerConfig, _, err := config.ReadConfig(assemblerNodeConfigPath, testutil.CreateLoggerForModule(t, "ReadConfigAssembler", zap.DebugLevel)) + require.NoError(t, err) + + var updatedPartyConfig *protos.PartyConfig + for _, partyConfig := range assemblerConfig.SharedConfig.GetPartiesConfig() { + if partyConfig.PartyID == uint32(partyToUpdate) { + updatedPartyConfig = partyConfig + break + } + } + require.NotNil(t, updatedPartyConfig, "Updated party config not found in the config") + + newTlsCertBytes, err := os.ReadFile(filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "router", "tls", "tls-cert.pem")) + require.NoError(t, err) + // Verify that the router TLS cert path is updated in the config + require.Equal(t, newTlsCertBytes, updatedPartyConfig.RouterConfig.GetTlsCert(), "Certificate path was not updated in the config") + + // Verify that the batcher TLS certs path are updated in the config + for _, shardConfig := range updatedPartyConfig.BatchersConfig { + newBatcherTlsCertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), fmt.Sprintf("batcher%d", shardConfig.ShardID), "tls") + newTlsCertBytes, err = os.ReadFile(filepath.Join(newBatcherTlsCertPath, "tls-cert.pem")) + require.NoError(t, err) + newBatcherSignCertPath := filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), fmt.Sprintf("batcher%d", shardConfig.ShardID), "msp", "signcerts") + newSignCertBytes, err := os.ReadFile(filepath.Join(newBatcherSignCertPath, "sign-cert.pem")) + require.NoError(t, err) + require.Equal(t, newTlsCertBytes, shardConfig.GetTlsCert(), "Batcher certificate path was not updated in the config") + require.Equal(t, newSignCertBytes, shardConfig.GetSignCert(), "Batcher signing certificate path was not updated in the config") + } + + newTlsCertBytes, err = os.ReadFile(filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "assembler", "tls", "tls-cert.pem")) + require.NoError(t, err) + // Verify that the assembler TLS cert path is updated in the config + require.Equal(t, newTlsCertBytes, updatedPartyConfig.AssemblerConfig.GetTlsCert(), "Certificate path was not updated in the config") + + newTlsCertBytes, err = os.ReadFile(filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "consenter", "tls", "tls-cert.pem")) + require.NoError(t, err) + newSignCertBytes, err := os.ReadFile(filepath.Join(dir, "crypto", "ordererOrganizations", updateOrg, "orderers", fmt.Sprintf("party%d", partyToUpdate), "consenter", "msp", "signcerts", "sign-cert.pem")) + require.NoError(t, err) + // Verify that the consenter TLS and signing cert paths are updated in the config + require.Equal(t, newTlsCertBytes, updatedPartyConfig.ConsenterConfig.GetTlsCert(), "Consenter certificate path was not updated in the config") + require.Equal(t, newSignCertBytes, updatedPartyConfig.ConsenterConfig.GetSignCert(), "Consenter signing certificate path was not updated in the config") + + // Restart Arma nodes + armaNetwork.Restart(t, readyChan) + defer armaNetwork.Stop() + + testutil.WaitReady(t, readyChan, numOfArmaNodes, 10) + + updatedRouterInfo := armaNetwork.GetRouter(t, partyToUpdate) + // Verify that the updated router TLS connection to a consenter node is successful by checking for successful pulls, + // if the TLS cert was not updated correctly, the router would fail to establish connections to the batchers and the pull would fail with TLS errors + updatedRouterInfo.RunInfo.Session.Err.Detect("pullAndProcessDecisions -> Pulled config block number") + + for shardToUpdate := types.ShardID(1); int(shardToUpdate) <= numOfShards; shardToUpdate++ { + updatedBatcherInfo := armaNetwork.GetBatcher(t, partyToUpdate, shardToUpdate) + // Verify that the updated batcher TLS connection to a consenter is successful by checking for successful pulls, + // if the TLS cert was not updated correctly, the batcher would fail to establish connections to the consenter and the pull would fail with TLS errors + updatedBatcherInfo.RunInfo.Session.Err.Detect("replicateDecision -> Got config block number") + + assemblerInfo := armaNetwork.GetAssembler(t, partyToUpdate) + // Verify that the assembler can pull blocks successfully with the updated certificates + assemblerInfo.RunInfo.Session.Err.Detect("pullBlocks -> Started pulling blocks from: shard%dparty%d", shardToUpdate, partyToUpdate) + } + + broadcastClient = client.NewBroadcastTxClient(uc, 10*time.Second) + signer, certBytes, err = testutil.LoadCryptoMaterialsFromDir(t, uc.MSPDir) + require.NoError(t, err) + + for i := range totalTxNumber { + txContent := tx.PrepareTxWithTimestamp(i+totalTxNumber, 64, []byte("sessionNumber")) + env := tx.CreateSignedStructuredEnvelope(txContent, signer, certBytes, submittingOrg) + err = broadcastClient.SendTx(env) + require.NoError(t, err) + } + + broadcastClient.Stop() + + pullRequestSigner = signutil.CreateTestSigner(t, submittingOrg, dir) + statusUnknown = common.Status_UNKNOWN + // Pull blocks to verify all transactions are included + PullFromAssemblers(t, &BlockPullerOptions{ + UserConfig: uc, + Parties: parties, + Transactions: totalTxNumber*2 + 1, // including config update tx + Timeout: 60, + ErrString: "cancelled pull from assembler: %d; pull ended: failed to receive a deliver response: rpc error: code = Canceled desc = grpc: the client connection is closing", + Status: &statusUnknown, + Signer: pullRequestSigner, + }) +}