Skip to content

Commit 85ad31e

Browse files
TheBlueMattTheBlueMatt
committed
Add partial-block block encodings API
1 parent 5249dac commit 85ad31e

File tree

3 files changed

+365
-0
lines changed

3 files changed

+365
-0
lines changed

‎src/Makefile.am

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ BITCOIN_CORE_H = \
7474
addrman.h \
7575
base58.h \
7676
bloom.h \
77+
blockencodings.h \
7778
chain.h \
7879
chainparams.h \
7980
chainparamsbase.h \
@@ -163,6 +164,7 @@ libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
163164
libbitcoin_server_a_SOURCES = \
164165
addrman.cpp \
165166
bloom.cpp \
167+
blockencodings.cpp \
166168
chain.cpp \
167169
checkpoints.cpp \
168170
httprpc.cpp \

‎src/blockencodings.cpp

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
// Copyright (c) 2016 The Bitcoin Core developers
2+
// Distributed under the MIT software license, see the accompanying
3+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4+
5+
#include "blockencodings.h"
6+
#include "consensus/consensus.h"
7+
#include "consensus/validation.h"
8+
#include "chainparams.h"
9+
#include "hash.h"
10+
#include "random.h"
11+
#include "streams.h"
12+
#include "txmempool.h"
13+
#include "main.h"
14+
15+
#include <unordered_map>
16+
17+
#define MIN_TRANSACTION_SIZE (::GetSerializeSize(CTransaction(), SER_NETWORK, PROTOCOL_VERSION))
18+
19+
CBlockHeaderAndShortTxIDs::CBlockHeaderAndShortTxIDs(const CBlock& block) :
20+
nonce(GetRand(std::numeric_limits<uint64_t>::max())),
21+
shorttxids(block.vtx.size() - 1), prefilledtxn(1), header(block) {
22+
FillShortTxIDSelector();
23+
//TODO: Use our mempool prior to block acceptance to predictively fill more than just the coinbase
24+
prefilledtxn[0] = {0, block.vtx[0]};
25+
for (size_t i = 1; i < block.vtx.size(); i++) {
26+
const CTransaction& tx = block.vtx[i];
27+
shorttxids[i - 1] = GetShortID(tx.GetHash());
28+
}
29+
}
30+
31+
void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const {
32+
CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
33+
stream << header << nonce;
34+
CSHA256 hasher;
35+
hasher.Write((unsigned char*)&(*stream.begin()), stream.end() - stream.begin());
36+
uint256 shorttxidhash;
37+
hasher.Finalize(shorttxidhash.begin());
38+
shorttxidk0 = shorttxidhash.GetUint64(0);
39+
shorttxidk1 = shorttxidhash.GetUint64(1);
40+
}
41+
42+
uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256& txhash) const {
43+
static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids calculation assumes 6-byte shorttxids");
44+
return SipHashUint256(shorttxidk0, shorttxidk1, txhash) & 0xffffffffffffL;
45+
}
46+
47+
48+
49+
ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock) {
50+
if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty()))
51+
return READ_STATUS_INVALID;
52+
if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > MAX_BLOCK_SIZE / MIN_TRANSACTION_SIZE)
53+
return READ_STATUS_INVALID;
54+
55+
assert(header.IsNull() && txn_available.empty());
56+
header = cmpctblock.header;
57+
txn_available.resize(cmpctblock.BlockTxCount());
58+
59+
int32_t lastprefilledindex = -1;
60+
for (size_t i = 0; i < cmpctblock.prefilledtxn.size(); i++) {
61+
if (cmpctblock.prefilledtxn[i].tx.IsNull())
62+
return READ_STATUS_INVALID;
63+
64+
lastprefilledindex += cmpctblock.prefilledtxn[i].index + 1; //index is a uint16_t, so cant overflow here
65+
if (lastprefilledindex > std::numeric_limits<uint16_t>::max())
66+
return READ_STATUS_INVALID;
67+
if ((uint32_t)lastprefilledindex > cmpctblock.shorttxids.size() + i) {
68+
// If we are inserting a tx at an index greater than our full list of shorttxids
69+
// plus the number of prefilled txn we've inserted, then we have txn for which we
70+
// have neither a prefilled txn or a shorttxid!
71+
return READ_STATUS_INVALID;
72+
}
73+
txn_available[lastprefilledindex] = std::make_shared<CTransaction>(cmpctblock.prefilledtxn[i].tx);
74+
}
75+
76+
// Calculate map of txids -> positions and check mempool to see what we have (or dont)
77+
// Because well-formed cmpctblock messages will have a (relatively) uniform distribution
78+
// of short IDs, any highly-uneven distribution of elements can be safely treated as a
79+
// READ_STATUS_FAILED.
80+
std::unordered_map<uint64_t, uint16_t> shorttxids(cmpctblock.shorttxids.size());
81+
uint16_t index_offset = 0;
82+
for (size_t i = 0; i < cmpctblock.shorttxids.size(); i++) {
83+
while (txn_available[i + index_offset])
84+
index_offset++;
85+
shorttxids[cmpctblock.shorttxids[i]] = i + index_offset;
86+
// Bucket selection is a simple Binomial distribution. If we assume blocks of
87+
// 10,000 transactions, allowing up to 12 elements per bucket should only fail
88+
// once every ~1.3 million blocks and once every 74,000 blocks in a worst-case
89+
// 16,000-transaction block.
90+
if (shorttxids.bucket_size(shorttxids.bucket(cmpctblock.shorttxids[i])) > 12)
91+
return READ_STATUS_FAILED;
92+
}
93+
// TODO: in the shortid-collision case, we should instead request both transactions
94+
// which collided. Falling back to full-block-request here is overkill.
95+
if (shorttxids.size() != cmpctblock.shorttxids.size())
96+
return READ_STATUS_FAILED; // Short ID collision
97+
98+
std::vector<bool> have_txn(txn_available.size());
99+
LOCK(pool->cs);
100+
for (CTxMemPool::txiter it = pool->mapTx.begin(); it != pool->mapTx.end(); it++) {
101+
std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(cmpctblock.GetShortID(it->GetTx().GetHash()));
102+
if (idit != shorttxids.end()) {
103+
if (!have_txn[idit->second]) {
104+
txn_available[idit->second] = it->GetSharedTx();
105+
have_txn[idit->second] = true;
106+
} else {
107+
// If we find two mempool txn that match the short id, just request it.
108+
// This should be rare enough that the extra bandwidth doesn't matter,
109+
// but eating a round-trip due to FillBlock failure would be annoying
110+
txn_available[idit->second].reset();
111+
}
112+
}
113+
// Though ideally we'd continue scanning for the two-txn-match-shortid case,
114+
// the performance win of an early exit here is too good to pass up and worth
115+
// the extra risk.
116+
if (mempool_count == shorttxids.size())
117+
break;
118+
}
119+
120+
return READ_STATUS_OK;
121+
}
122+
123+
bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const {
124+
assert(!header.IsNull());
125+
assert(index < txn_available.size());
126+
return txn_available[index] ? true : false;
127+
}
128+
129+
ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector<CTransaction>& vtx_missing) const {
130+
assert(!header.IsNull());
131+
block = header;
132+
block.vtx.resize(txn_available.size());
133+
134+
size_t tx_missing_offset = 0;
135+
for (size_t i = 0; i < txn_available.size(); i++) {
136+
if (!txn_available[i]) {
137+
if (vtx_missing.size() <= tx_missing_offset)
138+
return READ_STATUS_INVALID;
139+
block.vtx[i] = vtx_missing[tx_missing_offset++];
140+
} else
141+
block.vtx[i] = *txn_available[i];
142+
}
143+
if (vtx_missing.size() != tx_missing_offset)
144+
return READ_STATUS_INVALID;
145+
146+
CValidationState state;
147+
if (!CheckBlock(block, state, Params().GetConsensus())) {
148+
// TODO: We really want to just check merkle tree manually here,
149+
// but that is expensive, and CheckBlock caches a block's
150+
// "checked-status" (in the CBlock?). CBlock should be able to
151+
// check its own merkle root and cache that check.
152+
if (state.CorruptionPossible())
153+
return READ_STATUS_FAILED; // Possible Short ID collision
154+
return READ_STATUS_INVALID;
155+
}
156+
157+
return READ_STATUS_OK;
158+
}

‎src/blockencodings.h

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
// Copyright (c) 2016 The Bitcoin Core developers
2+
// Distributed under the MIT software license, see the accompanying
3+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4+
5+
#ifndef BITCOIN_BLOCK_ENCODINGS_H
6+
#define BITCOIN_BLOCK_ENCODINGS_H
7+
8+
#include "primitives/block.h"
9+
10+
#include <memory>
11+
12+
class CTxMemPool;
13+
14+
// Dumb helper to handle CTransaction compression at serialize-time
15+
struct TransactionCompressor {
16+
private:
17+
CTransaction& tx;
18+
public:
19+
TransactionCompressor(CTransaction& txIn) : tx(txIn) {}
20+
21+
ADD_SERIALIZE_METHODS;
22+
23+
template <typename Stream, typename Operation>
24+
inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) {
25+
READWRITE(tx); //TODO: Compress tx encoding
26+
}
27+
};
28+
29+
class BlockTransactionsRequest {
30+
public:
31+
// A BlockTransactionsRequest message
32+
uint256 blockhash;
33+
std::vector<uint16_t> indexes;
34+
35+
ADD_SERIALIZE_METHODS;
36+
37+
template <typename Stream, typename Operation>
38+
inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) {
39+
READWRITE(blockhash);
40+
uint64_t indexes_size = (uint64_t)indexes.size();
41+
READWRITE(COMPACTSIZE(indexes_size));
42+
if (ser_action.ForRead()) {
43+
size_t i = 0;
44+
while (indexes.size() < indexes_size) {
45+
indexes.resize(std::min((uint64_t)(1000 + indexes.size()), indexes_size));
46+
for (; i < indexes.size(); i++) {
47+
uint64_t index = 0;
48+
READWRITE(COMPACTSIZE(index));
49+
if (index > std::numeric_limits<uint16_t>::max())
50+
throw std::ios_base::failure("index overflowed 16 bits");
51+
indexes[i] = index;
52+
}
53+
}
54+
55+
uint16_t offset = 0;
56+
for (size_t i = 0; i < indexes.size(); i++) {
57+
if (uint64_t(indexes[i]) + uint64_t(offset) > std::numeric_limits<uint16_t>::max())
58+
throw std::ios_base::failure("indexes overflowed 16 bits");
59+
indexes[i] = indexes[i] + offset;
60+
offset = indexes[i] + 1;
61+
}
62+
} else {
63+
for (size_t i = 0; i < indexes.size(); i++) {
64+
uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1));
65+
READWRITE(COMPACTSIZE(index));
66+
}
67+
}
68+
}
69+
};
70+
71+
class BlockTransactions {
72+
public:
73+
// A BlockTransactions message
74+
uint256 blockhash;
75+
std::vector<CTransaction> txn;
76+
77+
BlockTransactions() {}
78+
BlockTransactions(const BlockTransactionsRequest& req) :
79+
blockhash(req.blockhash), txn(req.indexes.size()) {}
80+
81+
ADD_SERIALIZE_METHODS;
82+
83+
template <typename Stream, typename Operation>
84+
inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) {
85+
READWRITE(blockhash);
86+
uint64_t txn_size = (uint64_t)txn.size();
87+
READWRITE(COMPACTSIZE(txn_size));
88+
if (ser_action.ForRead()) {
89+
size_t i = 0;
90+
while (txn.size() < txn_size) {
91+
txn.resize(std::min((uint64_t)(1000 + txn.size()), txn_size));
92+
for (; i < txn.size(); i++)
93+
READWRITE(REF(TransactionCompressor(txn[i])));
94+
}
95+
} else {
96+
for (size_t i = 0; i < txn.size(); i++)
97+
READWRITE(REF(TransactionCompressor(txn[i])));
98+
}
99+
}
100+
};
101+
102+
// Dumb serialization/storage-helper for CBlockHeaderAndShortTxIDs and PartiallyDownlaodedBlock
103+
struct PrefilledTransaction {
104+
// Used as an offset since last prefilled tx in CBlockHeaderAndShortTxIDs,
105+
// as a proper transaction-in-block-index in PartiallyDownloadedBlock
106+
uint16_t index;
107+
CTransaction tx;
108+
109+
ADD_SERIALIZE_METHODS;
110+
111+
template <typename Stream, typename Operation>
112+
inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) {
113+
uint64_t idx = index;
114+
READWRITE(COMPACTSIZE(idx));
115+
if (idx > std::numeric_limits<uint16_t>::max())
116+
throw std::ios_base::failure("index overflowed 16-bits");
117+
index = idx;
118+
READWRITE(REF(TransactionCompressor(tx)));
119+
}
120+
};
121+
122+
typedef enum ReadStatus_t
123+
{
124+
READ_STATUS_OK,
125+
READ_STATUS_INVALID, // Invalid object, peer is sending bogus crap
126+
READ_STATUS_FAILED, // Failed to process object
127+
} ReadStatus;
128+
129+
class CBlockHeaderAndShortTxIDs {
130+
private:
131+
mutable uint64_t shorttxidk0, shorttxidk1;
132+
uint64_t nonce;
133+
134+
void FillShortTxIDSelector() const;
135+
136+
friend class PartiallyDownloadedBlock;
137+
138+
static const int SHORTTXIDS_LENGTH = 6;
139+
protected:
140+
std::vector<uint64_t> shorttxids;
141+
std::vector<PrefilledTransaction> prefilledtxn;
142+
143+
public:
144+
CBlockHeader header;
145+
146+
// Dummy for deserialization
147+
CBlockHeaderAndShortTxIDs() {}
148+
149+
CBlockHeaderAndShortTxIDs(const CBlock& block);
150+
151+
uint64_t GetShortID(const uint256& txhash) const;
152+
153+
size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); }
154+
155+
ADD_SERIALIZE_METHODS;
156+
157+
template <typename Stream, typename Operation>
158+
inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) {
159+
READWRITE(header);
160+
READWRITE(nonce);
161+
162+
uint64_t shorttxids_size = (uint64_t)shorttxids.size();
163+
READWRITE(COMPACTSIZE(shorttxids_size));
164+
if (ser_action.ForRead()) {
165+
size_t i = 0;
166+
while (shorttxids.size() < shorttxids_size) {
167+
shorttxids.resize(std::min((uint64_t)(1000 + shorttxids.size()), shorttxids_size));
168+
for (; i < shorttxids.size(); i++) {
169+
uint32_t lsb = 0; uint16_t msb = 0;
170+
READWRITE(lsb);
171+
READWRITE(msb);
172+
shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb);
173+
static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids");
174+
}
175+
}
176+
} else {
177+
for (size_t i = 0; i < shorttxids.size(); i++) {
178+
uint32_t lsb = shorttxids[i] & 0xffffffff;
179+
uint16_t msb = (shorttxids[i] >> 32) & 0xffff;
180+
READWRITE(lsb);
181+
READWRITE(msb);
182+
}
183+
}
184+
185+
READWRITE(prefilledtxn);
186+
187+
if (ser_action.ForRead())
188+
FillShortTxIDSelector();
189+
}
190+
};
191+
192+
class PartiallyDownloadedBlock {
193+
protected:
194+
std::vector<std::shared_ptr<const CTransaction> > txn_available;
195+
CTxMemPool* pool;
196+
public:
197+
CBlockHeader header;
198+
PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {}
199+
200+
ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock);
201+
bool IsTxAvailable(size_t index) const;
202+
ReadStatus FillBlock(CBlock& block, const std::vector<CTransaction>& vtx_missing) const;
203+
};
204+
205+
#endif

0 commit comments

Comments
 (0)