Skip to content

Commit 6824044

Browse files
vans163vans163
committed
finalize hashfilters; tx_count;
1 parent ede975d commit 6824044

File tree

6 files changed

+85
-37
lines changed

6 files changed

+85
-37
lines changed

‎ex/lib/api/api_chain.ex‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,14 +85,13 @@ defmodule API.Chain do
8585

8686
def kpi() do
8787
{_, uaw} = API.Contract.richlist()
88-
tx_cnt = RocksDB.get_cf_prop(:tx, "rocksdb.estimate-num-keys") |> :erlang.binary_to_integer()
8988
%{
9089
ama_burned: Float.round(API.Contract.total_burned().float, 2),
9190
fees_paid: Float.round(API.Contract.total_burned().float * 2, 2),
9291
active_validator_keys: 67 + length(API.Epoch.score()),
9392
active_peers: length(API.Peer.all()),
9493
block_time: 500,
95-
total_tx: tx_cnt,
94+
total_tx: DB.Chain.tx_count(),
9695
uaw: uaw,
9796
}
9897
end

‎ex/lib/api/db_chain.ex‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ defmodule DB.Chain do
4343
RocksDB.get("account:#{pk}:attribute:nonce", db_handle(db_opts, :contractstate, %{to_integer: true}))
4444
end
4545

46+
def tx_count(db_opts \\ %{}) do
47+
RocksDB.get("tx_count", db_handle(db_opts, :sysconf, %{to_integer: true})) || 0
48+
end
49+
4650
def balance(pk, symbol \\ "AMA", db_opts \\ %{}) do
4751
RocksDB.get("account:#{pk}:balance:#{symbol}", db_handle(db_opts, :contractstate, %{to_integer: true})) || 0
4852
end
@@ -82,7 +86,7 @@ defmodule DB.Chain do
8286
def validators_for_height(height, db_opts \\ %{}) do
8387
opts = db_handle(db_opts, :contractstate, %{})
8488
cond do
85-
height in 3195570..3195575 ->
89+
height in 3195570..3195575 and !Application.fetch_env!(:ama, :testnet) ->
8690
RocksDB.get("bic:epoch:validators:height:000000319557", opts)
8791
|> RDB.vecpak_decode()
8892
true ->

‎ex/lib/api/db_entry.ex‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -102,16 +102,16 @@ defmodule DB.Entry do
102102
RocksDB.put("entry:#{entry.hash}:root_receipts", root_receipts, db_handle(db_opts, :entry_meta, %{}))
103103
RocksDB.put("entry:#{entry.hash}:root_contractstate", root_contractstate, db_handle(db_opts, :entry_meta, %{}))
104104

105+
#Build hashfilter
105106
tx_filters = RDB.build_tx_hashfilters(entry.txs)
106107
Enum.each(tx_filters, fn {key, hash} ->
107108
RocksDB.put(key, hash, db_handle(db_opts, :tx_filter, %{}))
108109
end)
109110

110-
if entry.header.height >= 445_00000 do
111-
old_cnt = RocksDB.get("tx_count", db_handle(db_opts, :sysconf, %{})) || "0"
112-
new_cnt = :erlang.binary_to_integer(old_cnt) + length(entry.txs)
113-
RocksDB.put("tx_count", :erlang.integer_to_binary(new_cnt), db_handle(db_opts, :sysconf, %{}))
114-
end
111+
#Count tx
112+
old_cnt = RocksDB.get("tx_count", db_handle(db_opts, :sysconf, %{})) || "0"
113+
new_cnt = :erlang.binary_to_integer(old_cnt) + length(entry.txs)
114+
RocksDB.put("tx_count", :erlang.integer_to_binary(new_cnt), db_handle(db_opts, :sysconf, %{}))
115115

116116
receipts_by_txid = Map.new(receipts, fn r -> {r.txid, Map.drop(r, [:txid])} end)
117117
Enum.each(entry.txs, fn(txu)->
@@ -165,16 +165,16 @@ defmodule DB.Entry do
165165
RocksDB.delete_prefix("consensus:#{hash}:", db_handle(db_opts, :attestation, %{}))
166166
RocksDB.delete_prefix("attestation:#{height_padded}:#{hash}:", db_handle(db_opts, :attestation, %{}))
167167

168+
#Delete hashfilter
168169
tx_filters = RDB.build_tx_hashfilters(entry.txs)
169170
Enum.each(tx_filters, fn {key, _hash} ->
170171
RocksDB.delete(key, db_handle(db_opts, :tx_filter, %{}))
171172
end)
172173

173-
if entry.header.height >= 445_00000 do
174-
old_cnt = RocksDB.get("tx_count", db_handle(db_opts, :sysconf, %{})) || "0"
175-
new_cnt = :erlang.binary_to_integer(old_cnt) - length(entry.txs)
176-
RocksDB.put("tx_count", :erlang.integer_to_binary(new_cnt), db_handle(db_opts, :sysconf, %{}))
177-
end
174+
#Decrement tx
175+
old_cnt = RocksDB.get("tx_count", db_handle(db_opts, :sysconf, %{})) || "0"
176+
new_cnt = :erlang.binary_to_integer(old_cnt) - length(entry.txs)
177+
RocksDB.put("tx_count", :erlang.integer_to_binary(new_cnt), db_handle(db_opts, :sysconf, %{}))
178178

179179
Enum.each(entry.txs, fn(txu)->
180180
RocksDB.delete(txu.hash, db_handle(db_opts, :tx, %{}))
@@ -222,7 +222,7 @@ defmodule DB.Entry do
222222

223223
n = next(rebuilt_up_to)
224224
if n do
225-
rebuilt_up_to = RocksDB.put("filter_hashes_rebuilt_up_to", next(rebuilt_up_to), db_handle(%{}, :sysconf, %{})) || EntryGenesis.get().hash
225+
RocksDB.put("filter_hashes_rebuilt_up_to", n, db_handle(%{}, :sysconf, %{}))
226226
end
227227
end
228228
end

‎ex/lib/api/db_entry_hashbuilder.ex‎

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,30 +5,26 @@ defmodule DB.Entry.Hashbuilder do
55
GenServer.start_link(__MODULE__, %{}, name: __MODULE__)
66
end
77

8-
def init(state) do
9-
case RocksDB.get("filter_hashes_end_hash", DB.API.db_handle(%{}, :sysconf, %{})) do
10-
nil ->
11-
IO.inspect "starting hashfilter builder"
12-
RocksDB.put("filter_hashes_end_hash", DB.Chain.tip(), DB.API.db_handle(%{}, :sysconf, %{}))
13-
_ -> nil
14-
end
8+
def start() do :persistent_term.put(HashBuilder, true) end
9+
def start_counter(to_height) do :persistent_term.put(HashBuilderCounter, to_height) end
10+
def stop() do :persistent_term.put(HashBuilder, false) end
11+
def stop_counter() do :persistent_term.put(HashBuilderCounter, nil) end
1512

13+
def init(state) do
1614
:erlang.send_after(1000, self(), :tick)
1715
{:ok, state}
1816
end
1917

2018
def handle_info(:tick, state) do
21-
if RocksDB.get_cf_size(:tx_filter) >= 60 do
22-
clear()
23-
end
24-
tick()
19+
:persistent_term.get(HashBuilder, false) && tick()
20+
:persistent_term.get(HashBuilderCounter, nil) && tick_count()
2521
:erlang.send_after(1000, self(), :tick)
2622
{:noreply, state}
2723
end
2824

29-
def clear() do
25+
def clear_all() do
3026
IO.puts "clearing stale hashfilter"
31-
RocksDB.delete_range_cf_call(:tx_filter, true)
27+
RocksDB.delete_range_cf_call(:tx_filter, false)
3228
RocksDB.delete("filter_hashes_rebuilt_up_to", DB.API.db_handle(%{}, :sysconf, %{}))
3329
RocksDB.put("filter_hashes_end_hash", DB.Chain.tip(), DB.API.db_handle(%{}, :sysconf, %{}))
3430
end
@@ -42,4 +38,33 @@ defmodule DB.Entry.Hashbuilder do
4238
Enum.each(0..100_000, fn(_)-> DB.Entry.build_filter_hashes() end)
4339
end
4440
end
41+
42+
def tick_count() do
43+
count_up_to = RocksDB.get("txs_count_up_to", DB.API.db_handle(%{}, :sysconf, %{})) || EntryGenesis.get().hash
44+
count_up_to_height = DB.Entry.by_hash(count_up_to).header.height
45+
if count_up_to_height < :persistent_term.get(HashBuilderCounter) do
46+
Enum.each(0..100_000, fn(_)-> count_txs() end)
47+
end
48+
end
49+
50+
def count_txs() do
51+
count_up_to = RocksDB.get("txs_count_up_to", DB.API.db_handle(%{}, :sysconf, %{})) || EntryGenesis.get().hash
52+
entry = DB.Entry.by_hash(count_up_to)
53+
if entry.header.height >= :persistent_term.get(HashBuilderCounter) do
54+
throw %{error: :count_txs_finished, height: entry.header.height}
55+
end
56+
57+
if rem(entry.header.height, 10_000) == 0 do
58+
IO.inspect {:count_up_to, entry.header.height}
59+
end
60+
61+
old_cnt = RocksDB.get("tx_count_historic", DB.API.db_handle(%{}, :sysconf, %{})) || "0"
62+
new_cnt = :erlang.binary_to_integer(old_cnt) + length(entry.txs)
63+
RocksDB.put("tx_count_historic", :erlang.integer_to_binary(new_cnt), DB.API.db_handle(%{}, :sysconf, %{}))
64+
65+
n = DB.Entry.next(count_up_to)
66+
if n do
67+
RocksDB.put("txs_count_up_to", n, DB.API.db_handle(%{}, :sysconf, %{}))
68+
end
69+
end
4570
end

‎ex/lib/http/multiserver.ex‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,9 @@ defmodule Ama.MultiServer do
102102
r.method == "GET" and String.starts_with?(r.path, "/api/chain/stats") ->
103103
stats = API.Chain.stats()
104104
quick_reply(state, %{error: :ok, stats: stats})
105+
r.method == "GET" and String.starts_with?(r.path, "/api/chain/kpi") ->
106+
kpi = API.Chain.kpi()
107+
quick_reply(state, %{error: :ok, kpi: kpi})
105108
r.method == "GET" and String.starts_with?(r.path, "/api/chain/tip") ->
106109
result = API.Chain.entry_tip()
107110
quick_reply(state, result)
@@ -207,7 +210,7 @@ defmodule Ama.MultiServer do
207210

208211
limit: :erlang.binary_to_integer(query[:limit] || "100"),
209212
sort: case query[:sort] do "desc" -> :desc; _ -> :asc end,
210-
cursor: if query[:cursor_b58] do Base58.decode(query.cursor_b58) else query[:cursor] end,
213+
cursor: query[:cursor] && Base58.decode(query.cursor),
211214
}
212215
{cursor, txs} = API.TX.get_by_filter(filters)
213216
result = %{cursor: cursor, txs: txs}

‎ex/native/rdb/src/tx_filter.rs‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -79,14 +79,31 @@ pub fn build_tx_hashfilters<'a>(env: Env<'a>, txus: Vec<Term<'a>>) -> NifResult<
7979
all_filters.push((bin.into(), tx_hash8));
8080
};
8181

82-
push_key(&[signer, ZERO, ZERO, ZERO]);
83-
push_key(&[ZERO, arg0, ZERO, ZERO]);
84-
push_key(&[signer, arg0, ZERO, ZERO]);
85-
push_key(&[signer, ZERO, contract, ZERO]);
86-
push_key(&[signer, ZERO, contract, func]);
87-
push_key(&[ZERO, arg0, contract, ZERO]);
88-
push_key(&[ZERO, arg0, contract, func]);
89-
push_key(&[signer, arg0, contract, func]);
82+
match (contract, func) {
83+
(b"Epoch", b"submit_sol") => {
84+
push_key(&[signer, ZERO, ZERO, ZERO]);
85+
push_key(&[ZERO, arg0, ZERO, ZERO]);
86+
// Do we care about this?
87+
// tx_filter CF goes from 10G to 22G if we add this back
88+
// reevaluate later difference is OK
89+
push_key(&[signer, ZERO, contract, ZERO]);
90+
push_key(&[signer, ZERO, contract, func]);
91+
push_key(&[ZERO, ZERO, contract, ZERO]);
92+
push_key(&[ZERO, ZERO, contract, func]);
93+
},
94+
_ => {
95+
push_key(&[signer, ZERO, ZERO, ZERO]);
96+
push_key(&[ZERO, arg0, ZERO, ZERO]);
97+
push_key(&[signer, arg0, ZERO, ZERO]);
98+
push_key(&[ZERO, ZERO, contract, ZERO]);
99+
push_key(&[ZERO, ZERO, contract, func]);
100+
push_key(&[signer, ZERO, contract, ZERO]);
101+
push_key(&[signer, ZERO, contract, func]);
102+
push_key(&[ZERO, arg0, contract, ZERO]);
103+
push_key(&[ZERO, arg0, contract, func]);
104+
push_key(&[signer, arg0, contract, func]);
105+
}
106+
}
90107
}
91108

92109
Ok(all_filters)
@@ -157,7 +174,7 @@ pub fn query_tx_hashfilter<'a, 'db>(env: Env<'a>, db: &'db TransactionDB<MultiTh
157174

158175
let tx_prefix_8 = iter_txfilter.value().unwrap();
159176

160-
// --- Inner Scan: Resolve Tx Hash Collisions ---
177+
// TODO: check for collisions when signer/arg0/contract/function is used
161178
iter_tx.seek(tx_prefix_8);
162179
while iter_tx.valid() {
163180
match iter_tx.key() {

0 commit comments

Comments
 (0)