feat: normalized events & pwr collector (#70)

Author: fmorency

README.md

@@ -123,6 +123,22 @@ erDiagram
     serial id
     jsonb data
   }
+  "api.events_raw" {
+    varchar(64) id
+    bigint event_index
+    jsonb data
+  }
+  "api.events_main" {
+    varchar(64) id
+    bigint event_index
+    bigint attr_index
+    text event_type
+    text attr_key
+    text attr_value
+    bigint msg_index
+  }
+  "api.transactions_raw" ||--o{ "api.events_raw": "trigger insert/update"
+  "api.events_raw" ||--|| "api.events_main" : "trigger insert/update"
 ```
 
 #### Usage

cmd/yaci/postgres_test.go

@@ -60,8 +60,8 @@ func testExtractBlocksAndTxs(t *testing.T) {
 
 		transactions := getJSONResponse(t, RestTxEndpoint, nil)
 		require.NotEmpty(t, transactions)
-		// The number of transactions is 32 as defined in the `compose.yaml` file under the `manifest-ledger-tx` service
-		require.Len(t, transactions, 32)
+		// The number of transactions is 42 as defined in the `compose.yaml` file under the `manifest-ledger-tx` service
+		require.Len(t, transactions, 42)
 	})
 }
 
@@ -180,10 +180,14 @@ func testPrometheusMetrics(t *testing.T) {
 		require.Equal(t, 200, resp.StatusCode())
 		body := string(resp.Body())
 		require.Contains(t, body, "yaci_addresses_total_unique_user{source=\"postgres\"} 3")
-		require.Contains(t, body, "yaci_addresses_total_unique_group{source=\"postgres\"} 2")
-		require.Contains(t, body, "yaci_transactions_total_count{source=\"postgres\"} 32")
-		require.Contains(t, body, "yaci_tokenomics_total_burn_amount{source=\"postgres\"} 123")
+		require.Contains(t, body, "yaci_addresses_total_unique_group{source=\"postgres\"} 3")
+		require.Contains(t, body, "yaci_transactions_total_count{source=\"postgres\"} 42")
+		// 3000000umfx were burned by the MFX to PWR conversion
+		// 123umfx were burned by a POA proposal
+		require.Contains(t, body, "yaci_tokenomics_total_burn_amount{source=\"postgres\"} 3.000123e+06")
 		require.Contains(t, body, "yaci_tokenomics_total_payout_amount{source=\"postgres\"} 7.54321e+06")
+		// 6000000factory/.../upwr were minted by the MFX to PWR conversion
+		require.Contains(t, body, "yaci_tokenomics_total_pwr_minted_amount{source=\"postgres\"} 6e+06")
 		require.Contains(t, body, "yaci_locked_tokens_count{amount=\"2000000000\",denom=\"umfx\",source=\"postgres\"} 1")
 	})
 }

docker/.env

@@ -19,7 +19,7 @@ P2P="26656"
 GRPC="9090"
 GRPC_WEB="9091"
 ROSETTA="8080"
-TIMEOUT_COMMIT="2s"
+TIMEOUT_COMMIT="3s"
 VOTING_TIMEOUT="5s"
 DENOM=umfx
 BOND_DENOM=upoa
@@ -32,3 +32,4 @@ TX_COUNT_DIR=/tx-count
 TX_COUNT_FILE=tx-count.txt
 TX_COUNT_PATH="${TX_COUNT_DIR}/${TX_COUNT_FILE}"
 POSTGREST_HOST=postgrest
+CONVERT_WASM_ADDRESS="manifest14hj2tavq8fpesdwxxcu44rty3hh90vhujrvcmstl4zr3txmfvw9s4zfs7u"

docker/infra/compose.yaml

@@ -30,6 +30,7 @@ services:
         - ./scripts/init_manifest_ledger.sh:/usr/local/bin/init_manifest_ledger.sh:ro
     ports:
       - "9090:9090"
+      - "26657:26657"
     entrypoint: ["/bin/bash", "-c"]
     env_file: ../.env
     command: /usr/local/bin/init_manifest_ledger.sh

docker/infra/proposals/grant.template.json

@@ -0,0 +1,33 @@
+{
+  "group_policy_address": "${POA_ADMIN_ADDRESS}",
+  "messages": [
+    {
+      "@type": "/cosmos.authz.v1beta1.MsgGrant",
+      "granter": "${POA_ADMIN_ADDRESS}",
+      "grantee": "${CONVERT_WASM_ADDRESS}",
+      "grant": {
+        "authorization": {
+          "@type": "/cosmos.authz.v1beta1.GenericAuthorization",
+          "msg": "/osmosis.tokenfactory.v1beta1.MsgMint"
+        },
+        "expiration": "2030-01-01T00:00:00Z"
+      }
+    },
+    {
+      "@type": "/cosmos.authz.v1beta1.MsgGrant",
+      "granter": "${POA_ADMIN_ADDRESS}",
+      "grantee": "${CONVERT_WASM_ADDRESS}",
+      "grant": {
+        "authorization": {
+          "@type": "/cosmos.authz.v1beta1.GenericAuthorization",
+          "msg": "/liftedinit.manifest.v1.MsgBurnHeldBalance"
+        },
+        "expiration": "2030-01-01T00:00:00Z"
+      }
+    }
+  ],
+  "metadata": "",
+  "title": "Grant converter contract mint and burn permissions",
+  "summary": "We want to allow the converter contract to mint PWR and burn MFX for testing purposes",
+  "proposers": ["manifest1hj5fveer5cjtn4wd6wstzugjfdxzl0xp8ws9ct"]
+}

docker/infra/scripts/init_manifest_ledger.sh

@@ -3,15 +3,15 @@
 
 set -e
 
-update_test_genesis() {
-  cat $HOME_DIR/config/genesis.json | jq $1 > $HOME_DIR/config/tmp_genesis.json
-  mv $HOME_DIR/config/tmp_genesis.json $HOME_DIR/config/genesis.json
+update_test_genesis () {
+  cat $HOME_DIR/config/genesis.json | jq "$1" > $HOME_DIR/config/tmp_genesis.json && mv $HOME_DIR/config/tmp_genesis.json $HOME_DIR/config/genesis.json
 }
 
 echo "$MNEMO1" | $BINARY keys add "$KEY" --home="$HOME_DIR" --keyring-backend "$KEYRING" --recover
 echo "$MNEMO2" | $BINARY keys add "$KEY2" --home="$HOME_DIR" --keyring-backend "$KEYRING" --recover
 $BINARY init $MONIKER --home=$HOME_DIR --chain-id $CHAIN_ID
 update_test_genesis '.consensus["params"]["block"]["max_gas"]="1000000000"'
+update_test_genesis '.app_state["bank"]["denom_metadata"]=[{"base":"umfx","denom_units":[{"aliases":[],"denom":"umfx","exponent":0},{"aliases":[],"denom":"MFX","exponent":6}],"description":"MFX","display":"MFX","name":"MFX","symbol":"MFX","uri":"","uri_hash":""}]'
 update_test_genesis '.app_state["gov"]["params"]["min_deposit"]=[{"denom":"'$DENOM'","amount":"1000000"}]'
 update_test_genesis '.app_state["gov"]["params"]["voting_period"]="15s"'
 update_test_genesis '.app_state["gov"]["params"]["expedited_voting_period"]="10s"'
@@ -27,6 +27,10 @@ update_test_genesis '.app_state["group"]["groups"]=[{"id":"1","admin":"'${POA_AD
 update_test_genesis '.app_state["group"]["group_members"]=[{"group_id":"1","member":{"address":"'${ADDR1}'","weight":"1","metadata":"user1","added_at":"2024-05-16T15:10:54.372190727Z"}},{"group_id":"1","member":{"address":"'${ADDR2}'","weight":"1","metadata":"user2","added_at":"2024-05-16T15:10:54.372190727Z"}}]'
 update_test_genesis '.app_state["group"]["group_policy_seq"]="1"'
 update_test_genesis '.app_state["group"]["group_policies"]=[{"address":"'${POA_ADMIN_ADDRESS}'","group_id":"1","admin":"'${POA_ADMIN_ADDRESS}'","metadata":"AQ==","version":"2","decision_policy":{"@type":"/cosmos.group.v1.ThresholdDecisionPolicy","threshold":"1","windows":{"voting_period":"5s","min_execution_period":"0s"}},"created_at":"2024-05-16T15:10:54.372190727Z"}]'
+update_test_genesis '.app_state["tokenfactory"]["factory_denoms"]=[{"denom":"factory/manifest1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzsfmy9qj/upwr","authority_metadata":{"admin":"manifest1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzsfmy9qj"}}]'
+update_test_genesis '.app_state["bank"]["denom_metadata"] |= . + [{"description":"PWR","denom_units":[{"denom":"factory/manifest1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzsfmy9qj/upwr","exponent":0,"aliases":["PWR"]},{"denom":"PWR","exponent":6,"aliases":["factory/manifest1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzsfmy9qj/upwr"]}],"base":"factory/manifest1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzsfmy9qj/upwr","display":"PWR","name":"POWER","symbol":"PWR","uri":"","uri_hash":""}]'
+update_test_genesis '.app_state["wasm"]["params"]["code_upload_access"]["permission"]="Everybody"'
+update_test_genesis '.app_state["wasm"]["params"]["instantiate_default_permission"]="Everybody"'
 $BINARY genesis add-genesis-account $KEY 100000000000000000${BOND_DENOM},100000000000000000000000000000${DENOM} --keyring-backend $KEYRING --home=$HOME_DIR
 $BINARY genesis add-genesis-account $KEY2 100000000000000000${DENOM} --keyring-backend $KEYRING --home=$HOME_DIR
 $BINARY genesis gentx $KEY 1000000${BOND_DENOM} --keyring-backend $KEYRING --home=$HOME_DIR --chain-id $CHAIN_ID --commission-rate=0.0 --commission-max-rate=1.0 --commission-max-change-rate=0.1

docker/infra/scripts/init_manifest_ledger_txs.sh

@@ -5,6 +5,11 @@ set -e
 
 TX_COUNT=0
 PROPOSAL_ID=1
+CONTRACT_VERSION=v0.1.0
+
+# Download the wasm contract
+echo "Downloading converter.wasm contract version ${CONTRACT_VERSION}"
+curl -sL https://github.com/manifest-network/manifest-contracts/releases/download/${CONTRACT_VERSION}/converter.wasm --output /tmp/converter.wasm
 
 # We want to keep track of the transaction count in a file; check if the directory containing the file exists
 if [ -d "${TX_COUNT_DIR}" ]; then
@@ -30,6 +35,20 @@ function run_proposal() {
   TX_COUNT=$((TX_COUNT + 3))
 }
 
+## Wasm module
+echo "-> Storing converter.wasm contract"
+run_tx tx wasm store /tmp/converter.wasm --from $KEY --note "tx-store-converter"
+
+echo "-> Instantiating converter.wasm contract"
+run_tx tx wasm instantiate 1 '{"admin":"'${POA_ADMIN_ADDRESS}'","poa_admin":"'${POA_ADMIN_ADDRESS}'","rate":"2","source_denom":"umfx","target_denom":"factory/'${POA_ADMIN_ADDRESS}'/upwr","paused":false}' --from $KEY --admin $ADDR1 --label "converter" --note "tx-instantiate-converter"
+
+echo "-> Granting converter contract mint/burn permissions"
+run_proposal "grant.json" "$ADDR1" "tx-grant-proposal" --from $KEY
+
+# Converter contract address is `manifest14hj2tavq8fpesdwxxcu44rty3hh90vhujrvcmstl4zr3txmfvw9s4zfs7u`
+echo "-> Converting 3 MFX to 6 PWR"
+run_tx tx wasm execute ${CONVERT_WASM_ADDRESS} '{"convert":{}}' --from $KEY --note "tx-mint-from-converter" --amount 3000000umfx
+
 ## Bank module
 run_tx tx bank send $ADDR1 ${POA_ADMIN_ADDRESS} 10000${DENOM} --from $KEY --note "tx-send-to-poa-admin"
 run_tx tx bank multi-send $ADDR1 $ADDR2 ${POA_ADMIN_ADDRESS} 10000${DENOM} --from $KEY --note "tx-multi-send-to-poa-admin"
@@ -48,6 +67,14 @@ run_tx tx tokenfactory change-admin factory/$ADDR1/ufoobar $ADDR2 --from $KEY --
 run_proposal "payout.json" "$ADDR1" "tx-payout-proposal" --from $KEY
 run_proposal "burn.json" "$ADDR1" "tx-burn-proposal" --from $KEY
 
+# Submit and withdraw payout and burn proposals to make sure collector works properly
+run_tx tx group submit-proposal "/proposals/payout.json" --note "tx-payout-proposal-submit-2" --from $KEY
+run_tx tx group withdraw-proposal $PROPOSAL_ID $ADDR1 --note "tx-payout-proposal-withdraw" --from $KEY
+PROPOSAL_ID=$((PROPOSAL_ID + 1))
+run_tx tx group submit-proposal "/proposals/burn.json" --note "tx-burn-proposal-submit-2" --from $KEY
+run_tx tx group withdraw-proposal $PROPOSAL_ID $ADDR1 --note "tx-burn-proposal-withdraw" --from $KEY
+PROPOSAL_ID=$((PROPOSAL_ID + 1))
+
 ### Group module
 echo ${GROUP_MEMBERS} > members.json && cat members.json
 echo ${DECISION_POLICY} > policy.json && cat policy.json

docker/yaci/scripts/yaci_tx_check.sh

@@ -48,10 +48,10 @@ function check_addr_txs() {
 
 # Address-specific configurations
 declare -A ADDRESS_EXPECTED_TXS=(
-  ["${ADDR1}"]="33"
-  ["${ADDR2}"]="11"
+  ["${ADDR1}"]="43"
+  ["${ADDR2}"]="12"
   ["${USER_GROUP_ADDRESS}"]="15"
-  ["${POA_ADMIN_ADDRESS}"]="8"
+  ["${POA_ADMIN_ADDRESS}"]="14"
   ["${VESTING_ADDR}"]="1"
 )
 
@@ -92,6 +92,16 @@ declare -a ADDRESS1_TX_CHECKS=(
   "tx-send-proposal-error-vote:false"
   "tx-send-proposal-error-exec:false"
   "tx-create-periodic-vesting-account:false"
+  "tx-store-converter:false"
+  "tx-instantiate-converter:false"
+  "tx-grant-proposal-submit:false"
+  "tx-grant-proposal-vote:false"
+  "tx-grant-proposal-exec:false"
+  "tx-mint-from-converter:false"
+  "tx-payout-proposal-submit-2:false"
+  "tx-payout-proposal-withdraw:false"
+  "tx-burn-proposal-submit-2:false"
+  "tx-burn-proposal-withdraw:false"
 )
 
 declare -a ADDRESS2_TX_CHECKS=(
@@ -135,6 +145,10 @@ declare -a POA_ADMIN_ADDRESS_TX_CHECKS=(
   "tx-burn-proposal-submit:false"
   "tx-burn-proposal-submit:true"
   "tx-burn-proposal-exec:false"
+  "tx-grant-proposal-submit:false"
+  "tx-grant-proposal-submit:true"
+  "tx-payout-proposal-submit-2:false"
+  "tx-burn-proposal-submit-2:false"
 )
 
 declare -A ADDRESS_TX_CHECKS=(

internal/metrics/collectors/total_payout_burn.go

@@ -8,25 +8,65 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+// TotalPayoutBurnQuery expected format of the cosmwasm event attribute for burn amounts, e.g., 123456umfx
 const TotalPayoutBurnQuery = `
-		WITH payout_amount AS (
-		  SELECT
-			COALESCE(SUM((pp.element->'coin'->>'amount')::numeric), 0) AS amount
-		  FROM api.messages_main m
-		  CROSS JOIN LATERAL jsonb_array_elements(m.metadata->'payoutPairs') AS pp(element)
-		  WHERE m.type = '/liftedinit.manifest.v1.MsgPayout'
-		),
-		burn_amount AS (
-		  SELECT
-			COALESCE(SUM((pp.element->>'amount')::numeric), 0) AS amount
-		  FROM api.messages_main m
-		  CROSS JOIN LATERAL jsonb_array_elements(m.metadata->'burnCoins') AS pp(element)
-		  WHERE type = '/liftedinit.manifest.v1.MsgBurnHeldBalance'
+	WITH mfx_burned AS (
+	  WITH burn_events AS (
+		SELECT
+		  e.id,
+		  e.event_index,
+		  MAX(CASE WHEN e.attr_key = 'amount' THEN e.attr_value END) AS amount_raw,
+		  MAX(e.msg_index) AS msg_index
+		FROM api.events_main e
+		WHERE e.event_type = 'burn'
+		GROUP BY e.id, e.event_index
+	  )
+	  SELECT COALESCE(SUM((rm.captures)[1]::numeric), 0) AS amount
+	  FROM burn_events b
+	  JOIN api.messages_main m
+		ON m.id = b.id
+	   AND m.message_index = b.msg_index
+	  JOIN api.transactions_main tx
+		ON tx.id = b.id
+	   AND tx.error IS NULL
+	  JOIN LATERAL regexp_matches(
+		b.amount_raw,
+		'^([0-9]+)([[:alnum:]_\/\.]+)$'
+	  ) AS rm(captures) ON TRUE
+	  WHERE (rm.captures)[2] = 'umfx'
+	),
+	payout_amount AS (
+	  SELECT
+		COALESCE(SUM((pp.element->'coin'->>'amount')::numeric), 0) AS amount
+	  FROM api.messages_main m
+	  JOIN api.transactions_main t
+		ON t.id = m.id
+	   AND t.error IS NULL
+	  CROSS JOIN LATERAL jsonb_array_elements(
+		COALESCE(m.metadata->'payoutPairs', '[]'::jsonb)
+	  ) AS pp(element)
+	  WHERE m.type = '/liftedinit.manifest.v1.MsgPayout'
+		AND (pp.element->'coin'->>'denom') = 'umfx'           -- keep only umfx
+		AND (
+		  m.message_index < 10000
+		  OR (
+			m.message_index >= 10000
+			AND EXISTS (
+			  SELECT 1
+			  FROM api.transactions_main tx2
+			  JOIN api.messages_main m2 ON tx2.id = m2.id
+			  WHERE tx2.error IS NULL
+				AND m2.type = '/cosmos.group.v1.MsgExec'
+				AND tx2.proposal_ids IS NOT NULL
+            	AND tx2.proposal_ids && t.proposal_ids
+			)
+		  )
 		)
-		SELECT 
-		  (SELECT amount FROM payout_amount) AS payout_amount,
-		  (SELECT amount FROM burn_amount) AS burn_amount
-	`
+	)
+	SELECT
+	  (SELECT amount FROM payout_amount)  AS total_payout_umfx,
+	  (SELECT amount FROM mfx_burned)     AS total_burned_umfx;
+`
 
 type TotalPayoutBurnCollector struct {
 	db                *sql.DB

internal/metrics/collectors/total_pwr_minted.go

@@ -0,0 +1,76 @@
+//go:build manifest
+
+package collectors
+
+import (
+	"database/sql"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// TotalPwrMintAmountQuery expected format of the cosmwasm event attribute for mint amounts, e.g., 123456upwr
+// Note: only counts PWR minted via the tf_mint event (i.e., via the tf_mint function)
+const TotalPwrMintAmountQuery = `
+		WITH pwr_minted AS (
+		  WITH tf_mint_events AS (
+		    SELECT
+		  	e.id,
+		  	e.event_index,
+		  	MAX(CASE WHEN e.attr_key = 'amount' THEN e.attr_value END) AS amount_raw,
+		  	MAX(e.msg_index) AS msg_index
+		    FROM api.events_main e
+		    WHERE e.event_type = 'tf_mint'
+		    GROUP BY e.id, e.event_index
+		  )
+		  SELECT COALESCE(SUM((rm.captures)[1]::numeric), 0) AS amount
+		  FROM tf_mint_events b
+		  JOIN api.messages_main m
+		    ON m.id = b.id
+		   AND m.message_index = b.msg_index
+		   AND m.type = '/cosmwasm.wasm.v1.MsgExecuteContract'
+		  JOIN LATERAL regexp_matches(
+		    b.amount_raw,
+		    '^([0-9]+)([[:alnum:]_\/\.]+)$'
+		  ) AS rm(captures) ON TRUE
+		  WHERE (rm.captures)[2] = 'factory/manifest1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzsfmy9qj/upwr'
+		)
+		SELECT amount FROM pwr_minted;
+`
+
+type TotalPwrMintedCollector struct {
+	totalPwrMintedAmount *prometheus.Desc
+	db                   *sql.DB
+}
+
+func NewTotalPwrMintedCollector(db *sql.DB) *TotalPwrMintedCollector {
+	return &TotalPwrMintedCollector{
+		db: db,
+		totalPwrMintedAmount: prometheus.NewDesc(
+			prometheus.BuildFQName("yaci", "tokenomics", "total_pwr_minted_amount"),
+			"Total PWR minted",
+			nil,
+			prometheus.Labels{"source": "postgres"},
+		),
+	}
+}
+
+func (c *TotalPwrMintedCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.totalPwrMintedAmount
+}
+
+func (c *TotalPwrMintedCollector) Collect(ch chan<- prometheus.Metric) {
+	var mintAmount float64
+	err := c.db.QueryRow(TotalPwrMintAmountQuery).Scan(&mintAmount)
+	if err != nil {
+		ch <- prometheus.NewInvalidMetric(c.totalPwrMintedAmount, err)
+		return
+	}
+
+	ch <- prometheus.MustNewConstMetric(c.totalPwrMintedAmount, prometheus.CounterValue, mintAmount)
+}
+
+func init() {
+	RegisterCollectorFactory(func(db *sql.DB, extraParams ...interface{}) (prometheus.Collector, error) {
+		return NewTotalPwrMintedCollector(db), nil
+	})
+}