Merge branch 'main' into metadata-crawler-improvements

Author: Andrey

crawlers/deploy/deploy.bash

@@ -58,6 +58,10 @@ MUMBAI_SYNCHRONIZE_SERVICE="mumbai-synchronize.service"
 MUMBAI_MISSING_SERVICE_FILE="mumbai-missing.service"
 MUMBAI_MISSING_TIMER_FILE="mumbai-missing.timer"
 MUMBAI_MOONWORM_CRAWLER_SERVICE_FILE="mumbai-moonworm-crawler.service"
+MUMBAI_STATE_SERVICE_FILE="mumbai-state.service"
+MUMBAI_STATE_TIMER_FILE="mumbai-state.timer"
+MUMBAI_STATE_CLEAN_SERVICE_FILE="mumbai-state-clean.service"
+MUMBAI_STATE_CLEAN_TIMER_FILE="mumbai-state-clean.timer"
 
 # XDai service files
 XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service"
@@ -289,3 +293,23 @@ cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" "/etc/systemd
 cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" "/etc/systemd/system/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}"
 systemctl daemon-reload
 systemctl restart --no-block "${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}"
+
+
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing MUMBAI state service and timer with: ${MUMBAI_STATE_SERVICE_FILE}, ${MUMBAI_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${MUMBAI_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${MUMBAI_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${MUMBAI_STATE_SERVICE_FILE}" "/etc/systemd/system/${MUMBAI_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${MUMBAI_STATE_TIMER_FILE}" "/etc/systemd/system/${MUMBAI_STATE_TIMER_FILE}"
+systemctl daemon-reload
+systemctl restart --no-block "${MUMBAI_STATE_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing MUMBAI state clean service and timer with: ${MUMBAI_STATE_CLEAN_SERVICE_FILE}, ${MUMBAI_STATE_CLEAN_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${MUMBAI_STATE_CLEAN_SERVICE_FILE}" "${SCRIPT_DIR}/${MUMBAI_STATE_CLEAN_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${MUMBAI_STATE_CLEAN_SERVICE_FILE}" "/etc/systemd/system/${MUMBAI_STATE_CLEAN_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${MUMBAI_STATE_CLEAN_TIMER_FILE}" "/etc/systemd/system/${MUMBAI_STATE_CLEAN_TIMER_FILE}"
+systemctl daemon-reload
+systemctl restart --no-block "${MUMBAI_STATE_CLEAN_TIMER_FILE}"

crawlers/deploy/mumbai-state-clean.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Execute state clean labels crawler
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain mumbai -N 10000
+CPUWeight=60
+SyslogIdentifier=mumbai-state-clean

crawlers/deploy/mumbai-state-clean.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Mumbai state clean labels crawler each 25m
+
+[Timer]
+OnBootSec=50s
+OnUnitActiveSec=25m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/mumbai-state.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Execute state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain mumbai --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json
+CPUWeight=60
+SyslogIdentifier=mumbai-state

crawlers/deploy/mumbai-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Mumbai state crawler each 10m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/polygon-state.service

@@ -8,6 +8,6 @@ User=ubuntu
 Group=www-data
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json
 CPUWeight=60
 SyslogIdentifier=polygon-state

crawlers/mooncrawl/mooncrawl/settings.py

@@ -210,3 +210,24 @@
 MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX = os.environ.get(
     "MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX", "dev"
 )
+
+
+# infura config
+
+
+INFURA_PROJECT_ID = os.environ.get("INFURA_PROJECT_ID")
+
+infura_networks = {
+    AvailableBlockchainType.ETHEREUM: {
+        "name": "mainnet",
+        "url": f"https://mainnet.infura.io/v3/{INFURA_PROJECT_ID}",
+    },
+    AvailableBlockchainType.POLYGON: {
+        "name": "polygon",
+        "url": f"https://polygon-mainnet.infura.io/v3/{INFURA_PROJECT_ID}",
+    },
+    AvailableBlockchainType.MUMBAI: {
+        "name": "mumbai",
+        "url": f"https://polygon-mumbai.infura.io/v3/{INFURA_PROJECT_ID}",
+    },
+}