Merge pull request #737 from bugout-dev/state-crawler-jobs-from-cli

State crawler jobs from cli

Author: Andrei-Dolgolev

crawlers/deploy/mumbai-state-clean.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Execute state clean labels crawler
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain mumbai -N 10000
+CPUWeight=60
+SyslogIdentifier=mumbai-state-clean

crawlers/deploy/mumbai-state-clean.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Mumbai state clean labels crawler each 25m
+
+[Timer]
+OnBootSec=50s
+OnUnitActiveSec=25m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/mumbai-state.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Execute state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain mumbai --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json
+CPUWeight=60
+SyslogIdentifier=mumbai-state

crawlers/deploy/mumbai-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Mumbai state crawler each 10m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/polygon-state.service

@@ -8,6 +8,6 @@ User=ubuntu
 Group=www-data
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json
 CPUWeight=60
 SyslogIdentifier=polygon-state

crawlers/mooncrawl/mooncrawl/settings.py

@@ -210,3 +210,24 @@
 MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX = os.environ.get(
     "MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX", "dev"
 )
+
+
+# infura config
+
+
+INFURA_PROJECT_ID = os.environ.get("INFURA_PROJECT_ID")
+
+infura_networks = {
+    AvailableBlockchainType.ETHEREUM: {
+        "name": "mainnet",
+        "url": f"https://mainnet.infura.io/v3/{INFURA_PROJECT_ID}",
+    },
+    AvailableBlockchainType.POLYGON: {
+        "name": "polygon",
+        "url": f"https://polygon-mainnet.infura.io/v3/{INFURA_PROJECT_ID}",
+    },
+    AvailableBlockchainType.MUMBAI: {
+        "name": "mumbai",
+        "url": f"https://polygon-mumbai.infura.io/v3/{INFURA_PROJECT_ID}",
+    },
+}