Merge pull request #739 from bugout-dev/metadata-crawler-improvements

Add fixes for metadata crawler

Author: Andrei-Dolgolev

crawlers/deploy/deploy.bash

@@ -62,6 +62,8 @@ MUMBAI_STATE_SERVICE_FILE="mumbai-state.service"
 MUMBAI_STATE_TIMER_FILE="mumbai-state.timer"
 MUMBAI_STATE_CLEAN_SERVICE_FILE="mumbai-state-clean.service"
 MUMBAI_STATE_CLEAN_TIMER_FILE="mumbai-state-clean.timer"
+MUMBAI_METADATA_SERVICE_FILE="mumbai-metadata.service"
+MUMBAI_METADATA_TIMER_FILE="mumbai-metadata.timer"
 
 # XDai service files
 XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service"
@@ -312,4 +314,14 @@ chmod 644 "${SCRIPT_DIR}/${MUMBAI_STATE_CLEAN_SERVICE_FILE}" "${SCRIPT_DIR}/${MU
 cp "${SCRIPT_DIR}/${MUMBAI_STATE_CLEAN_SERVICE_FILE}" "/etc/systemd/system/${MUMBAI_STATE_CLEAN_SERVICE_FILE}"
 cp "${SCRIPT_DIR}/${MUMBAI_STATE_CLEAN_TIMER_FILE}" "/etc/systemd/system/${MUMBAI_STATE_CLEAN_TIMER_FILE}"
 systemctl daemon-reload
-systemctl restart --no-block "${MUMBAI_STATE_CLEAN_TIMER_FILE}"
+systemctl restart --no-block "${MUMBAI_STATE_CLEAN_TIMER_FILE}"
+
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing MUMBAI metadata service and timer with: ${MUMBAI_METADATA_SERVICE_FILE}, ${MUMBAI_METADATA_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${MUMBAI_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${MUMBAI_METADATA_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${MUMBAI_METADATA_SERVICE_FILE}" "/etc/systemd/system/${MUMBAI_METADATA_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${MUMBAI_METADATA_TIMER_FILE}" "/etc/systemd/system/${MUMBAI_METADATA_TIMER_FILE}"
+systemctl daemon-reload
+systemctl restart --no-block "${MUMBAI_METADATA_TIMER_FILE}"

crawlers/deploy/mumbai-metadata.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Execute metadata crawler
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.metadata_crawler.cli crawl --blockchain mumbai
+CPUWeight=60
+SyslogIdentifier=mumbai-metadata

crawlers/deploy/mumbai-metadata.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Mumbai metadata crawler each 10m
+
+[Timer]
+OnBootSec=20s
+OnUnitActiveSec=60m
+
+[Install]
+WantedBy=timers.target

crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py

@@ -39,7 +39,7 @@ def crawl_uri(metadata_uri: str) -> Any:
     while retry < 3:
         try:
 
-            response = urllib.request.urlopen(metadata_uri, timeout=5)
+            response = urllib.request.urlopen(metadata_uri, timeout=10)
 
             if response.status == 200:
                 result = json.loads(response.read())
@@ -52,6 +52,7 @@ def crawl_uri(metadata_uri: str) -> Any:
             continue
         except Exception as err:
             logger.error(err)
+            logger.error(f"request end with error for url: {metadata_uri}")
             retry += 1
             continue
     return result

crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py

@@ -96,12 +96,13 @@ def get_uris_of_tokens(
     results = [
         TokenURIs(
             token_id=data[0],
-            token_uri=data[1],
+            token_uri=data[1][0],
             block_number=data[2],
             block_timestamp=data[3],
             address=data[4],
         )
         for data in metadata_for_parsing
+        if data[1] is not None and len(data[1]) > 0
     ]
 
     return results