Remove remaining infrastructure for dev ZIM file polling. The call to kick this off was already removed in dba549e

Author: audiodude

supervisord.conf

@@ -198,30 +198,6 @@ stopsignal=TERM
 autostart=true
 autorestart=true
 
-[program:wp1-zimfile-polling]
-; In the docker-compose world, the redis host is just 'redis'
-command=/usr/local/bin/rq worker -u redis://redis zimfile-polling
-; process_num is required if you specify >1 numprocs
-process_name=zimfile-polling-%(process_num)s
-
-; If you want to run more than one zimfile-polling worker, increase this
-numprocs=1
-
-; This is the directory from which RQ is run. Be sure to point this to the
-; directory where your source code is importable from
-directory=/usr/src/app
-
-redirect_stderr=true
-stdout_logfile=/var/log/wp1bot/%(program_name)s-%(process_num)s.log
-stdout_logfile_maxbytes=100MB
-stdout_logfile_backups=5
-
-; RQ requires the TERM signal to perform a warm shutdown. If RQ does not die
-; within 10 seconds, supervisor will forcefully kill it
-stopsignal=TERM
-autostart=true
-autorestart=true
-
 [program:wp1-zimfile-scheduling]
 ; In the docker-compose world, the redis host is just 'redis'
 command=/usr/local/bin/rq worker -u redis://redis zimfile-scheduling

wp1/constants.py

@@ -84,7 +84,4 @@ class AssessmentKind(enum.Enum):
 
 WP1_USER_AGENT = "WP 1.0 bot 1.0.0/Audiodude <audiodude@gmail.com>"
 
-# 2 hours
-MAX_ZIM_FILE_POLL_TIME = 2 * 60 * 60
-
 SECONDS_PER_MONTH = 30 * 24 * 60 * 60  # Approximate seconds in a month

wp1/logic/builder.py

@@ -16,7 +16,6 @@
 from wp1.constants import (
     CONTENT_TYPE_TO_EXT,
     EXT_TO_CONTENT_TYPE,
-    MAX_ZIM_FILE_POLL_TIME,
     TS_FORMAT_WP10,
 )
 from wp1.credentials import CREDENTIALS, ENV
@@ -785,34 +784,6 @@ def _format_active_schedule_data(schedule):
     return data
 
 
-def on_zim_file_status_poll(task_id):
-    wp10db = wp10_connect()
-    redis = redis_connect()
-    app_logging.configure_logging()
-
-    result = zimfarm.is_zim_file_ready(task_id)
-    logging.info("Polled for ZIM file for task_id=%s, result: %s", task_id, result)
-    if result == "FILE_READY":
-        logic_selection.update_zimfarm_task(
-            wp10db, task_id, "FILE_READY", set_updated_now=True
-        )
-        update_version_for_finished_zim(wp10db, task_id)
-    elif result == "REQUESTED":
-        requested = logic_selection.zim_file_requested_at_for(wp10db, task_id)
-        if requested is not None:
-            now = utcnow().timestamp()
-            if now - requested > MAX_ZIM_FILE_POLL_TIME:
-                logic_selection.update_zimfarm_task(wp10db, task_id, "FAILED")
-                return
-
-        # There was no requested time, or the time hasn't expired, re-request
-        queues.poll_for_zim_file_status(redis, task_id)
-    elif result == "FAILED":
-        logic_selection.update_zimfarm_task(wp10db, task_id, "FAILED")
-
-    wp10db.close()
-
-
 def _get_builder_data(builder):
     return {
         "id": builder["b_id"].decode("utf-8"),

wp1/queues.py

@@ -35,10 +35,6 @@ def _get_materializer_queue(redis):
     return Queue("materializer", connection=redis)
 
 
-def _get_zimfile_poll_queue(redis):
-    return Queue("zimfile-polling", connection=redis)
-
-
 def _get_zimfile_scheduling_queue(redis):
     return Queue("zimfile-scheduling", connection=redis)
 
@@ -187,14 +183,6 @@ def enqueue_materialize(redis, builder_cls, builder, content_type):
     )
 
 
-def poll_for_zim_file_status(redis, task_id):
-    poll_q = _get_zimfile_poll_queue(redis)
-    scheduler = Scheduler(queue=poll_q, connection=redis)
-    scheduler.enqueue_in(
-        timedelta(minutes=2), logic_builder.on_zim_file_status_poll, task_id
-    )
-
-
 def schedule_recurring_zimfarm_task(
     redis: Redis, args, scheduled_time, interval_seconds, repeat_count
 ):

wp1/zimfarm.py

@@ -487,24 +487,6 @@ def _get_task_by_id(task_id):
     return r.json()
 
 
-def is_zim_file_ready(task_id):
-    data = _get_task_by_id(task_id)
-    if data is None:
-        # Some kind of HTTP error (could be 404 if the task hasn't moved
-        # from requested-tasks to tasks yet). Ignore and we'll retry.
-        return "REQUESTED"
-
-    if data.get("status") == "failed":
-        return "FAILED"
-
-    files = data.get("files", {})
-
-    for key, value in files.items():
-        if value.get("status") == "uploaded":
-            return "FILE_READY"
-    return "REQUESTED"
-
-
 def zim_file_url_for_task_id(task_id):
     data = _get_task_by_id(task_id)