Fix deadline in manage_vms (#4169)

Fix problem in oss-fuzz where manage_vms doesn't complete, causing old
VMs/containers to hang around.
Do this by increasing the deadline to 24 hours for manage_vms to
compelte. Also, temporarily change the order of manage_vms so that the
previously unresized groups will be resized first. This should hopefully
get rid of weird errors we are seeing.

Author: jonathanmetzman

infra/k8s/manage-vms.yaml

@@ -21,7 +21,7 @@ spec:
   concurrencyPolicy: Forbid
   jobTemplate:
     spec:
-      activeDeadlineSeconds: 43200  # 12 hours.
+      activeDeadlineSeconds: 86400  # 24 hours.
       template:
         spec:
           containers:

src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py

@@ -1193,9 +1193,13 @@ def run_engine_fuzzer(engine_impl, target_name, sync_corpus_directory,
   """Run engine for fuzzing."""
   if environment.is_trusted_host():
     from clusterfuzz._internal.bot.untrusted_runner import tasks_host
-    return tasks_host.engine_fuzz(engine_impl, target_name,
-                                  sync_corpus_directory, testcase_directory)
+    logs.info('Running remote engine fuzz.')
+    result = tasks_host.engine_fuzz(engine_impl, target_name,
+                                    sync_corpus_directory, testcase_directory)
+    logs.info('Done remote engine fuzz.')
+    return result
 
+  logs.info('Worker engine fuzz.')
   build_dir = environment.get_value('BUILD_DIR')
   target_path = engine_common.find_fuzzer_path(build_dir, target_name)
   if target_path is None:

src/clusterfuzz/_internal/cron/manage_vms.py

@@ -19,6 +19,7 @@
 import itertools
 import json
 import logging
+from typing import List
 
 from google.cloud import ndb
 
@@ -156,6 +157,21 @@ def _template_needs_update(current_template, new_template, resource_name):
   return False
 
 
+def reverse_pairs(l: List) -> List:
+  """Temporary function to speedup resizing of group 8 which hasn't been
+  updating due to timeouts."""
+  l = list(reversed(l))
+  length = len(l)
+  l_output = []
+  for idx in range(0, length, 2):
+    if idx + 1 >= length:
+      l_output.append(l[idx])
+      continue
+    l_output.append(l[idx + 1])
+    l_output.append(l[idx])
+  return l_output
+
+
 class ClustersManager:
   """Manager for clusters in a project."""
 
@@ -186,7 +202,10 @@ def update_clusters(self):
     """Update all clusters in a project."""
     self.start_thread_pool()
 
-    for cluster in self.gce_project.clusters:
+    # TODO(metzman): Remove reverse_pairs.
+    # Use this function so we don't introduce weird bugs by changing the order
+    # in which hosts and workers are reassigned.
+    for cluster in reverse_pairs(self.gce_project.clusters):
       self.pending_updates.append(
           self.thread_pool.submit(self.update_cluster, cluster, cluster.name,
                                   cluster.instance_count))

src/clusterfuzz/_internal/tests/appengine/handlers/cron/manage_vms_test.py

@@ -1084,6 +1084,14 @@ def test_primes(self):
     self.assertEqual(3571, sum(result))
 
 
+class ReversePairsTest(unittest.TestCase):
+  """Tests reverse_pairs."""
+
+  def test_reverse_pairs(self):
+    l = list(range(6))
+    self.assertEqual(manage_vms.reverse_pairs(l), [4, 5, 2, 3, 0, 1])
+
+
 @test_utils.with_cloud_emulators('datastore')
 class AssignHostWorkerTest(unittest.TestCase):
   """Tests host -> worker assignment."""