[Monitoring] Adding a blackbox fuzzer testcase generation time metric (#4414)

### Motivation

Chrome folks need to know how long on average a fuzzer takes to generate
a testcase. This PR implements that.

Part of #4271

Author: vitorguidi

src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py

@@ -1567,6 +1567,23 @@ def do_engine_fuzzing(self, engine_impl):
 
     return crashes, fuzzer_metadata
 
+  def _emit_testcase_generation_time_metric(self, start_time, testcase_count,
+                                            fuzzer, job):
+    testcase_generation_finish = time.time()
+    elapsed_testcase_generation_time = testcase_generation_finish
+    elapsed_testcase_generation_time -= start_time
+    # Avoid division by zero.
+    if testcase_count:
+      average_time_per_testcase = elapsed_testcase_generation_time
+      average_time_per_testcase = average_time_per_testcase / testcase_count
+      monitoring_metrics.TESTCASE_GENERATION_AVERAGE_TIME.add(
+          average_time_per_testcase,
+          labels={
+              'job': job,
+              'fuzzer': fuzzer,
+              'platform': environment.platform(),
+          })
+
   def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type):
     """Run blackbox fuzzing. Currently also used for engine fuzzing."""
     # Set the thread timeout values.
@@ -1590,11 +1607,15 @@ def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type):
 
     # Run the fuzzer to generate testcases. If error occurred while trying
     # to run the fuzzer, bail out.
+    testcase_generation_start = time.time()
     generate_result = self.generate_blackbox_testcases(
         fuzzer, job_type, fuzzer_directory, testcase_count)
     if not generate_result.success:
       return None, None, None, None
 
+    self._emit_testcase_generation_time_metric(
+        testcase_generation_start, testcase_count, fuzzer.name, job_type)
+
     environment.set_value('FUZZER_NAME', self.fully_qualified_fuzzer_name)
 
     # Initialize a list of crashes.

src/clusterfuzz/_internal/metrics/monitoring_metrics.py

@@ -140,6 +140,19 @@
     ],
 )
 
+TESTCASE_GENERATION_AVERAGE_TIME = monitor.CumulativeDistributionMetric(
+    'task/fuzz/fuzzer/testcase_generation_duration',
+    bucketer=monitor.GeometricBucketer(),
+    description=('Distribution of blackbox fuzzer average testcase '
+                 ' generation time, in seconds '
+                 '(grouped by fuzzer, job and platform).'),
+    field_spec=[
+        monitor.StringField('platform'),
+        monitor.StringField('job'),
+        monitor.StringField('fuzzer'),
+    ],
+)
+
 FUZZER_TESTCASE_COUNT_RATIO = monitor.CumulativeDistributionMetric(
     'task/fuzz/fuzzer/testcase_count_ratio',
     bucketer=monitor.FixedWidthBucketer(width=0.05, num_finite_buckets=20),