Merge branch 'master' into test-syncer-k8s

Author: ViniciustCosta

Pipfile

@@ -23,7 +23,7 @@ pipenv = "==2022.8.5"
 pyfakefs = "==4.4.0"
 pylint = "~=2.4"
 pylint-protobuf = "==0.21.0"
-pyright = "==1.1.378"
+pyright = "==1.1.393"
 selenium = "==3.141.0"
 twine = "*"
 WebTest = "==3.0.0"

Pipfile.lock

@@ -36,12 +36,12 @@ variable "ip_cidr_range" {
 }
 
 variable "gke_num_nodes" {
-  default     = 2
+  default     = 5
   description = "The number of gke nodes"
 }
 
 variable "machine_type" {
-  default     = "e2-standard-2"
+  default     = "e2-standard-4"
   description = "The machine type"
 }
 

infra/terraform/variables.tf

@@ -9,6 +9,6 @@
   "executionEnvironments": [{
     "root": "src",
     "extraPaths": [ "src/third_party" ],
-    "pythonVersion": "3.7"
+    "pythonVersion": "3.11"
   }]
 }

pyrightconfig.json

@@ -298,3 +298,22 @@ def post(self):
     """Get and render the jobs in JSON."""
     result, _ = get_results()
     return self.render_json(result)
+
+
+class GetEnvironmentHandler(base_handler.Handler):
+  """Handler that gets the computed environment for a job."""
+
+  @handler.get(handler.JSON)
+  @handler.check_user_access(need_privileged_access=True)
+  def get(self):
+    """Get and render the computed environment in JSON."""
+    name = request.args.get('name')
+    if not name:
+      raise helpers.EarlyExitError('No job name provided.', 400)
+
+    job = data_types.Job.query(data_types.Job.name == name).get()
+    if not job:
+      raise helpers.EarlyExitError('Job not found.', 404)
+
+    environment = job.get_environment()
+    return self.render_json({'environment': environment})

src/appengine/handlers/jobs.py

@@ -18,6 +18,8 @@
 <link rel="import" href="../../bower_components/iron-icon/iron-icon.html">
 <link rel="import" href="../../bower_components/paper-button/paper-button.html">
 <link rel="import" href="../../bower_components/paper-checkbox/paper-checkbox.html">
+<link rel="import" href="../../bower_components/paper-dialog/paper-dialog.html">
+<link rel="import" href="../../bower_components/paper-dialog-scrollable/paper-dialog-scrollable.html">
 <link rel="import" href="../../bower_components/paper-dropdown-menu/paper-dropdown-menu.html">
 <link rel="import" href="../../bower_components/paper-input/paper-input.html">
 <link rel="import" href="../../bower_components/paper-input/paper-textarea.html">
@@ -100,9 +102,30 @@
       on-response="handleResponse"
       debounce-duration="500"></iron-ajax>
 
+    <iron-ajax
+      id="environmentAjax"
+      url="/jobs/environment"
+      method="GET"
+      content-type="application/json"
+      loading="{{envLoading}}"
+      last-error="{{envError}}"
+      last-response="{{envResponse}}"
+      on-error="handleEnvError"
+      on-response="handleEnvResponse"></iron-ajax>
+
     <delete-job-dialog id="deleteJobDialog" field-values="[[fieldValues]]" job="[[toDelete]]">
     </delete-job-dialog>
 
+    <paper-dialog id="envDialog" with-backdrop>
+      <h2>Computed Environment</h2>
+      <paper-dialog-scrollable>
+        <pre id="envContent"></pre>
+      </paper-dialog-scrollable>
+      <div class="buttons">
+        <paper-button dialog-dismiss>Close</paper-button>
+      </div>
+    </paper-dialog>
+
     <h2>Templates</h2>
 
     <div class="section">
@@ -334,6 +357,7 @@ <h2>Jobs</h2>
 
                     <paper-button raised on-tap="submitForm" data-index$="[[index]]">Save</paper-button>
                     <paper-button class="info" job="[[item]]" raised on-tap="deleteJobTapped">Delete</paper-button>
+                    <paper-button raised on-tap="viewEnvironment" data-job-name$="[[item.name]]">View inherited environment</paper-button>
                   </div>
 
                   <div class="right">
@@ -368,6 +392,10 @@ <h2>Jobs</h2>
             type: Boolean,
             value: false
           },
+          envLoading: {
+            type: Boolean,
+            value: false
+          },
           submitId: {
             type: Number,
             value: 0
@@ -539,6 +567,30 @@ <h2>Jobs</h2>
       stopEventPropagation(ev) {
         ev.stopPropagation();
       }
+
+      handleEnvError(event) {
+        var response = event.detail.request.xhr.response;
+        if (response.error) {
+          this.$.envContent.textContent = 'Error: ' + response.error.message;
+        } else {
+          this.$.envContent.textContent = 'Error fetching environment';
+        }
+        this.$.envDialog.open();
+      }
+
+      handleEnvResponse(event) {
+        var response = event.detail.response;
+        this.$.envContent.textContent = Object.entries(response.environment)
+          .map(([key, value]) => `${key} = ${value}`)
+          .join('\n');
+        this.$.envDialog.open();
+      }
+
+      viewEnvironment(event) {
+        var jobName = event.target.dataset.jobName;
+        this.$.environmentAjax.params = {name: jobName};
+        this.$.environmentAjax.generateRequest();
+      }
     }
 
     customElements.define(JobsPage.is, JobsPage);

src/appengine/private/components/jobs/jobs.html

@@ -162,6 +162,7 @@ def register_routes(flask_app, routes):
     ('/jobs', jobs.Handler),
     ('/jobs/load', jobs.JsonHandler),
     ('/jobs/delete-job', jobs.DeleteJobHandler),
+    ('/jobs/environment', jobs.GetEnvironmentHandler),
     ('/login', login.Handler),
     ('/logout', login.LogoutHandler),
     ('/report-bug', help_redirector.ReportBugHandler),

src/appengine/server.py

@@ -183,6 +183,7 @@ def _deadline_exceeded(self, cleanup_function=None, soft_check=False):
     """Check to see if we have exceeded the deadline for execution."""
     if self.minimizer.deadline and time.time() > self.minimizer.deadline:
       if soft_check:
+        logs.warning(f'Minimization deadline exceeded. soft_check={soft_check}')
         return True
 
       # If we are here, we have exceeded the deadline on a hard check. Clean up.
@@ -192,6 +193,7 @@ def _deadline_exceeded(self, cleanup_function=None, soft_check=False):
       if self.minimizer.cleanup_function:
         self.minimizer.cleanup_function()
 
+      logs.warning('Minimization deadline exceeded.')
       # Raise an exception if this is not a soft deadline check.
       raise errors.MinimizationDeadlineExceededError(self)
 
@@ -558,6 +560,7 @@ def minimize(self, data):
       # minimized test case is stored with it so that we can recover the work
       # that had been done up to that point.
       testcase = error.testcase
+      logs.warning('Minimization Deadline Exceeded.')
     except errors.TokenizationFailureError:
       logs.info('Tokenized data did not match original data. Defaulting to line'
                 'minimization.')

src/clusterfuzz/_internal/bot/minimizer/minimizer.py

@@ -445,7 +445,10 @@ def utask_main(uworker_input: uworker_msg_pb2.Input):  # pylint: disable=no-memb
         ' should have been detected in preprocess.')
     return None
 
-  max_threads = utils.maximum_parallel_processes_allowed()
+  # TODO(alhijazi): re-install multithreaded runs
+  # max_threads = utils.maximum_parallel_processes_allowed()
+  # Temporarily run minimization single threaded.
+  max_threads = 1
 
   # Prepare the test case runner.
   crash_retries = environment.get_value('CRASH_RETRIES')
@@ -1748,8 +1751,10 @@ def do_html_minimization(test_function, get_temp_file, data, deadline, threads,
       delete_temp_files=delete_temp_files,
       progress_report_function=logs.info)
   try:
+    logs.info('Launching html minimization.')
     return current_minimizer.minimize(data)
   except minimizer_errors.AntlrDecodeError:
+    logs.info('Launching line minimization.')
     return do_line_minimization(test_function, get_temp_file, data, deadline,
                                 threads, cleanup_interval, delete_temp_files)
 

src/clusterfuzz/_internal/bot/tasks/utasks/minimize_task.py

@@ -522,7 +522,6 @@ def find_fixed_range(uworker_input):
 
   logs.info(
       f'min_revision is {min_revision} and max_revision is {max_revision}.')
-  logs.info(f'revision_list is {revision_list}.')
 
   min_index = revisions.find_min_revision_index(revision_list, min_revision)
   if min_index is None: