webapp: Add new api for custom type definition querying (#2166)

* webapp: Add new api for custom type definition querying

Signed-off-by: Arthur Chan <[email protected]>

* Fix formatting

Signed-off-by: Arthur Chan <[email protected]>

---------

Signed-off-by: Arthur Chan <[email protected]>

Author: arthurscchan

tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py

@@ -47,6 +47,11 @@ def get_introspector_report_url_report(project_name, datestr):
                                             datestr) + "fuzz_report.html"
 
 
+def get_introspector_report_url_typedef(project_name, datestr):
+    return get_introspector_report_url_base(project_name,
+                                            datestr) + "full_type_defs.json"
+
+
 def get_fuzzer_stats_fuzz_count_url(project_name, date_str):
     base_url = 'https://storage.googleapis.com/oss-fuzz-coverage/{0}/fuzzer_stats/{1}/coverage_targets.txt'
     coverage_targets = base_url.format(project_name, date_str)
@@ -233,6 +238,17 @@ def extract_local_introspector_report(project_name, oss_fuzz_folder):
     return json_dict
 
 
+def extract_local_introspector_typedef(project_name, oss_fuzz_folder):
+    summary_json = os.path.join(oss_fuzz_folder, 'build', 'out', project_name,
+                                'inspector', 'full_type_defs.json')
+    if not os.path.isfile(summary_json):
+        return {}
+
+    with open(summary_json, 'r') as f:
+        json_list = json.load(f)
+    return json_list
+
+
 def get_local_code_coverage_summary(project_name, oss_fuzz_folder):
     summary_json = os.path.join(oss_fuzz_folder, 'build', 'out', project_name,
                                 'report', 'linux', 'summary.json')
@@ -366,6 +382,24 @@ def extract_introspector_test_files_xref(project_name, date_str):
     return test_files
 
 
+def extract_introspector_typedef(project_name, date_str):
+    introspector_test_url = get_introspector_report_url_typedef(
+        project_name, date_str.replace("-", ""))
+
+    # Read the introspector atifact
+    try:
+        raw_introspector_json_request = requests.get(introspector_test_url,
+                                                     timeout=10)
+    except:
+        return []
+    try:
+        typedef_list = json.loads(raw_introspector_json_request.text)
+    except:
+        return []
+
+    return typedef_list
+
+
 def extract_introspector_report(project_name, date_str):
     introspector_summary_url = get_introspector_report_url_summary(
         project_name, date_str.replace("-", ""))

tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py

@@ -449,6 +449,10 @@ def extract_local_project_data(project_name, oss_fuzz_path,
     except:
         project_repository = 'N/A'
 
+    # Extract type definition
+    typedef_list = oss_fuzz.extract_local_introspector_typedef(
+        project_name, oss_fuzz_path)
+
     introspector_data_dict = {
         "introspector_report_url":
         'introspector_url',
@@ -469,7 +473,9 @@ def extract_local_project_data(project_name, oss_fuzz_path,
         'annotated_cfg':
         annotated_cfg,
         'project_name':
-        project_name
+        project_name,
+        'typedef_list':
+        typedef_list
     }
 
     code_coverage_data_dict = prepare_code_coverage_dict(
@@ -721,6 +727,10 @@ def extract_project_data(project_name, date_str, should_include_details,
         # Dump things we dont want to accummulate.
         save_branch_blockers(branch_pairs, project_name)
 
+        # Extract type definition
+        typedef_list = oss_fuzz.extract_introspector_typedef(
+            project_name, date_str)
+
         introspector_data_dict = {
             "introspector_report_url": introspector_report_url,
             "coverage_lines":
@@ -734,7 +744,8 @@ def extract_project_data(project_name, date_str, should_include_details,
             'refined_constructor_list': refined_constructor_list,
             'annotated_cfg': annotated_cfg,
             'optimal_targets': optimal_targets,
-            'project_name': project_name
+            'project_name': project_name,
+            'typedef_list': typedef_list
         }
 
     code_coverage_data_dict = prepare_code_coverage_dict(

tools/web-fuzzing-introspection/app/webapp/routes.py

@@ -1319,6 +1319,36 @@ def api_annotated_cfg(args):
         return {'result': 'error', 'msg': 'Found no introspector data.'}
 
 
+@api_blueprint.route('/api/full-type-definition')
+@api_blueprint.arguments(ProjectSchema, location='query')
+def api_full_type_definition(args):
+    """API that returns the full type definition of a project."""
+    project_name = args.get('project', '')
+    if not project_name:
+        return {'result': 'error', 'msg': 'Please provide project name'}
+
+    target_project = get_project_with_name(project_name)
+    if target_project is None:
+        return {'result': 'error', 'msg': 'Project not in the database'}
+
+    if target_project.introspector_data is None:
+        return {'result': 'error', 'msg': 'Found no introspector data.'}
+
+    try:
+        return {
+            'result': 'success',
+            'project': {
+                'name': project_name,
+                'typedef_list':
+                target_project.introspector_data['typedef_list'],
+            }
+        }
+    except KeyError:
+        return {'result': 'error', 'msg': 'Found no type definition data.'}
+    except TypeError:
+        return {'result': 'error', 'msg': 'Found no introspector data.'}
+
+
 @api_blueprint.route('/api/project-summary')
 @api_blueprint.arguments(ProjectSchema, location='query')
 def api_project_summary(args):