From b12ccd9e893dcae8268428bd2ca17cedf9237a42 Mon Sep 17 00:00:00 2001 From: Vinicius da Costa Date: Thu, 4 Sep 2025 19:07:30 +0000 Subject: [PATCH 1/5] Add engine field to coverage information entity. --- src/clusterfuzz/_internal/datastore/data_handler.py | 7 +++++-- src/clusterfuzz/_internal/datastore/data_types.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/clusterfuzz/_internal/datastore/data_handler.py b/src/clusterfuzz/_internal/datastore/data_handler.py index 2bd17653b59..e2c992b8d8a 100644 --- a/src/clusterfuzz/_internal/datastore/data_handler.py +++ b/src/clusterfuzz/_internal/datastore/data_handler.py @@ -1822,7 +1822,10 @@ def get_all_job_type_names(project=None): return sorted([job.name for job in query]) -def get_coverage_information(fuzzer_name, date, create_if_needed=False): +def get_coverage_information(fuzzer_name, + date, + engine=None, + create_if_needed=False): """Get coverage information, or create if it doesn't exist.""" coverage_info = ndb.Key( data_types.CoverageInformation, @@ -1830,7 +1833,7 @@ def get_coverage_information(fuzzer_name, date, create_if_needed=False): if not coverage_info and create_if_needed: coverage_info = data_types.CoverageInformation( - fuzzer=fuzzer_name, date=date) + fuzzer=fuzzer_name, date=date, engine=engine) return coverage_info diff --git a/src/clusterfuzz/_internal/datastore/data_types.py b/src/clusterfuzz/_internal/datastore/data_types.py index ca4bd4603ba..a8ee86c7776 100644 --- a/src/clusterfuzz/_internal/datastore/data_types.py +++ b/src/clusterfuzz/_internal/datastore/data_types.py @@ -1389,6 +1389,7 @@ class CoverageInformation(Model): """Coverage info.""" date = ndb.DateProperty(auto_now_add=True) fuzzer = ndb.StringProperty() + engine = ndb.StringProperty() # Function coverage information. functions_covered = ndb.IntegerProperty() From 394851826966aa73fbdef1f3de9a6c1c70153d0c Mon Sep 17 00:00:00 2001 From: Vinicius da Costa Date: Thu, 4 Sep 2025 19:17:55 +0000 Subject: [PATCH 2/5] Add fuzzer engine to filter the correct information in fuzzer stats page. --- .../_internal/metrics/fuzzer_stats.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/clusterfuzz/_internal/metrics/fuzzer_stats.py b/src/clusterfuzz/_internal/metrics/fuzzer_stats.py index 511f6877713..73bc4ca8339 100644 --- a/src/clusterfuzz/_internal/metrics/fuzzer_stats.py +++ b/src/clusterfuzz/_internal/metrics/fuzzer_stats.py @@ -368,10 +368,12 @@ def get_coverage_info(self, fuzzer, date=None): return get_coverage_info(project, date) fuzz_target = data_handler.get_fuzz_target(fuzzer) + engine = None if fuzz_target: fuzzer = fuzz_target.project_qualified_name() + engine = fuzz_target.engine - return get_coverage_info(fuzzer, date) + return get_coverage_info(fuzzer, date, engine) class BaseCoverageField: @@ -965,11 +967,17 @@ def build(self): return ' '.join(result) -def get_coverage_info(fuzzer, date=None): - """Returns a CoverageInformation entity for a given fuzzer and date. If date - is not specified, returns the latest entity available.""" +def get_coverage_info(fuzzer, date=None, engine=None): + """Returns a CoverageInformation entity for a given fuzzer, engine and date. + + If date is not specified, returns the latest entity available. + """ query = data_types.CoverageInformation.query( data_types.CoverageInformation.fuzzer == fuzzer) + + if engine: + query = query.filter(data_types.CoverageInformation.engine == engine) + if date: # Return info for specific date. query = query.filter(data_types.CoverageInformation.date == date) From 1a91aa32e293303b11229896838e3b2a22d5759c Mon Sep 17 00:00:00 2001 From: Vinicius da Costa Date: Thu, 4 Sep 2025 20:03:24 +0000 Subject: [PATCH 3/5] Add engine to cov info in corpus pruning. --- .../_internal/bot/tasks/utasks/corpus_pruning_task.py | 9 ++++++++- .../core/bot/tasks/utasks/corpus_pruning_task_test.py | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py index b42aa03b609..d7c18d7b167 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py @@ -750,9 +750,10 @@ def do_corpus_pruning(context, revision) -> CorpusPruningResult: # Store corpus stats into CoverageInformation entity. project_qualified_name = context.fuzz_target.project_qualified_name() + engine = context.fuzz_target.engine today = datetime.datetime.utcnow() coverage_info = data_types.CoverageInformation( - fuzzer=project_qualified_name, date=today) + fuzzer=project_qualified_name, date=today, engine=engine) quarantine_corpus_size = shell.get_directory_file_count( context.quarantine_corpus_path) @@ -996,6 +997,11 @@ def _save_coverage_information(output): return cov_info = output.corpus_pruning_task_output.coverage_info + # Retrieve fuzz target engine, as it is not in the cov_info proto. + fuzz_target = uworker_io.entity_from_protobuf( + output.uworker_input.corpus_pruning_task_input.fuzz_target, + data_types.FuzzTarget) + engine = fuzz_target.engine # Use ndb.transaction with retries below to mitigate risk of a race condition. def _try_save_coverage_information(): @@ -1003,6 +1009,7 @@ def _try_save_coverage_information(): coverage_info = data_handler.get_coverage_information( cov_info.project_name, cov_info.timestamp.ToDatetime().date(), + engine=engine, create_if_needed=True) # Intentionally skip edge and function coverage values as those would come diff --git a/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/corpus_pruning_task_test.py b/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/corpus_pruning_task_test.py index fcb5495c762..577614e70bb 100644 --- a/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/corpus_pruning_task_test.py +++ b/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/corpus_pruning_task_test.py @@ -283,6 +283,8 @@ def test_prune(self): None, 'fuzzer': 'test_fuzzer', + 'engine': + 'libFuzzer', 'html_report_url': None, 'quarantine_location': From a1e733ca74415afdf280be7d441b3fe249e3462c Mon Sep 17 00:00:00 2001 From: Vinicius da Costa Date: Fri, 5 Sep 2025 16:39:58 +0000 Subject: [PATCH 4/5] Fix redefining engine name. --- .../_internal/bot/tasks/utasks/corpus_pruning_task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py index d7c18d7b167..f1ec9ed3c8f 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py @@ -750,10 +750,10 @@ def do_corpus_pruning(context, revision) -> CorpusPruningResult: # Store corpus stats into CoverageInformation entity. project_qualified_name = context.fuzz_target.project_qualified_name() - engine = context.fuzz_target.engine + target_engine = context.fuzz_target.engine today = datetime.datetime.utcnow() coverage_info = data_types.CoverageInformation( - fuzzer=project_qualified_name, date=today, engine=engine) + fuzzer=project_qualified_name, date=today, engine=target_engine) quarantine_corpus_size = shell.get_directory_file_count( context.quarantine_corpus_path) @@ -1001,7 +1001,7 @@ def _save_coverage_information(output): fuzz_target = uworker_io.entity_from_protobuf( output.uworker_input.corpus_pruning_task_input.fuzz_target, data_types.FuzzTarget) - engine = fuzz_target.engine + target_engine = fuzz_target.engine # Use ndb.transaction with retries below to mitigate risk of a race condition. def _try_save_coverage_information(): @@ -1009,7 +1009,7 @@ def _try_save_coverage_information(): coverage_info = data_handler.get_coverage_information( cov_info.project_name, cov_info.timestamp.ToDatetime().date(), - engine=engine, + engine=target_engine, create_if_needed=True) # Intentionally skip edge and function coverage values as those would come From 6346c0d27505cb6af3fafa96e403228d438799a3 Mon Sep 17 00:00:00 2001 From: Vinicius da Costa Date: Fri, 5 Sep 2025 16:40:23 +0000 Subject: [PATCH 5/5] Fix tests. --- .../tests/appengine/handlers/fuzzer_stats_test.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/clusterfuzz/_internal/tests/appengine/handlers/fuzzer_stats_test.py b/src/clusterfuzz/_internal/tests/appengine/handlers/fuzzer_stats_test.py index 3aa275702b0..e64312f1e02 100644 --- a/src/clusterfuzz/_internal/tests/appengine/handlers/fuzzer_stats_test.py +++ b/src/clusterfuzz/_internal/tests/appengine/handlers/fuzzer_stats_test.py @@ -108,7 +108,9 @@ def setUp(self): fuzz_target_name='testFuzzer2_1_fuzzer', job='job', last_run=now).put() cov_info = data_types.CoverageInformation( - fuzzer='2_fuzzer', date=datetime.date(2016, 10, 19)) + fuzzer='2_fuzzer', + engine='testFuzzer', + date=datetime.date(2016, 10, 19)) cov_info.edges_covered = 11 cov_info.edges_total = 30 cov_info.functions_covered = 10 @@ -124,7 +126,9 @@ def setUp(self): cov_info.put() cov_info = data_types.CoverageInformation( - fuzzer='2_fuzzer', date=datetime.date(2016, 10, 21)) + fuzzer='2_fuzzer', + engine='testFuzzer', + date=datetime.date(2016, 10, 21)) cov_info.edges_covered = 15 cov_info.edges_total = 30 cov_info.functions_covered = 11 @@ -140,7 +144,9 @@ def setUp(self): cov_info.put() cov_info = data_types.CoverageInformation( - fuzzer='1_fuzzer', date=datetime.date(2016, 10, 20)) + fuzzer='1_fuzzer', + engine='testFuzzer', + date=datetime.date(2016, 10, 20)) cov_info.edges_covered = 17 cov_info.edges_total = 38 cov_info.functions_covered = 12