Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -750,9 +750,10 @@ def do_corpus_pruning(context, revision) -> CorpusPruningResult:

# Store corpus stats into CoverageInformation entity.
project_qualified_name = context.fuzz_target.project_qualified_name()
target_engine = context.fuzz_target.engine
today = datetime.datetime.utcnow()
coverage_info = data_types.CoverageInformation(
fuzzer=project_qualified_name, date=today)
fuzzer=project_qualified_name, date=today, engine=target_engine)

quarantine_corpus_size = shell.get_directory_file_count(
context.quarantine_corpus_path)
Expand Down Expand Up @@ -996,13 +997,19 @@ def _save_coverage_information(output):
return

cov_info = output.corpus_pruning_task_output.coverage_info
# Retrieve fuzz target engine, as it is not in the cov_info proto.
fuzz_target = uworker_io.entity_from_protobuf(
output.uworker_input.corpus_pruning_task_input.fuzz_target,
data_types.FuzzTarget)
target_engine = fuzz_target.engine

# Use ndb.transaction with retries below to mitigate risk of a race condition.
def _try_save_coverage_information():
"""Implements save_coverage_information function."""
coverage_info = data_handler.get_coverage_information(
cov_info.project_name,
cov_info.timestamp.ToDatetime().date(),
engine=target_engine,
create_if_needed=True)

# Intentionally skip edge and function coverage values as those would come
Expand Down
7 changes: 5 additions & 2 deletions src/clusterfuzz/_internal/datastore/data_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1822,15 +1822,18 @@ def get_all_job_type_names(project=None):
return sorted([job.name for job in query])


def get_coverage_information(fuzzer_name, date, create_if_needed=False):
def get_coverage_information(fuzzer_name,
date,
engine=None,
create_if_needed=False):
"""Get coverage information, or create if it doesn't exist."""
coverage_info = ndb.Key(
data_types.CoverageInformation,
data_types.coverage_information_key(fuzzer_name, date)).get()

if not coverage_info and create_if_needed:
coverage_info = data_types.CoverageInformation(
fuzzer=fuzzer_name, date=date)
fuzzer=fuzzer_name, date=date, engine=engine)

return coverage_info

Expand Down
1 change: 1 addition & 0 deletions src/clusterfuzz/_internal/datastore/data_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -1389,6 +1389,7 @@ class CoverageInformation(Model):
"""Coverage info."""
date = ndb.DateProperty(auto_now_add=True)
fuzzer = ndb.StringProperty()
engine = ndb.StringProperty()

# Function coverage information.
functions_covered = ndb.IntegerProperty()
Expand Down
16 changes: 12 additions & 4 deletions src/clusterfuzz/_internal/metrics/fuzzer_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -368,10 +368,12 @@ def get_coverage_info(self, fuzzer, date=None):
return get_coverage_info(project, date)

fuzz_target = data_handler.get_fuzz_target(fuzzer)
engine = None
if fuzz_target:
fuzzer = fuzz_target.project_qualified_name()
engine = fuzz_target.engine

return get_coverage_info(fuzzer, date)
return get_coverage_info(fuzzer, date, engine)


class BaseCoverageField:
Expand Down Expand Up @@ -965,11 +967,17 @@ def build(self):
return ' '.join(result)


def get_coverage_info(fuzzer, date=None):
"""Returns a CoverageInformation entity for a given fuzzer and date. If date
is not specified, returns the latest entity available."""
def get_coverage_info(fuzzer, date=None, engine=None):
"""Returns a CoverageInformation entity for a given fuzzer, engine and date.

If date is not specified, returns the latest entity available.
"""
query = data_types.CoverageInformation.query(
data_types.CoverageInformation.fuzzer == fuzzer)

if engine:
query = query.filter(data_types.CoverageInformation.engine == engine)

if date:
# Return info for specific date.
query = query.filter(data_types.CoverageInformation.date == date)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,9 @@ def setUp(self):
fuzz_target_name='testFuzzer2_1_fuzzer', job='job', last_run=now).put()

cov_info = data_types.CoverageInformation(
fuzzer='2_fuzzer', date=datetime.date(2016, 10, 19))
fuzzer='2_fuzzer',
engine='testFuzzer',
date=datetime.date(2016, 10, 19))
cov_info.edges_covered = 11
cov_info.edges_total = 30
cov_info.functions_covered = 10
Expand All @@ -124,7 +126,9 @@ def setUp(self):
cov_info.put()

cov_info = data_types.CoverageInformation(
fuzzer='2_fuzzer', date=datetime.date(2016, 10, 21))
fuzzer='2_fuzzer',
engine='testFuzzer',
date=datetime.date(2016, 10, 21))
cov_info.edges_covered = 15
cov_info.edges_total = 30
cov_info.functions_covered = 11
Expand All @@ -140,7 +144,9 @@ def setUp(self):
cov_info.put()

cov_info = data_types.CoverageInformation(
fuzzer='1_fuzzer', date=datetime.date(2016, 10, 20))
fuzzer='1_fuzzer',
engine='testFuzzer',
date=datetime.date(2016, 10, 20))
cov_info.edges_covered = 17
cov_info.edges_total = 38
cov_info.functions_covered = 12
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,8 @@ def test_prune(self):
None,
'fuzzer':
'test_fuzzer',
'engine':
'libFuzzer',
'html_report_url':
None,
'quarantine_location':
Expand Down
Loading