Skip to content

Commit dfb6870

Browse files
dgarijodgarijo
authored
Merge pull request #98 from andriumon/validation-update
Added validation test for indicators and dimensions consistency
2 parents 2666195 + 9aa934b commit dfb6870

8 files changed

+93
-11
lines changed

‎indicators/has_ci-tests.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
{
22
"@context": "https://w3id.org/everse/rsqi#",
3-
"@id": "https://w3id.org/everse/i/indicators/software_has_ci-tests",
3+
"@id": "https://w3id.org/everse/i/indicators/has_ci-tests",
44
"@type": "SoftwareQualityIndicator",
55
"name": "Software has continuous integration tests",
66
"abbreviation": "has_ci-tests",
77
"description": "This indicator aims to determine if the project runs tests before pull requests are merged.",
88
"keywords": [ "continuous", "integration", "test"],
9-
"identifier": { "@id": "https://w3id.org/everse/i/indicators/software_has_ci-tests" },
9+
"identifier": { "@id": "https://w3id.org/everse/i/indicators/has_ci-tests" },
1010
"status": "Active",
1111
"version": "1.0.0",
1212
"author" : {

‎indicators/no_critical_vulnerability.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"@context": "https://w3id.org/everse/rsqi#",
33
"@type": "SoftwareQualityIndicator",
4+
"@id": "https://w3id.org/everse/i/indicators/no_critical_vulnerability",
45
"name": "No critical vulnerabilities",
56
"abbreviation": "no_critical_vulnerability",
67
"description": "Checks if reported critical vulnerabilities have been fixed",

indicators/no_leaked_credential.json renamed to indicators/no_leaked_credentials.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"@context": "https://w3id.org/everse/rsqi#",
33
"@type": "SoftwareQualityIndicator",
4+
"@id": "https://w3id.org/everse/i/indicators/no_leaked_credentials",
45
"name": "No leaked credentials",
56
"abbreviation": "no_leaked_credentials",
67
"description": "Checks if hardcoded secrets like passwords, API keys, and tokens is stored in the public git repository",

‎indicators/software_has_documentation.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"@id": "https://w3id.org/everse/i/indicators/software_has_documentation",
44
"@type": "SoftwareQualityIndicator",
55
"name": "Software has documentation",
6-
"abbreviation": "software_documentation",
6+
"abbreviation": "software_has_documentation",
77
"alternateName": "FRSM-05",
88
"description": "This indicator aims to determine if a software project comes with many forms of documentation like readme or readthedocs",
99
"keywords": [ "documentation", "fair", "usage", "readme"],

‎indicators/uses_fuzzing.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
{
22
"@context": "https://w3id.org/everse/rsqi#",
3-
"@id": "https://w3id.org/everse/i/indicators/fuzzing_use",
3+
"@id": "https://w3id.org/everse/i/indicators/uses_fuzzing",
44
"@type": "SoftwareQualityIndicator",
55
"name": "Software uses fuzzing",
6-
"abbreviation": "fuzzing_use",
7-
"description": "This indicator checks that If the software produced by the project includes software written using a memory-unsafe language (e.g., C or C++), then at least one dynamic tool (e.g., a fuzzer or web application scanner) be routinely used in combination with a mechanism to detect memory safety problems such as buffer overwrites. ",
6+
"abbreviation": "uses_fuzzing",
7+
"description": "This indicator checks that the software produced by the project includes software written using a memory-unsafe language (e.g., C or C++), then at least one dynamic tool (e.g., a fuzzer or web application scanner) be routinely used in combination with a mechanism to detect memory safety problems such as buffer overwrites. ",
88
"keywords": [ "fuzzing", "test", "code"],
9-
"identifier": { "@id": "https://w3id.org/everse/i/indicators/fuzzing_use" },
9+
"identifier": { "@id": "https://w3id.org/everse/i/indicators/uses_fuzzing" },
1010
"status": "Active",
1111
"version": "1.0.0",
1212
"author" : {

indicators/uses_tool_for_warning_and_mistakes.json renamed to indicators/uses_tool_for_warnings_and_mistakes.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
{
22
"@context": "https://w3id.org/everse/rsqi#",
3-
"@id": "https://w3id.org/everse/i/indicators/uses_tool_for_warnings_or_mistakes",
3+
"@id": "https://w3id.org/everse/i/indicators/uses_tool_for_warnings_and_mistakes",
44
"@type": "SoftwareQualityIndicator",
55
"name": "Software uses a tool for warnings or mistakes",
6-
"abbreviation": "uses_tool_for_warnings_or_mistakes",
6+
"abbreviation": "uses_tool_for_warnings_and_mistakes",
77
"description": "This indicator checks that the project enables one or more compiler warning flags, a \"safe\" language mode, or uses a separate \"linter\" tool to look for code quality errors or common simple mistakes, if there is at least one FLOSS tool that can implement this criterion in the selected language.",
88
"keywords": [ "linting", "code analysis", "fair"],
9-
"identifier": { "@id": "https://w3id.org/everse/i/indicators/uses_tool_for_warnings_or_mistakes" },
9+
"identifier": { "@id": "https://w3id.org/everse/i/indicators/uses_tool_for_warnings_and_mistakes" },
1010
"status": "Active",
1111
"version": "1.0.0",
1212
"author" : {

‎tests/helpers.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,61 @@ def validate_json_files_using_schema(schema_file_path, json_file_path):
6969
"The schema validation failed for one or more files:\n"
7070
+ "\n".join(validation_errors)
7171
)
72+
73+
74+
def validate_json_files_consistency(json_file_path):
75+
76+
json_files = glob.glob(f"{json_file_path}/*.json")
77+
78+
if not json_files:
79+
pytest.skip(f"No json files found in {json_file_path}/ directory.")
80+
81+
validation_errors = []
82+
83+
for json_file_name in json_files:
84+
filename = os.path.basename(json_file_name)
85+
86+
print(f"\nChecking consistency for {filename}...")
87+
88+
try:
89+
with open(json_file_name, "r") as f:
90+
data = json.load(f)
91+
92+
if json_file_path == "indicators":
93+
identifier = data.get("identifier", {}).get("@id")
94+
elif json_file_path == "dimensions":
95+
identifier = data.get("identifier", "")
96+
97+
id = data.get("@id")
98+
99+
abbreviation = data.get("abbreviation")
100+
101+
if id != identifier:
102+
validation_errors.append(
103+
"@id and identifier fields are not equal"
104+
)
105+
106+
identifier_suffix = id.rstrip("/").split("/")[-1]
107+
108+
if abbreviation != identifier_suffix:
109+
validation_errors.append(
110+
f"{filename}: abbreviation ({abbreviation}) "
111+
f"!= identifier suffix ({identifier_suffix})"
112+
)
113+
114+
expected_filename = f"{abbreviation}.json"
115+
if filename != expected_filename:
116+
validation_errors.append(
117+
f"{filename}: filename does not match abbreviation "
118+
f"(expected {expected_filename})"
119+
)
120+
121+
except json.JSONDecodeError as e:
122+
validation_errors.append(f"{filename}: Invalid JSON - {e}")
123+
except Exception as e:
124+
validation_errors.append(f"{filename}: Unexpected error - {e}")
125+
126+
assert not validation_errors, (
127+
"JSON consistency validation failed for one or more files:\n"
128+
+ "\n".join(validation_errors)
129+
)

‎tests/test_dimensions_and_indicators.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from helpers import validate_json_files_using_schema
1+
from helpers import validate_json_files_using_schema, validate_json_files_consistency
22

33
def test_indicators_validation():
44
"""
@@ -20,3 +20,25 @@ def test_dimensions_validation():
2020
schema_file_path="tests/dimension_validation_schema.json",
2121
json_file_path="dimensions",
2222
)
23+
24+
25+
def test_indicators_json_files_consistency():
26+
"""
27+
Validates that all JSON files in the indicators/ directory follow these guidelines:
28+
29+
- identifier and @id fields have the same value
30+
- abbreviation field is the same as the identifier and @id suffix (abbreviation = /identifier_suffix)
31+
- json file is named after the abbreviation field (abbreviation + .json)
32+
"""
33+
validate_json_files_consistency(json_file_path="indicators")
34+
35+
36+
def test_dimensions_json_files_consistency():
37+
"""
38+
Validates that all JSON files in the dimensions/ directory follow these guidelines:
39+
40+
- identifier and @id fields have the same value
41+
- abbreviation field is the same as the identifier and @id suffix (abbreviation = /identifier_suffix)
42+
- json file is named after the abbreviation field (abbreviation + .json)
43+
"""
44+
validate_json_files_consistency(json_file_path="dimensions")

0 commit comments

Comments
 (0)