Skip to content
This repository was archived by the owner on Sep 27, 2019. It is now read-only.

Commit a902d24

Browse files
schedutronschedutron
committed
Modularize validator
1 parent fb0667a commit a902d24

File tree

3 files changed

+73
-92
lines changed

3 files changed

+73
-92
lines changed

‎script/formatting/formatter.py

Lines changed: 1 addition & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
import subprocess
1515

1616
from functools import reduce
17-
from script.helpers import CLANG_FORMAT
17+
from script.helpers import CLANG_FORMAT, PELOTON_DIR, CLANG_FORMAT_FILE, LOG
1818

1919
## ==============================================
2020
## CONFIGURATION
@@ -23,8 +23,6 @@
2323
# NOTE: absolute path to peloton directory is calculated from current directory
2424
# directory structure: peloton/scripts/formatting/<this_file>
2525
# PELOTON_DIR needs to be redefined if the directory structure is changed
26-
CODE_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__))
27-
PELOTON_DIR = reduce(os.path.join, [CODE_SOURCE_DIR, os.path.pardir, os.path.pardir])
2826

2927
#other directory paths used are relative to peloton_dir
3028
PELOTON_SRC_DIR = os.path.join(PELOTON_DIR, "src")
@@ -35,8 +33,6 @@
3533
DEFAULT_DIRS.append(PELOTON_SRC_DIR)
3634
DEFAULT_DIRS.append(PELOTON_TESTS_DIR)
3735

38-
CLANG_FORMAT_FILE = os.path.join(PELOTON_DIR, ".clang-format")
39-
4036
## ==============================================
4137
## HEADER CONFIGURATION
4238
## ==============================================
@@ -62,20 +58,6 @@
6258
#regular expresseion used to track header
6359
HEADER_REGEX = re.compile(r"((\/\/===-*===\/\/\n(\/\/.*\n)*\/\/===-*===\/\/[\n]*)\n\n)*")
6460

65-
## ==============================================
66-
## LOGGING CONFIGURATION
67-
## ==============================================
68-
69-
LOG = logging.getLogger(__name__)
70-
LOG_HANDLER = logging.StreamHandler()
71-
LOG_FORMATTER = logging.Formatter(
72-
fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
73-
datefmt='%m-%d-%Y %H:%M:%S'
74-
)
75-
LOG_HANDLER.setFormatter(LOG_FORMATTER)
76-
LOG.addHandler(LOG_HANDLER)
77-
LOG.setLevel(logging.INFO)
78-
7961
## ==============================================
8062
## UTILITY FUNCTION DEFINITIONS
8163
## ==============================================
@@ -128,7 +110,6 @@ def format_file(file_path, update_header, clang_format_code):
128110
#END FORMAT__FILE(FILE_NAME)
129111

130112

131-
132113
def format_dir(dir_path, update_header, clang_format_code):
133114
"""Formats all the files in the dir passed as argument."""
134115
for subdir, _, files in os.walk(dir_path): # _ is for directories.

‎script/helpers.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,34 @@
11
#!/usr/bin/env python3
22
"""Common helper functions to be used in different Python scripts."""
3+
import difflib
34
import distutils.spawn
5+
import logging
6+
import os
7+
import subprocess
8+
9+
from functools import reduce
10+
11+
CODE_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__))
12+
PELOTON_DIR = CODE_SOURCE_DIR.rstrip('/script')
13+
CLANG_FORMAT_FILE = os.path.join(PELOTON_DIR, ".clang-format")
14+
15+
FORMATTING_FILE_WHITELIST = [
16+
# Fill me
17+
]
18+
19+
## ==============================================
20+
## LOGGING CONFIGURATION
21+
## ==============================================
22+
23+
LOG = logging.getLogger(__name__)
24+
LOG_HANDLER = logging.StreamHandler()
25+
LOG_FORMATTER = logging.Formatter(
26+
fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
27+
datefmt='%m-%d-%Y %H:%M:%S'
28+
)
29+
LOG_HANDLER.setFormatter(LOG_FORMATTER)
30+
LOG.addHandler(LOG_HANDLER)
31+
LOG.setLevel(logging.INFO)
432

533
def find_clangformat():
634
"""Finds appropriate clang-format executable."""
@@ -14,3 +42,45 @@ def find_clangformat():
1442
return path
1543

1644
CLANG_FORMAT = find_clangformat()
45+
46+
47+
def clang_check(file_path):
48+
"""Checks and reports bad code formatting."""
49+
rel_path_from_peloton_dir = os.path.relpath(file_path, PELOTON_DIR)
50+
51+
if rel_path_from_peloton_dir in FORMATTING_FILE_WHITELIST:
52+
return True
53+
54+
file_status = True
55+
56+
# Run clang-format on the file
57+
if CLANG_FORMAT is None:
58+
LOG.error("clang-format seems not installed")
59+
exit()
60+
clang_format_cmd = [CLANG_FORMAT, "-style=file", file_path]
61+
formatted_src = subprocess.check_output(clang_format_cmd).splitlines(True)
62+
63+
# For Python 3, the above command gives a list of binary sequences, each
64+
# of which has to be converted to string for diff to operate correctly.
65+
# Otherwise, strings would be compared with binary sequences and there
66+
# will always be a big difference.
67+
formatted_src = [line.decode('utf-8') for line in formatted_src]
68+
# Load source file
69+
with open(file_path, "r") as file:
70+
src = file.readlines()
71+
72+
# Do the diff
73+
difference = difflib.Differ()
74+
diff = difference.compare(src, formatted_src)
75+
line_num = 0
76+
for line in diff:
77+
code = line[:2]
78+
if code in (" ", "- "):
79+
line_num += 1
80+
if code == '- ':
81+
if file_status:
82+
LOG.info("Invalid formatting in file : " + file_path)
83+
LOG.info("Line %d: %s", line_num, line[2:].strip())
84+
file_status = False
85+
86+
return file_status

‎script/validators/source_validator.py

Lines changed: 2 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -2,32 +2,13 @@
22
"""Validates source files and reports compliance anomalies."""
33

44
import argparse
5-
import logging
65
import os
76
import re
87
import sys
9-
import subprocess
10-
import difflib
118
import mmap
129
import glob
13-
import functools
14-
15-
from script.helpers import CLANG_FORMAT
16-
17-
## ==============================================
18-
## LOGGING CONFIGURATION
19-
## ==============================================
20-
21-
LOG = logging.getLogger(__name__)
22-
LOG_HANDLER = logging.StreamHandler()
23-
LOG_FORMATTER = logging.Formatter(
24-
fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
25-
datefmt='%m-%d-%Y %H:%M:%S'
26-
)
27-
LOG_HANDLER.setFormatter(LOG_FORMATTER)
28-
LOG.addHandler(LOG_HANDLER)
29-
LOG.setLevel(logging.INFO)
3010

11+
from script.helpers import clang_check, PELOTON_DIR, LOG
3112

3213
## ==============================================
3314
## CONFIGURATION
@@ -36,12 +17,6 @@
3617
# NOTE: absolute path to peloton directory is calculated from current directory
3718
# directory structure: peloton/scripts/formatting/<this_file>
3819
# PELOTON_DIR needs to be redefined if the directory structure is changed
39-
CODE_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__))
40-
PELOTON_DIR = os.path.abspath(
41-
functools.reduce(os.path.join, [CODE_SOURCE_DIR, os.path.pardir, os.path.pardir])
42-
)
43-
44-
CLANG_FORMAT_FILE = os.path.join(PELOTON_DIR, ".clang-format")
4520

4621
# Other directory paths used are relative to PELOTON_DIR
4722
DEFAULT_DIRS = [
@@ -87,10 +62,6 @@
8762
"src/codegen/util/cc_hash_table.cpp"
8863
]
8964

90-
FORMATTING_FILE_WHITELIST = [
91-
# Fill me
92-
]
93-
9465
## ==============================================
9566
## UTILITY FUNCTION DEFINITIONS
9667
## ==============================================
@@ -122,47 +93,6 @@ def check_common_patterns(file_path):
12293

12394
return file_status
12495

125-
def check_format(file_path):
126-
"""Checks and reports bad code formatting."""
127-
rel_path_from_peloton_dir = os.path.relpath(file_path, PELOTON_DIR)
128-
129-
if rel_path_from_peloton_dir in FORMATTING_FILE_WHITELIST:
130-
return True
131-
132-
file_status = True
133-
134-
# Run clang-format on the file
135-
if CLANG_FORMAT is None:
136-
LOG.error("clang-format seems not installed")
137-
exit()
138-
clang_format_cmd = [CLANG_FORMAT, "-style=file", file_path]
139-
formatted_src = subprocess.check_output(clang_format_cmd).splitlines(True)
140-
141-
# For Python 3, the above command gives a list of binary sequences, each
142-
# of which has to be converted to string for diff to operate correctly.
143-
# Otherwise, strings would be compared with binary sequences and there
144-
# will always be a big difference.
145-
formatted_src = [line.decode('utf-8') for line in formatted_src]
146-
# Load source file
147-
with open(file_path, "r") as file:
148-
src = file.readlines()
149-
150-
# Do the diff
151-
difference = difflib.Differ()
152-
diff = difference.compare(src, formatted_src)
153-
line_num = 0
154-
for line in diff:
155-
code = line[:2]
156-
if code in (" ", "- "):
157-
line_num += 1
158-
if code == '- ':
159-
if file_status:
160-
LOG.info("Invalid formatting in file : " + file_path)
161-
LOG.info("Line %d: %s", line_num, line[2:].strip())
162-
file_status = False
163-
164-
return file_status
165-
16696

16797
def check_namespaces(file_path):
16898
"""Scans namespace openings and closings."""
@@ -332,7 +262,7 @@ def validate_dir(dir_path):
332262
# Validate just the provided files.
333263

334264
# In this mode, we perform explicit clang-format checks
335-
VALIDATORS.append(check_format)
265+
VALIDATORS.append(clang_check)
336266
for each_file in ARGS.files:
337267
each_file = os.path.abspath(each_file.lower())
338268

0 commit comments

Comments
 (0)