Skip to content

Commit 9d723b8

Browse files
gongweibaogongweibao
authored
Add docstring checker. (#9848)
1 parent 34e093a commit 9d723b8

File tree

6 files changed

+598
-0
lines changed

6 files changed

+598
-0
lines changed

‎.pre-commit-config.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,14 @@ repos:
3434
entry: bash ./tools/codestyle/cpplint_pre_commit.hook
3535
language: system
3636
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx)$
37+
- repo: local
38+
hooks:
39+
- id: pylint-doc-string
40+
name: pylint
41+
description: Check python docstring style using docstring_checker.
42+
entry: bash ./tools/codestyle/pylint_pre_commit.hook
43+
language: system
44+
files: \.(py)$
3745
- repo: https://github.com/PaddlePaddle/pre-commit-golang
3846
sha: 8337620115c25ff8333f1b1a493bd031049bd7c0
3947
hooks:

‎.travis.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ env:
1818
addons:
1919
ssh_known_hosts: 13.229.163.131
2020
before_install:
21+
# For pylint dockstring checker
22+
- sudo pip install pylint pytest astroid isort
2123
- |
2224
function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }
2325
script:

‎Dockerfile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,9 @@ RUN pip install pre-commit 'ipython==5.3.0' && \
7979
pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \
8080
pip install opencv-python
8181

82+
#For docstring checker
83+
RUN pip install pylint pytest astroid isort
84+
8285
COPY ./python/requirements.txt /root/
8386
RUN pip install -r /root/requirements.txt
8487

‎tools/codestyle/docstring_checker.py

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
"""DocstringChecker is used to check python doc string's style."""
15+
16+
import six
17+
import astroid
18+
19+
from pylint.checkers import BaseChecker, utils
20+
from pylint.interfaces import IAstroidChecker
21+
22+
from collections import defaultdict
23+
import re
24+
25+
26+
def register(linter):
27+
"""Register checkers."""
28+
linter.register_checker(DocstringChecker(linter))
29+
30+
31+
class Docstring(object):
32+
"""Docstring class holds the parsed doc string elements.
33+
"""
34+
35+
def __init__(self):
36+
self.d = defaultdict(list) #name->[]
37+
self.clear()
38+
39+
def clear(self):
40+
self.d['Args'] = []
41+
self.d['Examples'] = []
42+
self.d['Returns'] = []
43+
self.d['Raises'] = []
44+
self.args = {} #arg_name->arg_type
45+
46+
def get_level(self, string, indent=' '):
47+
level = 0
48+
unit_size = len(indent)
49+
while string[:unit_size] == indent:
50+
string = string[unit_size:]
51+
level += 1
52+
53+
return level
54+
55+
def parse(self, doc):
56+
"""parse gets sections from doc
57+
Such as Args, Returns, Raises, Examples s
58+
Args:
59+
doc (string): is the astroid node doc string.
60+
Returns:
61+
True if doc is parsed successfully.
62+
"""
63+
self.clear()
64+
65+
lines = doc.splitlines()
66+
state = ("others", -1)
67+
for l in lines:
68+
c = l.strip()
69+
if len(c) <= 0:
70+
continue
71+
72+
level = self.get_level(l)
73+
if c.startswith("Args:"):
74+
state = ("Args", level)
75+
elif c.startswith("Returns:"):
76+
state = ("Returns", level)
77+
elif c.startswith("Raises:"):
78+
state = ("Raises", level)
79+
elif c.startswith("Examples:"):
80+
state = ("Examples", level)
81+
else:
82+
if level > state[1]:
83+
self.d[state[0]].append(c)
84+
continue
85+
86+
state = ("others", -1)
87+
self.d[state[0]].append(c)
88+
89+
self._arg_with_type()
90+
return True
91+
92+
def get_returns(self):
93+
return self.d['Returns']
94+
95+
def get_raises(self):
96+
return self.d['Raises']
97+
98+
def get_examples(self):
99+
return self.d['Examples']
100+
101+
def _arg_with_type(self):
102+
103+
for t in self.d['Args']:
104+
m = re.search('([A-Za-z0-9_-]+)\s{0,4}(\(.+\))\s{0,4}:', t)
105+
if m:
106+
self.args[m.group(1)] = m.group(2)
107+
108+
return self.args
109+
110+
111+
class DocstringChecker(BaseChecker):
112+
"""DosstringChecker is pylint checker to
113+
check docstring style.
114+
"""
115+
__implements__ = (IAstroidChecker, )
116+
117+
POSITIONAL_MESSAGE_ID = 'str-used-on-positional-format-argument'
118+
KEYWORD_MESSAGE_ID = 'str-used-on-keyword-format-argument'
119+
120+
name = 'doc-string-checker'
121+
symbol = "doc-string"
122+
priority = -1
123+
msgs = {
124+
'W9001': ('One line doc string on > 1 lines', symbol + "-one-line",
125+
'Used when a short doc string is on multiple lines'),
126+
'W9002':
127+
('Doc string does not end with "." period', symbol + "-end-with",
128+
'Used when a doc string does not end with a period'),
129+
'W9003': ('All args with their types must be mentioned in doc string',
130+
symbol + "-with-all-args",
131+
'Used when not all arguments are in the doc string '),
132+
'W9005': ('Missing docstring or docstring is too short',
133+
symbol + "-missing", 'Add docstring longer >=10'),
134+
'W9006': ('Docstring indent error, use 4 space for indent',
135+
symbol + "-indent-error", 'Use 4 space for indent'),
136+
'W9007': ('You should add `Returns` in comments',
137+
symbol + "-with-returns",
138+
'There should be a `Returns` section in comments'),
139+
'W9008': ('You should add `Raises` section in comments',
140+
symbol + "-with-raises",
141+
'There should be a `Raises` section in comments'),
142+
}
143+
options = ()
144+
145+
def visit_functiondef(self, node):
146+
"""visit_functiondef checks Function node docstring style.
147+
Args:
148+
node (astroid.node): The visiting node.
149+
Returns:
150+
True if successful other wise False.
151+
"""
152+
153+
self.check_doc_string(node)
154+
155+
if node.tolineno - node.fromlineno <= 10:
156+
return True
157+
158+
if not node.doc:
159+
return True
160+
161+
doc = Docstring()
162+
doc.parse(node.doc)
163+
164+
self.all_args_in_doc(node, doc)
165+
self.with_returns(node, doc)
166+
self.with_raises(node, doc)
167+
168+
def visit_module(self, node):
169+
self.check_doc_string(node)
170+
171+
def visit_classdef(self, node):
172+
self.check_doc_string(node)
173+
174+
def check_doc_string(self, node):
175+
self.missing_doc_string(node)
176+
self.one_line(node)
177+
self.has_period(node)
178+
self.indent_style(node)
179+
180+
def missing_doc_string(self, node):
181+
if node.tolineno - node.fromlineno <= 10:
182+
return True
183+
184+
if node.doc is None or len(node.doc) < 10:
185+
self.add_message('W9005', node=node, line=node.fromlineno)
186+
return False
187+
188+
# FIXME(gongwb): give the docstring line-no
189+
def indent_style(self, node, indent=4):
190+
"""indent_style checks docstring's indent style
191+
Args:
192+
node (astroid.node): The visiting node.
193+
indent (int): The default indent of style
194+
Returns:
195+
True if successful other wise False.
196+
"""
197+
if node.doc is None:
198+
return True
199+
200+
doc = node.doc
201+
lines = doc.splitlines()
202+
203+
for l in lines:
204+
cur_indent = len(l) - len(l.lstrip())
205+
if cur_indent % indent != 0:
206+
self.add_message('W9006', node=node, line=node.fromlineno)
207+
return False
208+
209+
return True
210+
211+
def one_line(self, node):
212+
"""one_line checks if docstring (len < 40) is on one line.
213+
Args:
214+
node (astroid.node): The node visiting.
215+
Returns:
216+
True if successful otherwise False.
217+
"""
218+
219+
doc = node.doc
220+
if doc is None:
221+
return True
222+
223+
if len(doc) > 40:
224+
return True
225+
elif sum(doc.find(nl) for nl in ('\n', '\r', '\n\r')) == -3:
226+
return True
227+
else:
228+
self.add_message('W9001', node=node, line=node.fromlineno)
229+
return False
230+
231+
return True
232+
233+
def has_period(self, node):
234+
"""has_period checks if one line doc end-with '.' .
235+
Args:
236+
node (astroid.node): the node is visiting.
237+
Returns:
238+
True if successful otherwise False.
239+
"""
240+
if node.doc is None:
241+
return True
242+
243+
if len(node.doc.splitlines()) > 1:
244+
return True
245+
246+
if not node.doc.strip().endswith('.'):
247+
self.add_message('W9002', node=node, line=node.fromlineno)
248+
return False
249+
250+
return True
251+
252+
def with_raises(self, node, doc):
253+
"""with_raises checks if one line doc end-with '.' .
254+
Args:
255+
node (astroid.node): the node is visiting.
256+
doc (Docstring): Docstring object.
257+
Returns:
258+
True if successful otherwise False.
259+
"""
260+
261+
find = False
262+
for t in node.body:
263+
if not isinstance(t, astroid.Raise):
264+
continue
265+
266+
find = True
267+
break
268+
269+
if not find:
270+
return True
271+
272+
if len(doc.get_raises()) == 0:
273+
self.add_message('W9008', node=node, line=node.fromlineno)
274+
return False
275+
276+
return True
277+
278+
def with_returns(self, node, doc):
279+
"""with_returns checks if docstring comments what are returned .
280+
Args:
281+
node (astroid.node): the node is visiting.
282+
doc (Docstring): Docstring object.
283+
Returns:
284+
True if successful otherwise False.
285+
"""
286+
287+
find = False
288+
for t in node.body:
289+
if not isinstance(t, astroid.Return):
290+
continue
291+
292+
find = True
293+
break
294+
295+
if not find:
296+
return True
297+
298+
if len(doc.get_returns()) == 0:
299+
self.add_message('W9007', node=node, line=node.fromlineno)
300+
return False
301+
302+
return True
303+
304+
def all_args_in_doc(self, node, doc):
305+
"""all_args_in_doc checks if arguments are mentioned in doc
306+
Args:
307+
node (astroid.node): the node is visiting.
308+
doc (Docstring): Docstring object
309+
Returns:
310+
True if successful otherwise False.
311+
"""
312+
args = []
313+
for arg in node.args.get_children():
314+
if (not isinstance(arg, astroid.AssignName)) \
315+
or arg.name == "self":
316+
continue
317+
args.append(arg.name)
318+
319+
if len(args) <= 0:
320+
return True
321+
322+
parsed_args = doc.args
323+
if len(args) > 0 and len(parsed_args) <= 0:
324+
print "debug:parsed args: ", parsed_args
325+
self.add_message('W9003', node=node, line=node.fromlineno)
326+
return False
327+
328+
for t in args:
329+
if t not in parsed_args:
330+
print t, " with (type) not in ", parsed_args
331+
self.add_message('W9003', node=node, line=node.fromlineno)
332+
return False
333+
334+
return True

‎tools/codestyle/pylint_pre_commit.hook

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
TOTAL_ERRORS=0
4+
5+
6+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
7+
export PYTHONPATH=$DIR:$PYTHONPATH
8+
9+
# The trick to remove deleted files: https://stackoverflow.com/a/2413151
10+
for file in $(git diff --cached --name-status | awk '$1 != "D" {print $2}'); do
11+
pylint --disable=all --load-plugins=docstring_checker \
12+
--enable=doc-string-one-line,doc-string-end-with,doc-string-with-all-args,doc-string-triple-quotes,doc-string-missing,doc-string-indent-error,doc-string-with-returns,doc-string-with-raises $file;
13+
TOTAL_ERRORS=$(expr $TOTAL_ERRORS + $?);
14+
done
15+
16+
#exit $TOTAL_ERRORS
17+
#For now, just warning:
18+
exit 0
19+

0 commit comments

Comments
 (0)