Improve imperative mood check using stemmer/wordlist

Author: lordmauve

requirements/runtime.txt

@@ -0,0 +1 @@
+snowballstemmer==1.2.1

setup.py

@@ -31,6 +31,10 @@
     keywords='pydocstyle, PEP 257, pep257, PEP 8, pep8, docstrings',
     packages=('pydocstyle',),
     package_dir={'': 'src'},
+    package_data={'pydocstyle': ['data/*.txt']},
+    install_requires=[
+        'snowballstemmer==1.2.1',
+    ],
     entry_points={
         'console_scripts': [
             'pydocstyle = pydocstyle.cli:main',

src/pydocstyle/checker.py

@@ -13,6 +13,7 @@
 from .parser import (Package, Module, Class, NestedClass, Definition, AllError,
                      Method, Function, NestedFunction, Parser, StringIO)
 from .utils import log, is_blank
+from .wordlists import IMPERATIVE_VERBS, IMPERATIVE_BLACKLIST, stem
 
 
 __all__ = ('check', )
@@ -361,12 +362,21 @@ def check_imperative_mood(self, function, docstring):  # def context
         "Returns the pathname ...".
 
         """
-        if docstring:
+        if docstring and not function.is_test:
             stripped = ast.literal_eval(docstring).strip()
             if stripped:
                 first_word = stripped.split()[0]
-                if first_word.endswith('s') and not first_word.endswith('ss'):
-                    return violations.D401(first_word[:-1], first_word)
+                check_word = first_word.lower()
+
+                if check_word in IMPERATIVE_BLACKLIST:
+                    return violations.D401b(first_word)
+
+                correct_form = IMPERATIVE_VERBS.get(stem(check_word))
+                if correct_form and correct_form != check_word:
+                    return violations.D401(
+                        correct_form.capitalize(),
+                        first_word
+                    )
 
     @check_for(Function)
     def check_no_signature(self, function, docstring):  # def context

src/pydocstyle/data/imperatives.txt

@@ -0,0 +1,232 @@
+# Imperative forms of verbs
+#
+# This file contains the imperative form of frequently encountered
+# docstring verbs. Some of these may be more commonly encountered as
+# nouns, but blacklisting them for this may cause false positives.
+accept
+access
+add
+adjust
+aggregate
+allow
+append
+apply
+archive
+assert
+assign
+attempt
+authenticate
+authorize
+break
+build
+cache
+calculate
+call
+cancel
+capture
+change
+check
+clean
+clear
+close
+collect
+combine
+commit
+compare
+compute
+configure
+confirm
+connect
+construct
+control
+convert
+copy
+count
+create
+customize
+declare
+decode
+decorate
+define
+delegate
+delete
+deprecate
+derive
+describe
+detect
+determine
+display
+download
+drop
+dump
+emit
+empty
+enable
+encapsulate
+encode
+end
+ensure
+enumerate
+establish
+evaluate
+examine
+execute
+exit
+expand
+expect
+export
+extend
+extract
+feed
+fetch
+fill
+filter
+finalize
+find
+fire
+fix
+flag
+force
+format
+forward
+generate
+get
+give
+go
+group
+handle
+help
+hold
+identify
+implement
+import
+indicate
+init
+initalise
+initialise
+initialize
+input
+insert
+instantiate
+intercept
+invoke
+iterate
+join
+keep
+launch
+list
+listen
+load
+log
+look
+make
+manage
+manipulate
+map
+mark
+match
+merge
+mock
+modify
+monitor
+move
+normalize
+note
+obtain
+open
+output
+override
+overwrite
+pad
+parse
+partial
+pass
+perform
+persist
+pick
+plot
+poll
+populate
+post
+prepare
+print
+process
+produce
+provide
+publish
+pull
+put
+query
+raise
+read
+record
+refer
+refresh
+register
+reload
+remove
+rename
+render
+replace
+reply
+report
+represent
+request
+require
+reset
+resolve
+retrieve
+return
+roll
+rollback
+round
+run
+sample
+save
+scan
+search
+select
+send
+serialise
+serialize
+serve
+set
+show
+simulate
+source
+specify
+split
+start
+step
+stop
+store
+strip
+submit
+subscribe
+sum
+swap
+sync
+synchronise
+synchronize
+take
+tear
+test
+time
+transform
+translate
+transmit
+truncate
+try
+turn
+tweak
+update
+upload
+use
+validate
+verify
+view
+wait
+walk
+wrap
+write
+yield

src/pydocstyle/data/imperatives_blacklist.txt

@@ -0,0 +1,109 @@
+# Blacklisted imperative words
+#
+# These are words that, if they begin a docstring, are a good indicator that
+# the docstring is not written in an imperative voice.
+#
+# The words included in this list fall into a number of categories:
+#
+# - Starting with a noun/pronoun indicates that the docstring is a noun phrase
+#   or a sentence but not in the imperative mood
+# - Adjectives are always followed by a noun, so same
+# - Particles are also followed by a noun
+# - Some adverbs don't really indicate an imperative sentence, for example
+#   "importantly" or "currently".
+# - Some irregular verb forms that don't stem to the same string as the
+#   imperative does (eg. 'does')
+:param
+:return
+:return:
+:returns
+:returns:
+:rtype
+a
+an
+the
+action
+always
+api
+base
+basic
+business
+calculation
+callback
+collection
+common
+constructor
+convenience
+convenient
+current
+currently
+custom
+data
+data
+default
+deprecated
+description
+description:
+dict
+dictionary
+does
+dummy
+example
+factory
+false
+final
+formula
+function
+generic
+handler
+handler
+helper
+here
+hook
+implementation
+importantly
+internal
+it
+main
+method
+module
+new
+number
+optional
+package
+parameters:
+placeholder
+reference
+result
+returns:
+same
+schema
+setup
+should
+simple
+some
+special
+sql
+standard
+static
+string
+subclasses
+that
+these
+this
+true
+unique
+unit
+utility
+what
+wrapper
+
+
+# These are nouns, but often used in the context of functions that act as
+# objects; thus we do not blacklist these.
+#
+# context  # as in context manager
+# decorator
+# class  # as in class decorator
+# property
+# generator