Skip to content

Commit bde143e

Browse files
joefarebrotherjoefarebrother
authored
Merge pull request #20038 from joefarebrother/python-qual-comparison
Python: Modernize 3 quality queries for comparison methods
2 parents 028f1cb + 3a27758 commit bde143e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+549
-556
lines changed

‎python/ql/integration-tests/query-suite/python-code-quality-extended.qls.expected

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
ql/python/ql/src/Classes/Comparisons/EqualsOrHash.ql
2+
ql/python/ql/src/Classes/Comparisons/EqualsOrNotEquals.ql
3+
ql/python/ql/src/Classes/Comparisons/IncompleteOrdering.ql
14
ql/python/ql/src/Classes/ConflictingAttributesInBaseClasses.ql
25
ql/python/ql/src/Classes/DefineEqualsWhenAddingAttributes.ql
3-
ql/python/ql/src/Classes/EqualsOrHash.ql
46
ql/python/ql/src/Classes/InconsistentMRO.ql
57
ql/python/ql/src/Classes/InitCallsSubclass/InitCallsSubclassMethod.ql
68
ql/python/ql/src/Classes/MissingCallToDel.ql

‎python/ql/integration-tests/query-suite/python-code-quality.qls.expected

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
ql/python/ql/src/Classes/Comparisons/EqualsOrHash.ql
2+
ql/python/ql/src/Classes/Comparisons/EqualsOrNotEquals.ql
3+
ql/python/ql/src/Classes/Comparisons/IncompleteOrdering.ql
14
ql/python/ql/src/Classes/ConflictingAttributesInBaseClasses.ql
25
ql/python/ql/src/Classes/DefineEqualsWhenAddingAttributes.ql
3-
ql/python/ql/src/Classes/EqualsOrHash.ql
46
ql/python/ql/src/Classes/InconsistentMRO.ql
57
ql/python/ql/src/Classes/InitCallsSubclass/InitCallsSubclassMethod.ql
68
ql/python/ql/src/Classes/MissingCallToDel.ql

‎python/ql/integration-tests/query-suite/python-security-and-quality.qls.expected

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1+
ql/python/ql/src/Classes/Comparisons/EqualsOrHash.ql
2+
ql/python/ql/src/Classes/Comparisons/EqualsOrNotEquals.ql
3+
ql/python/ql/src/Classes/Comparisons/IncompleteOrdering.ql
14
ql/python/ql/src/Classes/ConflictingAttributesInBaseClasses.ql
25
ql/python/ql/src/Classes/DefineEqualsWhenAddingAttributes.ql
3-
ql/python/ql/src/Classes/EqualsOrHash.ql
4-
ql/python/ql/src/Classes/EqualsOrNotEquals.ql
5-
ql/python/ql/src/Classes/IncompleteOrdering.ql
66
ql/python/ql/src/Classes/InconsistentMRO.ql
77
ql/python/ql/src/Classes/InitCallsSubclass/InitCallsSubclassMethod.ql
88
ql/python/ql/src/Classes/MissingCallToDel.ql

‎python/ql/lib/semmle/python/Class.qll

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,12 @@ class Class extends Class_, Scope, AstNode {
9191
/** Gets a method defined in this class */
9292
Function getAMethod() { result.getScope() = this }
9393

94+
/** Gets the method defined in this class with the specified name, if any. */
95+
Function getMethod(string name) {
96+
result = this.getAMethod() and
97+
result.getName() = name
98+
}
99+
94100
override Location getLocation() { py_scope_location(result, this) }
95101

96102
/** Gets the scope (module, class or function) in which this class is defined */

‎python/ql/src/Classes/Comparisons/EqualsOrHash.qhelp

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<!DOCTYPE qhelp PUBLIC
2+
"-//Semmle//qhelp//EN"
3+
"qhelp.dtd">
4+
<qhelp>
5+
6+
<overview>
7+
<p>A hashable class has an <code>__eq__</code> method, and a <code>__hash__</code> method that agrees with equality.
8+
When a hash method is defined, an equality method should also be defined; otherwise object identity is used for equality comparisons
9+
which may not be intended.
10+
</p>
11+
12+
<p>Note that defining an <code>__eq__</code> method without defining a <code>__hash__</code> method automatically makes the class unhashable in Python 3.
13+
(even if a superclass defines a hash method).</p>
14+
15+
</overview>
16+
<recommendation>
17+
18+
<p>
19+
If a <code>__hash__</code> method is defined, ensure a compatible <code>__eq__</code> method is also defined.
20+
</p>
21+
22+
<p>
23+
To explicitly declare a class as unhashable, set <code>__hash__ = None</code>, rather than defining a <code>__hash__</code> method that always
24+
raises an exception. Otherwise, the class would be incorrectly identified as hashable by an <code>isinstance(obj, collections.abc.Hashable)</code> call.
25+
</p>
26+
27+
</recommendation>
28+
<example>
29+
<p>In the following example, the <code>A</code> class defines an hash method but
30+
no equality method. Equality will be determined by object identity, which may not be the expected behaviour.
31+
</p>
32+
33+
<sample src="examples/EqualsOrHash.py" />
34+
35+
</example>
36+
<references>
37+
38+
39+
<li>Python Language Reference: <a href="http://docs.python.org/reference/datamodel.html#object.__hash__">object.__hash__</a>.</li>
40+
<li>Python Glossary: <a href="http://docs.python.org/3/glossary.html#term-hashable">hashable</a>.</li>
41+
42+
43+
</references>
44+
</qhelp>

‎python/ql/src/Classes/Comparisons/EqualsOrHash.ql

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/**
2+
* @name Inconsistent equality and hashing
3+
* @description Defining a hash operation without defining equality may be a mistake.
4+
* @kind problem
5+
* @tags quality
6+
* reliability
7+
* correctness
8+
* external/cwe/cwe-581
9+
* @problem.severity warning
10+
* @sub-severity high
11+
* @precision very-high
12+
* @id py/equals-hash-mismatch
13+
*/
14+
15+
import python
16+
17+
predicate missingEquality(Class cls, Function defined) {
18+
defined = cls.getMethod("__hash__") and
19+
not exists(cls.getMethod("__eq__"))
20+
// In python 3, the case of defined eq without hash automatically makes the class unhashable (even if a superclass defined hash)
21+
// So this is not an issue.
22+
}
23+
24+
from Class cls, Function defined
25+
where missingEquality(cls, defined)
26+
select cls, "This class implements $@, but does not implement __eq__.", defined, defined.getName()

‎python/ql/src/Classes/Comparisons/EqualsOrNotEquals.qhelp

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
<!DOCTYPE qhelp PUBLIC
2+
"-//Semmle//qhelp//EN"
3+
"qhelp.dtd">
4+
<qhelp>
5+
6+
<overview>
7+
<p>In order to ensure the <code>==</code> and <code>!=</code> operators behave consistently as expected (i.e. they should be negations of each other), care should be taken when implementing the
8+
<code>__eq__</code> and <code>__ne__</code> special methods.</p>
9+
10+
<p>In Python 3, if the <code>__eq__</code> method is defined in a class while the <code>__ne__</code> is not,
11+
then the <code>!=</code> operator will automatically delegate to the <code>__eq__</code> method in the expected way.
12+
</p>
13+
14+
<p>However, if the <code>__ne__</code> method is defined without a corresponding <code>__eq__</code> method,
15+
the <code>==</code> operator will still default to object identity (equivalent to the <code>is</code> operator), while the <code>!=</code>
16+
operator will use the <code>__ne__</code> method, which may be inconsistent.
17+
</p>
18+
19+
<p>Additionally, if the <code>__ne__</code> method is defined on a superclass, and the subclass defines its own <code>__eq__</code> method without overriding
20+
the superclass <code>__ne__</code> method, the <code>!=</code> operator will use this superclass <code>__ne__</code> method, rather than automatically delegating
21+
to <code>__eq__</code>, which may be incorrect.
22+
</p>
23+
24+
</overview>
25+
<recommendation>
26+
27+
<p>Ensure that when an <code>__ne__</code> method is defined, the <code>__eq__</code> method is also defined, and their results are consistent.
28+
In most cases, the <code>__ne__</code> method does not need to be defined at all, as the default behavior is to delegate to <code>__eq__</code> and negate the result. </p>
29+
30+
</recommendation>
31+
<example>
32+
<p>In the following example, <code>A</code> defines a <code>__ne__</code> method, but not an <code>__eq__</code> method.
33+
This leads to inconsistent results between equality and inequality operators.
34+
</p>
35+
36+
<sample src="examples/EqualsOrNotEquals1.py" />
37+
38+
<p>In the following example, <code>C</code> defines an <code>__eq__</code> method, but its <code>__ne__</code> implementation is inherited from <code>B</code>,
39+
which is not consistent with the equality operation.
40+
</p>
41+
42+
<sample src="examples/EqualsOrNotEquals2.py" />
43+
44+
</example>
45+
<references>
46+
47+
48+
<li>Python Language Reference: <a href="http://docs.python.org/3/reference/datamodel.html#object.__ne__">object.__ne__</a>,
49+
<a href="http://docs.python.org/3/reference/expressions.html#comparisons">Comparisons</a>.</li>
50+
51+
52+
</references>
53+
</qhelp>

‎python/ql/src/Classes/Comparisons/EqualsOrNotEquals.ql

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/**
2+
* @name Inconsistent equality and inequality
3+
* @description Class definitions of equality and inequality operators may be inconsistent.
4+
* @kind problem
5+
* @tags quality
6+
* reliability
7+
* correctness
8+
* @problem.severity warning
9+
* @sub-severity high
10+
* @precision very-high
11+
* @id py/inconsistent-equality
12+
*/
13+
14+
import python
15+
import semmle.python.dataflow.new.internal.DataFlowDispatch
16+
import Classes.Equality
17+
18+
predicate missingEquality(Class cls, Function defined, string missing) {
19+
defined = cls.getMethod("__ne__") and
20+
not exists(cls.getMethod("__eq__")) and
21+
missing = "__eq__"
22+
or
23+
// In python 3, __ne__ automatically delegates to __eq__ if its not defined in the hierarchy
24+
// However if it is defined in a superclass (and isn't a delegation method) then it will use the superclass method (which may be incorrect)
25+
defined = cls.getMethod("__eq__") and
26+
not exists(cls.getMethod("__ne__")) and
27+
exists(Function neMeth |
28+
neMeth = getADirectSuperclass+(cls).getMethod("__ne__") and
29+
not neMeth instanceof DelegatingEqualityMethod
30+
) and
31+
missing = "__ne__"
32+
}
33+
34+
from Class cls, Function defined, string missing
35+
where missingEquality(cls, defined, missing)
36+
select cls, "This class implements $@, but does not implement " + missing + ".", defined,
37+
defined.getName()

‎python/ql/src/Classes/Comparisons/IncompleteOrdering.qhelp

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<!DOCTYPE qhelp PUBLIC
2+
"-//Semmle//qhelp//EN"
3+
"qhelp.dtd">
4+
<qhelp>
5+
<overview>
6+
<p> A class that implements the rich comparison operators
7+
(<code>__lt__</code>, <code>__gt__</code>, <code>__le__</code>, or <code>__ge__</code>) should ensure that all four
8+
comparison operations <code>&lt;</code>, <code>&lt;=</code>, <code>&gt;</code>, and <code>&gt;=</code> function as expected, consistent
9+
with expected mathematical rules.
10+
In Python 3, this is ensured by implementing one of <code>__lt__</code> or <code>__gt__</code>, and one of <code>__le__</code> or <code>__ge__</code>.
11+
If the ordering is not consistent with default equality, then <code>__eq__</code> should also be implemented.
12+
</p>
13+
14+
</overview>
15+
<recommendation>
16+
<p>Ensure that at least one of <code>__lt__</code> or <code>__gt__</code> and at least one of <code>__le__</code> or <code>__ge__</code> is defined.
17+
</p>
18+
19+
<p>
20+
The <code>functools.total_ordering</code> class decorator can be used to automatically implement all four comparison methods from a
21+
single one,
22+
which is typically the cleanest way to ensure all necessary comparison methods are implemented consistently.</p>
23+
24+
</recommendation>
25+
<example>
26+
<p>In the following example, only the <code>__lt__</code> operator has been implemented, which would lead to unexpected
27+
errors if the <code>&lt;=</code> or <code>&gt;=</code> operators are used on <code>A</code> instances.
28+
The <code>__le__</code> method should also be defined, as well as <code>__eq__</code> in this case.</p>
29+
<sample src="examples/IncompleteOrdering.py" />
30+
31+
</example>
32+
<references>
33+
34+
<li>Python Language Reference: <a href="http://docs.python.org/3/reference/datamodel.html#object.__lt__">Rich comparisons in Python</a>.</li>
35+
36+
37+
</references>
38+
</qhelp>

‎python/ql/src/Classes/Comparisons/IncompleteOrdering.ql

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/**
2+
* @name Incomplete ordering
3+
* @description Class defines ordering comparison methods, but does not define both strict and nonstrict ordering methods, to ensure all four comparison operators behave as expected.
4+
* @kind problem
5+
* @tags quality
6+
* reliability
7+
* correctness
8+
* @problem.severity warning
9+
* @sub-severity low
10+
* @precision very-high
11+
* @id py/incomplete-ordering
12+
*/
13+
14+
import python
15+
import semmle.python.dataflow.new.internal.DataFlowDispatch
16+
import semmle.python.ApiGraphs
17+
18+
/** Holds if `cls` has the `functools.total_ordering` decorator. */
19+
predicate totalOrdering(Class cls) {
20+
API::moduleImport("functools")
21+
.getMember("total_ordering")
22+
.asSource()
23+
.flowsTo(DataFlow::exprNode(cls.getADecorator()))
24+
}
25+
26+
predicate definesStrictOrdering(Class cls, Function meth) {
27+
meth = cls.getMethod("__lt__")
28+
or
29+
not exists(cls.getMethod("__lt__")) and
30+
meth = cls.getMethod("__gt__")
31+
}
32+
33+
predicate definesNonStrictOrdering(Class cls, Function meth) {
34+
meth = cls.getMethod("__le__")
35+
or
36+
not exists(cls.getMethod("__le__")) and
37+
meth = cls.getMethod("__ge__")
38+
}
39+
40+
predicate missingComparison(Class cls, Function defined, string missing) {
41+
definesStrictOrdering(cls, defined) and
42+
not definesNonStrictOrdering(getADirectSuperclass*(cls), _) and
43+
missing = "__le__ or __ge__"
44+
or
45+
definesNonStrictOrdering(cls, defined) and
46+
not definesStrictOrdering(getADirectSuperclass*(cls), _) and
47+
missing = "__lt__ or __gt__"
48+
}
49+
50+
from Class cls, Function defined, string missing
51+
where
52+
not totalOrdering(cls) and
53+
missingComparison(cls, defined, missing)
54+
select cls, "This class implements $@, but does not implement " + missing + ".", defined,
55+
defined.getName()

0 commit comments

Comments
 (0)