Skip to content

Commit 8f68115

Browse files
yoffyoff
authored
Merge pull request github#17683 from yoff/python/flip-default-for-stdlib-extraction
Python: Flip default for stdlib extraction
2 parents 5b57826 + 5650694 commit 8f68115

File tree

37 files changed

+48
-93
lines changed

37 files changed

+48
-93
lines changed

‎python/extractor/cli-integration-test/extract-stdlib/test.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ rm -rf dbs
1313

1414
mkdir dbs
1515

16-
CODEQL_EXTRACTOR_PYTHON_DONT_EXTRACT_STDLIB=True $CODEQL database create dbs/without-stdlib --language python --source-root repo_dir/
16+
$CODEQL database create dbs/without-stdlib --language python --source-root repo_dir/
1717
$CODEQL query run --database dbs/without-stdlib query.ql > query.without-stdlib.actual
1818
diff query.without-stdlib.expected query.without-stdlib.actual
1919

20-
LGTM_INDEX_EXCLUDE="/usr/lib/**" $CODEQL database create dbs/with-stdlib --language python --source-root repo_dir/
20+
LGTM_INDEX_EXCLUDE="/usr/lib/**" CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB=True $CODEQL database create dbs/with-stdlib --language python --source-root repo_dir/
2121
$CODEQL query run --database dbs/with-stdlib query.ql > query.with-stdlib.actual
2222
diff query.with-stdlib.expected query.with-stdlib.actual

‎python/extractor/semmle/cmdline.py

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,10 @@ def make_parser():
102102
config_options.add_option("--colorize", dest="colorize", default=False, action="store_true",
103103
help = """Colorize the logging output.""")
104104

105-
config_options.add_option("--dont-extract-stdlib", dest="extract_stdlib", default=True, action="store_false",
106-
help="Do not extract the standard library.")
105+
config_options.add_option("--dont-extract-stdlib", dest="extract_stdlib", action="store_false",
106+
help="This flag is deprecated; not extracting the standard library is now the default.")
107+
config_options.add_option("--extract-stdlib", dest="extract_stdlib", default=False, action="store_true",
108+
help="Extract the standard library.")
107109

108110
parser.add_option_group(config_options)
109111

@@ -226,8 +228,18 @@ def parse(command_line):
226228

227229
if 'CODEQL_EXTRACTOR_PYTHON_DONT_EXTRACT_STDLIB' in os.environ:
228230
options.extract_stdlib = False
231+
print ("WARNING: CODEQL_EXTRACTOR_PYTHON_DONT_EXTRACT_STDLIB is deprecated; the default is now to not extract the standard library.")
232+
233+
if 'CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB' in os.environ:
234+
options.extract_stdlib = True
229235

230236
options.prune = True
237+
238+
if options.extract_stdlib:
239+
print ("WARNING: The analysis will extract the standard library. This behavior is deprecated and will be removed in a future release. We expect it to be gone in CLI version 2.20.0.")
240+
else:
241+
print ("INFO: The Python extractor has recently stopped extracting the standard library by default. If you encounter problems, please let us know by submitting an issue to https://github.com/github/codeql. It is possible to re-enable extraction of the standard library by setting the environment variable CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB.")
242+
231243
return options, args
232244

233245
def split_and_flatten(options_list, div):

‎python/extractor/semmle/populator.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def main(sys_path = sys.path[:]):
6767
update_analysis_version(last_version)
6868

6969
found_py2 = False
70-
if get_analysis_major_version() == 2:
70+
if get_analysis_major_version() == 2 and options.extract_stdlib:
7171
# Setup `sys_path` to use the Python 2 standard library
7272
sys_path, found_py2 = get_py2_sys_path(logger, sys_path)
7373

‎python/extractor/semmle/util.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
#Semantic version of extractor.
1212
#Update this if any changes are made
13-
VERSION = "6.1.2"
13+
VERSION = "7.0.0"
1414

1515
PY_EXTENSIONS = ".py", ".pyw"
1616

‎python/ql/test/2/library-tests/PointsTo/class_properties/ClassValues.expected

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
1-
| mapping | builtin-class collections.defaultdict |
21
| mapping | builtin-class dict |
32
| mapping | class MyDictSubclass |
4-
| mapping | class MyMappingABC |
5-
| mapping | class OrderedDict |
63
| neither sequence nor mapping | builtin-class set |
4+
| neither sequence nor mapping | class MyMappingABC |
5+
| neither sequence nor mapping | class MySequenceABC |
76
| sequence | builtin-class list |
87
| sequence | builtin-class str |
98
| sequence | builtin-class tuple |
109
| sequence | builtin-class unicode |
11-
| sequence | class MySequenceABC |
1210
| sequence | class MySequenceImpl |

‎python/ql/test/2/library-tests/PointsTo/imports2/Runtime.expected

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,10 @@
4444
| test.py | 15 | ControlFlowNode for moduleX | Module package.moduleX | Entry node for Module package.moduleX |
4545
| test.py | 16 | ControlFlowNode for Attribute | class Y | ControlFlowNode for ClassExpr |
4646
| test.py | 16 | ControlFlowNode for moduleX | Module package.moduleX | Entry node for Module package.moduleX |
47-
| test.py | 19 | ControlFlowNode for ImportExpr | Module tty | ControlFlowNode for ImportExpr |
48-
| test.py | 19 | ControlFlowNode for tty | Module tty | ControlFlowNode for ImportExpr |
4947
| test.py | 22 | ControlFlowNode for Attribute | Builtin-function exc_info | ControlFlowNode for from sys import * |
5048
| test.py | 22 | ControlFlowNode for x | Module package.x | Entry node for Module package.x |
5149
| test.py | 24 | ControlFlowNode for IntegerLiteral | int 0 | ControlFlowNode for IntegerLiteral |
5250
| test.py | 24 | ControlFlowNode for argv | int 0 | ControlFlowNode for IntegerLiteral |
5351
| test.py | 27 | ControlFlowNode for ImportExpr | Module sys | ControlFlowNode for ImportExpr |
5452
| test.py | 31 | ControlFlowNode for argv | list object | ControlFlowNode for from sys import * |
55-
| test.py | 33 | ControlFlowNode for ImportExpr | Module socket | ControlFlowNode for ImportExpr |
56-
| test.py | 34 | ControlFlowNode for timeout | builtin-class socket.timeout | ControlFlowNode for from _socket import * |
5753
| x.py | 2 | ControlFlowNode for ImportExpr | Module sys | ControlFlowNode for ImportExpr |

‎python/ql/test/2/library-tests/PointsTo/imports2/RuntimeWithType.expected

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,10 @@
4444
| test.py | 15 | ControlFlowNode for moduleX | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
4545
| test.py | 16 | ControlFlowNode for Attribute | class Y | builtin-class type | ControlFlowNode for ClassExpr |
4646
| test.py | 16 | ControlFlowNode for moduleX | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
47-
| test.py | 19 | ControlFlowNode for ImportExpr | Module tty | builtin-class module | ControlFlowNode for ImportExpr |
48-
| test.py | 19 | ControlFlowNode for tty | Module tty | builtin-class module | ControlFlowNode for ImportExpr |
4947
| test.py | 22 | ControlFlowNode for Attribute | Builtin-function exc_info | builtin-class builtin_function_or_method | ControlFlowNode for from sys import * |
5048
| test.py | 22 | ControlFlowNode for x | Module package.x | builtin-class module | Entry node for Module package.x |
5149
| test.py | 24 | ControlFlowNode for IntegerLiteral | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
5250
| test.py | 24 | ControlFlowNode for argv | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
5351
| test.py | 27 | ControlFlowNode for ImportExpr | Module sys | builtin-class module | ControlFlowNode for ImportExpr |
5452
| test.py | 31 | ControlFlowNode for argv | list object | builtin-class list | ControlFlowNode for from sys import * |
55-
| test.py | 33 | ControlFlowNode for ImportExpr | Module socket | builtin-class module | ControlFlowNode for ImportExpr |
56-
| test.py | 34 | ControlFlowNode for timeout | builtin-class socket.timeout | builtin-class type | ControlFlowNode for from _socket import * |
5753
| x.py | 2 | ControlFlowNode for ImportExpr | Module sys | builtin-class module | ControlFlowNode for ImportExpr |

‎python/ql/test/2/library-tests/six/pointsto.expected

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
| six | Package six |
22
| six.moves | Package six.moves |
3-
| six.moves.http_client | Module httplib |
4-
| six.moves.http_client.HTTPConnection | class HTTPConnection |
3+
| six.moves.http_client | Missing module httplib |
4+
| six.moves.http_client.HTTPConnection | Missing module attribute httplib.HTTPConnection |
55
| six.moves.range | builtin-class xrange |
66
| six.moves.urllib | Package six.moves.urllib |
77
| six.moves.urllib.parse | Module six.moves.urllib_parse |

‎python/ql/test/2/query-tests/Summary/LinesOfCode.expected

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
| 38 |
1+
| 11 |

‎python/ql/test/3/library-tests/PointsTo/class_properties/ClassValues.expected

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
1-
| mapping | builtin-class collections.OrderedDict |
2-
| mapping | builtin-class collections.defaultdict |
31
| mapping | builtin-class dict |
42
| mapping | class MyDictSubclass |
53
| mapping | class MyMappingABC |
6-
| mapping | class OrderedDict |
74
| neither sequence nor mapping | builtin-class set |
85
| sequence | builtin-class bytes |
96
| sequence | builtin-class list |

0 commit comments

Comments
 (0)