Python: Use shared prettyExpr in ConceptsTest.qll

This required quite some changes in the expected output. I think it's much more
clear what the selected nodes are now 👍 (but it was a bit boring work to fix
this up)

Author: RasmusWL

python/ql/test/experimental/meta/ConceptsTest.qll

@@ -2,19 +2,7 @@ import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.Concepts
 import TestUtilities.InlineExpectationsTest
-
-string value_from_expr(Expr e) {
-  // TODO: This one is starting to look like `repr` predicate from TestTaintLib
-  result =
-    e.(StrConst).getPrefix() + e.(StrConst).getText() +
-      e.(StrConst).getPrefix().regexpReplaceAll("[a-zA-Z]+", "")
-  or
-  result = e.(Name).getId()
-  or
-  not e instanceof StrConst and
-  not e instanceof Name and
-  result = e.toString()
-}
+import experimental.dataflow.TestUtil.PrintNode
 
 class SystemCommandExecutionTest extends InlineExpectationsTest {
   SystemCommandExecutionTest() { this = "SystemCommandExecutionTest" }
@@ -27,7 +15,7 @@ class SystemCommandExecutionTest extends InlineExpectationsTest {
       command = sce.getCommand() and
       location = command.getLocation() and
       element = command.toString() and
-      value = value_from_expr(command.asExpr()) and
+      value = prettyExpr(command.asExpr()) and
       tag = "getCommand"
     )
   }
@@ -46,7 +34,7 @@ class DecodingTest extends InlineExpectationsTest {
       exists(DataFlow::Node data |
         location = data.getLocation() and
         element = data.toString() and
-        value = value_from_expr(data.asExpr()) and
+        value = prettyExpr(data.asExpr()) and
         (
           data = d.getAnInput() and
           tag = "decodeInput"
@@ -84,7 +72,7 @@ class EncodingTest extends InlineExpectationsTest {
       exists(DataFlow::Node data |
         location = data.getLocation() and
         element = data.toString() and
-        value = value_from_expr(data.asExpr()) and
+        value = prettyExpr(data.asExpr()) and
         (
           data = e.getAnInput() and
           tag = "encodeInput"
@@ -117,7 +105,7 @@ class CodeExecutionTest extends InlineExpectationsTest {
       code = ce.getCode() and
       location = code.getLocation() and
       element = code.toString() and
-      value = value_from_expr(code.asExpr()) and
+      value = prettyExpr(code.asExpr()) and
       tag = "getCode"
     )
   }
@@ -135,7 +123,7 @@ class SqlExecutionTest extends InlineExpectationsTest {
       sql = e.getSql() and
       location = e.getLocation() and
       element = sql.toString() and
-      value = value_from_expr(sql.asExpr()) and
+      value = prettyExpr(sql.asExpr()) and
       tag = "getSql"
     )
   }
@@ -218,7 +206,7 @@ class HttpServerHttpResponseTest extends InlineExpectationsTest {
       exists(HTTP::Server::HttpResponse response |
         location = response.getLocation() and
         element = response.toString() and
-        value = value_from_expr(response.getBody().asExpr()) and
+        value = prettyExpr(response.getBody().asExpr()) and
         tag = "responseBody"
       )
       or
@@ -257,7 +245,7 @@ class HttpServerHttpRedirectResponseTest extends InlineExpectationsTest {
       exists(HTTP::Server::HttpRedirectResponse redirect |
         location = redirect.getLocation() and
         element = redirect.toString() and
-        value = value_from_expr(redirect.getRedirectLocation().asExpr()) and
+        value = prettyExpr(redirect.getRedirectLocation().asExpr()) and
         tag = "redirectLocation"
       )
     )
@@ -275,7 +263,7 @@ class FileSystemAccessTest extends InlineExpectationsTest {
       path = a.getAPathArgument() and
       location = a.getLocation() and
       element = path.toString() and
-      value = value_from_expr(path.asExpr()) and
+      value = prettyExpr(path.asExpr()) and
       tag = "getAPathArgument"
     )
   }
@@ -309,7 +297,7 @@ class SafeAccessCheckTest extends InlineExpectationsTest {
       location = c.getLocation() and
       (
         element = checks.toString() and
-        value = value_from_expr(checks.asExpr()) and
+        value = prettyExpr(checks.asExpr()) and
         tag = "checks"
         or
         element = branch.toString() and

python/ql/test/library-tests/frameworks/dill/Decoding.py

@@ -1,3 +1,3 @@
 import dill
 
-dill.loads(payload)  # $decodeInput=payload decodeOutput=Attribute() decodeFormat=dill decodeMayExecuteInput
+dill.loads(payload)  # $decodeInput=payload decodeOutput=dill.loads(..) decodeFormat=dill decodeMayExecuteInput

python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py

@@ -74,28 +74,28 @@ def get_redirect_url(self, foo): # $ requestHandler routedParameter=foo
 
 # Ensure that simple subclasses are still vuln to XSS
 def xss__not_found(request):
-    return HttpResponseNotFound(request.GET.get("name"))  # $HttpResponse mimetype=text/html responseBody=Attribute()
+    return HttpResponseNotFound(request.GET.get("name"))  # $HttpResponse mimetype=text/html responseBody=request.GET.get(..)
 
 # Ensure we still have an XSS sink when manually setting the content_type to HTML
 def xss__manual_response_type(request):
-    return HttpResponse(request.GET.get("name"), content_type="text/html; charset=utf-8")  # $HttpResponse mimetype=text/html responseBody=Attribute()
+    return HttpResponse(request.GET.get("name"), content_type="text/html; charset=utf-8")  # $HttpResponse mimetype=text/html responseBody=request.GET.get(..)
 
 def xss__write(request):
     response = HttpResponse()  # $HttpResponse mimetype=text/html
-    response.write(request.GET.get("name"))  # $HttpResponse mimetype=text/html responseBody=Attribute()
+    response.write(request.GET.get("name"))  # $HttpResponse mimetype=text/html responseBody=request.GET.get(..)
 
 # This is safe but probably a bug if the argument to `write` is not a result of `json.dumps` or similar.
 def safe__write_json(request):
     response = JsonResponse()  # $HttpResponse mimetype=application/json
-    response.write(request.GET.get("name"))  # $HttpResponse mimetype=application/json responseBody=Attribute()
+    response.write(request.GET.get("name"))  # $HttpResponse mimetype=application/json responseBody=request.GET.get(..)
 
 # Ensure manual subclasses are vulnerable
 class CustomResponse(HttpResponse):
     def __init__(self, banner, content, *args, **kwargs):
         super().__init__(content, *args, content_type="text/html", **kwargs)
 
 def xss__custom_response(request):
-    return CustomResponse("ACME Responses", request.GET("name"))  # $HttpResponse MISSING: mimetype=text/html responseBody=Attribute() SPURIOUS: responseBody="ACME Responses"
+    return CustomResponse("ACME Responses", request.GET("name"))  # $HttpResponse MISSING: mimetype=text/html responseBody=request.GET.get(..) SPURIOUS: responseBody="ACME Responses"
 
 class CustomJsonResponse(JsonResponse):
     def __init__(self, banner, content, *args, **kwargs):

python/ql/test/library-tests/frameworks/django-v2-v3/testproj/settings.py

@@ -13,7 +13,7 @@
 from pathlib import Path
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
-BASE_DIR = Path(__file__).resolve().parent.parent  #$ getAPathArgument=Path()
+BASE_DIR = Path(__file__).resolve().parent.parent  #$ getAPathArgument=Path(..)
 
 
 # Quick-start development settings - unsuitable for production

python/ql/test/library-tests/frameworks/idna/taint_test.py

@@ -5,9 +5,9 @@ def test_idna():
     tb = TAINTED_BYTES
 
     ensure_tainted(
-        idna.encode(ts),  # $ tainted encodeInput=ts encodeOutput=Attribute() encodeFormat=IDNA
-        idna.encode(s=ts),  # $ tainted encodeInput=ts encodeOutput=Attribute() encodeFormat=IDNA
+        idna.encode(ts),  # $ tainted encodeInput=ts encodeOutput=idna.encode(..) encodeFormat=IDNA
+        idna.encode(s=ts),  # $ tainted encodeInput=ts encodeOutput=idna.encode(..) encodeFormat=IDNA
 
-        idna.decode(tb),  # $ tainted decodeInput=tb decodeOutput=Attribute() decodeFormat=IDNA
-        idna.decode(s=tb),  # $ tainted decodeInput=tb decodeOutput=Attribute() decodeFormat=IDNA
+        idna.decode(tb),  # $ tainted decodeInput=tb decodeOutput=idna.decode(..) decodeFormat=IDNA
+        idna.decode(s=tb),  # $ tainted decodeInput=tb decodeOutput=idna.decode(..) decodeFormat=IDNA
     )

python/ql/test/library-tests/frameworks/simplejson/taint_test.py

@@ -5,14 +5,14 @@ def test():
     ts = TAINTED_STRING
     tainted_obj = {"foo": ts}
 
-    encoded = simplejson.dumps(tainted_obj) # $ encodeOutput=Attribute() encodeFormat=JSON encodeInput=tainted_obj
+    encoded = simplejson.dumps(tainted_obj) # $ encodeOutput=simplejson.dumps(..) encodeFormat=JSON encodeInput=tainted_obj
 
     ensure_tainted(
         encoded, # $ tainted
-        simplejson.dumps(tainted_obj), # $ tainted encodeOutput=Attribute() encodeFormat=JSON encodeInput=tainted_obj
-        simplejson.dumps(obj=tainted_obj), # $ tainted encodeOutput=Attribute() encodeFormat=JSON encodeInput=tainted_obj
-        simplejson.loads(encoded), # $ tainted decodeOutput=Attribute() decodeFormat=JSON decodeInput=encoded
-        simplejson.loads(s=encoded), # $ tainted decodeOutput=Attribute() decodeFormat=JSON decodeInput=encoded
+        simplejson.dumps(tainted_obj), # $ tainted encodeOutput=simplejson.dumps(..) encodeFormat=JSON encodeInput=tainted_obj
+        simplejson.dumps(obj=tainted_obj), # $ tainted encodeOutput=simplejson.dumps(..) encodeFormat=JSON encodeInput=tainted_obj
+        simplejson.loads(encoded), # $ tainted decodeOutput=simplejson.loads(..) decodeFormat=JSON decodeInput=encoded
+        simplejson.loads(s=encoded), # $ tainted decodeOutput=simplejson.loads(..) decodeFormat=JSON decodeInput=encoded
     )
 
     # load/dump with file-like
@@ -22,7 +22,7 @@ def test():
     tainted_filelike.seek(0)
     ensure_tainted(
         tainted_filelike, # $ MISSING: tainted
-        simplejson.load(tainted_filelike), # $ decodeOutput=Attribute() decodeFormat=JSON decodeInput=tainted_filelike MISSING: tainted
+        simplejson.load(tainted_filelike), # $ decodeOutput=simplejson.load(..) decodeFormat=JSON decodeInput=tainted_filelike MISSING: tainted
     )
 
     # load/dump with file-like using keyword-args
@@ -32,7 +32,7 @@ def test():
     tainted_filelike.seek(0)
     ensure_tainted(
         tainted_filelike, # $ MISSING: tainted
-        simplejson.load(fp=tainted_filelike), # $ decodeOutput=Attribute() decodeFormat=JSON decodeInput=tainted_filelike MISSING: tainted
+        simplejson.load(fp=tainted_filelike), # $ decodeOutput=simplejson.load(..) decodeFormat=JSON decodeInput=tainted_filelike MISSING: tainted
     )
 
 # To make things runable

python/ql/test/library-tests/frameworks/stdlib-py3/Decoding.py

@@ -1,6 +1,6 @@
 import base64
 
 # TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py
-base64.a85decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Ascii85
-base64.b85decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base85
-base64.decodebytes(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
+base64.a85decode(payload)  # $ decodeInput=payload decodeOutput=base64.a85decode(..) decodeFormat=Ascii85
+base64.b85decode(payload)  # $ decodeInput=payload decodeOutput=base64.b85decode(..) decodeFormat=Base85
+base64.decodebytes(payload)  # $ decodeInput=payload decodeOutput=base64.decodebytes(..) decodeFormat=Base64

python/ql/test/library-tests/frameworks/stdlib-py3/Encoding.py

@@ -1,6 +1,6 @@
 import base64
 
 # TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py
-base64.a85encode(bs) # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Ascii85
-base64.b85encode(bs)# $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base85
-base64.encodebytes(bs)# $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
+base64.a85encode(bs) # $ encodeInput=bs encodeOutput=base64.a85encode(..) encodeFormat=Ascii85
+base64.b85encode(bs)# $ encodeInput=bs encodeOutput=base64.b85encode(..) encodeFormat=Base85
+base64.encodebytes(bs)# $ encodeInput=bs encodeOutput=base64.encodebytes(..) encodeFormat=Base64

python/ql/test/library-tests/frameworks/stdlib/Decoding.py

@@ -2,14 +2,14 @@
 import marshal
 import base64
 
-pickle.loads(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=pickle decodeMayExecuteInput
-marshal.loads(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=marshal decodeMayExecuteInput
+pickle.loads(payload)  # $ decodeInput=payload decodeOutput=pickle.loads(..) decodeFormat=pickle decodeMayExecuteInput
+marshal.loads(payload)  # $ decodeInput=payload decodeOutput=marshal.loads(..) decodeFormat=marshal decodeMayExecuteInput
 
 # TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py
-base64.b64decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
-base64.standard_b64decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
-base64.urlsafe_b64decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
-base64.b32decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base32
-base64.b16decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base16
+base64.b64decode(payload)  # $ decodeInput=payload decodeOutput=base64.b64decode(..) decodeFormat=Base64
+base64.standard_b64decode(payload)  # $ decodeInput=payload decodeOutput=base64.standard_b64decode(..) decodeFormat=Base64
+base64.urlsafe_b64decode(payload)  # $ decodeInput=payload decodeOutput=base64.urlsafe_b64decode(..) decodeFormat=Base64
+base64.b32decode(payload)  # $ decodeInput=payload decodeOutput=base64.b32decode(..) decodeFormat=Base32
+base64.b16decode(payload)  # $ decodeInput=payload decodeOutput=base64.b16decode(..) decodeFormat=Base16
 # deprecated since Python 3.1, but still works
-base64.decodestring(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
+base64.decodestring(payload)  # $ decodeInput=payload decodeOutput=base64.decodestring(..) decodeFormat=Base64

python/ql/test/library-tests/frameworks/stdlib/Encoding.py

@@ -2,14 +2,14 @@
 import marshal
 import base64
 
-pickle.dumps(obj)  # $ MISSING: encodeInput=obj encodeOutput=Attribute() encodeFormat=pickle encodeMayExecuteInput
-marshal.dumps(obj)  # $ MISSING: encodeInput=obj encodeOutput=Attribute() encodeFormat=marshal encodeMayExecuteInput
+pickle.dumps(obj)  # $ MISSING: encodeInput=obj encodeOutput=pickle.dumps(..) encodeFormat=pickle encodeMayExecuteInput
+marshal.dumps(obj)  # $ MISSING: encodeInput=obj encodeOutput=marshal.dumps(..) encodeFormat=marshal encodeMayExecuteInput
 
 # TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py
-base64.b64encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
-base64.standard_b64encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
-base64.urlsafe_b64encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
-base64.b32encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base32
-base64.b16encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base16
+base64.b64encode(bs)  # $ encodeInput=bs encodeOutput=base64.b64encode(..) encodeFormat=Base64
+base64.standard_b64encode(bs)  # $ encodeInput=bs encodeOutput=base64.standard_b64encode(..) encodeFormat=Base64
+base64.urlsafe_b64encode(bs)  # $ encodeInput=bs encodeOutput=base64.urlsafe_b64encode(..) encodeFormat=Base64
+base64.b32encode(bs)  # $ encodeInput=bs encodeOutput=base64.b32encode(..) encodeFormat=Base32
+base64.b16encode(bs)  # $ encodeInput=bs encodeOutput=base64.b16encode(..) encodeFormat=Base16
 # deprecated since Python 3.1, but still works
-base64.encodestring(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
+base64.encodestring(bs)  # $ encodeInput=bs encodeOutput=base64.encodestring(..) encodeFormat=Base64