Python: Improve yarl.URL modeling

Author: RasmusWL

python/ql/src/semmle/python/frameworks/Yarl.qll

@@ -34,6 +34,11 @@ module Yarl {
      */
     abstract class InstanceSource extends DataFlow::LocalSourceNode { }
 
+    /** A direct instantiation of `yarl.URL`. */
+    private class ClassInstantiation extends InstanceSource, DataFlow::CallCfgNode {
+      ClassInstantiation() { this = API::moduleImport("yarl").getMember("URL").getACall() }
+    }
+
     /** Gets a reference to an instance of `yarl.URL`. */
     private DataFlow::LocalSourceNode instance(DataFlow::TypeTracker t) {
       t.start() and
@@ -52,6 +57,12 @@ module Yarl {
      */
     class YarlUrlAdditionalTaintStep extends TaintTracking::AdditionalTaintStep {
       override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
+        // class instantiation
+        exists(ClassInstantiation call |
+          nodeFrom in [call.getArg(0), call.getArgByName("val")] and
+          nodeTo = call
+        )
+        or
         // Methods
         //
         // TODO: When we have tools that make it easy, model these properly to handle

python/ql/test/library-tests/frameworks/aiohttp/taint_test.py

@@ -5,11 +5,12 @@ async def test_taint(request: web.Request): # $ requestHandler
     ensure_tainted(
         request, # $ tainted
 
-        # yarl.URL instances
+        # yarl.URL instances, see tests under `yarl` framework tests
         # https://yarl.readthedocs.io/en/stable/api.html#yarl.URL
-        # see below
         request.url, # $ tainted
+        request.url.human_repr(), # $ tainted
         request.rel_url, # $ tainted
+        request.rel_url.human_repr(), # $ tainted
 
         request.forwarded, # $ tainted
 
@@ -130,68 +131,6 @@ async def test_taint(request: web.Request): # $ requestHandler
         request.config_dict,
     )
 
-    # TODO: Should have a better way to capture that we in fact _do_ model this as a
-    # an instance of the right class, and have the actual taint_test for that in a
-    # different file!
-    import yarl
-
-    ensure_tainted(
-        # see https://yarl.readthedocs.io/en/stable/api.html#yarl.URL
-        request.url.user, # $ tainted
-        request.url.raw_user, # $ tainted
-
-        request.url.password, # $ tainted
-        request.url.raw_password, # $ tainted
-
-        request.url.host, # $ tainted
-        request.url.raw_host, # $ tainted
-
-        request.url.port, # $ tainted
-        request.url.explicit_port, # $ tainted
-
-        request.url.authority, # $ tainted
-        request.url.raw_authority, # $ tainted
-
-        request.url.path, # $ tainted
-        request.url.raw_path, # $ tainted
-
-        request.url.path_qs, # $ tainted
-        request.url.raw_path_qs, # $ tainted
-
-        request.url.query_string, # $ tainted
-        request.url.raw_query_string, # $ tainted
-
-        request.url.fragment, # $ tainted
-        request.url.raw_fragment, # $ tainted
-
-        request.url.parts, # $ tainted
-        request.url.raw_parts, # $ tainted
-
-        request.url.name, # $ tainted
-        request.url.raw_name, # $ tainted
-
-        # multidict.MultiDictProxy[str]
-        request.url.query, # $ tainted
-        request.url.query.getone("key"), # $ tainted
-
-        request.url.with_scheme("foo"), # $ tainted
-        request.url.with_user("foo"), # $ tainted
-        request.url.with_password("foo"), # $ tainted
-        request.url.with_host("foo"), # $ tainted
-        request.url.with_port("foo"), # $ tainted
-        request.url.with_path("foo"), # $ tainted
-        request.url.with_query({"foo": 42}), # $ tainted
-        request.url.with_query(foo=42), # $ tainted
-        request.url.update_query({"foo": 42}), # $ tainted
-        request.url.update_query(foo=42), # $ tainted
-        request.url.with_fragment("foo"), # $ tainted
-        request.url.with_name("foo"), # $ tainted
-
-        request.url.join(yarl.URL("wat.html")), # $ tainted
-
-        request.url.human_repr(), # $ tainted
-    )
-
 
 class TaintTestClass(web.View):
     def get(self): # $ requestHandler

python/ql/test/library-tests/frameworks/yarl/ConceptsTest.expected

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/yarl/ConceptsTest.ql

@@ -0,0 +1,3 @@
+argumentToEnsureNotTaintedNotMarkedAsSpurious
+untaintedArgumentToEnsureTaintedNotMarkedAsMissing
+failures

python/ql/test/library-tests/frameworks/yarl/InlineTaintTest.expected

@@ -0,0 +1 @@
+import experimental.meta.InlineTaintTest

python/ql/test/library-tests/frameworks/yarl/InlineTaintTest.ql

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=1 --lang=3

python/ql/test/library-tests/frameworks/yarl/options

@@ -0,0 +1,64 @@
+import yarl
+
+
+url = yarl.URL(TAINTED_STRING)
+
+
+ensure_tainted(
+    url, # $ tainted
+
+    # see https://yarl.readthedocs.io/en/stable/api.html#yarl.URL
+    url.user, # $ tainted
+    url.raw_user, # $ tainted
+
+    url.password, # $ tainted
+    url.raw_password, # $ tainted
+
+    url.host, # $ tainted
+    url.raw_host, # $ tainted
+
+    url.port, # $ tainted
+    url.explicit_port, # $ tainted
+
+    url.authority, # $ tainted
+    url.raw_authority, # $ tainted
+
+    url.path, # $ tainted
+    url.raw_path, # $ tainted
+
+    url.path_qs, # $ tainted
+    url.raw_path_qs, # $ tainted
+
+    url.query_string, # $ tainted
+    url.raw_query_string, # $ tainted
+
+    url.fragment, # $ tainted
+    url.raw_fragment, # $ tainted
+
+    url.parts, # $ tainted
+    url.raw_parts, # $ tainted
+
+    url.name, # $ tainted
+    url.raw_name, # $ tainted
+
+    # multidict.MultiDictProxy[str]
+    url.query, # $ tainted
+    url.query.getone("key"), # $ tainted
+
+    url.with_scheme("foo"), # $ tainted
+    url.with_user("foo"), # $ tainted
+    url.with_password("foo"), # $ tainted
+    url.with_host("foo"), # $ tainted
+    url.with_port("foo"), # $ tainted
+    url.with_path("foo"), # $ tainted
+    url.with_query({"foo": 42}), # $ tainted
+    url.with_query(foo=42), # $ tainted
+    url.update_query({"foo": 42}), # $ tainted
+    url.update_query(foo=42), # $ tainted
+    url.with_fragment("foo"), # $ tainted
+    url.with_name("foo"), # $ tainted
+
+    url.join(yarl.URL("wat.html")), # $ tainted
+
+    url.human_repr(), # $ tainted
+)