Python: fixup and fix expectations

Author: yoff

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -929,7 +929,7 @@ predicate readStep(Node nodeFrom, ContentSet c, Node nodeTo) {
   or
   VariableCapture::readStep(nodeFrom, c, nodeTo)
   or
-  Decodings::decoderReadStep(nodeFrom, c, nodeTo)
+  Conversions::readStep(nodeFrom, c, nodeTo)
 }
 
 /** Data flows from a sequence to a subscript of the sequence. */
@@ -985,16 +985,38 @@ predicate attributeReadStep(Node nodeFrom, AttributeContent c, AttrRead nodeTo)
   nodeTo.accesses(nodeFrom, c.getAttribute())
 }
 
-module Decodings {
+module Conversions {
   private import semmle.python.Concepts
 
   predicate decoderReadStep(Node nodeFrom, ContentSet c, Node nodeTo) {
     exists(Decoding decoding |
       nodeFrom = decoding.getAnInput() and
-      nodeTo = decoding.getOutput() and
+      nodeTo = decoding.getOutput()
+    ) and
+    (
+      c instanceof TupleElementContent
+      or
       c instanceof DictionaryElementContent
     )
   }
+
+  predicate encoderReadStep(Node nodeFrom, ContentSet c, Node nodeTo) {
+    exists(Encoding encoding |
+      nodeFrom = encoding.getAnInput() and
+      nodeTo = encoding.getOutput()
+    ) and
+    (
+      c instanceof TupleElementContent
+      or
+      c instanceof DictionaryElementContent
+    )
+  }
+
+  predicate readStep(Node nodeFrom, ContentSet c, Node nodeTo) {
+    decoderReadStep(nodeFrom, c, nodeTo)
+    or
+    encoderReadStep(nodeFrom, c, nodeTo)
+  }
 }
 
 /**

python/ql/test/library-tests/dataflow/sensitive-data/test.py

@@ -131,6 +131,5 @@ def call_wrapper(func):
 print(password) # $ SensitiveUse=password
 _config = {"sleep_timer": 5, "mysql_password": password}
 
-# since we have taint-step from store of `password`, we will consider any item in the
-# dictionary to be a password :(
-print(_config["sleep_timer"]) # $ SPURIOUS: SensitiveUse=password
+# since we have precise dictionary content, other items of the config are not tainted
+print(_config["sleep_timer"])

python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py

@@ -17,7 +17,7 @@ def str_methods():
         ts.casefold(), # $ tainted
 
         ts.format_map({}), # $ tainted
-        "{unsafe}".format_map({"unsafe": ts}), # $ tainted
+        "{unsafe}".format_map({"unsafe": ts}), # $ MISSING: tainted
     )
 
 

python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_collections.py

@@ -29,10 +29,10 @@ def test_construction():
 
     ensure_tainted(
         list(tainted_list), # $ tainted
-        list(tainted_tuple), # $ tainted
+        list(tainted_tuple), # $ MISSING: tainted
         list(tainted_set), # $ tainted
-        list(tainted_dict.values()), # $ tainted
-        list(tainted_dict.items()), # $ tainted
+        list(tainted_dict.values()), # $ MISSING: tainted
+        list(tainted_dict.items()), # $ MISSING: tainted
 
         tuple(tainted_list), # $ tainted
         set(tainted_list), # $ tainted

python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py

@@ -115,7 +115,7 @@ def percent_fmt():
     ensure_tainted(
         tainted_fmt % (1, 2), # $ tainted
         "%s foo bar" % ts, # $ tainted
-        "%s %s %s" % (1, 2, ts), # $ tainted
+        "%s %s %s" % (1, 2, ts), # $ MISSING: tainted
     )
 
 

python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_unpacking.py

@@ -53,7 +53,7 @@ def contrived_1():
 
     (a, b, c), (d, e, f) = tainted_list, no_taint_list
     ensure_tainted(a, b, c) # $ tainted
-    ensure_not_tainted(d, e, f) # $ SPURIOUS: tainted
+    ensure_not_tainted(d, e, f)
 
 
 def contrived_2():

python/ql/test/library-tests/frameworks/stdlib/test_re.py

@@ -74,9 +74,9 @@
 )
 
 ensure_not_tainted(
-    re.subn(pat, repl="safe", string=ts),
     re.subn(pat, repl="safe", string=ts)[1], # // the number of substitutions made
 )
 ensure_tainted(
+    re.subn(pat, repl="safe", string=ts), # tainted // implicit read at sink
     re.subn(pat, repl="safe", string=ts)[0], # $ tainted // the string
 )

python/ql/test/query-tests/Security/CWE-020-ExternalAPIs/ExternalAPIsUsedWithUntrustedData.expected

@@ -1,3 +1,4 @@
+| hmac.new [**] | 1 | 1 |
 | hmac.new [keyword msg] | 1 | 1 |
 | hmac.new [position 1] | 1 | 1 |
 | unknown.lib.func [keyword kw] | 2 | 1 |

python/ql/test/query-tests/Security/CWE-020-ExternalAPIs/UntrustedDataToExternalAPI.expected

@@ -15,6 +15,8 @@ edges
 | test.py:23:16:23:27 | ControlFlowNode for Attribute | test.py:23:16:23:39 | ControlFlowNode for Attribute() | provenance | dict.get |
 | test.py:23:16:23:39 | ControlFlowNode for Attribute() | test.py:23:5:23:12 | ControlFlowNode for data_raw | provenance |  |
 | test.py:24:5:24:8 | ControlFlowNode for data | test.py:25:44:25:47 | ControlFlowNode for data | provenance |  |
+| test.py:24:5:24:8 | ControlFlowNode for data | test.py:25:44:25:47 | ControlFlowNode for data | provenance |  |
+| test.py:25:44:25:47 | ControlFlowNode for data | test.py:25:15:25:74 | SynthDictSplatArgumentNode | provenance |  |
 | test.py:34:5:34:8 | ControlFlowNode for data | test.py:35:10:35:13 | ControlFlowNode for data | provenance |  |
 | test.py:34:5:34:8 | ControlFlowNode for data | test.py:36:13:36:16 | ControlFlowNode for data | provenance |  |
 | test.py:34:12:34:18 | ControlFlowNode for request | test.py:34:12:34:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
@@ -45,6 +47,8 @@ nodes
 | test.py:23:16:23:27 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
 | test.py:23:16:23:39 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
 | test.py:24:5:24:8 | ControlFlowNode for data | semmle.label | ControlFlowNode for data |
+| test.py:25:15:25:74 | SynthDictSplatArgumentNode | semmle.label | SynthDictSplatArgumentNode |
+| test.py:25:44:25:47 | ControlFlowNode for data | semmle.label | ControlFlowNode for data |
 | test.py:25:44:25:47 | ControlFlowNode for data | semmle.label | ControlFlowNode for data |
 | test.py:34:5:34:8 | ControlFlowNode for data | semmle.label | ControlFlowNode for data |
 | test.py:34:12:34:18 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
@@ -68,6 +72,7 @@ nodes
 subpaths
 #select
 | test.py:15:36:15:39 | ControlFlowNode for data | test.py:5:26:5:32 | ControlFlowNode for ImportMember | test.py:15:36:15:39 | ControlFlowNode for data | Call to hmac.new [position 1] with untrusted data from $@. | test.py:5:26:5:32 | ControlFlowNode for ImportMember | ControlFlowNode for ImportMember |
+| test.py:25:15:25:74 | SynthDictSplatArgumentNode | test.py:5:26:5:32 | ControlFlowNode for ImportMember | test.py:25:15:25:74 | SynthDictSplatArgumentNode | Call to hmac.new [**] with untrusted data from $@. | test.py:5:26:5:32 | ControlFlowNode for ImportMember | ControlFlowNode for ImportMember |
 | test.py:25:44:25:47 | ControlFlowNode for data | test.py:5:26:5:32 | ControlFlowNode for ImportMember | test.py:25:44:25:47 | ControlFlowNode for data | Call to hmac.new [keyword msg] with untrusted data from $@. | test.py:5:26:5:32 | ControlFlowNode for ImportMember | ControlFlowNode for ImportMember |
 | test.py:35:10:35:13 | ControlFlowNode for data | test.py:5:26:5:32 | ControlFlowNode for ImportMember | test.py:35:10:35:13 | ControlFlowNode for data | Call to unknown.lib.func [position 0] with untrusted data from $@. | test.py:5:26:5:32 | ControlFlowNode for ImportMember | ControlFlowNode for ImportMember |
 | test.py:36:13:36:16 | ControlFlowNode for data | test.py:5:26:5:32 | ControlFlowNode for ImportMember | test.py:36:13:36:16 | ControlFlowNode for data | Call to unknown.lib.func [keyword kw] with untrusted data from $@. | test.py:5:26:5:32 | ControlFlowNode for ImportMember | ControlFlowNode for ImportMember |

python/ql/test/query-tests/Security/CWE-312-CleartextLogging/CleartextLogging.expected

@@ -20,8 +20,6 @@ edges
 | test.py:67:38:67:48 | ControlFlowNode for bank_number | test.py:70:15:70:25 | ControlFlowNode for bank_number | provenance |  |
 | test.py:67:76:67:78 | ControlFlowNode for ccn | test.py:73:15:73:17 | ControlFlowNode for ccn | provenance |  |
 | test.py:67:81:67:88 | ControlFlowNode for user_ccn | test.py:74:15:74:22 | ControlFlowNode for user_ccn | provenance |  |
-| test.py:101:5:101:10 | ControlFlowNode for config | test.py:105:11:105:31 | ControlFlowNode for Subscript | provenance |  |
-| test.py:103:21:103:37 | ControlFlowNode for Attribute | test.py:101:5:101:10 | ControlFlowNode for config | provenance |  |
 nodes
 | test.py:19:5:19:12 | ControlFlowNode for password | semmle.label | ControlFlowNode for password |
 | test.py:19:16:19:29 | ControlFlowNode for get_password() | semmle.label | ControlFlowNode for get_password() |
@@ -62,9 +60,6 @@ nodes
 | test.py:70:15:70:25 | ControlFlowNode for bank_number | semmle.label | ControlFlowNode for bank_number |
 | test.py:73:15:73:17 | ControlFlowNode for ccn | semmle.label | ControlFlowNode for ccn |
 | test.py:74:15:74:22 | ControlFlowNode for user_ccn | semmle.label | ControlFlowNode for user_ccn |
-| test.py:101:5:101:10 | ControlFlowNode for config | semmle.label | ControlFlowNode for config |
-| test.py:103:21:103:37 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
-| test.py:105:11:105:31 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
 subpaths
 #select
 | test.py:20:48:20:55 | ControlFlowNode for password | test.py:19:16:19:29 | ControlFlowNode for get_password() | test.py:20:48:20:55 | ControlFlowNode for password | This expression logs $@ as clear text. | test.py:19:16:19:29 | ControlFlowNode for get_password() | sensitive data (password) |
@@ -89,4 +84,3 @@ subpaths
 | test.py:70:15:70:25 | ControlFlowNode for bank_number | test.py:67:38:67:48 | ControlFlowNode for bank_number | test.py:70:15:70:25 | ControlFlowNode for bank_number | This expression logs $@ as clear text. | test.py:67:38:67:48 | ControlFlowNode for bank_number | sensitive data (private) |
 | test.py:73:15:73:17 | ControlFlowNode for ccn | test.py:67:76:67:78 | ControlFlowNode for ccn | test.py:73:15:73:17 | ControlFlowNode for ccn | This expression logs $@ as clear text. | test.py:67:76:67:78 | ControlFlowNode for ccn | sensitive data (private) |
 | test.py:74:15:74:22 | ControlFlowNode for user_ccn | test.py:67:81:67:88 | ControlFlowNode for user_ccn | test.py:74:15:74:22 | ControlFlowNode for user_ccn | This expression logs $@ as clear text. | test.py:67:81:67:88 | ControlFlowNode for user_ccn | sensitive data (private) |
-| test.py:105:11:105:31 | ControlFlowNode for Subscript | test.py:103:21:103:37 | ControlFlowNode for Attribute | test.py:105:11:105:31 | ControlFlowNode for Subscript | This expression logs $@ as clear text. | test.py:103:21:103:37 | ControlFlowNode for Attribute | sensitive data (password) |