Merge pull request #57 from amyreese/onnx

More robust evaluation of assignments

Author: thatch

dowsing/setuptools/setup_py_parsing.py

@@ -8,7 +8,12 @@
 from typing import Any, Dict, Optional
 
 import libcst as cst
-from libcst.metadata import ParentNodeProvider, QualifiedNameProvider, ScopeProvider
+from libcst.metadata import (
+    ParentNodeProvider,
+    PositionProvider,
+    QualifiedNameProvider,
+    ScopeProvider,
+)
 
 from ..types import Distribution
 from .setup_and_metadata import SETUP_ARGS
@@ -124,7 +129,12 @@ def leave_Call(
 
 
 class SetupCallAnalyzer(cst.CSTVisitor):
-    METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider, QualifiedNameProvider)
+    METADATA_DEPENDENCIES = (
+        ScopeProvider,
+        ParentNodeProvider,
+        QualifiedNameProvider,
+        PositionProvider,
+    )
 
     # TODO names resulting from other than 'from setuptools import setup'
     # TODO wrapper funcs that modify args
@@ -178,7 +188,9 @@ def visit_Call(self, node: cst.Call) -> Optional[bool]:
     BOOL_NAMES = {"True": True, "False": False, "None": None}
     PRETEND_ARGV = ["setup.py", "bdist_wheel"]
 
-    def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any:
+    def evaluate_in_scope(
+        self, item: cst.CSTNode, scope: Any, target_line: int = 0
+    ) -> Any:
         qnames = self.get_metadata(QualifiedNameProvider, item)
 
         if isinstance(item, cst.SimpleString):
@@ -190,19 +202,36 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any:
         elif isinstance(item, cst.Name):
             name = item.value
             assignments = scope[name]
-            for a in assignments:
-                # TODO: Only assignments "before" this node matter if in the
-                # same scope; really if we had a call graph and walked the other
-                # way, we could have a better idea of what has already happened.
+            assignment_nodes = sorted(
+                (
+                    (self.get_metadata(PositionProvider, a.node).start.line, a.node)
+                    for a in assignments
+                    if a.node
+                ),
+                reverse=True,
+            )
+            # Walk assignments from bottom to top, evaluating them recursively.
+            for lineno, node in assignment_nodes:
+
+                # When recursing, only look at assignments above the "target line".
+                if target_line and lineno >= target_line:
+                    continue
 
                 # Assign(
                 #   targets=[AssignTarget(target=Name(value="v"))],
                 #   value=SimpleString(value="'x'"),
                 # )
+                #
+                # AugAssign(
+                #   target=Name(value="v"),
+                #   operator=AddAssign(...),
+                #   value=SimpleString(value="'x'"),
+                # )
+                #
                 # TODO or an import...
                 # TODO builtins have BuiltinAssignment
+
                 try:
-                    node = a.node
                     if node:
                         parent = self.get_metadata(ParentNodeProvider, node)
                         if parent:
@@ -212,25 +241,37 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any:
                     else:
                         raise KeyError
                 except (KeyError, AttributeError):
-                    return "??"
-
-                # This presumes a single assignment
-                if not isinstance(gp, cst.Assign) or len(gp.targets) != 1:
-                    return "??"  # TooComplicated(repr(gp))
+                    continue
 
                 try:
                     scope = self.get_metadata(ScopeProvider, gp)
                 except KeyError:
                     # module scope isn't in the dict
-                    return "??"
+                    continue
+
+                # This presumes a single assignment
+                if isinstance(gp, cst.Assign) and len(gp.targets) == 1:
+                    result = self.evaluate_in_scope(gp.value, scope, lineno)
+                elif isinstance(parent, cst.AugAssign):
+                    result = self.evaluate_in_scope(parent, scope, lineno)
+                else:
+                    # too complicated?
+                    continue
 
-                return self.evaluate_in_scope(gp.value, scope)
+                # keep trying assignments until we get something other than ??
+                if result != "??":
+                    return result
+
+            # give up
+            return "??"
         elif isinstance(item, (cst.Tuple, cst.List)):
             lst = []
             for el in item.elements:
                 lst.append(
                     self.evaluate_in_scope(
-                        el.value, self.get_metadata(ScopeProvider, el)
+                        el.value,
+                        self.get_metadata(ScopeProvider, el),
+                        target_line,
                     )
                 )
             if isinstance(item, cst.Tuple):
@@ -248,10 +289,10 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any:
             for arg in item.args:
                 if isinstance(arg.keyword, cst.Name):
                     args[names.index(arg.keyword.value)] = self.evaluate_in_scope(
-                        arg.value, scope
+                        arg.value, scope, target_line
                     )
                 else:
-                    args[i] = self.evaluate_in_scope(arg.value, scope)
+                    args[i] = self.evaluate_in_scope(arg.value, scope, target_line)
                     i += 1
 
             # TODO clear ones that are still default
@@ -264,26 +305,30 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any:
             d = {}
             for arg in item.args:
                 if isinstance(arg.keyword, cst.Name):
-                    d[arg.keyword.value] = self.evaluate_in_scope(arg.value, scope)
+                    d[arg.keyword.value] = self.evaluate_in_scope(
+                        arg.value, scope, target_line
+                    )
                 # TODO something with **kwargs
             return d
         elif isinstance(item, cst.Dict):
             d = {}
             for el2 in item.elements:
                 if isinstance(el2, cst.DictElement):
                     d[self.evaluate_in_scope(el2.key, scope)] = self.evaluate_in_scope(
-                        el2.value, scope
+                        el2.value, scope, target_line
                     )
             return d
         elif isinstance(item, cst.Subscript):
-            lhs = self.evaluate_in_scope(item.value, scope)
+            lhs = self.evaluate_in_scope(item.value, scope, target_line)
             if isinstance(lhs, str):
                 # A "??" entry, propagate
                 return "??"
 
             # TODO: Figure out why this is Sequence
             if isinstance(item.slice[0].slice, cst.Index):
-                rhs = self.evaluate_in_scope(item.slice[0].slice.value, scope)
+                rhs = self.evaluate_in_scope(
+                    item.slice[0].slice.value, scope, target_line
+                )
                 try:
                     if isinstance(lhs, dict):
                         return lhs.get(rhs, "??")
@@ -296,15 +341,29 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any:
                 # LOG.warning(f"Omit2 {type(item.slice[0].slice)!r}")
                 return "??"
         elif isinstance(item, cst.BinaryOperation):
-            lhs = self.evaluate_in_scope(item.left, scope)
-            rhs = self.evaluate_in_scope(item.right, scope)
+            lhs = self.evaluate_in_scope(item.left, scope, target_line)
+            rhs = self.evaluate_in_scope(item.right, scope, target_line)
+            if lhs == "??" or rhs == "??":
+                return "??"
             if isinstance(item.operator, cst.Add):
                 try:
                     return lhs + rhs
                 except Exception:
                     return "??"
             else:
                 return "??"
+        elif isinstance(item, cst.AugAssign):
+            lhs = self.evaluate_in_scope(item.target, scope, target_line)
+            rhs = self.evaluate_in_scope(item.value, scope, target_line)
+            if lhs == "??" or rhs == "??":
+                return "??"
+            if isinstance(item.operator, cst.AddAssign):
+                try:
+                    return lhs + rhs
+                except Exception:
+                    return "??"
+            else:
+                return "??"
         else:
             # LOG.warning(f"Omit1 {type(item)!r}")
             return "??"

dowsing/tests/setuptools.py

@@ -344,3 +344,76 @@ def test_add_items(self) -> None:
         self.assertEqual(d.name, "aaaa1111")
         self.assertEqual(d.packages, ["a", "b", "c"])
         self.assertEqual(d.classifiers, "??")
+
+    def test_self_reference_assignments(self) -> None:
+        d = self._read(
+            """\
+from setuptools import setup
+
+version = "base"
+name = "foo"
+name += "bar"
+version = version + ".suffix"
+
+classifiers = [
+    "123",
+    "abc",
+]
+
+if True:
+    classifiers = classifiers + ["xyz"]
+
+setup(
+    name=name,
+    version=version,
+    classifiers=classifiers,
+)
+            """
+        )
+        self.assertEqual(d.name, "foobar")
+        self.assertEqual(d.version, "base.suffix")
+        self.assertSequenceEqual(d.classifiers, ["123", "abc", "xyz"])
+
+    def test_circular_references(self) -> None:
+        d = self._read(
+            """\
+from setuptools import setup
+
+name = "foo"
+
+foo = bar
+bar = version
+version = foo
+
+classifiers = classifiers
+
+setup(
+    name=name,
+    version=version,
+)
+            """
+        )
+        self.assertEqual(d.name, "foo")
+        self.assertEqual(d.version, "??")
+        self.assertEqual(d.classifiers, ())
+
+    def test_redefines_builtin(self) -> None:
+        d = self._read(
+            """\
+import setuptools
+with open("CREDITS.txt", "r", encoding="utf-8") as fp:
+    credits = fp.read()
+
+long_desc = "a" + credits + "b"
+name = "foo"
+
+kwargs = dict(
+    long_description = long_desc,
+    name = name,
+)
+
+setuptools.setup(**kwargs)
+"""
+        )
+        self.assertEqual(d.name, "foo")
+        self.assertEqual(d.description, "??")