fix in-place set bitwise assignment and del of closure variable

Author: dylan-sutton-chavez

bugs.json

@@ -2,8 +2,8 @@
   "_meta": {
     "source": "MicroPython tests/basics differential vs CPython 3.13",
     "verified": "each repro re-run through VM + python3",
-    "count": 9,
-    "fixed": "91 bugs fixed and migrated to vm.json (try/finally + with unwinding cluster; 11 builtin/parser/format divergences from rows 1-11 of the low-NEWLINE triage; generator_return, string_format_modulo_int %*d, seq_unpack list targets, class_getattr dunder-on-type, builtin_slice type identity, containment dict/set-led tuple, range sequence-unpack; yield-from-as-expression cluster: gen_yield_from + gen_yield_from_ducktype + gen_yield_from_exc + gen_yield_from_stopped via a parse_atom Yield arm plus a LoadYieldFrom opcode that carries the subiterator return/StopIteration value, 40 regression cases; range_iter_overflow: AFL sig:06 abort on a range whose step increment crossed the i64 edge, fixed by checked_add in the three range walkers (iter_to_vec_for_spread, IterFrame::next_item, IterCursor::Range::next), 10 regression cases; range_iter_truncation: for-loop/comprehension (IterFrame::next_item) and spread/unpack (iter_to_vec_for_spread) yielded Val::int past the 47-bit inline range instead of promoting, routed both through range_int (pub(crate)) so values auto-promote to LongInt, 10 regression cases; range_contains: `x in range(...)` always returned False (contains had a catch-all Ok(false) for Range) — replaced with an O(1) bounds+step check over i128 that also matches integral floats like CPython, 10 regression cases)"
+    "count": 7,
+    "fixed": "93 bugs fixed and migrated to vm.json (try/finally + with unwinding cluster; 11 builtin/parser/format divergences from rows 1-11 of the low-NEWLINE triage; generator_return, string_format_modulo_int %*d, seq_unpack list targets, class_getattr dunder-on-type, builtin_slice type identity, containment dict/set-led tuple, range sequence-unpack; yield-from-as-expression cluster: gen_yield_from + gen_yield_from_ducktype + gen_yield_from_exc + gen_yield_from_stopped via a parse_atom Yield arm plus a LoadYieldFrom opcode that carries the subiterator return/StopIteration value, 40 regression cases; range_iter_overflow: AFL sig:06 abort on a range whose step increment crossed the i64 edge, fixed by checked_add in the three range walkers (iter_to_vec_for_spread, IterFrame::next_item, IterCursor::Range::next), 10 regression cases; range_iter_truncation: for-loop/comprehension (IterFrame::next_item) and spread/unpack (iter_to_vec_for_spread) yielded Val::int past the 47-bit inline range instead of promoting, routed both through range_int (pub(crate)) so values auto-promote to LongInt, 10 regression cases; range_contains: `x in range(...)` always returned False (contains had a catch-all Ok(false) for Range) — replaced with an O(1) bounds+step check over i128 that also matches integral floats like CPython, 10 regression cases; set_binop: augmented set `|=`/`&=`/`^=` rebound to a new set instead of mutating in place — added InPlaceBitOr/And/Xor opcodes (parser maps the augmented ops, handle_bitwise routes to set_iop_and_push) that rewrite the left set in place, frozenset still rebinds, 10 regression cases; del_deref: `del` of a closed-over variable cleared only the local slot, not the shared cell, so closures still read the old value — the Del handler now also unbinds the frame's cell for that slot, 10 regression cases)"
   },
   "bugs": [
     {
@@ -14,14 +14,6 @@
       "expected": "8",
       "actual": "19"
     },
-    {
-      "id": "del_deref",
-      "category": "scoping",
-      "summary": "del of a closed-over (nonlocal) variable does not unbind it; subsequent read returns the old value instead of raising NameError.",
-      "repro": "def f():\n    x = 1\n    def g():\n        nonlocal x\n        try:\n            print(x)\n        except NameError:\n            print(\"NameError\")\n    del x\n    g()\nf()",
-      "expected": "NameError",
-      "actual": "1"
-    },
     {
       "id": "gen_yield_from_executing",
       "category": "scoping",
@@ -62,14 +54,6 @@
       "expected": "[1, 2, 3]",
       "actual": "__VMERR__ TypeError: '<' not supported between instances of 'object' and 'object'"
     },
-    {
-      "id": "set_binop",
-      "category": "wrong-value",
-      "summary": "Augmented set assignment (|= etc.) rebinds to a new set instead of mutating in place, so an aliasing identity check is False instead of True.",
-      "repro": "s1 = s2 = set('abc')\ns1 |= set('ad')\nprint(s1 is s2, len(s1))",
-      "expected": "True 4",
-      "actual": "False 4"
-    },
     {
       "id": "gen_next_nested_arg",
       "category": "stack-corruption",

compiler/src/modules/parser/stmt.rs

@@ -380,8 +380,15 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
                 self.advance();
                 self.emit_load_ssa(name.clone());
                 self.expr();
-                // `+=` on a name uses the in-place variant so list targets mutate the shared object instead of rebinding (alias-visible, like CPython).
-                self.chunk.emit(if op == OpCode::Add { OpCode::InPlaceAdd } else { op }, 0);
+                // In-place variants so list `+=` and set `|=`/`&=`/`^=` mutate the shared object (alias-visible).
+                let op = match op {
+                    OpCode::Add => OpCode::InPlaceAdd,
+                    OpCode::BitOr => OpCode::InPlaceBitOr,
+                    OpCode::BitAnd => OpCode::InPlaceBitAnd,
+                    OpCode::BitXor => OpCode::InPlaceBitXor,
+                    other => other,
+                };
+                self.chunk.emit(op, 0);
                 self.store_name(name);
                 false
             }

compiler/src/modules/parser/types.rs

@@ -45,6 +45,8 @@ pub enum OpCode {
     Pos,
     /* Pushes the value `yield from` produces: the exhausted subiterator's return / StopIteration value. */
     LoadYieldFrom,
+    // Augmented set bitwise `|=` `&=` `^=`: mutate the left set in place.
+    InPlaceBitOr, InPlaceBitAnd, InPlaceBitXor,
 }
 
 // Python builtin name -> (specialised OpCode, `leaves_value_on_stack`).

compiler/src/modules/vm/dispatch.rs

@@ -560,7 +560,8 @@ impl<'a> VM<'a> {
     fn dispatch_generic(&mut self, opcode: OpCode, operand: u16, chunk: &SSAChunk, slots: &mut [Val], ip: &mut usize, exc_base: usize) -> Result<Option<Val>, VmErr> {
         match opcode {
             OpCode::BitAnd | OpCode::BitOr | OpCode::BitXor
-            | OpCode::BitNot | OpCode::Shl | OpCode::Shr => self.handle_bitwise(opcode, chunk, slots)?,
+            | OpCode::BitNot | OpCode::Shl | OpCode::Shr
+            | OpCode::InPlaceBitOr | OpCode::InPlaceBitAnd | OpCode::InPlaceBitXor => self.handle_bitwise(opcode, chunk, slots)?,
             OpCode::In | OpCode::NotIn | OpCode::Is | OpCode::IsNot => self.handle_identity(opcode, chunk, slots)?,
 
             OpCode::BuildList | OpCode::BuildTuple | OpCode::BuildDict

compiler/src/modules/vm/handlers/arith.rs

@@ -269,6 +269,14 @@ impl<'a> VM<'a> {
 
     /* i128 bitwise + Shl/Shr (overflow trap); BitNot unary. Set/Set on |/&/^ means union/intersection/symmetric-diff; other types use the bitwise path. */
     pub(crate) fn handle_bitwise(&mut self, op: OpCode, chunk: &SSAChunk, slots: &mut [Val]) -> Result<(), VmErr> {
+        // Augmented set bitwise reuses the plain path but mutates the left set in place.
+        let inplace = matches!(op, OpCode::InPlaceBitOr | OpCode::InPlaceBitAnd | OpCode::InPlaceBitXor);
+        let op = match op {
+            OpCode::InPlaceBitOr => OpCode::BitOr,
+            OpCode::InPlaceBitAnd => OpCode::BitAnd,
+            OpCode::InPlaceBitXor => OpCode::BitXor,
+            other => other,
+        };
         if op == OpCode::BitNot {
             let v = self.pop()?;
             let i = self.as_i128(v).ok_or(cold_type("~ requires an integer"))?;
@@ -289,7 +297,7 @@ impl<'a> VM<'a> {
 
         if self.is_set_like(a) && self.is_set_like(b)
             && matches!(op, OpCode::BitAnd | OpCode::BitOr | OpCode::BitXor) {
-            return self.set_binop_and_push(a, b, op);
+            return if inplace { self.set_iop_and_push(a, b, op) } else { self.set_binop_and_push(a, b, op) };
         }
         // `dict | dict` (and `|=`) merges, right operand winning.
         if op == OpCode::BitOr && a.is_heap() && b.is_heap()

compiler/src/modules/vm/handlers/data.rs

@@ -199,6 +199,12 @@ impl<'a> VM<'a> {
                 if core::ptr::eq(chunk, self.chunk) && let Some(n) = chunk.names.get(slot) {
                     self.module_state.remove(ssa_strip(n));
                 }
+                // Unbind the shared closure cell too, so closures over this name see the deletion.
+                let cell = self.call_stack.last()
+                    .and_then(|f| f.cells.iter().find(|(s, _)| *s == slot).map(|&(_, c)| c));
+                if let Some(cell) = cell && cell.is_heap() && let HeapObj::List(rc) = self.heap.get(cell) {
+                    rc.borrow_mut()[0] = Val::undef(); // cells are 1-element boxes
+                }
             }
             OpCode::Global | OpCode::Nonlocal => self.mark_impure(),
             OpCode::Raise | OpCode::RaiseFrom => {

compiler/src/modules/vm/ops.rs

@@ -158,24 +158,40 @@ impl<'a> VM<'a> {
     }
 
     /* Set bitwise ops (|, &, ^) over set/frozenset; result frozen iff `a` is frozen. */
-    pub(crate) fn set_binop_and_push(&mut self, a: Val, b: Val, op: OpCode) -> Result<(), VmErr> {
+    // Union/intersection/symmetric-diff items; content membership lets alloc dedup distinct-handle equals.
+    fn set_binop_items(&self, a: Val, b: Val, op: OpCode) -> Result<Vec<Val>, VmErr> {
         let (sa, sb) = match (self.clone_set_items(a), self.clone_set_items(b)) {
             (Some(x), Some(y)) => (x, y),
             _ => return Err(cold_runtime("set_binop on non-set operands")),
         };
-        // Content membership so distinct-handle equal elements combine correctly; alloc dedups.
-        let items: Vec<Val> = match op {
+        Ok(match op {
             OpCode::BitOr => sa.iter().chain(sb.iter()).copied().collect(),
             OpCode::BitAnd => sa.iter().filter(|&&v| sb.contains(v, &self.heap)).copied().collect(),
             OpCode::BitXor => sa.iter().filter(|&&v| !sb.contains(v, &self.heap))
                 .chain(sb.iter().filter(|&&v| !sa.contains(v, &self.heap))).copied().collect(),
             _ => return Err(cold_runtime("set_binop with non-bitwise opcode")),
-        };
+        })
+    }
+
+    pub(crate) fn set_binop_and_push(&mut self, a: Val, b: Val, op: OpCode) -> Result<(), VmErr> {
+        let items = self.set_binop_items(a, b, op)?;
         let frozen = matches!(self.heap.get(a), HeapObj::FrozenSet(_));
         let v = self.alloc_set_result(items, frozen)?;
         self.push(v); Ok(())
     }
 
+    // Augmented set bitwise: rewrite the left set in place (identity preserved); frozenset rebinds.
+    pub(crate) fn set_iop_and_push(&mut self, a: Val, b: Val, op: OpCode) -> Result<(), VmErr> {
+        if !matches!(self.heap.get(a), HeapObj::Set(_)) {
+            return self.set_binop_and_push(a, b, op);
+        }
+        let items = self.set_binop_items(a, b, op)?;
+        let mut s = ValSet::with_capacity(items.len());
+        for v in items { s.insert(v, &self.heap); }
+        if let HeapObj::Set(rc) = self.heap.get(a) { *rc.borrow_mut() = s; }
+        self.push(a); Ok(())
+    }
+
     /* Set comparisons with subset/superset semantics over set/frozenset. */
     pub(crate) fn set_compare_and_push(&mut self, a: Val, b: Val, op: OpCode) -> Result<(), VmErr> {
         let (sa, sb) = match (self.clone_set_items(a), self.clone_set_items(b)) {

compiler/tests/cases/parser.json

@@ -1229,21 +1229,21 @@
     "src": "x = 7\nx &= 3", 
     "constants": ["7", "3"], 
     "names": ["x_1", "x_2"], 
-    "instructions": [["LoadConst",0], ["StoreName",0], ["LoadName",0], ["LoadConst",1], ["BitAnd",0], ["StoreName",0], ["ReturnValue",0]], 
+    "instructions": [["LoadConst",0], ["StoreName",0], ["LoadName",0], ["LoadConst",1], ["InPlaceBitAnd",0], ["StoreName",0], ["ReturnValue",0]],
     "annotations": {}
   },
   {
     "src": "x = 5\nx |= 3", 
     "constants": ["5", "3"], 
     "names": ["x_1", "x_2"], 
-    "instructions": [["LoadConst",0], ["StoreName",0], ["LoadName",0], ["LoadConst",1], ["BitOr",0], ["StoreName",0], ["ReturnValue",0]], 
+    "instructions": [["LoadConst",0], ["StoreName",0], ["LoadName",0], ["LoadConst",1], ["InPlaceBitOr",0], ["StoreName",0], ["ReturnValue",0]],
     "annotations": {}
   },
   {
     "src": "x = 7\nx ^= 3", 
     "constants": ["7", "3"], 
     "names": ["x_1", "x_2"], 
-    "instructions": [["LoadConst",0], ["StoreName",0], ["LoadName",0], ["LoadConst",1], ["BitXor",0], ["StoreName",0], ["ReturnValue",0]], 
+    "instructions": [["LoadConst",0], ["StoreName",0], ["LoadName",0], ["LoadConst",1], ["InPlaceBitXor",0], ["StoreName",0], ["ReturnValue",0]],
     "annotations": {}
   },
   {

compiler/tests/cases/vm.json

@@ -3566,5 +3566,26 @@
   {"src": "print(sorted(['bb', 'a', 'ccc'], key=len))", "output": ["['a', 'bb', 'ccc']"]},
   {"src": "xs = [3, 1, 2]\nxs.sort(reverse=True)\nprint(xs)", "output": ["[3, 2, 1]"]},
   {"src": "try:\n    print('x', bad=1)\nexcept TypeError:\n    print('te')", "output": ["te"]},
-  {"src": "try:\n    list(5)\nexcept TypeError:\n    print('ni')", "output": ["ni"]}
+  {"src": "try:\n    list(5)\nexcept TypeError:\n    print('ni')", "output": ["ni"]},
+  {"src": "s1 = s2 = set('abc')\ns1 |= set('ad')\nprint(s1 is s2, len(s1))", "output": ["True 4"]},
+  {"src": "s1 = s2 = set('abc')\ns1 &= set('ab')\nprint(s1 is s2, len(s1))", "output": ["True 2"]},
+  {"src": "s1 = s2 = set('abc')\ns1 ^= set('ad')\nprint(s1 is s2, len(s1))", "output": ["True 3"]},
+  {"src": "s = {1, 2}\ns |= {2, 3}\nprint(sorted(s))", "output": ["[1, 2, 3]"]},
+  {"src": "s = {1, 2, 3}\ns &= {2, 3, 4}\nprint(sorted(s))", "output": ["[2, 3]"]},
+  {"src": "s = {1, 2, 3}\ns ^= {3, 4}\nprint(sorted(s))", "output": ["[1, 2, 4]"]},
+  {"src": "s1 = {1}\ns2 = s1 | {2}\nprint(s1 is s2, sorted(s2))", "output": ["False [1, 2]"]},
+  {"src": "f = frozenset({1})\ng = f\nf |= {2}\nprint(f is g, sorted(f))", "output": ["False [1, 2]"]},
+  {"src": "s = {1, 2}\ns |= set()\nprint(sorted(s))", "output": ["[1, 2]"]},
+  {"src": "a = b = c = {1}\na |= {2}\nprint(b is c, sorted(a))", "output": ["True [1, 2]"]},
+  {"src":"def f():\n    x = 1\n    def g():\n        nonlocal x\n        try:\n            print(x)\n        except NameError:\n            print(\"NameError\")\n    del x\n    g()\nf()","output":["NameError"]},
+  {"src":"def f():\n    x = 5\n    def g():\n        return x\n    return g()\nprint(f())","output":["5"]},
+  {"src":"def f():\n    x = 1\n    def g():\n        nonlocal x\n        x = 9\n    g()\n    return x\nprint(f())","output":["9"]},
+  {"src":"def f():\n    y = 3\n    del y\n    try:\n        return y\n    except NameError:\n        return 'NE'\nprint(f())","output":["NE"]},
+  {"src":"def f():\n    z = 1\n    del z\n    try:\n        del z\n    except NameError:\n        print('twice')\nf()","output":["twice"]},
+  {"src":"g = 5\ndel g\ntry:\n    print(g)\nexcept NameError:\n    print('gone')","output":["gone"]},
+  {"src":"def f():\n    x = 1\n    def g():\n        return x\n    a = g()\n    del x\n    return a\nprint(f())","output":["1"]},
+  {"src":"def f():\n    x = 1\n    y = 2\n    def g():\n        nonlocal x, y\n        try:\n            print(x)\n        except NameError:\n            print('x gone')\n        print(y)\n    del x\n    g()\nf()","output":["x gone","2"]},
+  {"src":"def f():\n    x = 1\n    def g():\n        nonlocal x\n        try:\n            return x\n        except NameError:\n            return 'NE'\n    def h():\n        nonlocal x\n        try:\n            return x\n        except NameError:\n            return 'NE'\n    del x\n    print(g(), h())\nf()","output":["NE NE"]},
+  {"src":"def f():\n    x = 1\n    def g():\n        nonlocal x\n        return x\n    del x\n    try:\n        g()\n    except NameError:\n        return 'caught'\nprint(f())","output":["caught"]},
+  {"src":"def f():\n    x = 1\n    def g():\n        return x\n    del x\n    x = 99\n    return g()\nprint(f())","output":["99"]}
 ]

docs/content/language/data-types.md

@@ -302,7 +302,7 @@ y
 
 ## Set
 
-Unordered, no duplicates, hashable values. Mutators (`add`, `remove`, `discard`, `pop`, `clear`, `update`) and algebraic operators (`|`, `&`, `-`, `^` and named methods). See [Methods](/reference/methods).
+Unordered, no duplicates, hashable values. Mutators (`add`, `remove`, `discard`, `pop`, `clear`, `update`) and algebraic operators (`|`, `&`, `-`, `^` and named methods). See [Methods](/reference/methods). Augmented `|=` `&=` `^=` mutate the set in place (aliases observe it); plain `|` `&` `^` build a new set.
 
 ```python
 s = {1, 2, 2, 3}