From bf3b04d4825ccd0180cbf22163a0c4ebdea41525 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 17:46:13 +0000 Subject: [PATCH 1/3] Add collections module with deque Introduce an importable `collections` module and implement `collections.deque`, the first of the commonly-used container datatypes. - New `Deque` heap type (`VecDeque`-backed) with maxlen support, threaded through the heap/GC/repr/eq/iteration/host-boundary dispatch. - `collections` module registration (StandardLib, StaticStrings, module attrs). - deque exposed as `collections.deque`; supports append/appendleft/pop/popleft/ extend/extendleft/insert/remove/clear/copy/count/index/reverse/rotate, integer indexing, iteration (with mutation detection), membership, ordering, and CPython-matching error messages. - Consolidated test_cases coverage and limitations/collections.md. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015FUUYYxP68sbHXsdr2qGE8 --- crates/monty/src/exception_private.rs | 8 + crates/monty/src/heap.rs | 20 +- crates/monty/src/heap_data.rs | 20 +- crates/monty/src/intern.rs | 52 ++ crates/monty/src/modules/collections.rs | 33 + crates/monty/src/modules/mod.rs | 6 + crates/monty/src/object.rs | 16 + crates/monty/src/types/deque.rs | 713 ++++++++++++++++++ crates/monty/src/types/iter.rs | 18 + crates/monty/src/types/mod.rs | 2 + crates/monty/src/types/type.rs | 6 +- crates/monty/src/value.rs | 13 + crates/monty/test_cases/collections__deque.py | 207 +++++ limitations/collections.md | 40 + limitations/modules.md | 10 +- 15 files changed, 1152 insertions(+), 12 deletions(-) create mode 100644 crates/monty/src/modules/collections.rs create mode 100644 crates/monty/src/types/deque.rs create mode 100644 crates/monty/test_cases/collections__deque.py create mode 100644 limitations/collections.md diff --git a/crates/monty/src/exception_private.rs b/crates/monty/src/exception_private.rs index 14ed1fdd0..d4cb1b4f3 100644 --- a/crates/monty/src/exception_private.rs +++ b/crates/monty/src/exception_private.rs @@ -1103,6 +1103,14 @@ impl ExcType { SimpleException::new_msg(Self::RuntimeError, "Set changed size during iteration").into() } + /// Creates a RuntimeError for deque mutation during iteration. + /// + /// Matches CPython's format: `RuntimeError: deque mutated during iteration` + #[must_use] + pub(crate) fn runtime_error_deque_changed_size() -> RunError { + SimpleException::new_msg(Self::RuntimeError, "deque mutated during iteration").into() + } + /// Creates a TypeError for functions that don't accept keyword arguments. /// /// Matches CPython's format: `TypeError: {name}() takes no keyword arguments` diff --git a/crates/monty/src/heap.rs b/crates/monty/src/heap.rs index 459897b7d..04f23d796 100644 --- a/crates/monty/src/heap.rs +++ b/crates/monty/src/heap.rs @@ -19,9 +19,9 @@ use crate::{ heap_data::{CellValue, Closure, FunctionDefaults}, resource::{ResourceError, ResourceTracker}, types::{ - BoundMethod, Bytes, Class, Dataclass, Dict, DictItemsView, DictKeysView, DictValuesView, FrozenSet, Instance, - List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, Range, ReMatch, RePattern, Set, Slice, Str, - TimeZone, Tuple, date, datetime, timedelta, timezone, + BoundMethod, Bytes, Class, Dataclass, Deque, Dict, DictItemsView, DictKeysView, DictValuesView, FrozenSet, + Instance, List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, Range, ReMatch, RePattern, Set, Slice, + Str, TimeZone, Tuple, date, datetime, timedelta, timezone, }, value::Value, }; @@ -207,6 +207,7 @@ pub enum HeapReadOutput<'a> { List(HeapRead<'a, List>), Tuple(HeapRead<'a, Tuple>), NamedTuple(HeapRead<'a, NamedTuple>), + Deque(HeapRead<'a, Deque>), Dict(HeapRead<'a, Dict>), DictItemsView(HeapRead<'a, DictItemsView>), DictKeysView(HeapRead<'a, DictKeysView>), @@ -592,6 +593,7 @@ impl<'a> HeapPtr<'a> { HeapData::List(list) => HeapReadOutput::List(heap_read(base, list, readers)), HeapData::Tuple(tuple) => HeapReadOutput::Tuple(heap_read(base, tuple, readers)), HeapData::NamedTuple(named_tuple) => HeapReadOutput::NamedTuple(heap_read(base, named_tuple, readers)), + HeapData::Deque(deque) => HeapReadOutput::Deque(heap_read(base, deque, readers)), HeapData::Dict(dict) => HeapReadOutput::Dict(heap_read(base, dict, readers)), HeapData::DictItemsView(v) => HeapReadOutput::DictItemsView(heap_read(base, v, readers)), HeapData::DictKeysView(v) => HeapReadOutput::DictKeysView(heap_read(base, v, readers)), @@ -1446,6 +1448,17 @@ fn for_each_child_id(data: &HeapData, mut on_child: F) { } } } + HeapData::Deque(deque) => { + // Skip iteration if no refs - GC optimization for deques of primitives + if !deque.contains_refs() { + return; + } + for value in deque.as_deque() { + if let Value::Ref(id) = value { + on_child(*id); + } + } + } HeapData::Dict(dict) => { // Skip iteration if no refs - major GC optimization for dicts of primitives if !dict.has_refs() { @@ -1647,6 +1660,7 @@ fn py_dec_ref_ids_for_data(data: &mut HeapData, stack: &mut Vec) { HeapData::List(l) => l.py_dec_ref_ids(stack), HeapData::Tuple(t) => t.py_dec_ref_ids(stack), HeapData::NamedTuple(nt) => nt.py_dec_ref_ids(stack), + HeapData::Deque(dq) => dq.py_dec_ref_ids(stack), HeapData::Dict(d) => d.py_dec_ref_ids(stack), HeapData::DictKeysView(view) => view.py_dec_ref_ids(stack), HeapData::DictItemsView(view) => view.py_dec_ref_ids(stack), diff --git a/crates/monty/src/heap_data.rs b/crates/monty/src/heap_data.rs index 84fe0848a..4aeea1c57 100644 --- a/crates/monty/src/heap_data.rs +++ b/crates/monty/src/heap_data.rs @@ -18,9 +18,9 @@ use crate::{ heap::{DropWithHeap, HeapId, HeapItem, HeapReadOutput}, intern::FunctionId, types::{ - BoundMethod, Bytes, Class, Dataclass, Dict, DictItemsView, DictKeysView, DictValuesView, FrozenSet, Instance, - LazyHeapSet, List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, PyTrait, Range, ReMatch, RePattern, - Set, Slice, Str, Tuple, Type, date, datetime, str::allocate_string, timedelta, timezone, + BoundMethod, Bytes, Class, Dataclass, Deque, Dict, DictItemsView, DictKeysView, DictValuesView, FrozenSet, + Instance, LazyHeapSet, List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, PyTrait, Range, ReMatch, + RePattern, Set, Slice, Str, Tuple, Type, date, datetime, str::allocate_string, timedelta, timezone, }, value::{EitherStr, Value, eq_bigint, eq_bytes, eq_ext_function, eq_str}, }; @@ -37,6 +37,8 @@ pub(crate) enum HeapData { List(List), Tuple(Tuple), NamedTuple(NamedTuple), + /// A `collections.deque` double-ended queue. + Deque(Deque), Dict(Dict), DictKeysView(DictKeysView), DictItemsView(DictItemsView), @@ -169,6 +171,7 @@ impl HeapData { Self::List(_) | Self::Tuple(_) | Self::NamedTuple(_) + | Self::Deque(_) | Self::Dict(_) | Self::DictKeysView(_) | Self::DictItemsView(_) @@ -207,6 +210,7 @@ impl HeapData { Self::Bytes(_) => Type::Bytes, Self::List(_) => Type::List, Self::Tuple(_) | Self::NamedTuple(_) => Type::Tuple, + Self::Deque(_) => Type::Deque, Self::Dict(_) => Type::Dict, Self::DictKeysView(_) => Type::DictKeys, Self::DictItemsView(_) => Type::DictItems, @@ -245,6 +249,7 @@ impl HeapData { Self::List(l) => l.py_estimate_size(), Self::Tuple(t) => t.py_estimate_size(), Self::NamedTuple(nt) => nt.py_estimate_size(), + Self::Deque(dq) => dq.py_estimate_size(), Self::Dict(d) => d.py_estimate_size(), Self::DictKeysView(view) => view.py_estimate_size(), Self::DictItemsView(view) => view.py_estimate_size(), @@ -461,6 +466,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_bool(vm), Self::Tuple(t) => t.py_bool(vm), Self::NamedTuple(nt) => nt.py_bool(vm), + Self::Deque(dq) => dq.py_bool(vm), Self::Dict(d) => d.py_bool(vm), Self::DictKeysView(view) => view.py_bool(vm), Self::DictItemsView(view) => view.py_bool(vm), @@ -502,6 +508,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { HeapReadOutput::Bytes(b) => Ok(b.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::List(list) => Ok(list.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::Tuple(t) => Ok(t.py_call_attr(self_id, vm, attr, args)?), + HeapReadOutput::Deque(dq) => Ok(dq.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::Dict(dict) => Ok(dict.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::DictKeysView(view) => Ok(view.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::DictItemsView(view) => Ok(view.py_call_attr(self_id, vm, attr, args)?), @@ -578,6 +585,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_type(vm), Self::Tuple(t) => t.py_type(vm), Self::NamedTuple(nt) => nt.py_type(vm), + Self::Deque(dq) => dq.py_type(vm), Self::Dict(d) => d.py_type(vm), Self::DictKeysView(v) => v.py_type(vm), Self::DictItemsView(v) => v.py_type(vm), @@ -615,6 +623,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_len(vm), Self::Tuple(t) => t.py_len(vm), Self::NamedTuple(nt) => nt.py_len(vm), + Self::Deque(dq) => dq.py_len(vm), Self::Dict(d) => d.py_len(vm), Self::DictKeysView(view) => view.py_len(vm), Self::DictItemsView(view) => view.py_len(vm), @@ -655,6 +664,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { HeapReadOutput::List(a) => a.py_eq_impl(other, vm), HeapReadOutput::Tuple(a) => a.py_eq_impl(other, vm), HeapReadOutput::NamedTuple(a) => a.py_eq_impl(other, vm), + HeapReadOutput::Deque(a) => a.py_eq_impl(other, vm), HeapReadOutput::Dict(a) => a.py_eq_impl(other, vm), HeapReadOutput::Set(a) => a.py_eq_impl(other, vm), HeapReadOutput::FrozenSet(a) => a.py_eq_impl(other, vm), @@ -750,6 +760,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_repr_fmt(f, vm, heap_ids), Self::Tuple(t) => t.py_repr_fmt(f, vm, heap_ids), Self::NamedTuple(nt) => nt.py_repr_fmt(f, vm, heap_ids), + Self::Deque(dq) => dq.py_repr_fmt(f, vm, heap_ids), Self::Dict(d) => d.py_repr_fmt(f, vm, heap_ids), Self::DictKeysView(view) => view.py_repr_fmt(f, vm, heap_ids), Self::DictItemsView(view) => view.py_repr_fmt(f, vm, heap_ids), @@ -954,6 +965,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_getitem(key, vm), Self::Tuple(t) => t.py_getitem(key, vm), Self::NamedTuple(nt) => nt.py_getitem(key, vm), + Self::Deque(dq) => dq.py_getitem(key, vm), Self::Dict(d) => d.py_getitem(key, vm), Self::Range(r) => r.py_getitem(key, vm), Self::ReMatch(m) => m.py_getitem(key, vm), @@ -964,6 +976,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { fn py_setitem(&mut self, key: Value, value: Value, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult<()> { match self { Self::List(l) => l.py_setitem(key, value, vm), + Self::Deque(dq) => dq.py_setitem(key, value, vm), Self::Dict(d) => d.py_setitem(key, value, vm), _ => { key.drop_with_heap(vm); @@ -982,6 +995,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_getattr(attr, vm), Self::Tuple(t) => t.py_getattr(attr, vm), Self::NamedTuple(nt) => nt.py_getattr(attr, vm), + Self::Deque(dq) => dq.py_getattr(attr, vm), Self::Dict(d) => d.py_getattr(attr, vm), Self::DictKeysView(view) => view.py_getattr(attr, vm), Self::DictItemsView(view) => view.py_getattr(attr, vm), diff --git a/crates/monty/src/intern.rs b/crates/monty/src/intern.rs index d53084956..6339a4444 100644 --- a/crates/monty/src/intern.rs +++ b/crates/monty/src/intern.rs @@ -756,6 +756,58 @@ pub enum StaticStrings { FalseRepr, #[strum(serialize = "Ellipsis")] EllipsisRepr, + + // ========================== + // collections module strings. Appended at the enum end: StaticStrings + // discriminants are serialized `StringId`s, so mid-enum insertion would + // shift every later id. + /// Module name for `import collections`. + Collections, + /// `collections.deque` type. + Deque, + /// `collections.defaultdict` type. + Defaultdict, + /// `collections.Counter` type. + #[strum(serialize = "Counter")] + Counter, + /// `collections.namedtuple` factory function. + Namedtuple, + /// `deque.appendleft()` method. + Appendleft, + /// `deque.popleft()` method. + Popleft, + /// `deque.extendleft()` method. + Extendleft, + /// `deque.rotate()` method. + Rotate, + /// `deque.maxlen` attribute and constructor kwarg. + Maxlen, + /// `Counter.most_common()` method. + MostCommon, + /// `Counter.elements()` method. + Elements, + /// `Counter.subtract()` method. + Subtract, + /// `defaultdict.default_factory` attribute. + DefaultFactory, + /// `namedtuple` class `_fields` attribute. + #[strum(serialize = "_fields")] + Fields, + /// `namedtuple` class `_make()` classmethod. + #[strum(serialize = "_make")] + Make, + /// `namedtuple` instance `_asdict()` method. + #[strum(serialize = "_asdict")] + Asdict, + /// `namedtuple` instance `_replace()` method. + #[strum(serialize = "_replace")] + ReplaceMethod, + /// `namedtuple` class `_field_defaults` attribute. + #[strum(serialize = "_field_defaults")] + FieldDefaults, + /// `namedtuple(defaults=...)` keyword argument. (`rename` reuses the + /// existing `Rename` variant.) + Defaults, } impl StaticStrings { diff --git a/crates/monty/src/modules/collections.rs b/crates/monty/src/modules/collections.rs new file mode 100644 index 000000000..7bf68c676 --- /dev/null +++ b/crates/monty/src/modules/collections.rs @@ -0,0 +1,33 @@ +//! Implementation of the `collections` module. +//! +//! Exposes the most commonly used container datatypes: +//! - `deque` — a double-ended queue ([`Type::Deque`]) +//! +//! Types are exposed as callable `Value::Builtin(Builtins::Type(...))` module +//! attributes; their construction and behavior live in the corresponding +//! runtime types under `crate::types`. + +use crate::{ + builtins::Builtins, + bytecode::VM, + heap::{HeapData, HeapId}, + intern::StaticStrings, + resource::{ResourceError, ResourceTracker}, + types::{Module, Type}, + value::Value, +}; + +/// Creates the `collections` module and allocates it on the heap. +/// +/// Returns a `HeapId` pointing to the newly allocated module. +/// +/// # Panics +/// +/// Panics if the required strings have not been pre-interned during prepare phase. +pub fn create_module(vm: &mut VM<'_, impl ResourceTracker>) -> Result { + let mut module = Module::new(StaticStrings::Collections); + + module.set_attr(StaticStrings::Deque, Value::Builtin(Builtins::Type(Type::Deque)), vm); + + vm.heap.allocate(HeapData::Module(module)) +} diff --git a/crates/monty/src/modules/mod.rs b/crates/monty/src/modules/mod.rs index 988f12603..13ac39572 100644 --- a/crates/monty/src/modules/mod.rs +++ b/crates/monty/src/modules/mod.rs @@ -17,6 +17,7 @@ use crate::{ }; pub(crate) mod asyncio; +pub(crate) mod collections; pub(crate) mod datetime; #[cfg(feature = "test-hooks")] pub(crate) mod gc; @@ -53,6 +54,9 @@ pub(crate) enum StandardLib { Datetime, /// The `unicodedata` module providing Unicode Character Database access. Unicodedata, + /// The `collections` module providing specialized container datatypes + /// (`deque`, `defaultdict`, `Counter`, `namedtuple`). + Collections, /// The `gc` module exposing a single `collect()` for tests. Only present /// under the `test-hooks` feature so production sandboxes never see it. /// @@ -78,6 +82,7 @@ impl StandardLib { StaticStrings::Re => Some(Self::Re), StaticStrings::Datetime => Some(Self::Datetime), StaticStrings::Unicodedata => Some(Self::Unicodedata), + StaticStrings::Collections => Some(Self::Collections), #[cfg(feature = "test-hooks")] StaticStrings::Gc => Some(Self::Gc), _ => None, @@ -103,6 +108,7 @@ impl StandardLib { Self::Re => re::create_module(vm), Self::Datetime => datetime::create_module(vm), Self::Unicodedata => unicodedata::create_module(vm), + Self::Collections => collections::create_module(vm), #[cfg(feature = "test-hooks")] Self::Gc => gc::create_module(vm), } diff --git a/crates/monty/src/object.rs b/crates/monty/src/object.rs index 9c96ef413..55bff9654 100644 --- a/crates/monty/src/object.rs +++ b/crates/monty/src/object.rs @@ -360,6 +360,7 @@ pub enum MontyType { List, Tuple, NamedTuple, + Deque, Dict, DictKeys, DictItems, @@ -449,6 +450,7 @@ impl MontyType { Self::List => Some(Type::List), Self::Tuple => Some(Type::Tuple), Self::NamedTuple => Some(Type::NamedTuple), + Self::Deque => Some(Type::Deque), Self::Dict => Some(Type::Dict), Self::DictKeys => Some(Type::DictKeys), Self::DictItems => Some(Type::DictItems), @@ -502,6 +504,7 @@ impl MontyType { Type::List => Self::List, Type::Tuple => Self::Tuple, Type::NamedTuple => Self::NamedTuple, + Type::Deque => Self::Deque, Type::Dict => Self::Dict, Type::DictKeys => Self::DictKeys, Type::DictItems => Self::DictItems, @@ -842,6 +845,7 @@ impl MontyObject { return match vm.heap.get(*id) { HeapData::List(_) => Self::Cycle(id.index(), "[...]".to_owned()), HeapData::Tuple(_) | HeapData::NamedTuple(_) => Self::Cycle(id.index(), "(...)".to_owned()), + HeapData::Deque(_) => Self::Cycle(id.index(), "deque([...])".to_owned()), HeapData::Dict(_) => Self::Cycle(id.index(), "{...}".to_owned()), _ => Self::Cycle(id.index(), "...".to_owned()), }; @@ -894,6 +898,18 @@ impl MontyObject { values, } } + HeapReadOutput::Deque(deque) => { + // Represent a deque to the host as a list of its items — + // the wire boundary has no dedicated deque type. + let len = deque.get(vm.heap).as_deque().len(); + let mut items = Vec::with_capacity(len); + for i in 0..len { + let item = deque.get(vm.heap).as_deque()[i].clone_with_heap(vm.heap); + defer_drop!(item, vm); + items.push(Self::from_value_inner(item, vm, visited)); + } + Self::List(items) + } HeapReadOutput::Dict(dict) => { let len = dict.get(vm.heap).len(); let mut pairs = Vec::with_capacity(len); diff --git a/crates/monty/src/types/deque.rs b/crates/monty/src/types/deque.rs new file mode 100644 index 000000000..babdab332 --- /dev/null +++ b/crates/monty/src/types/deque.rs @@ -0,0 +1,713 @@ +use std::{cmp::Ordering, collections::VecDeque, fmt::Write, mem}; + +use super::{CmpOrder, MontyIter, PyTrait}; +use crate::{ + args::{ArgValues, FromArgs}, + bytecode::{CallResult, ContainsVM, DropWithVM, RecursionToken, VM}, + defer_drop, defer_drop_mut, defer_drop_vm_mut, + exception_private::{ExcType, RunError, RunResult, SimpleException}, + heap::{DropWithHeap, HeapData, HeapId, HeapItem, HeapRead, HeapReadOutput}, + intern::StaticStrings, + resource::ResourceTracker, + types::{LazyHeapSet, Type, slice::normalize_sequence_index}, + value::{EitherStr, VALUE_SIZE, Value}, +}; + +/// Python `collections.deque` type, a double-ended queue backed by a `VecDeque`. +/// +/// Supports O(1) appends and pops from both ends. An optional `maxlen` bounds +/// the queue: once full, an append on one end discards an item from the other, +/// matching CPython's "bounded deque" semantics (e.g. a fixed-size ring buffer). +/// +/// Unlike `list`, deques do **not** support slicing (`d[1:2]` raises +/// `TypeError`), only integer indexing. Equality compares element-wise against +/// another deque; a deque never compares equal to a `list`. +/// +/// # Reference counting +/// Items transferred into the deque have already had their refcounts handled by +/// the caller (as with [`List`](super::List)); items discarded due to `maxlen` +/// overflow are dropped via `drop_with_heap`. +/// +/// # GC optimization +/// `contains_refs` tracks whether any item is a `Value::Ref`, letting +/// `py_dec_ref_ids` skip iteration for deques of primitives. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct Deque { + items: VecDeque, + /// Maximum length; `None` means unbounded. + maxlen: Option, + /// True if any item is a `Value::Ref` (GC fast-path flag). + contains_refs: bool, +} + +impl Deque { + /// Creates a deque from a `VecDeque` of values and an optional `maxlen`. + /// + /// Does NOT enforce `maxlen` on the passed items or manage refcounts — the + /// caller must ensure `items.len() <= maxlen` and that refcounts are owned. + #[must_use] + pub fn new(items: VecDeque, maxlen: Option) -> Self { + let contains_refs = items.iter().any(|v| matches!(v, Value::Ref(_))); + Self { + items, + maxlen, + contains_refs, + } + } + + /// Returns a reference to the underlying `VecDeque`. + #[must_use] + pub fn as_deque(&self) -> &VecDeque { + &self.items + } + + /// Returns whether the deque contains any heap references. + #[inline] + #[must_use] + pub fn contains_refs(&self) -> bool { + self.contains_refs + } +} + +impl Deque { + /// Implements the `deque(iterable=(), maxlen=None)` constructor. + pub fn init(vm: &mut VM<'_, impl ResourceTracker>, args: ArgValues) -> RunResult { + let DequeInitArgs { iterable, maxlen } = DequeInitArgs::from_args(args, vm)?; + defer_drop!(iterable, vm); + defer_drop!(maxlen, vm); + + let maxlen = parse_maxlen(maxlen, vm)?; + + let mut items: VecDeque = VecDeque::new(); + if !matches!(*iterable, Value::None) { + let collected: Vec = MontyIter::new(iterable.clone_with_heap(vm.heap), vm)?.collect(vm)?; + vm.heap.track_growth(collected.len() * VALUE_SIZE)?; + items.extend(collected); + // Enforce maxlen by discarding from the front, matching CPython. + if let Some(max) = maxlen { + while items.len() > max { + if let Some(dropped) = items.pop_front() { + dropped.drop_with_heap(vm.heap); + } + } + } + } + + let heap_id = vm.heap.allocate(HeapData::Deque(Self::new(items, maxlen)))?; + Ok(Value::Ref(heap_id)) + } +} + +/// Parses the `maxlen` constructor argument into `Option`. +/// +/// `None` stays unbounded; an integer must be non-negative (CPython raises +/// `ValueError: maxlen must be non-negative` otherwise), and non-int types +/// raise `TypeError`. +fn parse_maxlen(maxlen: &Value, vm: &VM<'_, impl ResourceTracker>) -> RunResult> { + if matches!(maxlen, Value::None) { + Ok(None) + } else { + let n = maxlen.as_int(vm)?; + if n < 0 { + Err(SimpleException::new_msg(ExcType::ValueError, "maxlen must be non-negative").into()) + } else { + Ok(Some(usize::try_from(n).expect("non-negative i64 fits usize on 64-bit"))) + } + } +} + +/// Constructor arguments for `deque(iterable=(), maxlen=None)`. +/// +/// Both are `Value` so the body can coerce `maxlen` and produce CPython-matching +/// errors (deque is C-implemented, so a bad `maxlen` type is reported by the +/// body, not during binding). +#[derive(FromArgs)] +#[from_args(name = "deque")] +struct DequeInitArgs { + #[from_args(default = Value::None)] + iterable: Value, + #[from_args(static_string = "Maxlen", default = Value::None)] + maxlen: Value, +} + +impl<'h> HeapRead<'h, Deque> { + /// Appends an item to the right end, enforcing `maxlen` by discarding from + /// the left. Ownership of `item` transfers to the deque (refcount already + /// handled by the caller). + pub fn append(&mut self, vm: &mut VM<'h, impl ResourceTracker>, item: Value) -> RunResult<()> { + if matches!(item, Value::Ref(_)) { + self.get_mut(vm.heap).contains_refs = true; + } + let maxlen = self.get(vm.heap).maxlen; + if maxlen == Some(0) { + // A zero-maxlen deque silently discards everything. + item.drop_with_heap(vm.heap); + return Ok(()); + } + vm.heap.track_growth(VALUE_SIZE)?; + self.get_mut(vm.heap).items.push_back(item); + if let Some(max) = maxlen + && self.get(vm.heap).items.len() > max + && let Some(dropped) = self.get_mut(vm.heap).items.pop_front() + { + dropped.drop_with_heap(vm.heap); + } + Ok(()) + } + + /// Appends an item to the left end, enforcing `maxlen` by discarding from + /// the right. Ownership of `item` transfers to the deque. + pub fn appendleft(&mut self, vm: &mut VM<'h, impl ResourceTracker>, item: Value) -> RunResult<()> { + if matches!(item, Value::Ref(_)) { + self.get_mut(vm.heap).contains_refs = true; + } + let maxlen = self.get(vm.heap).maxlen; + if maxlen == Some(0) { + item.drop_with_heap(vm.heap); + return Ok(()); + } + vm.heap.track_growth(VALUE_SIZE)?; + self.get_mut(vm.heap).items.push_front(item); + if let Some(max) = maxlen + && self.get(vm.heap).items.len() > max + && let Some(dropped) = self.get_mut(vm.heap).items.pop_back() + { + dropped.drop_with_heap(vm.heap); + } + Ok(()) + } + + /// Clones the item at `index` with proper refcount management. + pub(crate) fn clone_item(&self, index: usize, vm: &mut VM<'h, impl ResourceTracker>) -> Value { + self.get(vm.heap).items[index].clone_with_heap(vm.heap) + } + + /// Normalizes a possibly-negative index against the current length, + /// returning `Some(usize)` if in bounds. Uses `index + len` (not `-index`) + /// to avoid overflow on `i64::MIN`. + fn normalize_index(&self, index: i64, vm: &VM<'h, impl ResourceTracker>) -> Option { + let len = i64::try_from(self.get(vm.heap).items.len()).ok()?; + let normalized = if index < 0 { index + len } else { index }; + if normalized < 0 || normalized >= len { + None + } else { + usize::try_from(normalized).ok() + } + } + + /// Returns a stack-borrowed lending iterator over the deque's items, + /// holding a recursion-depth token for its lifetime. See [`DequeIter`]. + #[expect(clippy::iter_not_returning_iterator)] + pub(crate) fn iter(&self, vm: &mut VM<'h, R>) -> RunResult> { + DequeIter::new(self, vm) + } +} + +/// Stack-borrowed lending iterator over a [`Deque`]'s items. +/// +/// Same lending shape and recursion-token discipline as +/// [`ListIter`](super::list::ListIter): [`next`](Self::next) returns +/// `Option<&Value>`, owning the yielded item in `current` and dropping the +/// prior one on each call. MUST be wrapped in [`defer_drop_vm_mut!`] so the +/// token and in-flight item are released on every exit path. The length is +/// re-read each call so concurrent mutation cannot cause an out-of-bounds +/// panic. +pub(crate) struct DequeIter<'a, 'h> { + deque: &'a HeapRead<'h, Deque>, + index: usize, + token: RecursionToken, + current: Value, +} + +impl<'a, 'h> DequeIter<'a, 'h> { + fn new(deque: &'a HeapRead<'h, Deque>, vm: &mut VM<'h, R>) -> RunResult { + let token = vm.recursion_token()?; + Ok(Self { + deque, + index: 0, + token, + current: Value::Undefined, + }) + } + + pub(crate) fn next<'i, R: ResourceTracker>(&'i mut self, vm: &mut VM<'h, R>) -> RunResult> { + mem::replace(&mut self.current, Value::Undefined).drop_with_heap(vm.heap); + vm.heap.check_time()?; + if self.index >= self.deque.get(vm.heap).items.len() { + return Ok(None); + } + self.current = self.deque.get(vm.heap).items[self.index].clone_with_heap(vm.heap); + self.index += 1; + Ok(Some(&self.current)) + } + + pub(crate) fn next_with_index<'i, R: ResourceTracker>( + &'i mut self, + vm: &mut VM<'h, R>, + ) -> RunResult> { + let position = self.index; + Ok(self.next(vm)?.map(|item| (position, item))) + } +} + +impl<'h> DropWithVM<'h> for DequeIter<'_, 'h> { + fn drop_with_vm(self, container: &mut impl ContainsVM<'h>) { + self.current.drop_with_heap(container); + self.token.drop_with_vm(container); + } +} + +impl<'h> PyTrait<'h> for HeapRead<'h, Deque> { + fn py_type(&self, _vm: &VM<'h, impl ResourceTracker>) -> Type { + Type::Deque + } + + fn py_len(&self, vm: &VM<'h, impl ResourceTracker>) -> Option { + Some(self.get(vm.heap).items.len()) + } + + fn py_bool(&self, vm: &mut VM<'h, impl ResourceTracker>) -> bool { + !self.get(vm.heap).items.is_empty() + } + + fn py_getitem(&self, key: &Value, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let index = deque_key_to_index(key, vm)?; + match self.normalize_index(index, vm) { + Some(idx) => Ok(self.get(vm.heap).items[idx].clone_with_heap(vm.heap)), + None => Err(deque_index_error()), + } + } + + fn py_setitem(&mut self, key: Value, value: Value, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult<()> { + defer_drop!(key, vm); + defer_drop_mut!(value, vm); + let index = deque_key_to_index(key, vm)?; + let Some(idx) = self.normalize_index(index, vm) else { + return Err(deque_index_error()); + }; + if matches!(*value, Value::Ref(_)) { + self.get_mut(vm.heap).contains_refs = true; + } + mem::swap(&mut self.get_mut(vm.heap).items[idx], value); + Ok(()) + } + + fn py_eq_impl(&self, other: &Value, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult> { + let Some(HeapReadOutput::Deque(other)) = other.read_heap(vm) else { + return Ok(None); + }; + if self.get(vm.heap).items.len() != other.get(vm.heap).items.len() { + return Ok(Some(false)); + } + let iter = self.iter(vm)?; + defer_drop_vm_mut!(iter, vm); + while let Some((i, a)) = iter.next_with_index(vm)? { + let b = other.clone_item(i, vm); + defer_drop!(b, vm); + if !a.py_eq(b, vm)? { + return Ok(Some(false)); + } + } + Ok(Some(true)) + } + + fn py_cmp(&self, other: &Self, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let a_len = self.get(vm.heap).items.len(); + let b_len = other.get(vm.heap).items.len(); + let min_len = a_len.min(b_len); + let iter = self.iter(vm)?; + defer_drop_vm_mut!(iter, vm); + while let Some((i, av)) = iter.next_with_index(vm)? { + if i >= min_len { + break; + } + let bv = other.clone_item(i, vm); + defer_drop!(bv, vm); + match av.py_cmp(bv, vm)? { + CmpOrder::Ordered(Ordering::Equal) => {} + CmpOrder::Ordered(ord) => return Ok(CmpOrder::Ordered(ord)), + CmpOrder::Unordered => return Ok(CmpOrder::Unordered), + CmpOrder::Incomparable => { + if !av.py_eq(bv, vm)? { + return Ok(CmpOrder::Incomparable); + } + } + } + } + Ok(CmpOrder::Ordered(a_len.cmp(&b_len))) + } + + fn py_repr_fmt( + &self, + f: &mut impl Write, + vm: &mut VM<'h, impl ResourceTracker>, + heap_ids: &mut LazyHeapSet, + ) -> RunResult<()> { + let Ok(mut guard) = vm.recursion_guard() else { + return Ok(f.write_str("...")?); + }; + let vm = &mut *guard; + + f.write_str("deque([")?; + let len = self.get(vm.heap).items.len(); + for i in 0..len { + if i > 0 { + if vm.heap.check_time().is_err() { + f.write_str(", ...[timeout]")?; + break; + } + f.write_str(", ")?; + } + let item = self.clone_item(i, vm); + defer_drop!(item, vm); + item.py_repr_fmt(f, vm, heap_ids)?; + } + f.write_char(']')?; + if let Some(max) = self.get(vm.heap).maxlen { + write!(f, ", maxlen={max}")?; + } + f.write_char(')')?; + Ok(()) + } + + fn py_call_attr( + &mut self, + _self_id: HeapId, + vm: &mut VM<'h, impl ResourceTracker>, + attr: &EitherStr, + args: ArgValues, + ) -> RunResult { + let Some(method) = attr.static_string() else { + args.drop_with_heap(vm); + return Err(ExcType::attribute_error(Type::Deque, attr.as_str(vm.interns))); + }; + call_deque_method(self, method, args, vm).map(CallResult::Value) + } + + fn py_getattr(&self, attr: &EitherStr, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult> { + // `maxlen` is a read-only attribute; everything else is a method + // (resolved via py_call_attr) or unknown. + if attr.static_string() == Some(StaticStrings::Maxlen) { + let value = match self.get(vm.heap).maxlen { + Some(max) => Value::Int(i64::try_from(max).expect("maxlen fits i64")), + None => Value::None, + }; + Ok(Some(CallResult::Value(value))) + } else { + Ok(None) + } + } +} + +impl HeapItem for Deque { + fn py_estimate_size(&self) -> usize { + mem::size_of::() + self.items.len() * VALUE_SIZE + } + + fn py_dec_ref_ids(&mut self, stack: &mut Vec) { + if !self.contains_refs { + return; + } + for obj in &mut self.items { + if let Value::Ref(id) = obj { + stack.push(*id); + #[cfg(feature = "memory-model-checks")] + obj.dec_ref_forget(); + } + } + } +} + +/// Builds the `IndexError: deque index out of range` exception. +fn deque_index_error() -> RunError { + SimpleException::new_msg(ExcType::IndexError, "deque index out of range").into() +} + +/// Converts a subscript key into an `i64` index for a deque. +/// +/// Unlike lists, deques accept only integers (no slices): every non-integer +/// key — including `slice` — raises CPython's +/// `TypeError: sequence index must be integer, not '{type}'`. A `LongInt` +/// that overflows `i64` can never be in range, so it reports the same +/// out-of-range `IndexError` as any other oversized index. +fn deque_key_to_index(key: &Value, vm: &VM<'_, impl ResourceTracker>) -> RunResult { + match key { + Value::Int(i) => Ok(*i), + Value::Bool(b) => Ok(i64::from(*b)), + Value::Ref(id) => match vm.heap.get(*id) { + HeapData::LongInt(li) => li.to_i64().ok_or_else(deque_index_error), + _ => Err(deque_type_index_error(key, vm)), + }, + _ => Err(deque_type_index_error(key, vm)), + } +} + +/// Builds the `TypeError: sequence index must be integer, not '{type}'` raised +/// for a non-integer deque subscript. +fn deque_type_index_error(key: &Value, vm: &VM<'_, impl ResourceTracker>) -> RunError { + SimpleException::new_msg( + ExcType::TypeError, + format!("sequence index must be integer, not '{}'", key.py_type_name(vm)), + ) + .into() +} + +/// Dispatches a method call on a deque value. +fn call_deque_method<'h>( + deque: &mut HeapRead<'h, Deque>, + method: StaticStrings, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + match method { + StaticStrings::Append => { + let item = args.get_one_arg("append", vm.heap)?; + deque.append(vm, item)?; + Ok(Value::None) + } + StaticStrings::Appendleft => { + let item = args.get_one_arg("appendleft", vm.heap)?; + deque.appendleft(vm, item)?; + Ok(Value::None) + } + StaticStrings::Pop => { + args.check_zero_args("pop", vm.heap)?; + match deque.get_mut(vm.heap).items.pop_back() { + Some(item) => Ok(item), + None => Err(SimpleException::new_msg(ExcType::IndexError, "pop from an empty deque").into()), + } + } + StaticStrings::Popleft => { + args.check_zero_args("popleft", vm.heap)?; + match deque.get_mut(vm.heap).items.pop_front() { + Some(item) => Ok(item), + None => Err(SimpleException::new_msg(ExcType::IndexError, "pop from an empty deque").into()), + } + } + StaticStrings::Extend => deque_extend(deque, args, vm, false), + StaticStrings::Extendleft => deque_extend(deque, args, vm, true), + StaticStrings::Clear => { + args.check_zero_args("clear", vm.heap)?; + let items = mem::take(&mut deque.get_mut(vm.heap).items); + Vec::from(items).drop_with_heap(vm.heap); + Ok(Value::None) + } + StaticStrings::Copy => { + args.check_zero_args("copy", vm.heap)?; + deque_copy(deque, vm) + } + StaticStrings::Count => deque_count(deque, args, vm), + StaticStrings::Index => deque_index_method(deque, args, vm), + StaticStrings::Insert => deque_insert(deque, args, vm), + StaticStrings::Remove => deque_remove(deque, args, vm), + StaticStrings::Reverse => { + args.check_zero_args("reverse", vm.heap)?; + deque.get_mut(vm.heap).items = mem::take(&mut deque.get_mut(vm.heap).items).into_iter().rev().collect(); + Ok(Value::None) + } + StaticStrings::Rotate => deque_rotate(deque, args, vm), + _ => { + args.drop_with_heap(vm.heap); + Err(ExcType::attribute_error(Type::Deque, method.into())) + } + } +} + +/// Implements `deque.extend(iterable)` / `deque.extendleft(iterable)`. +/// +/// `extendleft` appends each item to the left, which reverses their order in +/// the deque (matching CPython). +fn deque_extend<'h>( + deque: &mut HeapRead<'h, Deque>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, + left: bool, +) -> RunResult { + let iterable = args.get_one_arg(if left { "extendleft" } else { "extend" }, vm.heap)?; + let items: Vec = MontyIter::new(iterable, vm)?.collect(vm)?; + for item in items { + if left { + deque.appendleft(vm, item)?; + } else { + deque.append(vm, item)?; + } + } + Ok(Value::None) +} + +/// Implements `deque.copy()`, returning a new deque with the same `maxlen`. +fn deque_copy<'h>(deque: &HeapRead<'h, Deque>, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let len = deque.get(vm.heap).items.len(); + let mut items = VecDeque::with_capacity(len); + for i in 0..len { + items.push_back(deque.clone_item(i, vm)); + } + let maxlen = deque.get(vm.heap).maxlen; + let heap_id = vm.heap.allocate(HeapData::Deque(Deque::new(items, maxlen)))?; + Ok(Value::Ref(heap_id)) +} + +/// Implements `deque.count(value)`. +fn deque_count<'h>( + deque: &HeapRead<'h, Deque>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + let value = args.get_one_arg("count", vm.heap)?; + defer_drop!(value, vm); + let mut count: i64 = 0; + let iter = deque.iter(vm)?; + defer_drop_vm_mut!(iter, vm); + while let Some(item) = iter.next(vm)? { + if value.py_eq(item, vm)? { + count += 1; + } + } + Ok(Value::Int(count)) +} + +/// Implements `deque.index(value[, start[, stop]])`, raising `ValueError` when +/// the value is not found in the `[start, stop)` window. +fn deque_index_method<'h>( + deque: &HeapRead<'h, Deque>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + let pos_args = args.into_pos_only("index", vm.heap)?; + defer_drop!(pos_args, vm); + + let len = deque.get(vm.heap).items.len(); + let (value, start, end) = match pos_args.as_slice() { + [] => return Err(ExcType::type_error_at_least("index", 1, 0)), + [value] => (value, 0, len), + [value, start_arg] => { + let start = normalize_sequence_index(start_arg.as_int(vm)?, len); + (value, start, len) + } + [value, start_arg, end_arg] => { + let start = normalize_sequence_index(start_arg.as_int(vm)?, len); + let end = normalize_sequence_index(end_arg.as_int(vm)?, len).max(start); + (value, start, end) + } + other => return Err(ExcType::type_error_at_most("index", 3, other.len())), + }; + + let iter = deque.iter(vm)?; + defer_drop_vm_mut!(iter, vm); + while let Some((i, item)) = iter.next_with_index(vm)? { + if i >= end { + break; + } + if i >= start && value.py_eq(item, vm)? { + return Ok(Value::Int(i64::try_from(i).expect("index fits i64"))); + } + } + Err(SimpleException::new_msg( + ExcType::ValueError, + format!("{} is not in deque", repr_value(value, vm)?), + ) + .into()) +} + +/// Implements `deque.insert(index, value)`, honoring `maxlen`. +fn deque_insert<'h>( + deque: &mut HeapRead<'h, Deque>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + let (index_obj, item) = args.get_two_args("insert", vm.heap)?; + defer_drop!(index_obj, vm); + defer_drop_mut!(item, vm); + + // A full bounded deque rejects insert, matching CPython. + let len = deque.get(vm.heap).items.len(); + if let Some(max) = deque.get(vm.heap).maxlen + && len >= max + { + return Err(SimpleException::new_msg(ExcType::IndexError, "deque already at its maximum size").into()); + } + + let index_i64 = index_obj.as_int(vm)?; + let len_i64 = i64::try_from(len).expect("deque length fits i64"); + // Clamp per CPython: negatives add len then clamp to 0, large values append. + let idx = if index_i64 < 0 { + usize::try_from(index_i64 + len_i64).unwrap_or(0) + } else { + usize::try_from(index_i64).unwrap_or(len).min(len) + }; + if matches!(*item, Value::Ref(_)) { + deque.get_mut(vm.heap).contains_refs = true; + } + vm.heap.track_growth(VALUE_SIZE)?; + let value = mem::replace(&mut *item, Value::Undefined); + deque.get_mut(vm.heap).items.insert(idx, value); + Ok(Value::None) +} + +/// Implements `deque.remove(value)`, removing the first match. +fn deque_remove<'h>( + deque: &mut HeapRead<'h, Deque>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + let value = args.get_one_arg("remove", vm.heap)?; + defer_drop!(value, vm); + + let mut found = None; + { + let iter = deque.iter(vm)?; + defer_drop_vm_mut!(iter, vm); + while let Some((i, item)) = iter.next_with_index(vm)? { + if value.py_eq(item, vm)? { + found = Some(i); + break; + } + } + } + match found { + Some(idx) => { + if let Some(removed) = deque.get_mut(vm.heap).items.remove(idx) { + removed.drop_with_heap(vm.heap); + } + Ok(Value::None) + } + None => Err(SimpleException::new_msg( + ExcType::ValueError, + format!("{} is not in deque", repr_value(value, vm)?), + ) + .into()), + } +} + +/// Implements `deque.rotate(n=1)`: rotate `n` steps to the right (negative +/// rotates left). Rotations are taken modulo the length so large `n` is cheap. +fn deque_rotate<'h>( + deque: &mut HeapRead<'h, Deque>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + let n = match args.get_zero_one_arg("rotate", vm.heap)? { + Some(v) => { + let result = v.as_int(vm); + v.drop_with_heap(vm.heap); + result? + } + None => 1, + }; + let len = deque.get(vm.heap).items.len(); + if len > 0 { + let len_i64 = i64::try_from(len).expect("deque length fits i64"); + let shift = n.rem_euclid(len_i64); + let shift = usize::try_from(shift).expect("shift in [0, len)"); + deque.get_mut(vm.heap).items.rotate_right(shift); + } + Ok(Value::None) +} + +/// Renders `value`'s `repr()` as an owned `String` for error messages. +fn repr_value(value: &Value, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult { + let repr = value.py_repr(vm)?; + defer_drop!(repr, vm); + let owned = repr.to_str(vm).map(str::to_owned).unwrap_or_default(); + Ok(owned) +} diff --git a/crates/monty/src/types/iter.rs b/crates/monty/src/types/iter.rs index fbe46d9a4..94e7690d4 100644 --- a/crates/monty/src/types/iter.rs +++ b/crates/monty/src/types/iter.rs @@ -424,6 +424,18 @@ fn get_heap_item( } HeapData::Tuple(tuple) => Ok(Some(tuple.as_slice()[index].clone_with_heap(vm))), HeapData::NamedTuple(namedtuple) => Ok(Some(namedtuple.as_vec()[index].clone_with_heap(vm))), + HeapData::Deque(deque) => { + // Check for deque mutation during iteration (matches CPython) + if let Some(expected) = expected_len + && deque.as_deque().len() != expected + { + return Err(ExcType::runtime_error_deque_changed_size()); + } + match deque.as_deque().get(index) { + Some(item) => Ok(Some(item.clone_with_heap(vm))), + None => Ok(None), + } + } HeapData::Dict(dict) => { // Check for dict mutation if let Some(expected) = expected_len @@ -649,6 +661,12 @@ impl IterValue { len: Some(namedtuple.len()), checks_mutation: false, }), + // Deque: captured len, WITH mutation check (like dict/set) + HeapData::Deque(deque) => Some(Self::HeapRef { + heap_id, + len: Some(deque.as_deque().len()), + checks_mutation: true, + }), HeapData::Bytes(b) => Some(Self::HeapRef { heap_id, len: Some(b.len()), diff --git a/crates/monty/src/types/mod.rs b/crates/monty/src/types/mod.rs index 4e8f4ad97..698a17c59 100644 --- a/crates/monty/src/types/mod.rs +++ b/crates/monty/src/types/mod.rs @@ -10,6 +10,7 @@ pub mod class; pub mod dataclass; pub mod date; pub mod datetime; +pub mod deque; pub mod dict; pub mod dict_view; pub mod file; @@ -36,6 +37,7 @@ pub mod r#type; pub(crate) use bytes::Bytes; pub(crate) use class::Class; pub(crate) use dataclass::Dataclass; +pub(crate) use deque::Deque; pub(crate) use dict::Dict; pub(crate) use dict_view::{DictItemsView, DictKeysView, DictValuesView}; pub(crate) use file::OpenFile; diff --git a/crates/monty/src/types/type.rs b/crates/monty/src/types/type.rs index 74c3bd3dc..2818fdcc1 100644 --- a/crates/monty/src/types/type.rs +++ b/crates/monty/src/types/type.rs @@ -11,7 +11,7 @@ use crate::{ intern::{Interns, StaticStrings, StringId}, resource::ResourceTracker, types::{ - AttrCallResult, Bytes, Dict, FrozenSet, List, LongInt, MontyIter, Path, PyTrait, Range, Set, Slice, Str, + AttrCallResult, Bytes, Deque, Dict, FrozenSet, List, LongInt, MontyIter, Path, PyTrait, Range, Set, Slice, Str, TimeZone, Tuple, bytes::bytes_fromhex, date, datetime, dict::dict_fromkeys, instance::class_name, long_int::INT_MAX_STR_DIGITS, str::StringRepr, timedelta, }, @@ -61,6 +61,9 @@ pub enum Type { List, Tuple, NamedTuple, + /// `collections.deque` — a double-ended queue. Only reachable via import. + #[strum(serialize = "collections.deque")] + Deque, Dict, #[strum(serialize = "dict_keys")] DictKeys, @@ -375,6 +378,7 @@ impl Type { // Container types - delegate to init methods Self::List => List::init(vm, args), Self::Tuple => Tuple::init(vm, args), + Self::Deque => Deque::init(vm, args), Self::Dict => Dict::init(vm, args), Self::Set => Set::init(vm, args), Self::FrozenSet => FrozenSet::init(vm, args), diff --git a/crates/monty/src/value.rs b/crates/monty/src/value.rs index 01c8a7324..2d2ab90a8 100644 --- a/crates/monty/src/value.rs +++ b/crates/monty/src/value.rs @@ -265,6 +265,7 @@ impl<'h> PyTrait<'h> for Value { } (HeapReadOutput::Tuple(a), HeapReadOutput::Tuple(b)) => a.py_cmp(&b, vm), (HeapReadOutput::List(a), HeapReadOutput::List(b)) => a.py_cmp(&b, vm), + (HeapReadOutput::Deque(a), HeapReadOutput::Deque(b)) => a.py_cmp(&b, vm), (HeapReadOutput::Date(a), HeapReadOutput::Date(b)) => { Ok(CmpOrder::from_total(a.get(vm.heap).partial_cmp(b.get(vm.heap)))) } @@ -1715,6 +1716,18 @@ impl Value { } Ok(false) } + HeapReadOutput::Deque(deque) => { + let len = deque.get(vm.heap).as_deque().len(); + for i in 0..len { + let el = deque.clone_item(i, vm); + let eq = item.py_eq(&el, vm); + el.drop_with_heap(vm); + if eq? { + return Ok(true); + } + } + Ok(false) + } HeapReadOutput::Dict(dict) => dict.contains_key(item, vm), HeapReadOutput::DictKeysView(view) => { let dict_id = view.get(vm.heap).dict_id(); diff --git a/crates/monty/test_cases/collections__deque.py b/crates/monty/test_cases/collections__deque.py new file mode 100644 index 000000000..8c9cbd123 --- /dev/null +++ b/crates/monty/test_cases/collections__deque.py @@ -0,0 +1,207 @@ +# Tests for collections.deque + +from collections import deque + +# === Construction and iteration === +assert list(deque()) == [], 'empty deque' +assert list(deque([1, 2, 3])) == [1, 2, 3], 'deque from list' +assert list(deque((1, 2, 3))) == [1, 2, 3], 'deque from tuple' +assert list(deque('abc')) == ['a', 'b', 'c'], 'deque from str' +assert list(deque(range(3))) == [0, 1, 2], 'deque from range' +assert len(deque([1, 2, 3])) == 3, 'deque len' + +# === append / appendleft === +d = deque([1, 2, 3]) +d.append(4) +assert list(d) == [1, 2, 3, 4], 'append adds to right' +d.appendleft(0) +assert list(d) == [0, 1, 2, 3, 4], 'appendleft adds to left' + +# === pop / popleft === +assert d.pop() == 4, 'pop returns rightmost' +assert d.popleft() == 0, 'popleft returns leftmost' +assert list(d) == [1, 2, 3], 'after pops' + +# === indexing === +d = deque([10, 20, 30]) +assert d[0] == 10, 'index 0' +assert d[2] == 30, 'index 2' +assert d[-1] == 30, 'negative index' +assert d[-3] == 10, 'negative index full' +d[1] = 99 +assert list(d) == [10, 99, 30], 'setitem' +d[-1] = 77 +assert list(d) == [10, 99, 77], 'setitem negative' + +# === extend / extendleft === +d = deque([1, 2]) +d.extend([3, 4]) +assert list(d) == [1, 2, 3, 4], 'extend appends in order' +d.extendleft([0, -1]) +assert list(d) == [-1, 0, 1, 2, 3, 4], 'extendleft reverses order' + +# === insert === +d = deque([1, 2, 4]) +d.insert(2, 3) +assert list(d) == [1, 2, 3, 4], 'insert at index' +d.insert(0, 0) +assert list(d) == [0, 1, 2, 3, 4], 'insert at front' +d.insert(100, 5) +assert list(d) == [0, 1, 2, 3, 4, 5], 'insert past end appends' + +# === remove === +d = deque([1, 2, 3, 2]) +d.remove(2) +assert list(d) == [1, 3, 2], 'remove first occurrence' + +# === count / index === +d = deque([1, 2, 2, 3, 2]) +assert d.count(2) == 3, 'count occurrences' +assert d.count(9) == 0, 'count absent' +assert d.index(2) == 1, 'index first occurrence' +assert d.index(2, 2) == 2, 'index with start' +assert d.index(3, 0, 4) == 3, 'index with start and stop' + +# === reverse === +d = deque([1, 2, 3]) +d.reverse() +assert list(d) == [3, 2, 1], 'reverse in place' + +# === rotate === +d = deque([1, 2, 3, 4, 5]) +d.rotate(1) +assert list(d) == [5, 1, 2, 3, 4], 'rotate right by 1' +d.rotate(-1) +assert list(d) == [1, 2, 3, 4, 5], 'rotate left by 1' +d.rotate(2) +assert list(d) == [4, 5, 1, 2, 3], 'rotate right by 2' +d.rotate() +assert list(d) == [3, 4, 5, 1, 2], 'rotate default is 1' +d = deque([1, 2, 3]) +d.rotate(7) +assert list(d) == [3, 1, 2], 'rotate wraps modulo length' + +# === clear / copy === +d = deque([1, 2, 3]) +c = d.copy() +assert list(c) == [1, 2, 3], 'copy has same items' +c.append(4) +assert list(d) == [1, 2, 3], 'copy is independent' +d.clear() +assert list(d) == [], 'clear empties deque' +assert len(d) == 0, 'clear len is zero' + +# === maxlen === +assert deque([1, 2, 3]).maxlen is None, 'default maxlen is None' +b = deque([1, 2, 3], maxlen=3) +assert b.maxlen == 3, 'maxlen attribute' +b.append(4) +assert list(b) == [2, 3, 4], 'append drops from left when full' +b.appendleft(1) +assert list(b) == [1, 2, 3], 'appendleft drops from right when full' +b2 = deque(maxlen=2) +b2.extend([1, 2, 3, 4]) +assert list(b2) == [3, 4], 'extend respects maxlen' +z = deque(maxlen=0) +z.append(1) +assert list(z) == [], 'maxlen 0 discards everything' +over = deque([1, 2, 3, 4, 5], maxlen=3) +assert list(over) == [3, 4, 5], 'constructor truncates to maxlen' + +# === membership === +d = deque([1, 2, 3]) +assert 2 in d, 'membership present' +assert 9 not in d, 'membership absent' + +# === reversed === +assert list(reversed(deque([1, 2, 3]))) == [3, 2, 1], 'reversed deque' + +# === bool === +assert bool(deque([1])) is True, 'non-empty deque is truthy' +assert bool(deque()) is False, 'empty deque is falsy' + +# === equality === +assert deque([1, 2, 3]) == deque([1, 2, 3]), 'equal deques' +assert deque([1, 2]) != deque([2, 1]), 'order matters' +assert deque([1, 2]) != deque([1, 2, 3]), 'length matters' +assert (deque([1, 2]) == [1, 2]) is False, 'deque never equals list' + +# === ordering === +assert deque([1, 2]) < deque([1, 3]), 'less than' +assert deque([1, 2, 3]) > deque([1, 2]), 'longer is greater when prefix equal' +assert deque([1, 2]) <= deque([1, 2]), 'less than or equal' + +# === repr === +assert repr(deque([1, 2, 3])) == 'deque([1, 2, 3])', 'repr without maxlen' +assert repr(deque([1, 2], maxlen=5)) == 'deque([1, 2], maxlen=5)', 'repr with maxlen' +assert repr(deque()) == 'deque([])', 'repr empty' + +# === nested references === +d = deque([[1], [2]]) +d[0].append(9) +assert list(d) == [[1, 9], [2]], 'nested list mutation' + +# === error cases === +try: + deque().pop() + assert False, 'expected pop from empty to raise' +except IndexError as e: + assert str(e) == 'pop from an empty deque', 'pop empty message' + +try: + deque().popleft() + assert False, 'expected popleft from empty to raise' +except IndexError as e: + assert str(e) == 'pop from an empty deque', 'popleft empty message' + +try: + deque([1, 2])[5] + assert False, 'expected out of range to raise' +except IndexError as e: + assert str(e) == 'deque index out of range', 'index out of range message' + +try: + deque([1, 2])['a'] + assert False, 'expected non-int index to raise' +except TypeError as e: + assert str(e) == "sequence index must be integer, not 'str'", 'str index message' + +try: + deque([1, 2])[1:2] + assert False, 'expected slice index to raise' +except TypeError as e: + assert str(e) == "sequence index must be integer, not 'slice'", 'slice index message' + +try: + deque(maxlen=-1) + assert False, 'expected negative maxlen to raise' +except ValueError as e: + assert str(e) == 'maxlen must be non-negative', 'negative maxlen message' + +try: + deque([1, 2]).index(9) + assert False, 'expected index of absent to raise' +except ValueError as e: + assert str(e) == '9 is not in deque', 'index absent message' + +try: + deque(['x']).remove('y') + assert False, 'expected remove of absent to raise' +except ValueError as e: + assert str(e) == "'y' is not in deque", 'remove absent message' + +try: + full = deque([1, 2, 3], maxlen=3) + full.insert(0, 9) + assert False, 'expected insert on full bounded deque to raise' +except IndexError as e: + assert str(e) == 'deque already at its maximum size', 'insert full message' + +# === mutation during iteration === +try: + d = deque([1, 2, 3]) + for x in d: + d.append(x) + assert False, 'expected mutation during iteration to raise' +except RuntimeError as e: + assert str(e) == 'deque mutated during iteration', 'mutation message' diff --git a/limitations/collections.md b/limitations/collections.md new file mode 100644 index 000000000..d09db29d5 --- /dev/null +++ b/limitations/collections.md @@ -0,0 +1,40 @@ +# `collections` + +Monty implements the most commonly used members of `collections`. Only the +following names are importable; everything else in CPython's `collections` +(`OrderedDict`, `ChainMap`, `UserDict`, `UserList`, `UserString`, and the +`collections.abc` re-exports) is **not** provided and fails at type-check time +(the custom stub only exposes the implemented names) and at runtime: + +- `deque` + +The dict-like members (`defaultdict`, `Counter`) and `namedtuple` are +documented in their own sections below as they land. + +## `deque` + +Supported: construction from any iterable, `maxlen`, `append`, `appendleft`, +`pop`, `popleft`, `extend`, `extendleft`, `insert`, `remove`, `clear`, `copy`, +`count`, `index` (with optional `start`/`stop`), `reverse`, `rotate`, integer +indexing and assignment, iteration, `reversed()`, `in`, `len()`, `==`/`!=`, +ordering (`<`, `<=`, `>`, `>=`), `bool()`, and `repr()`. + +Divergences from CPython: + +- **No `+`, `*`, `+=`, or `*=`.** Deque concatenation (`d1 + d2`) and repetition + (`d * n`) raise `TypeError: unsupported operand type(s)`. Use `.extend()` / + `.extendleft()` instead. (CPython supports all four.) +- **No slicing.** `d[1:2]` raises `TypeError: sequence index must be integer, + not 'slice'` (CPython raises the same error — deque genuinely does not support + slices — but note the deque also has no `__getitem__` slice fallback here). +- **No `del d[i]`.** Item deletion by index is not supported (the `del` + statement is unimplemented Monty-wide, not deque-specific). +- **No `__reversed__`-based reverse iterator object.** `reversed(d)` still works + (it goes through indexing + length), but there is no distinct + `_collections._deque_reverse_iterator` type. +- **No `__copy__` / pickling / `__reduce__`.** `copy.copy(d)` and pickling are + unavailable (the `copy` and `pickle` modules are not importable anyway); use + the `.copy()` method. +- **`maxlen` argument coercion.** A non-integer `maxlen` is reported by the + constructor body; a `maxlen` that overflows the platform integer range is not + specially handled. diff --git a/limitations/modules.md b/limitations/modules.md index fe768514a..3c4a4f60d 100644 --- a/limitations/modules.md +++ b/limitations/modules.md @@ -9,6 +9,7 @@ and no way for sandboxed code to load additional modules. | Module | See | | ---------- | ------------------------------------ | | `asyncio` | [asyncio.md](asyncio.md) | +| `collections` | [collections.md](collections.md) | | `datetime` | [datetime.md](datetime.md) | | `json` | [json.md](json.md) | | `math` | [math.md](math.md) | @@ -25,9 +26,7 @@ suite; production sandboxes never see it. ## Notable modules NOT available Common modules that are *not* importable in Monty (non-exhaustive): -`abc`, `argparse`, `array`, `base64`, `bisect`, `collections` (no -`defaultdict`, `Counter`, `OrderedDict`, `deque`; `namedtuple` is exposed -as a builtin, not via `collections`), `contextlib`, `copy`, `csv`, +`abc`, `argparse`, `array`, `base64`, `bisect`, `contextlib`, `copy`, `csv`, `ctypes`, `dataclasses` (the `@dataclass` decorator is built in; the module is not importable), `decimal`, `enum`, `fractions`, `functools`, `hashlib`, `heapq`, `hmac`, `http`, `inspect`, `io`, `itertools`, @@ -38,5 +37,6 @@ module is not importable), `decimal`, `enum`, `fractions`, `functools`, Many of these are deliberately excluded (`socket`, `subprocess`, `multiprocessing`, `threading`, `ctypes`) because they would breach the -sandbox. Others (`itertools`, `functools`, `collections`, `enum`) are -simply unimplemented; they may appear over time. +sandbox. Others (`itertools`, `functools`, `enum`) are simply +unimplemented; they may appear over time. `collections` is available but +only implements a subset — see [collections.md](collections.md). From f49b7095d72d97d6afdecb05a8063b194eb076b1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 18:25:50 +0000 Subject: [PATCH 2/3] Add collections.defaultdict and Counter Implement the two dict-like collections members via a single new heap variant `DictSubclass` that wraps a backing `dict` (referenced by id) plus per-kind state, keeping central-dispatch churn low. - `defaultdict`: default_factory attribute, missing-key insertion via builtin factories (int/list/set/dict/...), dict-compatible construction and methods, copy preserving the subclass. - `Counter`: tallying construction from iterables/mappings/kwargs, `c[missing]` returning 0, most_common, elements, update/subtract, and the +/-/&/| arithmetic operators (wired through binary.rs), count-descending repr. - Backing-dict-by-id design lets views (keys/values/items) reference the real dict and lets __getitem__ mutate on a defaultdict miss. - dict()/dict.update() now treat a dict subclass as a mapping; membership, iteration, ordering, and host-boundary conversion handle the new variant. - Consolidated test_cases for both types and limitations/collections.md. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015FUUYYxP68sbHXsdr2qGE8 --- crates/monty/src/bytecode/vm/binary.rs | 54 +- crates/monty/src/heap.rs | 19 +- crates/monty/src/heap_data.rs | 26 +- crates/monty/src/modules/collections.rs | 12 + crates/monty/src/object.rs | 16 +- crates/monty/src/types/dict.rs | 25 +- crates/monty/src/types/dict_subclass.rs | 859 ++++++++++++++++++ crates/monty/src/types/iter.rs | 14 + crates/monty/src/types/mod.rs | 2 + crates/monty/src/types/type.rs | 14 +- crates/monty/src/value.rs | 9 + .../monty/test_cases/collections__counter.py | 106 +++ .../test_cases/collections__defaultdict.py | 111 +++ limitations/collections.md | 57 +- 14 files changed, 1305 insertions(+), 19 deletions(-) create mode 100644 crates/monty/src/types/dict_subclass.rs create mode 100644 crates/monty/test_cases/collections__counter.py create mode 100644 crates/monty/test_cases/collections__defaultdict.py diff --git a/crates/monty/src/bytecode/vm/binary.rs b/crates/monty/src/bytecode/vm/binary.rs index 93c56294f..b8e37a82c 100644 --- a/crates/monty/src/bytecode/vm/binary.rs +++ b/crates/monty/src/bytecode/vm/binary.rs @@ -3,10 +3,15 @@ use super::VM; use crate::{ defer_drop, - exception_private::{ExcType, RunError}, + exception_private::{ExcType, RunError, RunResult}, heap::{HeapData, HeapGuard, HeapReadOutput}, resource::ResourceTracker, - types::{PyTrait, Set, dict_view::collect_iterable_to_set, set::SetBinaryOp}, + types::{ + PyTrait, Set, + dict_subclass::{CounterOp, counter_binop}, + dict_view::collect_iterable_to_set, + set::SetBinaryOp, + }, value::{BitwiseOp, Value}, }; @@ -23,6 +28,11 @@ impl VM<'_, T> { let lhs = this.pop(); defer_drop!(lhs, this); + if let Some(result) = this.try_counter_binop(lhs, rhs, CounterOp::Add)? { + this.push(result); + return Ok(()); + } + match lhs.py_add(rhs, this) { Ok(Some(v)) => { this.push(v); @@ -56,6 +66,11 @@ impl VM<'_, T> { let lhs = this.pop(); defer_drop!(lhs, this); + if let Some(result) = this.try_counter_binop(lhs, rhs, CounterOp::Sub)? { + this.push(result); + return Ok(()); + } + if let Some(result) = this.binary_dict_view_op(lhs, rhs, DictViewBinaryOp::Sub)? { this.push(result); return Ok(()); @@ -283,6 +298,11 @@ impl VM<'_, T> { let lhs = this.pop(); defer_drop!(lhs, this); + if let Some(result) = this.try_counter_binop(lhs, rhs, CounterOp::And)? { + this.push(result); + return Ok(()); + } + if let Some(result) = this.binary_dict_view_op(lhs, rhs, DictViewBinaryOp::And)? { this.push(result); return Ok(()); @@ -307,6 +327,11 @@ impl VM<'_, T> { let lhs = this.pop(); defer_drop!(lhs, this); + if let Some(result) = this.try_counter_binop(lhs, rhs, CounterOp::Or)? { + this.push(result); + return Ok(()); + } + if let Some(result) = this.binary_dict_view_op(lhs, rhs, DictViewBinaryOp::Or)? { this.push(result); return Ok(()); @@ -454,6 +479,31 @@ impl VM<'_, T> { let result_id = this.heap.allocate(result)?; Ok(Some(Value::Ref(result_id))) } + + /// Attempts a `Counter` arithmetic operation (`+`, `-`, `&`, `|`). + /// + /// Returns `Ok(None)` unless both operands are `Counter`s, in which case it + /// returns the resulting new `Counter`. Handled here (rather than via + /// `py_add`/`py_sub`) so the full `RunError` channel is available — the + /// `PyTrait` arithmetic hooks only surface `ResourceError`. + fn try_counter_binop(&mut self, lhs: &Value, rhs: &Value, op: CounterOp) -> RunResult> { + let this = self; + let (Value::Ref(l), Value::Ref(r)) = (lhs, rhs) else { + return Ok(None); + }; + if !matches!(this.heap.get(*l), HeapData::DictSubclass(_)) + || !matches!(this.heap.get(*r), HeapData::DictSubclass(_)) + { + return Ok(None); + } + let HeapReadOutput::DictSubclass(a) = this.heap.read(*l) else { + unreachable!("DictSubclass") + }; + let HeapReadOutput::DictSubclass(b) = this.heap.read(*r) else { + unreachable!("DictSubclass") + }; + counter_binop(&a, &b, op, this) + } } /// Supported dict-view set-like operators. diff --git a/crates/monty/src/heap.rs b/crates/monty/src/heap.rs index 04f23d796..0fcf3d3cb 100644 --- a/crates/monty/src/heap.rs +++ b/crates/monty/src/heap.rs @@ -19,9 +19,9 @@ use crate::{ heap_data::{CellValue, Closure, FunctionDefaults}, resource::{ResourceError, ResourceTracker}, types::{ - BoundMethod, Bytes, Class, Dataclass, Deque, Dict, DictItemsView, DictKeysView, DictValuesView, FrozenSet, - Instance, List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, Range, ReMatch, RePattern, Set, Slice, - Str, TimeZone, Tuple, date, datetime, timedelta, timezone, + BoundMethod, Bytes, Class, Dataclass, Deque, Dict, DictItemsView, DictKeysView, DictSubclass, DictSubclassKind, + DictValuesView, FrozenSet, Instance, List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, Range, + ReMatch, RePattern, Set, Slice, Str, TimeZone, Tuple, date, datetime, timedelta, timezone, }, value::Value, }; @@ -209,6 +209,7 @@ pub enum HeapReadOutput<'a> { NamedTuple(HeapRead<'a, NamedTuple>), Deque(HeapRead<'a, Deque>), Dict(HeapRead<'a, Dict>), + DictSubclass(HeapRead<'a, DictSubclass>), DictItemsView(HeapRead<'a, DictItemsView>), DictKeysView(HeapRead<'a, DictKeysView>), DictValuesView(HeapRead<'a, DictValuesView>), @@ -595,6 +596,7 @@ impl<'a> HeapPtr<'a> { HeapData::NamedTuple(named_tuple) => HeapReadOutput::NamedTuple(heap_read(base, named_tuple, readers)), HeapData::Deque(deque) => HeapReadOutput::Deque(heap_read(base, deque, readers)), HeapData::Dict(dict) => HeapReadOutput::Dict(heap_read(base, dict, readers)), + HeapData::DictSubclass(sub) => HeapReadOutput::DictSubclass(heap_read(base, sub, readers)), HeapData::DictItemsView(v) => HeapReadOutput::DictItemsView(heap_read(base, v, readers)), HeapData::DictKeysView(v) => HeapReadOutput::DictKeysView(heap_read(base, v, readers)), HeapData::DictValuesView(v) => HeapReadOutput::DictValuesView(heap_read(base, v, readers)), @@ -1459,6 +1461,16 @@ fn for_each_child_id(data: &HeapData, mut on_child: F) { } } } + HeapData::DictSubclass(sub) => { + // The backing dict is always a heap ref; the factory may be too. + on_child(sub.dict_id()); + if let DictSubclassKind::DefaultDict { + factory: Value::Ref(id), + } = sub.kind() + { + on_child(*id); + } + } HeapData::Dict(dict) => { // Skip iteration if no refs - major GC optimization for dicts of primitives if !dict.has_refs() { @@ -1662,6 +1674,7 @@ fn py_dec_ref_ids_for_data(data: &mut HeapData, stack: &mut Vec) { HeapData::NamedTuple(nt) => nt.py_dec_ref_ids(stack), HeapData::Deque(dq) => dq.py_dec_ref_ids(stack), HeapData::Dict(d) => d.py_dec_ref_ids(stack), + HeapData::DictSubclass(sub) => sub.py_dec_ref_ids(stack), HeapData::DictKeysView(view) => view.py_dec_ref_ids(stack), HeapData::DictItemsView(view) => view.py_dec_ref_ids(stack), HeapData::DictValuesView(view) => view.py_dec_ref_ids(stack), diff --git a/crates/monty/src/heap_data.rs b/crates/monty/src/heap_data.rs index 4aeea1c57..99b5f75f7 100644 --- a/crates/monty/src/heap_data.rs +++ b/crates/monty/src/heap_data.rs @@ -18,9 +18,10 @@ use crate::{ heap::{DropWithHeap, HeapId, HeapItem, HeapReadOutput}, intern::FunctionId, types::{ - BoundMethod, Bytes, Class, Dataclass, Deque, Dict, DictItemsView, DictKeysView, DictValuesView, FrozenSet, - Instance, LazyHeapSet, List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, PyTrait, Range, ReMatch, - RePattern, Set, Slice, Str, Tuple, Type, date, datetime, str::allocate_string, timedelta, timezone, + BoundMethod, Bytes, Class, Dataclass, Deque, Dict, DictItemsView, DictKeysView, DictSubclass, DictSubclassKind, + DictValuesView, FrozenSet, Instance, LazyHeapSet, List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, + PyTrait, Range, ReMatch, RePattern, Set, Slice, Str, Tuple, Type, date, datetime, str::allocate_string, + timedelta, timezone, }, value::{EitherStr, Value, eq_bigint, eq_bytes, eq_ext_function, eq_str}, }; @@ -40,6 +41,9 @@ pub(crate) enum HeapData { /// A `collections.deque` double-ended queue. Deque(Deque), Dict(Dict), + /// A `dict` subclass — `collections.defaultdict` or `collections.Counter`. + /// Wraps a backing `Dict` (referenced by id) plus per-kind state. + DictSubclass(DictSubclass), DictKeysView(DictKeysView), DictItemsView(DictItemsView), DictValuesView(DictValuesView), @@ -173,6 +177,7 @@ impl HeapData { | Self::NamedTuple(_) | Self::Deque(_) | Self::Dict(_) + | Self::DictSubclass(_) | Self::DictKeysView(_) | Self::DictItemsView(_) | Self::DictValuesView(_) @@ -212,6 +217,11 @@ impl HeapData { Self::Tuple(_) | Self::NamedTuple(_) => Type::Tuple, Self::Deque(_) => Type::Deque, Self::Dict(_) => Type::Dict, + // A dict subclass reports its concrete Python type by kind. + Self::DictSubclass(sub) => match sub.kind() { + DictSubclassKind::DefaultDict { .. } => Type::DefaultDict, + DictSubclassKind::Counter => Type::Counter, + }, Self::DictKeysView(_) => Type::DictKeys, Self::DictItemsView(_) => Type::DictItems, Self::DictValuesView(_) => Type::DictValues, @@ -251,6 +261,7 @@ impl HeapData { Self::NamedTuple(nt) => nt.py_estimate_size(), Self::Deque(dq) => dq.py_estimate_size(), Self::Dict(d) => d.py_estimate_size(), + Self::DictSubclass(sub) => sub.py_estimate_size(), Self::DictKeysView(view) => view.py_estimate_size(), Self::DictItemsView(view) => view.py_estimate_size(), Self::DictValuesView(view) => view.py_estimate_size(), @@ -468,6 +479,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::NamedTuple(nt) => nt.py_bool(vm), Self::Deque(dq) => dq.py_bool(vm), Self::Dict(d) => d.py_bool(vm), + Self::DictSubclass(sub) => sub.py_bool(vm), Self::DictKeysView(view) => view.py_bool(vm), Self::DictItemsView(view) => view.py_bool(vm), Self::DictValuesView(view) => view.py_bool(vm), @@ -510,6 +522,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { HeapReadOutput::Tuple(t) => Ok(t.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::Deque(dq) => Ok(dq.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::Dict(dict) => Ok(dict.py_call_attr(self_id, vm, attr, args)?), + HeapReadOutput::DictSubclass(sub) => Ok(sub.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::DictKeysView(view) => Ok(view.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::DictItemsView(view) => Ok(view.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::DictValuesView(view) => Ok(view.py_call_attr(self_id, vm, attr, args)?), @@ -587,6 +600,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::NamedTuple(nt) => nt.py_type(vm), Self::Deque(dq) => dq.py_type(vm), Self::Dict(d) => d.py_type(vm), + Self::DictSubclass(sub) => sub.py_type(vm), Self::DictKeysView(v) => v.py_type(vm), Self::DictItemsView(v) => v.py_type(vm), Self::DictValuesView(v) => v.py_type(vm), @@ -625,6 +639,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::NamedTuple(nt) => nt.py_len(vm), Self::Deque(dq) => dq.py_len(vm), Self::Dict(d) => d.py_len(vm), + Self::DictSubclass(sub) => sub.py_len(vm), Self::DictKeysView(view) => view.py_len(vm), Self::DictItemsView(view) => view.py_len(vm), Self::DictValuesView(view) => view.py_len(vm), @@ -666,6 +681,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { HeapReadOutput::NamedTuple(a) => a.py_eq_impl(other, vm), HeapReadOutput::Deque(a) => a.py_eq_impl(other, vm), HeapReadOutput::Dict(a) => a.py_eq_impl(other, vm), + HeapReadOutput::DictSubclass(a) => a.py_eq_impl(other, vm), HeapReadOutput::Set(a) => a.py_eq_impl(other, vm), HeapReadOutput::FrozenSet(a) => a.py_eq_impl(other, vm), HeapReadOutput::DictKeysView(a) => a.py_eq_impl(other, vm), @@ -762,6 +778,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::NamedTuple(nt) => nt.py_repr_fmt(f, vm, heap_ids), Self::Deque(dq) => dq.py_repr_fmt(f, vm, heap_ids), Self::Dict(d) => d.py_repr_fmt(f, vm, heap_ids), + Self::DictSubclass(sub) => sub.py_repr_fmt(f, vm, heap_ids), Self::DictKeysView(view) => view.py_repr_fmt(f, vm, heap_ids), Self::DictItemsView(view) => view.py_repr_fmt(f, vm, heap_ids), Self::DictValuesView(view) => view.py_repr_fmt(f, vm, heap_ids), @@ -967,6 +984,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::NamedTuple(nt) => nt.py_getitem(key, vm), Self::Deque(dq) => dq.py_getitem(key, vm), Self::Dict(d) => d.py_getitem(key, vm), + Self::DictSubclass(sub) => sub.py_getitem(key, vm), Self::Range(r) => r.py_getitem(key, vm), Self::ReMatch(m) => m.py_getitem(key, vm), _ => Err(ExcType::type_error_not_sub(&self.py_type(vm).name(vm.heap, vm.interns))), @@ -978,6 +996,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_setitem(key, value, vm), Self::Deque(dq) => dq.py_setitem(key, value, vm), Self::Dict(d) => d.py_setitem(key, value, vm), + Self::DictSubclass(sub) => sub.py_setitem(key, value, vm), _ => { key.drop_with_heap(vm); value.drop_with_heap(vm); @@ -997,6 +1016,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::NamedTuple(nt) => nt.py_getattr(attr, vm), Self::Deque(dq) => dq.py_getattr(attr, vm), Self::Dict(d) => d.py_getattr(attr, vm), + Self::DictSubclass(sub) => sub.py_getattr(attr, vm), Self::DictKeysView(view) => view.py_getattr(attr, vm), Self::DictItemsView(view) => view.py_getattr(attr, vm), Self::DictValuesView(view) => view.py_getattr(attr, vm), diff --git a/crates/monty/src/modules/collections.rs b/crates/monty/src/modules/collections.rs index 7bf68c676..5c3c94c2b 100644 --- a/crates/monty/src/modules/collections.rs +++ b/crates/monty/src/modules/collections.rs @@ -2,6 +2,8 @@ //! //! Exposes the most commonly used container datatypes: //! - `deque` — a double-ended queue ([`Type::Deque`]) +//! - `defaultdict` — a dict with a default factory ([`Type::DefaultDict`]) +//! - `Counter` — a dict for counting ([`Type::Counter`]) //! //! Types are exposed as callable `Value::Builtin(Builtins::Type(...))` module //! attributes; their construction and behavior live in the corresponding @@ -28,6 +30,16 @@ pub fn create_module(vm: &mut VM<'_, impl ResourceTracker>) -> Result Some(Type::Tuple), Self::NamedTuple => Some(Type::NamedTuple), Self::Deque => Some(Type::Deque), + Self::DefaultDict => Some(Type::DefaultDict), + Self::Counter => Some(Type::Counter), Self::Dict => Some(Type::Dict), Self::DictKeys => Some(Type::DictKeys), Self::DictItems => Some(Type::DictItems), @@ -505,6 +509,8 @@ impl MontyType { Type::Tuple => Self::Tuple, Type::NamedTuple => Self::NamedTuple, Type::Deque => Self::Deque, + Type::DefaultDict => Self::DefaultDict, + Type::Counter => Self::Counter, Type::Dict => Self::Dict, Type::DictKeys => Self::DictKeys, Type::DictItems => Self::DictItems, @@ -846,7 +852,7 @@ impl MontyObject { HeapData::List(_) => Self::Cycle(id.index(), "[...]".to_owned()), HeapData::Tuple(_) | HeapData::NamedTuple(_) => Self::Cycle(id.index(), "(...)".to_owned()), HeapData::Deque(_) => Self::Cycle(id.index(), "deque([...])".to_owned()), - HeapData::Dict(_) => Self::Cycle(id.index(), "{...}".to_owned()), + HeapData::Dict(_) | HeapData::DictSubclass(_) => Self::Cycle(id.index(), "{...}".to_owned()), _ => Self::Cycle(id.index(), "...".to_owned()), }; } @@ -910,6 +916,14 @@ impl MontyObject { } Self::List(items) } + HeapReadOutput::DictSubclass(sub) => { + // Represent a defaultdict/Counter to the host as its + // backing dict's contents. + let dict_id = sub.get(vm.heap).dict_id(); + let item = Value::Ref(dict_id).clone_with_heap(vm.heap); + defer_drop!(item, vm); + Self::from_value_inner(item, vm, visited) + } HeapReadOutput::Dict(dict) => { let len = dict.get(vm.heap).len(); let mut pairs = Vec::with_capacity(len); diff --git a/crates/monty/src/types/dict.rs b/crates/monty/src/types/dict.rs index 732e260ce..4aef33b87 100644 --- a/crates/monty/src/types/dict.rs +++ b/crates/monty/src/types/dict.rs @@ -1077,13 +1077,28 @@ struct DictUpdateArgs { /// /// This is shared between `dict()` construction and `dict.update()` so both /// entry points follow identical positional-source semantics. -fn dict_merge_from_value(dict: &mut Dict, other_value: Value, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult<()> { +pub(crate) fn dict_merge_from_value( + dict: &mut Dict, + other_value: Value, + vm: &mut VM<'_, impl ResourceTracker>, +) -> RunResult<()> { let mut other_value_guard = HeapGuard::new(other_value, vm); { let (other_value, vm) = other_value_guard.as_parts(); - if let Value::Ref(id) = other_value - && let HeapData::Dict(src_dict) = vm.heap.get(*id) - { + // Treat a plain dict — or a dict subclass (defaultdict/Counter), via its + // backing dict — as a mapping, cloning its key/value pairs. + let mapping_id = match other_value { + Value::Ref(id) => match vm.heap.get(*id) { + HeapData::Dict(_) => Some(*id), + HeapData::DictSubclass(sub) => Some(sub.dict_id()), + _ => None, + }, + _ => None, + }; + if let Some(mapping_id) = mapping_id { + let HeapData::Dict(src_dict) = vm.heap.get(mapping_id) else { + unreachable!("mapping id references a dict") + }; // Clone key-value pairs from the source dict. let pairs: Vec<(Value, Value)> = src_dict .iter() @@ -1157,7 +1172,7 @@ fn dict_merge_from_iterable_pairs( /// /// This helper drains `kwargs` safely on error so all values are dropped /// correctly, then inserts each key-value pair into `dict`. -fn dict_merge_from_kwargs( +pub(crate) fn dict_merge_from_kwargs( dict: &mut Dict, kwargs: KwargsValues, vm: &mut VM<'_, impl ResourceTracker>, diff --git a/crates/monty/src/types/dict_subclass.rs b/crates/monty/src/types/dict_subclass.rs new file mode 100644 index 000000000..b84eb2d8a --- /dev/null +++ b/crates/monty/src/types/dict_subclass.rs @@ -0,0 +1,859 @@ +use std::{cmp::Reverse, fmt::Write, mem}; + +use super::{Dict, LazyHeapSet, MontyIter, PyTrait, allocate_tuple}; +use crate::{ + args::{ArgValues, KwargsValues}, + bytecode::{CallResult, VM}, + defer_drop, defer_drop_mut, + exception_private::{ExcType, RunResult, SimpleException}, + heap::{DropWithHeap, HeapData, HeapGuard, HeapId, HeapItem, HeapRead, HeapReadOutput}, + intern::StaticStrings, + resource::ResourceTracker, + types::{ + List, Type, + dict::{dict_merge_from_kwargs, dict_merge_from_value}, + }, + value::{EitherStr, VALUE_SIZE, Value}, +}; + +/// A `dict` subclass that wraps a backing `dict` heap entry plus a small amount +/// of extra state. Covers `collections.defaultdict` and `collections.Counter` +/// in a single heap variant to keep central dispatch churn low. +/// +/// The backing dict is a **separate** `HeapData::Dict` entry referenced by +/// `dict_id` (owned: one ref held for the wrapper's lifetime). Keeping it as its +/// own entry — rather than embedding a `Dict` field — lets `keys()`/`values()`/ +/// `items()` views reference the real dict, and lets `__getitem__` (which takes +/// `&self`) still mutate the backing dict on a `defaultdict` miss, since the +/// backing dict is a distinct heap entry. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct DictSubclass { + /// HeapId of the backing `HeapData::Dict`. + dict_id: HeapId, + /// Which subclass this is, plus any per-kind state. + kind: DictSubclassKind, +} + +/// The concrete `dict` subclass, with per-kind state. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub(crate) enum DictSubclassKind { + /// `defaultdict` — `factory` is the `default_factory` (may be `Value::None`). + DefaultDict { factory: Value }, + /// `Counter`. + Counter, +} + +impl DictSubclass { + /// Returns the backing dict's HeapId. + #[must_use] + pub fn dict_id(&self) -> HeapId { + self.dict_id + } + + /// Returns a reference to the subclass kind. + #[must_use] + pub fn kind(&self) -> &DictSubclassKind { + &self.kind + } + + /// Implements the `defaultdict([default_factory[, ...]], **kwargs)` constructor. + /// + /// The first positional argument (if any) is the `default_factory`, which + /// must be callable or `None`; the remaining positional (an optional + /// mapping/iterable) and keyword arguments initialize the dict exactly like + /// `dict(...)`. + pub fn init_defaultdict(vm: &mut VM<'_, impl ResourceTracker>, args: ArgValues) -> RunResult { + let (pos, kwargs) = args.into_parts(); + let mut pos: Vec = pos.collect(); + + // First positional is the factory; validate callable-or-None. + let factory = if pos.is_empty() { Value::None } else { pos.remove(0) }; + if !matches!(factory, Value::None) && !is_callable(&factory, vm) { + factory.drop_with_heap(vm); + pos.drop_with_heap(vm); + kwargs.drop_with_heap(vm); + return Err(SimpleException::new_msg(ExcType::TypeError, "first argument must be callable or None").into()); + } + if pos.len() > 1 { + // The excess positionals would be handed to `dict(...)`, so CPython + // surfaces the *dict* arity error (e.g. "dict expected at most 1 + // argument, got 2"), counting only the non-factory positionals. + let got = pos.len(); + factory.drop_with_heap(vm); + pos.drop_with_heap(vm); + kwargs.drop_with_heap(vm); + return Err(ExcType::type_error_at_most("dict", 1, got)); + } + let source = pos.into_iter().next(); + + // Build the backing dict off-heap (via the scoped guard pattern), then + // allocate it and wrap it in the DictSubclass. + let mut dict_guard = HeapGuard::new(Dict::new(), vm); + { + let (dict, vm) = dict_guard.as_parts_mut(); + if let Some(source) = source { + dict_merge_from_value(dict, source, vm)?; + } + dict_merge_from_kwargs(dict, kwargs, vm)?; + } + let dict = dict_guard.into_inner(); + Self::finish(vm, dict, DictSubclassKind::DefaultDict { factory }) + } + + /// Implements the `Counter([iterable_or_mapping], **kwargs)` constructor, + /// tallying counts from the source and keyword arguments. + pub fn init_counter(vm: &mut VM<'_, impl ResourceTracker>, args: ArgValues) -> RunResult { + let (pos, kwargs) = args.into_parts(); + let pos: Vec = pos.collect(); + + if pos.len() > 1 { + // CPython's C `Counter.__init__` reports a clinic-style range error + // counting the implicit `self` (min 1, max 2). + let actual = pos.len() + 1; + pos.drop_with_heap(vm); + kwargs.drop_with_heap(vm); + return Err(ExcType::type_error_too_many_positional_range( + "Counter.__init__", + 1, + 2, + actual, + 0, + )); + } + let source = pos.into_iter().next(); + + // Allocate the empty backing dict first so tallying can use the + // `HeapRead` get/set API (which needs a heap-resident dict). Guard + // its owned ref so it is freed if tallying fails partway. + let dict_id = vm.heap.allocate(HeapData::Dict(Dict::new()))?; + let mut dict_guard = HeapGuard::new(Value::Ref(dict_id), vm); + { + let (_backing, vm) = dict_guard.as_parts_mut(); + if let Some(source) = source + && let Err(e) = counter_add_from_source(dict_id, source, 1, vm) + { + kwargs.drop_with_heap(vm); + return Err(e); + } + counter_add_from_kwargs(dict_id, kwargs, 1, vm)?; + } + // Transfer the backing dict's owned ref to the wrapper: `forget` the + // guard's `Value::Ref` so its count is not decremented here — the + // wrapper now owns it and releases it via `py_dec_ref_ids`. + mem::forget(dict_guard.into_inner()); + Self::wrap(vm, dict_id, DictSubclassKind::Counter) + } + + /// Allocates the backing `dict` from an already-built `Dict`, then wraps it. + fn finish(vm: &mut VM<'_, impl ResourceTracker>, dict: Dict, kind: DictSubclassKind) -> RunResult { + let dict_id = vm.heap.allocate(HeapData::Dict(dict))?; + Self::wrap(vm, dict_id, kind) + } + + /// Wraps an already-allocated backing dict (whose owned ref is transferred + /// into the wrapper) in a `DictSubclass` heap entry. + /// + /// A failure here is a terminal `ResourceError` (allocator over budget), so + /// the leaked backing-dict/factory refs on that path are acceptable per the + /// project's resource-exhaustion contract. + fn wrap(vm: &mut VM<'_, impl ResourceTracker>, dict_id: HeapId, kind: DictSubclassKind) -> RunResult { + let id = vm.heap.allocate(HeapData::DictSubclass(Self { dict_id, kind }))?; + Ok(Value::Ref(id)) + } +} + +impl<'h> HeapRead<'h, DictSubclass> { + /// Returns the backing dict's HeapId. + fn inner_id(&self, vm: &VM<'h, impl ResourceTracker>) -> HeapId { + self.get(vm.heap).dict_id + } + + /// Reads the backing dict as a `HeapRead`. + fn inner<'a>(&self, vm: &'a mut VM<'h, impl ResourceTracker>) -> HeapRead<'h, Dict> { + let dict_id = self.inner_id(vm); + match vm.heap.read(dict_id) { + HeapReadOutput::Dict(dict) => dict, + _ => unreachable!("DictSubclass.dict_id must reference a Dict"), + } + } + + /// Whether this is a `Counter` (vs `defaultdict`). + fn is_counter(&self, vm: &VM<'h, impl ResourceTracker>) -> bool { + matches!(self.get(vm.heap).kind, DictSubclassKind::Counter) + } +} + +impl<'h> PyTrait<'h> for HeapRead<'h, DictSubclass> { + fn py_type(&self, vm: &VM<'h, impl ResourceTracker>) -> Type { + match self.get(vm.heap).kind { + DictSubclassKind::DefaultDict { .. } => Type::DefaultDict, + DictSubclassKind::Counter => Type::Counter, + } + } + + fn py_len(&self, vm: &VM<'h, impl ResourceTracker>) -> Option { + let dict_id = self.get(vm.heap).dict_id; + match vm.heap.read(dict_id) { + HeapReadOutput::Dict(dict) => Some(dict.get(vm.heap).len()), + _ => None, + } + } + + fn py_bool(&self, vm: &mut VM<'h, impl ResourceTracker>) -> bool { + let dict = self.inner(vm); + !dict.get(vm.heap).is_empty() + } + + fn py_getitem(&self, key: &Value, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let dict = self.inner(vm); + if let Some(value) = dict.dict_get(key, vm)? { + return Ok(value); + } + // Miss: Counter returns 0; defaultdict calls its factory (builtins only). + match &self.get(vm.heap).kind { + DictSubclassKind::Counter => Ok(Value::Int(0)), + DictSubclassKind::DefaultDict { factory } => { + if matches!(factory, Value::None) { + return Err(ExcType::key_error(key, vm)); + } + let factory = factory.clone_with_heap(vm.heap); + defer_drop!(factory, vm); + let default = call_default_factory(factory, vm)?; + // Insert key -> default into the backing dict and return default. + let mut dict = self.inner(vm); + let key_clone = key.clone_with_heap(vm.heap); + let default_clone = default.clone_with_heap(vm.heap); + if let Some(old) = dict.set(key_clone, default_clone, vm)? { + old.drop_with_heap(vm); + } + Ok(default) + } + } + } + + fn py_setitem(&mut self, key: Value, value: Value, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult<()> { + let mut dict = self.inner(vm); + if let Some(old_value) = dict.set(key, value, vm)? { + old_value.drop_with_heap(vm); + } + Ok(()) + } + + fn py_eq_impl(&self, other: &Value, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult> { + // A dict subclass compares equal to a plain dict or another subclass with + // the same items (kind/factory are ignored), matching CPython. + let other_dict_id = match other { + Value::Ref(oid) => match vm.heap.get(*oid) { + HeapData::Dict(_) => *oid, + HeapData::DictSubclass(sub) => sub.dict_id(), + _ => return Ok(None), + }, + _ => return Ok(None), + }; + let self_dict_id = self.get(vm.heap).dict_id; + let HeapReadOutput::Dict(a) = vm.heap.read(self_dict_id) else { + unreachable!("backing dict") + }; + let HeapReadOutput::Dict(b) = vm.heap.read(other_dict_id) else { + unreachable!("backing dict") + }; + Ok(Some(a.eq_dict(&b, vm)?)) + } + + fn py_repr_fmt( + &self, + f: &mut impl Write, + vm: &mut VM<'h, impl ResourceTracker>, + heap_ids: &mut LazyHeapSet, + ) -> RunResult<()> { + match &self.get(vm.heap).kind { + DictSubclassKind::DefaultDict { factory } => { + let factory = factory.clone_with_heap(vm.heap); + defer_drop!(factory, vm); + f.write_str("defaultdict(")?; + factory.py_repr_fmt(f, vm, heap_ids)?; + f.write_str(", ")?; + let dict = self.inner(vm); + dict.py_repr_fmt(f, vm, heap_ids)?; + f.write_char(')')?; + Ok(()) + } + DictSubclassKind::Counter => counter_repr_fmt(self, f, vm, heap_ids), + } + } + + fn py_getattr(&self, attr: &EitherStr, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult> { + // `defaultdict.default_factory` is the only attribute; everything else is + // a method (via py_call_attr) or absent. + if attr.static_string() == Some(StaticStrings::DefaultFactory) + && let DictSubclassKind::DefaultDict { factory } = &self.get(vm.heap).kind + { + let value = factory.clone_with_heap(vm.heap); + Ok(Some(CallResult::Value(value))) + } else { + Ok(None) + } + } + + fn py_call_attr( + &mut self, + _self_id: HeapId, + vm: &mut VM<'h, impl ResourceTracker>, + attr: &EitherStr, + args: ArgValues, + ) -> RunResult { + let Some(method) = attr.static_string() else { + args.drop_with_heap(vm); + return Err(ExcType::attribute_error(self.py_type(vm), attr.as_str(vm.interns))); + }; + let counter = self.is_counter(vm); + match method { + // Same-kind-preserving copy. + StaticStrings::Copy => { + args.check_zero_args("copy", vm.heap)?; + self.copy_subclass(vm).map(CallResult::Value) + } + StaticStrings::MostCommon if counter => counter_most_common(self, args, vm).map(CallResult::Value), + StaticStrings::Elements if counter => counter_elements(self, args, vm).map(CallResult::Value), + StaticStrings::Subtract if counter => counter_subtract(self, args, vm).map(CallResult::Value), + StaticStrings::Update if counter => counter_update(self, args, vm).map(CallResult::Value), + // Shared dict methods delegate to the backing dict. Views reference + // the backing dict id, so they iterate correctly. + StaticStrings::Get + | StaticStrings::Keys + | StaticStrings::Values + | StaticStrings::Items + | StaticStrings::Pop + | StaticStrings::Popitem + | StaticStrings::Clear + | StaticStrings::Setdefault + | StaticStrings::Update => { + let dict_id = self.inner_id(vm); + let HeapReadOutput::Dict(mut dict) = vm.heap.read(dict_id) else { + unreachable!("backing dict") + }; + dict.py_call_attr(dict_id, vm, attr, args) + } + _ => { + args.drop_with_heap(vm); + Err(ExcType::attribute_error(self.py_type(vm), attr.as_str(vm.interns))) + } + } + } +} + +impl<'h> HeapRead<'h, DictSubclass> { + /// Returns a shallow copy preserving the subclass kind (and factory). + fn copy_subclass(&self, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + // Clone the backing dict's entries into a fresh dict. + let dict = self.inner(vm); + let len = dict.get(vm.heap).len(); + let mut pairs = Vec::with_capacity(len); + for i in 0..len { + let key = dict + .get(vm.heap) + .key_at(i) + .expect("index in range") + .clone_with_heap(vm.heap); + let value = dict + .get(vm.heap) + .value_at(i) + .expect("index in range") + .clone_with_heap(vm.heap); + pairs.push((key, value)); + } + let new_dict = Dict::from_pairs(pairs, vm)?; + let kind = match &self.get(vm.heap).kind { + DictSubclassKind::DefaultDict { factory } => DictSubclassKind::DefaultDict { + factory: factory.clone_with_heap(vm.heap), + }, + DictSubclassKind::Counter => DictSubclassKind::Counter, + }; + DictSubclass::finish(vm, new_dict, kind) + } +} + +impl HeapItem for DictSubclass { + fn py_estimate_size(&self) -> usize { + mem::size_of::() + VALUE_SIZE + } + + fn py_dec_ref_ids(&mut self, stack: &mut Vec) { + stack.push(self.dict_id); + if let DictSubclassKind::DefaultDict { factory } = &mut self.kind { + factory.py_dec_ref_ids(stack); + } + } +} + +/// Calls a `defaultdict` factory to produce a default value. +/// +/// Only builtin callables (e.g. `int`, `list`, `set`, `dict`) are supported — +/// they can be invoked synchronously. User-defined function factories cannot be +/// called from `__getitem__` (which cannot push a VM frame), so they raise a +/// `TypeError`; this divergence is documented in `limitations/collections.md`. +fn call_default_factory(factory: &Value, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult { + match factory { + Value::Builtin(builtin) => match builtin.call(vm, ArgValues::Empty)? { + CallResult::Value(v) => Ok(v), + _ => Err(SimpleException::new_msg( + ExcType::TypeError, + "defaultdict default_factory returned a non-value result", + ) + .into()), + }, + _ => Err(SimpleException::new_msg( + ExcType::TypeError, + "defaultdict with a non-builtin default_factory is not supported in Monty", + ) + .into()), + } +} + +/// Returns whether `value` is callable (usable as a `defaultdict` factory). +fn is_callable(value: &Value, vm: &VM<'_, impl ResourceTracker>) -> bool { + match value { + Value::DefFunction(_) | Value::Builtin(_) | Value::ExtFunction(_) | Value::ModuleFunction(_) => true, + Value::Ref(id) => matches!( + vm.heap.get(*id), + HeapData::Closure(_) | HeapData::FunctionDefaults(_) | HeapData::ExtFunction(_) | HeapData::Class(_) + ), + _ => false, + } +} + +/// Writes `Counter`'s repr: `Counter()` when empty, otherwise +/// `Counter({k: v, ...})` with entries ordered by count descending (ties keep +/// insertion order), matching CPython. +fn counter_repr_fmt<'h>( + counter: &HeapRead<'h, DictSubclass>, + f: &mut impl Write, + vm: &mut VM<'h, impl ResourceTracker>, + heap_ids: &mut LazyHeapSet, +) -> RunResult<()> { + let order = counter_indices_by_count(counter, vm); + if order.is_empty() { + return Ok(f.write_str("Counter()")?); + } + let Ok(mut guard) = vm.recursion_guard() else { + return Ok(f.write_str("Counter({...})")?); + }; + let vm = &mut *guard; + f.write_str("Counter({")?; + for (i, &idx) in order.iter().enumerate() { + if i > 0 { + f.write_str(", ")?; + } + let dict = counter.inner(vm); + let key = dict + .get(vm.heap) + .key_at(idx) + .expect("index in range") + .clone_with_heap(vm.heap); + defer_drop!(key, vm); + key.py_repr_fmt(f, vm, heap_ids)?; + f.write_str(": ")?; + let dict = counter.inner(vm); + let value = dict + .get(vm.heap) + .value_at(idx) + .expect("index in range") + .clone_with_heap(vm.heap); + defer_drop!(value, vm); + value.py_repr_fmt(f, vm, heap_ids)?; + } + f.write_str("})")?; + Ok(()) +} + +/// Implements `Counter.most_common([n])`, returning a list of `(elem, count)` +/// tuples ordered by count descending (ties keep insertion order). `n` limits +/// the result to the top-n; omitting it returns all entries. +fn counter_most_common<'h>( + counter: &HeapRead<'h, DictSubclass>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + let n = match args.get_zero_one_arg("most_common", vm.heap)? { + Some(v) if matches!(v, Value::None) => { + v.drop_with_heap(vm.heap); + None + } + Some(v) => { + let result = v.as_int(vm); + v.drop_with_heap(vm.heap); + Some(result?) + } + None => None, + }; + let order = counter_indices_by_count(counter, vm); + let limit = match n { + Some(n) if n < 0 => 0, + Some(n) => usize::try_from(n).unwrap_or(usize::MAX).min(order.len()), + None => order.len(), + }; + let mut items = Vec::with_capacity(limit); + for &idx in order.iter().take(limit) { + let dict = counter.inner(vm); + let key = dict + .get(vm.heap) + .key_at(idx) + .expect("index in range") + .clone_with_heap(vm.heap); + let value = dict + .get(vm.heap) + .value_at(idx) + .expect("index in range") + .clone_with_heap(vm.heap); + let pair = allocate_tuple(smallvec::smallvec![key, value], vm.heap)?; + items.push(pair); + } + let id = vm.heap.allocate(HeapData::List(List::new(items)))?; + Ok(Value::Ref(id)) +} + +/// Implements `Counter.elements()`, returning a list that repeats each element +/// by its count (elements with count <= 0 are skipped), in insertion order. +/// +/// CPython returns a lazy `itertools.chain` iterator; Monty returns a `list` +/// (documented in `limitations/collections.md`). +fn counter_elements<'h>( + counter: &HeapRead<'h, DictSubclass>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + args.check_zero_args("elements", vm.heap)?; + let dict = counter.inner(vm); + let len = dict.get(vm.heap).len(); + let mut items: Vec = Vec::new(); + for i in 0..len { + let count = dict + .get(vm.heap) + .value_at(i) + .expect("index in range") + .as_int(vm) + .unwrap_or(0); + for _ in 0..count.max(0) { + let key = dict + .get(vm.heap) + .key_at(i) + .expect("index in range") + .clone_with_heap(vm.heap); + vm.heap.track_growth(VALUE_SIZE)?; + items.push(key); + } + } + let id = vm.heap.allocate(HeapData::List(List::new(items)))?; + Ok(Value::Ref(id)) +} + +/// Implements `Counter.update(other)`, adding counts from another mapping, +/// Counter, or iterable (and keyword arguments). +fn counter_update<'h>( + counter: &mut HeapRead<'h, DictSubclass>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + counter_update_signed(counter, args, 1, "Counter.update", vm) +} + +/// Implements `Counter.subtract(other)`, subtracting counts (results may be +/// zero or negative). +fn counter_subtract<'h>( + counter: &mut HeapRead<'h, DictSubclass>, + args: ArgValues, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + counter_update_signed(counter, args, -1, "Counter.subtract", vm) +} + +/// Shared body for `Counter.update`/`Counter.subtract`: applies `sign * count` +/// from the source and keyword arguments to the backing dict. +fn counter_update_signed<'h>( + counter: &mut HeapRead<'h, DictSubclass>, + args: ArgValues, + sign: i64, + name: &str, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult { + let dict_id = counter.inner_id(vm); + let (pos, kwargs) = args.into_parts(); + let pos: Vec = pos.collect(); + if pos.len() > 1 { + let actual = pos.len() + 1; + pos.drop_with_heap(vm); + kwargs.drop_with_heap(vm); + return Err(ExcType::type_error_too_many_positional_range(name, 1, 2, actual, 0)); + } + if let Some(source) = pos.into_iter().next() + && let Err(e) = counter_add_from_source(dict_id, source, sign, vm) + { + kwargs.drop_with_heap(vm); + return Err(e); + } + counter_add_from_kwargs(dict_id, kwargs, sign, vm)?; + Ok(Value::None) +} + +/// Adds `sign * count` for each element of `source` into the backing dict. +/// +/// A mapping (`dict`/`Counter`) contributes each value as the count; any other +/// iterable contributes 1 per element. +fn counter_add_from_source( + dict_id: HeapId, + source: Value, + sign: i64, + vm: &mut VM<'_, impl ResourceTracker>, +) -> RunResult<()> { + let mut source_guard = HeapGuard::new(source, vm); + // Detect a mapping: a plain dict or another dict subclass. + let mapping_dict_id = { + let (source_ref, vm) = source_guard.as_parts(); + match source_ref { + Value::Ref(id) => match vm.heap.get(*id) { + HeapData::Dict(_) => Some(*id), + HeapData::DictSubclass(sub) => Some(sub.dict_id()), + _ => None, + }, + _ => None, + } + }; + + if let Some(map_id) = mapping_dict_id { + let vm = source_guard.heap(); + // Snapshot (key, count) pairs from the mapping first, so updating a + // Counter from itself (`c.update(c)`) sees a stable source. + let HeapReadOutput::Dict(map) = vm.heap.read(map_id) else { + unreachable!("mapping dict") + }; + let len = map.get(vm.heap).len(); + let mut pairs = Vec::with_capacity(len); + for i in 0..len { + let key = map + .get(vm.heap) + .key_at(i) + .expect("index in range") + .clone_with_heap(vm.heap); + let count = map + .get(vm.heap) + .value_at(i) + .expect("index in range") + .as_int(vm) + .unwrap_or(0); + pairs.push((key, count)); + } + drop(map); + for (key, count) in pairs { + counter_add_one(dict_id, key, sign * count, vm)?; + } + Ok(()) + } else { + // Iterable of hashable elements: `sign` per occurrence. + let source = source_guard.into_inner(); + let items: Vec = MontyIter::new(source, vm)?.collect(vm)?; + for item in items { + counter_add_one(dict_id, item, sign, vm)?; + } + Ok(()) + } +} + +/// Adds `sign * count` from keyword arguments into the backing dict. +fn counter_add_from_kwargs( + dict_id: HeapId, + kwargs: KwargsValues, + sign: i64, + vm: &mut VM<'_, impl ResourceTracker>, +) -> RunResult<()> { + let kwargs_iter = kwargs.into_iter(); + defer_drop_mut!(kwargs_iter, vm); + for (key, value) in kwargs_iter { + let count = value.as_int(vm).unwrap_or(0); + value.drop_with_heap(vm); + counter_add_one(dict_id, key, sign * count, vm)?; + } + Ok(()) +} + +/// Adds `delta` to the count for `key` in the backing dict (consuming `key`'s +/// ownership). The stored value stays an `int`; zero/negative results are kept +/// (only the arithmetic operators drop non-positive counts). +fn counter_add_one(dict_id: HeapId, key: Value, delta: i64, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult<()> { + let HeapReadOutput::Dict(mut dict) = vm.heap.read(dict_id) else { + unreachable!("backing dict") + }; + let current = match dict.dict_get(&key, vm)? { + Some(v) => { + let n = v.as_int(vm).unwrap_or(0); + v.drop_with_heap(vm); + n + } + None => 0, + }; + if let Some(old) = dict.set(key, Value::Int(current + delta), vm)? { + old.drop_with_heap(vm); + } + Ok(()) +} + +/// The four `Counter` arithmetic operators. All produce a new `Counter` and +/// discard non-positive results. +#[derive(Debug, Clone, Copy)] +pub(crate) enum CounterOp { + /// `+` — add counts. + Add, + /// `-` — subtract counts. + Sub, + /// `&` — minimum of counts (multiset intersection). + And, + /// `|` — maximum of counts (multiset union). + Or, +} + +/// Computes a `Counter` binary operation (`+`, `-`, `&`, `|`). +/// +/// Returns `Ok(None)` if either operand is not a `Counter` (so the caller falls +/// back to the standard `TypeError`). Follows CPython's algorithm: results with +/// a non-positive count are dropped, and iteration order is `self`'s keys first, +/// then `other`'s new keys. +pub(crate) fn counter_binop<'h>( + a: &HeapRead<'h, DictSubclass>, + b: &HeapRead<'h, DictSubclass>, + op: CounterOp, + vm: &mut VM<'h, impl ResourceTracker>, +) -> RunResult> { + if !a.is_counter(vm) || !b.is_counter(vm) { + return Ok(None); + } + let a_id = a.inner_id(vm); + let b_id = b.inner_id(vm); + let a_entries = snapshot_counts(a_id, vm); + + let result_id = vm.heap.allocate(HeapData::Dict(Dict::new()))?; + let mut result_guard = HeapGuard::new(Value::Ref(result_id), vm); + { + let (_backing, vm) = result_guard.as_parts_mut(); + + // First pass: every key of `self`. + for (key, self_count) in a_entries { + let other_count = counter_lookup(b_id, &key, vm)?; + let newcount = match op { + CounterOp::Add => self_count + other_count, + CounterOp::Sub => self_count - other_count, + CounterOp::And => self_count.min(other_count), + CounterOp::Or => self_count.max(other_count), + }; + if newcount > 0 { + counter_set(result_id, key, newcount, vm)?; + } else { + key.drop_with_heap(vm); + } + } + + // Second pass: keys only in `other`. `&` is intersection, so it skips this. + if !matches!(op, CounterOp::And) { + let b_entries = snapshot_counts(b_id, vm); + for (key, other_count) in b_entries { + let in_self = counter_contains(a_id, &key, vm)?; + let newcount = match op { + CounterOp::Sub => -other_count, // negatives become positive magnitudes + _ => other_count, + }; + if !in_self && newcount > 0 { + counter_set(result_id, key, newcount, vm)?; + } else { + key.drop_with_heap(vm); + } + } + } + } + + // Transfer the result dict's owned ref to the new Counter wrapper: `forget` + // the guard's `Value::Ref` so its count is not decremented here. + mem::forget(result_guard.into_inner()); + DictSubclass::wrap(vm, result_id, DictSubclassKind::Counter).map(Some) +} + +/// Snapshots a backing dict's `(key_clone, count)` entries. +fn snapshot_counts(dict_id: HeapId, vm: &mut VM<'_, impl ResourceTracker>) -> Vec<(Value, i64)> { + let HeapReadOutput::Dict(dict) = vm.heap.read(dict_id) else { + unreachable!("backing dict") + }; + let len = dict.get(vm.heap).len(); + let mut out = Vec::with_capacity(len); + for i in 0..len { + let key = dict + .get(vm.heap) + .key_at(i) + .expect("index in range") + .clone_with_heap(vm.heap); + let count = dict + .get(vm.heap) + .value_at(i) + .expect("index in range") + .as_int(vm) + .unwrap_or(0); + out.push((key, count)); + } + out +} + +/// Looks up `key`'s count in a backing dict, returning 0 if absent. +fn counter_lookup(dict_id: HeapId, key: &Value, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult { + let HeapReadOutput::Dict(dict) = vm.heap.read(dict_id) else { + unreachable!("backing dict") + }; + match dict.dict_get(key, vm)? { + Some(v) => { + let n = v.as_int(vm).unwrap_or(0); + v.drop_with_heap(vm); + Ok(n) + } + None => Ok(0), + } +} + +/// Returns whether `key` is present in a backing dict. +fn counter_contains(dict_id: HeapId, key: &Value, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult { + let HeapReadOutput::Dict(dict) = vm.heap.read(dict_id) else { + unreachable!("backing dict") + }; + dict.contains_key(key, vm) +} + +/// Sets `key -> count` (an `int`) in a backing dict, consuming `key`. +fn counter_set(dict_id: HeapId, key: Value, count: i64, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult<()> { + let HeapReadOutput::Dict(mut dict) = vm.heap.read(dict_id) else { + unreachable!("backing dict") + }; + if let Some(old) = dict.set(key, Value::Int(count), vm)? { + old.drop_with_heap(vm); + } + Ok(()) +} + +/// Returns the backing-dict entry indices of a Counter ordered by count +/// descending, with ties in insertion order. Returns indices (not clones) so +/// callers only clone the entries they actually use. Used by repr and +/// `most_common`. +fn counter_indices_by_count<'h>( + counter: &HeapRead<'h, DictSubclass>, + vm: &mut VM<'h, impl ResourceTracker>, +) -> Vec { + let dict = counter.inner(vm); + let len = dict.get(vm.heap).len(); + let mut idx_count: Vec<(usize, i64)> = Vec::with_capacity(len); + for i in 0..len { + let count = dict + .get(vm.heap) + .value_at(i) + .expect("index in range") + .as_int(vm) + .unwrap_or(0); + idx_count.push((i, count)); + } + // Stable sort by count descending; the original index as a tiebreaker + // preserves insertion order for equal counts. + idx_count.sort_by_key(|(idx, count)| (Reverse(*count), *idx)); + idx_count.into_iter().map(|(i, _)| i).collect() +} diff --git a/crates/monty/src/types/iter.rs b/crates/monty/src/types/iter.rs index 94e7690d4..7193a33a7 100644 --- a/crates/monty/src/types/iter.rs +++ b/crates/monty/src/types/iter.rs @@ -667,6 +667,20 @@ impl IterValue { len: Some(deque.as_deque().len()), checks_mutation: true, }), + // Dict subclass (defaultdict/Counter): iterate the backing dict's + // keys by pointing the iterator directly at the backing dict id. + HeapData::DictSubclass(sub) => { + let dict_id = sub.dict_id(); + let len = match heap.get(dict_id) { + HeapData::Dict(dict) => dict.len(), + _ => 0, + }; + Some(Self::HeapRef { + heap_id: dict_id, + len: Some(len), + checks_mutation: true, + }) + } HeapData::Bytes(b) => Some(Self::HeapRef { heap_id, len: Some(b.len()), diff --git a/crates/monty/src/types/mod.rs b/crates/monty/src/types/mod.rs index 698a17c59..23a929d53 100644 --- a/crates/monty/src/types/mod.rs +++ b/crates/monty/src/types/mod.rs @@ -12,6 +12,7 @@ pub mod date; pub mod datetime; pub mod deque; pub mod dict; +pub mod dict_subclass; pub mod dict_view; pub mod file; pub mod instance; @@ -39,6 +40,7 @@ pub(crate) use class::Class; pub(crate) use dataclass::Dataclass; pub(crate) use deque::Deque; pub(crate) use dict::Dict; +pub(crate) use dict_subclass::{DictSubclass, DictSubclassKind}; pub(crate) use dict_view::{DictItemsView, DictKeysView, DictValuesView}; pub(crate) use file::OpenFile; pub(crate) use instance::{BoundMethod, Instance}; diff --git a/crates/monty/src/types/type.rs b/crates/monty/src/types/type.rs index 2818fdcc1..6b6ce8383 100644 --- a/crates/monty/src/types/type.rs +++ b/crates/monty/src/types/type.rs @@ -11,9 +11,9 @@ use crate::{ intern::{Interns, StaticStrings, StringId}, resource::ResourceTracker, types::{ - AttrCallResult, Bytes, Deque, Dict, FrozenSet, List, LongInt, MontyIter, Path, PyTrait, Range, Set, Slice, Str, - TimeZone, Tuple, bytes::bytes_fromhex, date, datetime, dict::dict_fromkeys, instance::class_name, - long_int::INT_MAX_STR_DIGITS, str::StringRepr, timedelta, + AttrCallResult, Bytes, Deque, Dict, DictSubclass, FrozenSet, List, LongInt, MontyIter, Path, PyTrait, Range, + Set, Slice, Str, TimeZone, Tuple, bytes::bytes_fromhex, date, datetime, dict::dict_fromkeys, + instance::class_name, long_int::INT_MAX_STR_DIGITS, str::StringRepr, timedelta, }, value::Value, }; @@ -64,6 +64,12 @@ pub enum Type { /// `collections.deque` — a double-ended queue. Only reachable via import. #[strum(serialize = "collections.deque")] Deque, + /// `collections.defaultdict` — a dict subclass with a default factory. + #[strum(serialize = "collections.defaultdict")] + DefaultDict, + /// `collections.Counter` — a dict subclass for counting. + #[strum(serialize = "collections.Counter")] + Counter, Dict, #[strum(serialize = "dict_keys")] DictKeys, @@ -379,6 +385,8 @@ impl Type { Self::List => List::init(vm, args), Self::Tuple => Tuple::init(vm, args), Self::Deque => Deque::init(vm, args), + Self::DefaultDict => DictSubclass::init_defaultdict(vm, args), + Self::Counter => DictSubclass::init_counter(vm, args), Self::Dict => Dict::init(vm, args), Self::Set => Set::init(vm, args), Self::FrozenSet => FrozenSet::init(vm, args), diff --git a/crates/monty/src/value.rs b/crates/monty/src/value.rs index 2d2ab90a8..1aa0b9856 100644 --- a/crates/monty/src/value.rs +++ b/crates/monty/src/value.rs @@ -1729,6 +1729,15 @@ impl Value { Ok(false) } HeapReadOutput::Dict(dict) => dict.contains_key(item, vm), + HeapReadOutput::DictSubclass(sub) => { + // Membership on a defaultdict/Counter checks the backing + // dict's keys (a Counter miss is not counted as present). + let dict_id = sub.get(vm.heap).dict_id(); + let HeapReadOutput::Dict(dict) = vm.heap.read(dict_id) else { + panic!("dict subclass must reference a dict"); + }; + dict.contains_key(item, vm) + } HeapReadOutput::DictKeysView(view) => { let dict_id = view.get(vm.heap).dict_id(); let HeapReadOutput::Dict(dict) = vm.heap.read(dict_id) else { diff --git a/crates/monty/test_cases/collections__counter.py b/crates/monty/test_cases/collections__counter.py new file mode 100644 index 000000000..d513bdc9f --- /dev/null +++ b/crates/monty/test_cases/collections__counter.py @@ -0,0 +1,106 @@ +# Tests for collections.Counter + +from collections import Counter + +# === Construction from an iterable (tallying) === +c = Counter('aabbbc') +assert c == {'a': 2, 'b': 3, 'c': 1}, 'counts characters' +assert Counter([1, 1, 2, 3, 3, 3]) == {1: 2, 2: 1, 3: 3}, 'counts list elements' +assert Counter() == {}, 'empty Counter' + +# === Construction from a mapping or kwargs === +assert Counter({'a': 2, 'b': 3}) == {'a': 2, 'b': 3}, 'from mapping uses values as counts' +assert Counter(a=2, b=3) == {'a': 2, 'b': 3}, 'from kwargs' +assert Counter({'a': 2}, b=3) == {'a': 2, 'b': 3}, 'mapping plus kwargs' + +# === Missing keys return 0 without inserting === +c = Counter('aab') +assert c['z'] == 0, 'missing key returns 0' +assert 'z' not in c, 'missing-key access does not insert' +assert len(c) == 2, 'length unchanged after missing access' + +# === most_common === +c = Counter('aaabbc') +assert c.most_common() == [('a', 3), ('b', 2), ('c', 1)], 'most_common all, count-descending' +assert c.most_common(2) == [('a', 3), ('b', 2)], 'most_common top-n' +assert c.most_common(0) == [], 'most_common zero' +# ties preserve insertion order +ct = Counter() +ct['x'] = 1 +ct['y'] = 1 +ct['z'] = 1 +assert ct.most_common() == [('x', 1), ('y', 1), ('z', 1)], 'ties keep insertion order' + +# === elements === +c = Counter('abcabc') +assert list(c.elements()) == ['a', 'a', 'b', 'b', 'c', 'c'], 'elements repeats in insertion order' +# non-positive counts are skipped +cz = Counter(a=2, b=0, c=-1) +assert list(cz.elements()) == ['a', 'a'], 'elements skips non-positive counts' +assert list(Counter().elements()) == [], 'elements of empty Counter' + +# === update (adds counts) === +cu = Counter(a=1) +cu.update(Counter(a=2, b=1)) +assert cu == {'a': 3, 'b': 1}, 'update from Counter adds' +cu.update('aa') +assert cu == {'a': 5, 'b': 1}, 'update from iterable adds' +cu.update({'c': 4}) +assert cu == {'a': 5, 'b': 1, 'c': 4}, 'update from mapping adds' +cu.update(b=10) +assert cu == {'a': 5, 'b': 11, 'c': 4}, 'update from kwargs adds' + +# === subtract (may go zero or negative) === +cs = Counter(a=5) +cs.subtract(Counter(a=2, b=3)) +assert cs == {'a': 3, 'b': -3}, 'subtract keeps negatives' +cs2 = Counter(a=1) +cs2.subtract('aabb') +assert cs2 == {'a': -1, 'b': -2}, 'subtract from iterable' + +# === Arithmetic operators (drop non-positive) === +assert Counter(a=3, b=1) + Counter(a=1, c=1) == Counter({'a': 4, 'b': 1, 'c': 1}), 'addition' +assert Counter(a=3, b=1) - Counter(a=1, b=2) == Counter({'a': 2}), 'subtraction drops non-positive' +assert Counter(a=3, b=1) & Counter(a=1, b=2) == Counter({'a': 1, 'b': 1}), 'intersection is min' +assert Counter(a=3, b=1) | Counter(a=1, b=2) == Counter({'a': 3, 'b': 2}), 'union is max' +assert Counter(a=1) + Counter(a=-1) == Counter(), 'addition to zero drops key' + +# === Equality with plain dict === +assert Counter(a=1, b=2) == {'a': 1, 'b': 2}, 'Counter equals plain dict' +assert {'a': 1, 'b': 2} == Counter(a=1, b=2), 'plain dict equals Counter' + +# === copy preserves subclass === +cc = Counter('aab').copy() +assert cc == {'a': 2, 'b': 1}, 'copy has same counts' +assert cc['z'] == 0, 'copy still returns 0 for missing' +ccorig = Counter(a=1) +cccopy = ccorig.copy() +cccopy['b'] = 5 +assert ccorig == {'a': 1}, 'copy is independent' + +# === Standard dict methods === +c = Counter('aab') +assert sorted(c.keys()) == ['a', 'b'], 'keys view' +assert sorted(c.values()) == [1, 2], 'values view' +assert dict(c.items()) == {'a': 2, 'b': 1}, 'items view' +assert c.get('a') == 2, 'get existing' +assert c.get('z') is None, 'get missing returns None (not 0)' +c['a'] = 10 +assert c['a'] == 10, 'setitem' +del_count = c.pop('a') +assert del_count == 10, 'pop existing' +assert 'a' not in c, 'state after pop' + +# === Iteration === +c = Counter('aabbbc') +assert sorted(c) == ['a', 'b', 'c'], 'iterating yields keys' +assert dict(c) == {'a': 2, 'b': 3, 'c': 1}, 'dict() copies mapping' + +# === bool === +assert bool(Counter()) is False, 'empty Counter is falsy' +assert bool(Counter(a=1)) is True, 'non-empty is truthy' + +# === repr (count-descending) === +assert repr(Counter('aabbbc')) == "Counter({'b': 3, 'a': 2, 'c': 1})", 'repr sorts by count desc' +assert repr(Counter()) == 'Counter()', 'empty repr' +assert repr(Counter(a=1)) == "Counter({'a': 1})", 'single-entry repr' diff --git a/crates/monty/test_cases/collections__defaultdict.py b/crates/monty/test_cases/collections__defaultdict.py new file mode 100644 index 000000000..72860bf0c --- /dev/null +++ b/crates/monty/test_cases/collections__defaultdict.py @@ -0,0 +1,111 @@ +# Tests for collections.defaultdict + +from collections import defaultdict + +# === Basic factory behavior === +d = defaultdict(int) +assert d['a'] == 0, 'missing key uses int factory' +assert d == {'a': 0}, 'missing-key access inserts the default' +d['b'] += 1 +assert d['b'] == 1, 'augmented assignment on missing key' +assert d == {'a': 0, 'b': 1}, 'state after augmented assignment' + +# === list factory === +dl = defaultdict(list) +dl['x'].append(1) +dl['x'].append(2) +dl['y'].append(3) +assert dl == {'x': [1, 2], 'y': [3]}, 'list factory accumulates' + +# === set factory === +ds = defaultdict(set) +ds['k'].add(1) +ds['k'].add(2) +ds['k'].add(1) +assert ds['k'] == {1, 2}, 'set factory dedupes' + +# === dict factory (nested) === +dd = defaultdict(dict) +dd['a']['x'] = 1 +assert dd == {'a': {'x': 1}}, 'dict factory nests' + +# === default_factory attribute === +assert defaultdict(int).default_factory is int, 'default_factory is int' +assert defaultdict(list).default_factory is list, 'default_factory is list' +assert defaultdict().default_factory is None, 'no factory means None' + +# === None factory raises KeyError === +dn = defaultdict() +try: + dn['missing'] + assert False, 'expected KeyError with no factory' +except KeyError as e: + assert str(e) == "'missing'", 'KeyError message is the key repr' + +# === get() does not use the factory or insert === +dg = defaultdict(int) +assert dg.get('missing') is None, 'get returns None for missing' +assert dg.get('missing', 5) == 5, 'get returns provided default' +assert 'missing' not in dg, 'get does not insert' +assert len(dg) == 0, 'get leaves defaultdict empty' + +# === Construction with a mapping and kwargs === +dm = defaultdict(int, {'a': 1}) +assert dm == {'a': 1}, 'defaultdict from mapping' +dmk = defaultdict(int, {'a': 1}, b=2) +assert dmk == {'a': 1, 'b': 2}, 'defaultdict from mapping + kwargs' +di = defaultdict(int, [('a', 1), ('b', 2)]) +assert di == {'a': 1, 'b': 2}, 'defaultdict from iterable of pairs' + +# === Equality with plain dict === +assert defaultdict(int, {'a': 1}) == {'a': 1}, 'defaultdict equals plain dict' +assert {'a': 1} == defaultdict(int, {'a': 1}), 'plain dict equals defaultdict' + +# === copy preserves subclass, factory, and contents === +cp = defaultdict(int, {'a': 1}).copy() +assert cp == {'a': 1}, 'copy has same items' +assert cp.default_factory is int, 'copy preserves factory' +assert cp['new'] == 0, 'copy factory still works' +cp2 = defaultdict(int, {'a': 1}) +cp2copy = cp2.copy() +cp2copy['z'] = 9 +assert cp2 == {'a': 1}, 'copy is independent of original' + +# === Standard dict methods === +dmeth = defaultdict(int, {'a': 1, 'b': 2}) +assert sorted(dmeth.keys()) == ['a', 'b'], 'keys view' +assert sorted(dmeth.values()) == [1, 2], 'values view' +assert dict(dmeth.items()) == {'a': 1, 'b': 2}, 'items view' +assert dmeth.pop('a') == 1, 'pop existing key' +assert dmeth == {'b': 2}, 'state after pop' +dmeth.setdefault('c', 3) +assert dmeth['c'] == 3, 'setdefault inserts' +dmeth.update({'d': 4}) +assert dmeth == {'b': 2, 'c': 3, 'd': 4}, 'update from mapping' +dmeth.clear() +assert dmeth == {}, 'clear empties' + +# === Iteration === +diter = defaultdict(int) +diter['a'] = 1 +diter['b'] = 2 +assert sorted(diter) == ['a', 'b'], 'iterating yields keys' +assert sorted(list(diter)) == ['a', 'b'], 'list() yields keys' +assert dict(diter) == {'a': 1, 'b': 2}, 'dict() copies mapping' + +# === bool === +assert bool(defaultdict(int)) is False, 'empty defaultdict is falsy' +assert bool(defaultdict(int, {'a': 1})) is True, 'non-empty is truthy' + +# === repr === +assert repr(defaultdict(int)) == "defaultdict(, {})", 'empty int repr' +assert repr(defaultdict(int, {'a': 1})) == "defaultdict(, {'a': 1})", 'int repr with item' +assert repr(defaultdict(list, {'x': [1]})) == "defaultdict(, {'x': [1]})", 'list repr' +assert repr(defaultdict()) == 'defaultdict(None, {})', 'no-factory repr' + +# === Constructor errors === +try: + defaultdict(5) + assert False, 'expected non-callable factory to raise' +except TypeError as e: + assert str(e) == 'first argument must be callable or None', 'non-callable factory message' diff --git a/limitations/collections.md b/limitations/collections.md index d09db29d5..7f6924930 100644 --- a/limitations/collections.md +++ b/limitations/collections.md @@ -7,9 +7,19 @@ following names are importable; everything else in CPython's `collections` (the custom stub only exposes the implemented names) and at runtime: - `deque` +- `defaultdict` +- `Counter` -The dict-like members (`defaultdict`, `Counter`) and `namedtuple` are -documented in their own sections below as they land. +### General + +- **`type(obj).__name__` returns the qualified name.** For all three types + `type(x).__name__` is `"collections.defaultdict"` / `"collections.Counter"` / + `"collections.deque"` rather than the bare `"defaultdict"` / `"Counter"` / + `"deque"`. This is a pre-existing Monty behavior for qualified builtin types + (e.g. `datetime.datetime.__name__` is likewise qualified). `str(type(x))` + (``) matches CPython. +- **No subclassing** of these types, and no `copy.copy` / `pickle` support (the + `copy` and `pickle` modules are not importable); use the `.copy()` method. ## `deque` @@ -38,3 +48,46 @@ Divergences from CPython: - **`maxlen` argument coercion.** A non-integer `maxlen` is reported by the constructor body; a `maxlen` that overflows the platform integer range is not specially handled. + +## `defaultdict` + +Supported: `default_factory` (attribute), missing-key insertion via the factory, +construction from a mapping/iterable plus keyword arguments, all standard `dict` +methods (`get`, `keys`, `values`, `items`, `pop`, `popitem`, `setdefault`, +`update`, `clear`, `copy`), iteration, membership, equality with plain dicts, +`repr`, and `bool`. + +Divergences from CPython: + +- **`default_factory` must be a builtin callable.** On a missing key the factory + is only invoked for builtin callables (`int`, `list`, `set`, `dict`, `tuple`, + `str`, `float`, `bool`, `frozenset`, …). A **user-defined** factory (a `lambda` + or `def` function) constructs fine but raises + `TypeError: defaultdict with a non-builtin default_factory is not supported in + Monty` on the missing-key access. This is because `__getitem__` cannot invoke a + Python callback that needs its own VM frame. `.get()`, explicit assignment, and + iteration are unaffected. +- **`default_factory` is not reassignable** after construction (`dd.default_factory + = f` is unsupported). + +## `Counter` + +Supported: construction from an iterable (tallying), a mapping, or keyword +arguments; `c[missing]` returning `0` without inserting; `most_common([n])`; +`elements()`; `update()`; `subtract()`; the arithmetic operators `+`, `-`, `&`, +`|` (each dropping non-positive results); all standard `dict` methods; iteration; +equality with plain dicts; count-descending `repr`; and `bool`. + +Divergences from CPython: + +- **`elements()` returns a `list`, not a lazy iterator.** CPython returns an + `itertools.chain` object; Monty materializes a `list`. Iterating or calling + `list(...)` on the result is identical, but `type(c.elements())` differs. +- **No in-place operators** `+=`, `-=`, `&=`, `|=` between two Counters (the + binary forms are supported; use `.update()` / `.subtract()` for in-place count + changes). +- **No unary `+c` / `-c`** (which in CPython keep positive / negative counts). +- **`total()`, `fromkeys`** are not implemented (`Counter.fromkeys` raises in + CPython too, but `total()` is missing here). +- **Counts are integers.** Arithmetic assumes integer counts; non-integer values + are treated as `0` for count purposes rather than raising. From 6d7d66b7a5f86b8d3522160905e1ceed909bc858 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 19:04:05 +0000 Subject: [PATCH 3/3] Add collections.namedtuple factory and narrow the type-checker stub Implement `collections.namedtuple(...)` as a module function returning a callable class (`NamedTupleClass` heap type) that constructs `NamedTuple` instances, and narrow the type checker's `collections` surface to the implemented members. - namedtuple factory: field-name parsing (string or iterable), CPython-matching identifier/keyword/underscore/duplicate validation, rename=True, defaults, and an ignored module kwarg. - NamedTupleClass: callable (via call_heap_callable), _fields/_field_defaults/ _make/__name__ class surface, `` repr. - NamedTuple instances gain _fields/_asdict/_replace/_make plus inherited count/index; sequence unpacking now handles namedtuple and deque; namedtuple is a subclass of tuple for isinstance. type() identity is a documented divergence. - Narrowed collections stub (custom/collections/__init__.pyi + vendored) exposes only deque/defaultdict/Counter/namedtuple so type-checking matches runtime; update.py now copies custom package overrides. - Consolidated namedtuple + import test_cases; limitations updated. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015FUUYYxP68sbHXsdr2qGE8 --- .../custom/collections/__init__.pyi | 201 +++++++ crates/monty-typeshed/update.py | 11 + .../typeshed/stdlib/collections/__init__.pyi | 336 +---------- crates/monty/src/bytecode/vm/call.rs | 7 +- crates/monty/src/bytecode/vm/collections.rs | 14 + crates/monty/src/heap.rs | 15 +- crates/monty/src/heap_data.rs | 19 +- crates/monty/src/modules/collections.rs | 33 +- crates/monty/src/modules/mod.rs | 3 + crates/monty/src/object.rs | 5 + crates/monty/src/types/mod.rs | 2 + crates/monty/src/types/namedtuple.rs | 190 +++++- crates/monty/src/types/namedtuple_class.rs | 563 ++++++++++++++++++ crates/monty/src/types/str.rs | 2 +- crates/monty/src/types/type.rs | 3 + .../test_cases/collections__namedtuple.py | 148 +++++ .../monty/test_cases/import__collections.py | 23 + limitations/collections.md | 28 + limitations/namedtuple.md | 42 +- 19 files changed, 1280 insertions(+), 365 deletions(-) create mode 100644 crates/monty-typeshed/custom/collections/__init__.pyi create mode 100644 crates/monty/src/types/namedtuple_class.rs create mode 100644 crates/monty/test_cases/collections__namedtuple.py create mode 100644 crates/monty/test_cases/import__collections.py diff --git a/crates/monty-typeshed/custom/collections/__init__.pyi b/crates/monty-typeshed/custom/collections/__init__.pyi new file mode 100644 index 000000000..31202e678 --- /dev/null +++ b/crates/monty-typeshed/custom/collections/__init__.pyi @@ -0,0 +1,201 @@ +import sys +from _collections_abc import dict_items, dict_keys, dict_values +from types import GenericAlias +from typing import Any, ClassVar, Generic, NoReturn, SupportsIndex, TypeVar, final, overload, type_check_only + +from _typeshed import SupportsItems, SupportsKeysAndGetItem, SupportsRichComparison, SupportsRichComparisonT +from typing_extensions import Self, disjoint_base + +if sys.version_info >= (3, 10): + from collections.abc import ( + Callable, + ItemsView, + Iterable, + Iterator, + KeysView, + Mapping, + MutableMapping, + MutableSequence, + Sequence, + ValuesView, + ) +else: + from _collections_abc import * + +__all__ = [ + 'Counter', + 'defaultdict', + 'deque', + 'namedtuple', +] + +_S = TypeVar('_S') +_T = TypeVar('_T') +_T1 = TypeVar('_T1') +_T2 = TypeVar('_T2') +_KT = TypeVar('_KT') +_VT = TypeVar('_VT') +_KT_co = TypeVar('_KT_co', covariant=True) +_VT_co = TypeVar('_VT_co', covariant=True) + +# namedtuple is special-cased in the type checker; the initializer is ignored. +def namedtuple( + typename: str, + field_names: str | Iterable[str], + *, + rename: bool = False, + module: str | None = None, + defaults: Iterable[Any] | None = None, +) -> type[tuple[Any, ...]]: ... + + +# NOTE: This is a Monty-narrowed copy of the upstream typeshed stub. Only the +# members implemented by Monty's runtime are exposed (deque, defaultdict, +# Counter, namedtuple); OrderedDict, ChainMap, UserDict, UserList and +# UserString are intentionally omitted so type-checking matches the runtime. +# The source of truth is custom/collections/__init__.pyi (see custom/README.md). + +@disjoint_base +class deque(MutableSequence[_T]): + @property + def maxlen(self) -> int | None: ... + @overload + def __init__(self, *, maxlen: int | None = None) -> None: ... + @overload + def __init__(self, iterable: Iterable[_T], maxlen: int | None = None) -> None: ... + def append(self, x: _T, /) -> None: ... + def appendleft(self, x: _T, /) -> None: ... + def copy(self) -> Self: ... + def count(self, x: _T, /) -> int: ... + def extend(self, iterable: Iterable[_T], /) -> None: ... + def extendleft(self, iterable: Iterable[_T], /) -> None: ... + def insert(self, i: int, x: _T, /) -> None: ... + def index(self, x: _T, start: int = 0, stop: int = ..., /) -> int: ... + def pop(self) -> _T: ... # type: ignore[override] + def popleft(self) -> _T: ... + def remove(self, value: _T, /) -> None: ... + def rotate(self, n: int = 1, /) -> None: ... + def __copy__(self) -> Self: ... + def __len__(self) -> int: ... + __hash__: ClassVar[None] # type: ignore[assignment] + # These methods of deque don't take slices, unlike MutableSequence, hence the type: ignores + def __getitem__(self, key: SupportsIndex, /) -> _T: ... # type: ignore[override] + def __setitem__(self, key: SupportsIndex, value: _T, /) -> None: ... # type: ignore[override] + def __delitem__(self, key: SupportsIndex, /) -> None: ... # type: ignore[override] + def __contains__(self, key: object, /) -> bool: ... + def __reduce__(self) -> tuple[type[Self], tuple[()], None, Iterator[_T]]: ... + def __iadd__(self, value: Iterable[_T], /) -> Self: ... + def __add__(self, value: Self, /) -> Self: ... + def __mul__(self, value: int, /) -> Self: ... + def __imul__(self, value: int, /) -> Self: ... + def __lt__(self, value: deque[_T], /) -> bool: ... + def __le__(self, value: deque[_T], /) -> bool: ... + def __gt__(self, value: deque[_T], /) -> bool: ... + def __ge__(self, value: deque[_T], /) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class Counter(dict[_T, int], Generic[_T]): + @overload + def __init__(self, iterable: None = None, /) -> None: ... + @overload + def __init__(self: Counter[str], iterable: None = None, /, **kwargs: int) -> None: ... + @overload + def __init__(self, mapping: SupportsKeysAndGetItem[_T, int], /) -> None: ... + @overload + def __init__(self, iterable: Iterable[_T], /) -> None: ... + def copy(self) -> Self: ... + def elements(self) -> Iterator[_T]: ... + def most_common(self, n: int | None = None) -> list[tuple[_T, int]]: ... + @classmethod + def fromkeys(cls, iterable: Any, v: int | None = None) -> NoReturn: ... # type: ignore[override] + @overload + def subtract(self, iterable: None = None, /) -> None: ... + @overload + def subtract(self, mapping: Mapping[_T, int], /) -> None: ... + @overload + def subtract(self, iterable: Iterable[_T], /) -> None: ... + # Unlike dict.update(), use Mapping instead of SupportsKeysAndGetItem for the first overload + # (source code does an `isinstance(other, Mapping)` check) + # + # The second overload is also deliberately different to dict.update() + # (if it were `Iterable[_T] | Iterable[tuple[_T, int]]`, + # the tuples would be added as keys, breaking type safety) + @overload # type: ignore[override] + def update(self, m: Mapping[_T, int], /, **kwargs: int) -> None: ... + @overload + def update(self, iterable: Iterable[_T], /, **kwargs: int) -> None: ... + @overload + def update(self, iterable: None = None, /, **kwargs: int) -> None: ... + def __missing__(self, key: _T) -> int: ... + def __delitem__(self, elem: object) -> None: ... + if sys.version_info >= (3, 10): + def __eq__(self, other: object) -> bool: ... + def __ne__(self, other: object) -> bool: ... + + def __add__(self, other: Counter[_S]) -> Counter[_T | _S]: ... + def __sub__(self, other: Counter[_T]) -> Counter[_T]: ... + def __and__(self, other: Counter[_T]) -> Counter[_T]: ... + def __or__(self, other: Counter[_S]) -> Counter[_T | _S]: ... # type: ignore[override] + def __pos__(self) -> Counter[_T]: ... + def __neg__(self) -> Counter[_T]: ... + # several type: ignores because __iadd__ is supposedly incompatible with __add__, etc. + def __iadd__(self, other: SupportsItems[_T, int]) -> Self: ... # type: ignore[misc] + def __isub__(self, other: SupportsItems[_T, int]) -> Self: ... + def __iand__(self, other: SupportsItems[_T, int]) -> Self: ... + def __ior__(self, other: SupportsItems[_T, int]) -> Self: ... # type: ignore[override,misc] + if sys.version_info >= (3, 10): + def total(self) -> int: ... + def __le__(self, other: Counter[Any]) -> bool: ... + def __lt__(self, other: Counter[Any]) -> bool: ... + def __ge__(self, other: Counter[Any]) -> bool: ... + def __gt__(self, other: Counter[Any]) -> bool: ... + +@disjoint_base +class defaultdict(dict[_KT, _VT]): + default_factory: Callable[[], _VT] | None + @overload + def __init__(self) -> None: ... + @overload + def __init__(self: defaultdict[str, _VT], **kwargs: _VT) -> None: ... # pyright: ignore[reportInvalidTypeVarUse] #11780 + @overload + def __init__(self, default_factory: Callable[[], _VT] | None, /) -> None: ... + @overload + def __init__( + self: defaultdict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + default_factory: Callable[[], _VT] | None, + /, + **kwargs: _VT, + ) -> None: ... + @overload + def __init__(self, default_factory: Callable[[], _VT] | None, map: SupportsKeysAndGetItem[_KT, _VT], /) -> None: ... + @overload + def __init__( + self: defaultdict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + default_factory: Callable[[], _VT] | None, + map: SupportsKeysAndGetItem[str, _VT], + /, + **kwargs: _VT, + ) -> None: ... + @overload + def __init__(self, default_factory: Callable[[], _VT] | None, iterable: Iterable[tuple[_KT, _VT]], /) -> None: ... + @overload + def __init__( + self: defaultdict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + default_factory: Callable[[], _VT] | None, + iterable: Iterable[tuple[str, _VT]], + /, + **kwargs: _VT, + ) -> None: ... + def __missing__(self, key: _KT, /) -> _VT: ... + def __copy__(self) -> Self: ... + def copy(self) -> Self: ... + @overload + def __or__(self, value: dict[_KT, _VT], /) -> Self: ... + @overload + def __or__(self, value: dict[_T1, _T2], /) -> defaultdict[_KT | _T1, _VT | _T2]: ... + @overload + def __ror__(self, value: dict[_KT, _VT], /) -> Self: ... + @overload + def __ror__(self, value: dict[_T1, _T2], /) -> defaultdict[_KT | _T1, _VT | _T2]: ... # type: ignore[misc] + diff --git a/crates/monty-typeshed/update.py b/crates/monty-typeshed/update.py index b6dce1ddd..aac55e1ea 100644 --- a/crates/monty-typeshed/update.py +++ b/crates/monty-typeshed/update.py @@ -334,6 +334,17 @@ def main() -> int: for file in CUSTOM_DIR.glob('*.pyi'): shutil.copy2(file, STDLIB_DIR) custom_count += 1 + # Package overrides: a `custom//` directory replaces the matching + # module file within the vendored `stdlib//` package (e.g. + # `custom/collections/__init__.pyi` narrows `collections` to Monty's + # implemented members while leaving `collections/abc.pyi` untouched). + for pkg_dir in CUSTOM_DIR.iterdir(): + if not pkg_dir.is_dir(): + continue + dest_pkg = STDLIB_DIR / pkg_dir.name + for file in pkg_dir.glob('*.pyi'): + shutil.copy2(file, dest_pkg / file.name) + custom_count += 1 print(f'Copied {custom_count} custom typeshed files') return 0 diff --git a/crates/monty-typeshed/vendor/typeshed/stdlib/collections/__init__.pyi b/crates/monty-typeshed/vendor/typeshed/stdlib/collections/__init__.pyi index cd2908d20..31202e678 100644 --- a/crates/monty-typeshed/vendor/typeshed/stdlib/collections/__init__.pyi +++ b/crates/monty-typeshed/vendor/typeshed/stdlib/collections/__init__.pyi @@ -23,12 +23,7 @@ else: from _collections_abc import * __all__ = [ - 'ChainMap', 'Counter', - 'OrderedDict', - 'UserDict', - 'UserList', - 'UserString', 'defaultdict', 'deque', 'namedtuple', @@ -53,199 +48,12 @@ def namedtuple( defaults: Iterable[Any] | None = None, ) -> type[tuple[Any, ...]]: ... -class UserDict(MutableMapping[_KT, _VT]): - data: dict[_KT, _VT] - # __init__ should be kept roughly in line with `dict.__init__`, which has the same semantics - @overload - def __init__(self, dict: None = None, /) -> None: ... - @overload - def __init__( - self: UserDict[str, _VT], - dict: None = None, - /, - **kwargs: _VT, # pyright: ignore[reportInvalidTypeVarUse] #11780 - ) -> None: ... - @overload - def __init__(self, dict: SupportsKeysAndGetItem[_KT, _VT], /) -> None: ... - @overload - def __init__( - self: UserDict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 - dict: SupportsKeysAndGetItem[str, _VT], - /, - **kwargs: _VT, - ) -> None: ... - @overload - def __init__(self, iterable: Iterable[tuple[_KT, _VT]], /) -> None: ... - @overload - def __init__( - self: UserDict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 - iterable: Iterable[tuple[str, _VT]], - /, - **kwargs: _VT, - ) -> None: ... - @overload - def __init__(self: UserDict[str, str], iterable: Iterable[list[str]], /) -> None: ... - @overload - def __init__(self: UserDict[bytes, bytes], iterable: Iterable[list[bytes]], /) -> None: ... - def __len__(self) -> int: ... - def __getitem__(self, key: _KT) -> _VT: ... - def __setitem__(self, key: _KT, item: _VT) -> None: ... - def __delitem__(self, key: _KT) -> None: ... - def __iter__(self) -> Iterator[_KT]: ... - def __contains__(self, key: object) -> bool: ... - def copy(self) -> Self: ... - def __copy__(self) -> Self: ... - # `UserDict.fromkeys` has the same semantics as `dict.fromkeys`, so should be kept in line with `dict.fromkeys`. - # TODO: Much like `dict.fromkeys`, the true signature of `UserDict.fromkeys` is inexpressible in the current type system. - # See #3800 & https://github.com/python/typing/issues/548#issuecomment-683336963. - @classmethod - @overload - def fromkeys(cls, iterable: Iterable[_T], value: None = None) -> UserDict[_T, Any | None]: ... - @classmethod - @overload - def fromkeys(cls, iterable: Iterable[_T], value: _S) -> UserDict[_T, _S]: ... - @overload - def __or__(self, other: UserDict[_KT, _VT] | dict[_KT, _VT]) -> Self: ... - @overload - def __or__(self, other: UserDict[_T1, _T2] | dict[_T1, _T2]) -> UserDict[_KT | _T1, _VT | _T2]: ... - @overload - def __ror__(self, other: UserDict[_KT, _VT] | dict[_KT, _VT]) -> Self: ... - @overload - def __ror__(self, other: UserDict[_T1, _T2] | dict[_T1, _T2]) -> UserDict[_KT | _T1, _VT | _T2]: ... - # UserDict.__ior__ should be kept roughly in line with MutableMapping.update() - @overload # type: ignore[misc] - def __ior__(self, other: SupportsKeysAndGetItem[_KT, _VT]) -> Self: ... - @overload - def __ior__(self, other: Iterable[tuple[_KT, _VT]]) -> Self: ... - if sys.version_info >= (3, 12): - @overload - def get(self, key: _KT, default: None = None) -> _VT | None: ... - @overload - def get(self, key: _KT, default: _VT) -> _VT: ... - @overload - def get(self, key: _KT, default: _T) -> _VT | _T: ... - -class UserList(MutableSequence[_T]): - data: list[_T] - @overload - def __init__(self, initlist: None = None) -> None: ... - @overload - def __init__(self, initlist: Iterable[_T]) -> None: ... - __hash__: ClassVar[None] # type: ignore[assignment] - def __lt__(self, other: list[_T] | UserList[_T]) -> bool: ... - def __le__(self, other: list[_T] | UserList[_T]) -> bool: ... - def __gt__(self, other: list[_T] | UserList[_T]) -> bool: ... - def __ge__(self, other: list[_T] | UserList[_T]) -> bool: ... - def __eq__(self, other: object) -> bool: ... - def __contains__(self, item: object) -> bool: ... - def __len__(self) -> int: ... - @overload - def __getitem__(self, i: SupportsIndex) -> _T: ... - @overload - def __getitem__(self, i: slice[SupportsIndex | None]) -> Self: ... - @overload - def __setitem__(self, i: SupportsIndex, item: _T) -> None: ... - @overload - def __setitem__(self, i: slice[SupportsIndex | None], item: Iterable[_T]) -> None: ... - def __delitem__(self, i: SupportsIndex | slice[SupportsIndex | None]) -> None: ... - def __add__(self, other: Iterable[_T]) -> Self: ... - def __radd__(self, other: Iterable[_T]) -> Self: ... - def __iadd__(self, other: Iterable[_T]) -> Self: ... - def __mul__(self, n: int) -> Self: ... - def __rmul__(self, n: int) -> Self: ... - def __imul__(self, n: int) -> Self: ... - def append(self, item: _T) -> None: ... - def insert(self, i: int, item: _T) -> None: ... - def pop(self, i: int = -1) -> _T: ... - def remove(self, item: _T) -> None: ... - def copy(self) -> Self: ... - def __copy__(self) -> Self: ... - def count(self, item: _T) -> int: ... - # The runtime signature is "item, *args", and the arguments are then passed - # to `list.index`. In order to give more precise types, we pretend that the - # `item` argument is positional-only. - def index(self, item: _T, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int: ... - # All arguments are passed to `list.sort` at runtime, so the signature should be kept in line with `list.sort`. - @overload - def sort(self: UserList[SupportsRichComparisonT], *, key: None = None, reverse: bool = False) -> None: ... - @overload - def sort(self, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = False) -> None: ... - def extend(self, other: Iterable[_T]) -> None: ... - -class UserString(Sequence[UserString]): - data: str - def __init__(self, seq: object) -> None: ... - def __int__(self) -> int: ... - def __float__(self) -> float: ... - def __complex__(self) -> complex: ... - def __getnewargs__(self) -> tuple[str]: ... - def __lt__(self, string: str | UserString) -> bool: ... - def __le__(self, string: str | UserString) -> bool: ... - def __gt__(self, string: str | UserString) -> bool: ... - def __ge__(self, string: str | UserString) -> bool: ... - def __eq__(self, string: object) -> bool: ... - def __hash__(self) -> int: ... - def __contains__(self, char: object) -> bool: ... - def __len__(self) -> int: ... - def __getitem__(self, index: SupportsIndex | slice[SupportsIndex | None]) -> Self: ... - def __iter__(self) -> Iterator[Self]: ... - def __reversed__(self) -> Iterator[Self]: ... - def __add__(self, other: object) -> Self: ... - def __radd__(self, other: object) -> Self: ... - def __mul__(self, n: int) -> Self: ... - def __rmul__(self, n: int) -> Self: ... - def __mod__(self, args: Any) -> Self: ... - def __rmod__(self, template: object) -> Self: ... - def capitalize(self) -> Self: ... - def casefold(self) -> Self: ... - def center(self, width: int, *args: Any) -> Self: ... - def count(self, sub: str | UserString, start: int = 0, end: int = sys.maxsize) -> int: ... - def encode(self: UserString, encoding: str | None = 'utf-8', errors: str | None = 'strict') -> bytes: ... - def endswith(self, suffix: str | tuple[str, ...], start: int | None = 0, end: int | None = sys.maxsize) -> bool: ... - def expandtabs(self, tabsize: int = 8) -> Self: ... - def find(self, sub: str | UserString, start: int = 0, end: int = sys.maxsize) -> int: ... - def format(self, *args: Any, **kwds: Any) -> str: ... - def format_map(self, mapping: Mapping[str, Any]) -> str: ... - def index(self, sub: str, start: int = 0, end: int = sys.maxsize) -> int: ... - def isalpha(self) -> bool: ... - def isalnum(self) -> bool: ... - def isdecimal(self) -> bool: ... - def isdigit(self) -> bool: ... - def isidentifier(self) -> bool: ... - def islower(self) -> bool: ... - def isnumeric(self) -> bool: ... - def isprintable(self) -> bool: ... - def isspace(self) -> bool: ... - def istitle(self) -> bool: ... - def isupper(self) -> bool: ... - def isascii(self) -> bool: ... - def join(self, seq: Iterable[str]) -> str: ... - def ljust(self, width: int, *args: Any) -> Self: ... - def lower(self) -> Self: ... - def lstrip(self, chars: str | None = None) -> Self: ... - maketrans = str.maketrans - def partition(self, sep: str) -> tuple[str, str, str]: ... - def removeprefix(self, prefix: str | UserString, /) -> Self: ... - def removesuffix(self, suffix: str | UserString, /) -> Self: ... - def replace(self, old: str | UserString, new: str | UserString, maxsplit: int = -1) -> Self: ... - def rfind(self, sub: str | UserString, start: int = 0, end: int = sys.maxsize) -> int: ... - def rindex(self, sub: str | UserString, start: int = 0, end: int = sys.maxsize) -> int: ... - def rjust(self, width: int, *args: Any) -> Self: ... - def rpartition(self, sep: str) -> tuple[str, str, str]: ... - def rstrip(self, chars: str | None = None) -> Self: ... - def split(self, sep: str | None = None, maxsplit: int = -1) -> list[str]: ... - def rsplit(self, sep: str | None = None, maxsplit: int = -1) -> list[str]: ... - def splitlines(self, keepends: bool = False) -> list[str]: ... - def startswith( - self, prefix: str | tuple[str, ...], start: int | None = 0, end: int | None = sys.maxsize - ) -> bool: ... - def strip(self, chars: str | None = None) -> Self: ... - def swapcase(self) -> Self: ... - def title(self) -> Self: ... - def translate(self, *args: Any) -> Self: ... - def upper(self) -> Self: ... - def zfill(self, width: int) -> Self: ... +# NOTE: This is a Monty-narrowed copy of the upstream typeshed stub. Only the +# members implemented by Monty's runtime are exposed (deque, defaultdict, +# Counter, namedtuple); OrderedDict, ChainMap, UserDict, UserList and +# UserString are intentionally omitted so type-checking matches the runtime. +# The source of truth is custom/collections/__init__.pyi (see custom/README.md). @disjoint_base class deque(MutableSequence[_T]): @@ -343,76 +151,6 @@ class Counter(dict[_T, int], Generic[_T]): def __ge__(self, other: Counter[Any]) -> bool: ... def __gt__(self, other: Counter[Any]) -> bool: ... -# The pure-Python implementations of the "views" classes -# These are exposed at runtime in `collections/__init__.py` -class _OrderedDictKeysView(KeysView[_KT_co]): - def __reversed__(self) -> Iterator[_KT_co]: ... - -class _OrderedDictItemsView(ItemsView[_KT_co, _VT_co]): - def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... - -class _OrderedDictValuesView(ValuesView[_VT_co]): - def __reversed__(self) -> Iterator[_VT_co]: ... - -# The C implementations of the "views" classes -# (At runtime, these are called `odict_keys`, `odict_items` and `odict_values`, -# but they are not exposed anywhere) -# pyright doesn't have a specific error code for subclassing error! -@final -@type_check_only -class _odict_keys(dict_keys[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] - def __reversed__(self) -> Iterator[_KT_co]: ... - -@final -@type_check_only -class _odict_items(dict_items[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] - def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... - -@final -@type_check_only -class _odict_values(dict_values[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] - def __reversed__(self) -> Iterator[_VT_co]: ... - -@disjoint_base -class OrderedDict(dict[_KT, _VT]): - def popitem(self, last: bool = True) -> tuple[_KT, _VT]: ... - def move_to_end(self, key: _KT, last: bool = True) -> None: ... - def copy(self) -> Self: ... - def __reversed__(self) -> Iterator[_KT]: ... - def keys(self) -> _odict_keys[_KT, _VT]: ... - def items(self) -> _odict_items[_KT, _VT]: ... - def values(self) -> _odict_values[_KT, _VT]: ... - # The signature of OrderedDict.fromkeys should be kept in line with `dict.fromkeys`, modulo positional-only differences. - # Like dict.fromkeys, its true signature is not expressible in the current type system. - # See #3800 & https://github.com/python/typing/issues/548#issuecomment-683336963. - @classmethod - @overload - def fromkeys(cls, iterable: Iterable[_T], value: None = None) -> OrderedDict[_T, Any | None]: ... - @classmethod - @overload - def fromkeys(cls, iterable: Iterable[_T], value: _S) -> OrderedDict[_T, _S]: ... - # Keep OrderedDict.setdefault in line with MutableMapping.setdefault, modulo positional-only differences. - @overload - def setdefault(self: OrderedDict[_KT, _T | None], key: _KT, default: None = None) -> _T | None: ... - @overload - def setdefault(self, key: _KT, default: _VT) -> _VT: ... - # Same as dict.pop, but accepts keyword arguments - @overload - def pop(self, key: _KT) -> _VT: ... - @overload - def pop(self, key: _KT, default: _VT) -> _VT: ... - @overload - def pop(self, key: _KT, default: _T) -> _VT | _T: ... - def __eq__(self, value: object, /) -> bool: ... - @overload - def __or__(self, value: dict[_KT, _VT], /) -> Self: ... - @overload - def __or__(self, value: dict[_T1, _T2], /) -> OrderedDict[_KT | _T1, _VT | _T2]: ... - @overload - def __ror__(self, value: dict[_KT, _VT], /) -> Self: ... - @overload - def __ror__(self, value: dict[_T1, _T2], /) -> OrderedDict[_KT | _T1, _VT | _T2]: ... # type: ignore[misc] - @disjoint_base class defaultdict(dict[_KT, _VT]): default_factory: Callable[[], _VT] | None @@ -461,67 +199,3 @@ class defaultdict(dict[_KT, _VT]): @overload def __ror__(self, value: dict[_T1, _T2], /) -> defaultdict[_KT | _T1, _VT | _T2]: ... # type: ignore[misc] -class ChainMap(MutableMapping[_KT, _VT]): - maps: list[MutableMapping[_KT, _VT]] - def __init__(self, *maps: MutableMapping[_KT, _VT]) -> None: ... - def new_child(self, m: MutableMapping[_KT, _VT] | None = None) -> Self: ... - @property - def parents(self) -> Self: ... - def __setitem__(self, key: _KT, value: _VT) -> None: ... - def __delitem__(self, key: _KT) -> None: ... - def __getitem__(self, key: _KT) -> _VT: ... - def __iter__(self) -> Iterator[_KT]: ... - def __len__(self) -> int: ... - def __contains__(self, key: object) -> bool: ... - @overload - def get(self, key: _KT, default: None = None) -> _VT | None: ... - @overload - def get(self, key: _KT, default: _VT) -> _VT: ... - @overload - def get(self, key: _KT, default: _T) -> _VT | _T: ... - def __missing__(self, key: _KT) -> _VT: ... # undocumented - def __bool__(self) -> bool: ... - # Keep ChainMap.setdefault in line with MutableMapping.setdefault, modulo positional-only differences. - @overload - def setdefault(self: ChainMap[_KT, _T | None], key: _KT, default: None = None) -> _T | None: ... - @overload - def setdefault(self, key: _KT, default: _VT) -> _VT: ... - @overload - def pop(self, key: _KT) -> _VT: ... - @overload - def pop(self, key: _KT, default: _VT) -> _VT: ... - @overload - def pop(self, key: _KT, default: _T) -> _VT | _T: ... - def copy(self) -> Self: ... - __copy__ = copy - # All arguments to `fromkeys` are passed to `dict.fromkeys` at runtime, - # so the signature should be kept in line with `dict.fromkeys`. - if sys.version_info >= (3, 13): - @classmethod - @overload - def fromkeys(cls, iterable: Iterable[_T], /) -> ChainMap[_T, Any | None]: ... - else: - @classmethod - @overload - def fromkeys(cls, iterable: Iterable[_T]) -> ChainMap[_T, Any | None]: ... - - @classmethod - @overload - # Special-case None: the user probably wants to add non-None values later. - def fromkeys(cls, iterable: Iterable[_T], value: None, /) -> ChainMap[_T, Any | None]: ... - @classmethod - @overload - def fromkeys(cls, iterable: Iterable[_T], value: _S, /) -> ChainMap[_T, _S]: ... - @overload - def __or__(self, other: Mapping[_KT, _VT]) -> Self: ... - @overload - def __or__(self, other: Mapping[_T1, _T2]) -> ChainMap[_KT | _T1, _VT | _T2]: ... - @overload - def __ror__(self, other: Mapping[_KT, _VT]) -> Self: ... - @overload - def __ror__(self, other: Mapping[_T1, _T2]) -> ChainMap[_KT | _T1, _VT | _T2]: ... - # ChainMap.__ior__ should be kept roughly in line with MutableMapping.update() - @overload # type: ignore[misc] - def __ior__(self, other: SupportsKeysAndGetItem[_KT, _VT]) -> Self: ... - @overload - def __ior__(self, other: Iterable[tuple[_KT, _VT]]) -> Self: ... diff --git a/crates/monty/src/bytecode/vm/call.rs b/crates/monty/src/bytecode/vm/call.rs index b0636a032..ee609198a 100644 --- a/crates/monty/src/bytecode/vm/call.rs +++ b/crates/monty/src/bytecode/vm/call.rs @@ -20,7 +20,10 @@ use crate::{ intern::{FunctionId, StaticStrings, StringId}, os::OsFunctionCall, resource::ResourceTracker, - types::{Dict, Instance, PyTrait, Type, bytes::call_bytes_method, instance::class_name, str::call_str_method}, + types::{ + Dict, Instance, PyTrait, Type, bytes::call_bytes_method, instance::class_name, namedtuple_class, + str::call_str_method, + }, value::{EitherStr, Value}, }; @@ -446,6 +449,8 @@ impl VM<'_, T> { let (func_id, cells, defaults) = match self.heap.get(heap_id) { HeapData::Class(_) => return self.instantiate_class(heap_id, args), + // Calling a namedtuple class constructs a NamedTuple instance. + HeapData::NamedTupleClass(_) => return namedtuple_class::instantiate(self, heap_id, args), HeapData::BoundMethod(bm) => { let instance = bm.instance.clone_with_heap(self); let func = bm.func.clone_with_heap(self); diff --git a/crates/monty/src/bytecode/vm/collections.rs b/crates/monty/src/bytecode/vm/collections.rs index 41947d8c8..e0f2b74a1 100644 --- a/crates/monty/src/bytecode/vm/collections.rs +++ b/crates/monty/src/bytecode/vm/collections.rs @@ -561,6 +561,20 @@ impl VM<'_, T> { } tuple.as_slice().iter().map(|v| v.clone_with_heap(this)).collect() } + HeapData::NamedTuple(nt) => { + let nt_len = nt.as_vec().len(); + if nt_len != count { + return Err(unpack_size_error(count, nt_len)); + } + nt.as_vec().iter().map(|v| v.clone_with_heap(this)).collect() + } + HeapData::Deque(deque) => { + let deque_len = deque.as_deque().len(); + if deque_len != count { + return Err(unpack_size_error(count, deque_len)); + } + deque.as_deque().iter().map(|v| v.clone_with_heap(this)).collect() + } HeapData::Str(s) => { let str_len = s.as_str().chars().count(); if str_len != count { diff --git a/crates/monty/src/heap.rs b/crates/monty/src/heap.rs index 0fcf3d3cb..e7aa63d82 100644 --- a/crates/monty/src/heap.rs +++ b/crates/monty/src/heap.rs @@ -20,8 +20,8 @@ use crate::{ resource::{ResourceError, ResourceTracker}, types::{ BoundMethod, Bytes, Class, Dataclass, Deque, Dict, DictItemsView, DictKeysView, DictSubclass, DictSubclassKind, - DictValuesView, FrozenSet, Instance, List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, Range, - ReMatch, RePattern, Set, Slice, Str, TimeZone, Tuple, date, datetime, timedelta, timezone, + DictValuesView, FrozenSet, Instance, List, LongInt, Module, MontyIter, NamedTuple, NamedTupleClass, OpenFile, + Path, Range, ReMatch, RePattern, Set, Slice, Str, TimeZone, Tuple, date, datetime, timedelta, timezone, }, value::Value, }; @@ -207,6 +207,7 @@ pub enum HeapReadOutput<'a> { List(HeapRead<'a, List>), Tuple(HeapRead<'a, Tuple>), NamedTuple(HeapRead<'a, NamedTuple>), + NamedTupleClass(HeapRead<'a, NamedTupleClass>), Deque(HeapRead<'a, Deque>), Dict(HeapRead<'a, Dict>), DictSubclass(HeapRead<'a, DictSubclass>), @@ -594,6 +595,7 @@ impl<'a> HeapPtr<'a> { HeapData::List(list) => HeapReadOutput::List(heap_read(base, list, readers)), HeapData::Tuple(tuple) => HeapReadOutput::Tuple(heap_read(base, tuple, readers)), HeapData::NamedTuple(named_tuple) => HeapReadOutput::NamedTuple(heap_read(base, named_tuple, readers)), + HeapData::NamedTupleClass(class) => HeapReadOutput::NamedTupleClass(heap_read(base, class, readers)), HeapData::Deque(deque) => HeapReadOutput::Deque(heap_read(base, deque, readers)), HeapData::Dict(dict) => HeapReadOutput::Dict(heap_read(base, dict, readers)), HeapData::DictSubclass(sub) => HeapReadOutput::DictSubclass(heap_read(base, sub, readers)), @@ -1461,6 +1463,14 @@ fn for_each_child_id(data: &HeapData, mut on_child: F) { } } } + HeapData::NamedTupleClass(class) => { + // The only heap refs a namedtuple class holds are its default values. + for value in class.defaults() { + if let Value::Ref(id) = value { + on_child(*id); + } + } + } HeapData::DictSubclass(sub) => { // The backing dict is always a heap ref; the factory may be too. on_child(sub.dict_id()); @@ -1672,6 +1682,7 @@ fn py_dec_ref_ids_for_data(data: &mut HeapData, stack: &mut Vec) { HeapData::List(l) => l.py_dec_ref_ids(stack), HeapData::Tuple(t) => t.py_dec_ref_ids(stack), HeapData::NamedTuple(nt) => nt.py_dec_ref_ids(stack), + HeapData::NamedTupleClass(c) => c.py_dec_ref_ids(stack), HeapData::Deque(dq) => dq.py_dec_ref_ids(stack), HeapData::Dict(d) => d.py_dec_ref_ids(stack), HeapData::DictSubclass(sub) => sub.py_dec_ref_ids(stack), diff --git a/crates/monty/src/heap_data.rs b/crates/monty/src/heap_data.rs index 99b5f75f7..13c17959b 100644 --- a/crates/monty/src/heap_data.rs +++ b/crates/monty/src/heap_data.rs @@ -19,9 +19,9 @@ use crate::{ intern::FunctionId, types::{ BoundMethod, Bytes, Class, Dataclass, Deque, Dict, DictItemsView, DictKeysView, DictSubclass, DictSubclassKind, - DictValuesView, FrozenSet, Instance, LazyHeapSet, List, LongInt, Module, MontyIter, NamedTuple, OpenFile, Path, - PyTrait, Range, ReMatch, RePattern, Set, Slice, Str, Tuple, Type, date, datetime, str::allocate_string, - timedelta, timezone, + DictValuesView, FrozenSet, Instance, LazyHeapSet, List, LongInt, Module, MontyIter, NamedTuple, + NamedTupleClass, OpenFile, Path, PyTrait, Range, ReMatch, RePattern, Set, Slice, Str, Tuple, Type, date, + datetime, str::allocate_string, timedelta, timezone, }, value::{EitherStr, Value, eq_bigint, eq_bytes, eq_ext_function, eq_str}, }; @@ -38,6 +38,8 @@ pub(crate) enum HeapData { List(List), Tuple(Tuple), NamedTuple(NamedTuple), + /// A class object produced by `collections.namedtuple(...)`. + NamedTupleClass(NamedTupleClass), /// A `collections.deque` double-ended queue. Deque(Deque), Dict(Dict), @@ -175,6 +177,7 @@ impl HeapData { Self::List(_) | Self::Tuple(_) | Self::NamedTuple(_) + | Self::NamedTupleClass(_) | Self::Deque(_) | Self::Dict(_) | Self::DictSubclass(_) @@ -215,6 +218,8 @@ impl HeapData { Self::Bytes(_) => Type::Bytes, Self::List(_) => Type::List, Self::Tuple(_) | Self::NamedTuple(_) => Type::Tuple, + // A namedtuple *class* object's type is `type`. + Self::NamedTupleClass(_) => Type::Type, Self::Deque(_) => Type::Deque, Self::Dict(_) => Type::Dict, // A dict subclass reports its concrete Python type by kind. @@ -259,6 +264,7 @@ impl HeapData { Self::List(l) => l.py_estimate_size(), Self::Tuple(t) => t.py_estimate_size(), Self::NamedTuple(nt) => nt.py_estimate_size(), + Self::NamedTupleClass(c) => c.py_estimate_size(), Self::Deque(dq) => dq.py_estimate_size(), Self::Dict(d) => d.py_estimate_size(), Self::DictSubclass(sub) => sub.py_estimate_size(), @@ -477,6 +483,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_bool(vm), Self::Tuple(t) => t.py_bool(vm), Self::NamedTuple(nt) => nt.py_bool(vm), + Self::NamedTupleClass(_) => true, Self::Deque(dq) => dq.py_bool(vm), Self::Dict(d) => d.py_bool(vm), Self::DictSubclass(sub) => sub.py_bool(vm), @@ -520,7 +527,9 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { HeapReadOutput::Bytes(b) => Ok(b.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::List(list) => Ok(list.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::Tuple(t) => Ok(t.py_call_attr(self_id, vm, attr, args)?), + HeapReadOutput::NamedTuple(nt) => Ok(nt.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::Deque(dq) => Ok(dq.py_call_attr(self_id, vm, attr, args)?), + HeapReadOutput::NamedTupleClass(c) => Ok(c.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::Dict(dict) => Ok(dict.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::DictSubclass(sub) => Ok(sub.py_call_attr(self_id, vm, attr, args)?), HeapReadOutput::DictKeysView(view) => Ok(view.py_call_attr(self_id, vm, attr, args)?), @@ -598,6 +607,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_type(vm), Self::Tuple(t) => t.py_type(vm), Self::NamedTuple(nt) => nt.py_type(vm), + Self::NamedTupleClass(c) => c.py_type(vm), Self::Deque(dq) => dq.py_type(vm), Self::Dict(d) => d.py_type(vm), Self::DictSubclass(sub) => sub.py_type(vm), @@ -679,6 +689,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { HeapReadOutput::List(a) => a.py_eq_impl(other, vm), HeapReadOutput::Tuple(a) => a.py_eq_impl(other, vm), HeapReadOutput::NamedTuple(a) => a.py_eq_impl(other, vm), + HeapReadOutput::NamedTupleClass(a) => a.py_eq_impl(other, vm), HeapReadOutput::Deque(a) => a.py_eq_impl(other, vm), HeapReadOutput::Dict(a) => a.py_eq_impl(other, vm), HeapReadOutput::DictSubclass(a) => a.py_eq_impl(other, vm), @@ -776,6 +787,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_repr_fmt(f, vm, heap_ids), Self::Tuple(t) => t.py_repr_fmt(f, vm, heap_ids), Self::NamedTuple(nt) => nt.py_repr_fmt(f, vm, heap_ids), + Self::NamedTupleClass(c) => c.py_repr_fmt(f, vm, heap_ids), Self::Deque(dq) => dq.py_repr_fmt(f, vm, heap_ids), Self::Dict(d) => d.py_repr_fmt(f, vm, heap_ids), Self::DictSubclass(sub) => sub.py_repr_fmt(f, vm, heap_ids), @@ -1014,6 +1026,7 @@ impl<'h> PyTrait<'h> for HeapReadOutput<'h> { Self::List(l) => l.py_getattr(attr, vm), Self::Tuple(t) => t.py_getattr(attr, vm), Self::NamedTuple(nt) => nt.py_getattr(attr, vm), + Self::NamedTupleClass(c) => c.py_getattr(attr, vm), Self::Deque(dq) => dq.py_getattr(attr, vm), Self::Dict(d) => d.py_getattr(attr, vm), Self::DictSubclass(sub) => sub.py_getattr(attr, vm), diff --git a/crates/monty/src/modules/collections.rs b/crates/monty/src/modules/collections.rs index 5c3c94c2b..ccc371e85 100644 --- a/crates/monty/src/modules/collections.rs +++ b/crates/monty/src/modules/collections.rs @@ -4,21 +4,41 @@ //! - `deque` — a double-ended queue ([`Type::Deque`]) //! - `defaultdict` — a dict with a default factory ([`Type::DefaultDict`]) //! - `Counter` — a dict for counting ([`Type::Counter`]) +//! - `namedtuple` — a factory function producing named-tuple classes //! -//! Types are exposed as callable `Value::Builtin(Builtins::Type(...))` module -//! attributes; their construction and behavior live in the corresponding -//! runtime types under `crate::types`. +//! The container types are exposed as callable +//! `Value::Builtin(Builtins::Type(...))` module attributes; `namedtuple` is a +//! module-level function. Their behavior lives in the corresponding runtime +//! types under `crate::types`. use crate::{ + args::ArgValues, builtins::Builtins, bytecode::VM, + exception_private::RunResult, heap::{HeapData, HeapId}, intern::StaticStrings, + modules::ModuleFunctions, resource::{ResourceError, ResourceTracker}, - types::{Module, Type}, + types::{Module, Type, namedtuple_class::make_namedtuple}, value::Value, }; +/// Module-level functions of the `collections` module. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum::Display, serde::Serialize, serde::Deserialize)] +#[strum(serialize_all = "lowercase")] +pub(crate) enum CollectionsFunctions { + /// `collections.namedtuple(typename, field_names, *, rename, defaults, module)`. + Namedtuple, +} + +/// Dispatches a `collections` module function call. +pub fn call(vm: &mut VM<'_, impl ResourceTracker>, func: CollectionsFunctions, args: ArgValues) -> RunResult { + match func { + CollectionsFunctions::Namedtuple => make_namedtuple(vm, args), + } +} + /// Creates the `collections` module and allocates it on the heap. /// /// Returns a `HeapId` pointing to the newly allocated module. @@ -40,6 +60,11 @@ pub fn create_module(vm: &mut VM<'_, impl ResourceTracker>) -> Result) -> fmt::Result { match self { Self::Asyncio(func) => write!(f, "{func}"), + Self::Collections(func) => write!(f, "{func}"), Self::Json(func) => write!(f, "{func}"), Self::Math(func) => write!(f, "{func}"), Self::Os(func) => write!(f, "{func}"), @@ -161,6 +163,7 @@ impl ModuleFunctions { pub fn call(self, vm: &mut VM<'_, impl ResourceTracker>, args: ArgValues) -> RunResult { match self { Self::Asyncio(functions) => asyncio::call(vm, functions, args), + Self::Collections(functions) => collections::call(vm, functions, args).map(CallResult::Value), Self::Json(functions) => json::call(vm, functions, args).map(CallResult::Value), Self::Math(functions) => math::call(vm, functions, args).map(CallResult::Value), Self::Os(functions) => os::call(vm, functions, args), diff --git a/crates/monty/src/object.rs b/crates/monty/src/object.rs index ac539e798..9a71b87d4 100644 --- a/crates/monty/src/object.rs +++ b/crates/monty/src/object.rs @@ -904,6 +904,11 @@ impl MontyObject { values, } } + HeapReadOutput::NamedTupleClass(class) => { + // A namedtuple class is an opaque class object at the host + // boundary; represent it by its repr. + Self::Repr(format!("", class.get(vm.heap).typename(vm))) + } HeapReadOutput::Deque(deque) => { // Represent a deque to the host as a list of its items — // the wire boundary has no dedicated deque type. diff --git a/crates/monty/src/types/mod.rs b/crates/monty/src/types/mod.rs index 23a929d53..7b7980246 100644 --- a/crates/monty/src/types/mod.rs +++ b/crates/monty/src/types/mod.rs @@ -21,6 +21,7 @@ pub mod list; pub mod long_int; pub mod module; pub mod namedtuple; +pub mod namedtuple_class; pub mod path; pub mod property; pub mod py_trait; @@ -49,6 +50,7 @@ pub(crate) use list::List; pub(crate) use long_int::LongInt; pub(crate) use module::Module; pub(crate) use namedtuple::NamedTuple; +pub(crate) use namedtuple_class::NamedTupleClass; pub(crate) use path::Path; pub(crate) use property::Property; pub(crate) use py_trait::{AttrCallResult, CmpOrder, LazyHeapSet, PyTrait}; diff --git a/crates/monty/src/types/namedtuple.rs b/crates/monty/src/types/namedtuple.rs index fe79876a9..2953a1b2e 100644 --- a/crates/monty/src/types/namedtuple.rs +++ b/crates/monty/src/types/namedtuple.rs @@ -25,14 +25,18 @@ use std::{ use super::PyTrait; use crate::{ + args::ArgValues, bytecode::{CallResult, ContainsVM, DropWithVM, RecursionToken, VM}, defer_drop, defer_drop_vm_mut, - exception_private::{ExcType, RunResult}, + exception_private::{ExcType, RunResult, SimpleException}, hash::HashValue, - heap::{HeapId, HeapItem, HeapRead, HeapReadOutput}, - intern::{Interns, StringId}, + heap::{DropWithHeap, HeapData, HeapId, HeapItem, HeapRead, HeapReadOutput}, + intern::{Interns, StaticStrings, StringId}, resource::ResourceTracker, - types::{Type, py_trait::LazyHeapSet}, + types::{ + Dict, MontyIter, Type, allocate_tuple, py_trait::LazyHeapSet, slice::normalize_sequence_index, + str::allocate_string, + }, value::{EitherStr, Value}, }; @@ -376,6 +380,10 @@ impl<'h> PyTrait<'h> for HeapRead<'h, NamedTuple> { } fn py_getattr(&self, attr: &EitherStr, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult> { + // `_fields` is the tuple of field names (accessible on instances too). + if attr.static_string() == Some(StaticStrings::Fields) { + return Ok(Some(CallResult::Value(self.fields_tuple(vm)?))); + } let attr_name = attr.as_str(vm.interns); if let Some(value) = self.get(vm.heap).get_by_name(attr_name, vm.interns) { Ok(Some(CallResult::Value(value.clone_with_heap(vm.heap)))) @@ -384,6 +392,180 @@ impl<'h> PyTrait<'h> for HeapRead<'h, NamedTuple> { Err(ExcType::attribute_error(self.get(vm.heap).name(vm.interns), attr_name)) } } + + fn py_call_attr( + &mut self, + _self_id: HeapId, + vm: &mut VM<'h, impl ResourceTracker>, + attr: &EitherStr, + args: ArgValues, + ) -> RunResult { + match attr.static_string() { + Some(StaticStrings::Asdict) => { + args.check_zero_args("_asdict", vm.heap)?; + Ok(CallResult::Value(self.asdict(vm)?)) + } + Some(StaticStrings::ReplaceMethod) => Ok(CallResult::Value(self.replace(args, vm)?)), + Some(StaticStrings::Make) => Ok(CallResult::Value(self.make(args, vm)?)), + // Inherited tuple methods. + Some(StaticStrings::Count) => Ok(CallResult::Value(self.tuple_count(args, vm)?)), + Some(StaticStrings::Index) => Ok(CallResult::Value(self.tuple_index(args, vm)?)), + _ => { + let name = self.get(vm.heap).name(vm.interns).to_owned(); + args.drop_with_heap(vm); + Err(ExcType::attribute_error(&name, attr.as_str(vm.interns))) + } + } + } +} + +impl<'h> HeapRead<'h, NamedTuple> { + /// Builds the `_fields` tuple of field-name strings. + fn fields_tuple(&self, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let names: Vec = self + .get(vm.heap) + .field_names + .iter() + .map(|f| f.as_str(vm.interns).to_owned()) + .collect(); + let mut items = smallvec::SmallVec::<[Value; 3]>::new(); + for name in names { + items.push(allocate_string(name, vm.heap)?); + } + Ok(allocate_tuple(items, vm.heap)?) + } + + /// Implements `_asdict()`, returning an insertion-ordered `dict` of + /// field-name → value. + fn asdict(&self, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let len = self.get(vm.heap).items.len(); + let mut pairs = Vec::with_capacity(len); + for i in 0..len { + let name = self.get(vm.heap).field_names[i].as_str(vm.interns).to_owned(); + let key = allocate_string(name, vm.heap)?; + let value = self.get(vm.heap).items[i].clone_with_heap(vm.heap); + pairs.push((key, value)); + } + let dict = Dict::from_pairs(pairs, vm)?; + let id = vm.heap.allocate(HeapData::Dict(dict))?; + Ok(Value::Ref(id)) + } + + /// Implements `_replace(**kwargs)`, returning a new named tuple with the + /// named fields overridden. + fn replace(&self, args: ArgValues, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let (pos, kwargs) = args.into_parts(); + let pos: Vec = pos.collect(); + if !pos.is_empty() { + let n = pos.len(); + pos.drop_with_heap(vm); + kwargs.drop_with_heap(vm); + return Err(ExcType::type_error_too_many_positional_range( + "_replace", + 1, + 1, + n + 1, + 0, + )); + } + // Start from a clone of the current items, then override by field name. + let len = self.get(vm.heap).items.len(); + let mut items: Vec = (0..len) + .map(|i| self.get(vm.heap).items[i].clone_with_heap(vm.heap)) + .collect(); + for (key, value) in kwargs { + let key_str = key.to_str(vm).map(str::to_owned).unwrap_or_default(); + let position = self + .get(vm.heap) + .field_names + .iter() + .position(|f| f.as_str(vm.interns) == key_str); + if let Some(idx) = position { + let old = mem::replace(&mut items[idx], value); + old.drop_with_heap(vm); + } else { + value.drop_with_heap(vm); + items.drop_with_heap(vm); + return Err(SimpleException::new_msg( + ExcType::ValueError, + format!("Got unexpected field names: [{}]", py_repr_str(&key_str)), + ) + .into()); + } + } + let name = self.get(vm.heap).name.clone(); + let field_names = self.get(vm.heap).field_names.clone(); + let nt = NamedTuple::new(name, field_names, items); + let id = vm.heap.allocate(HeapData::NamedTuple(nt))?; + Ok(Value::Ref(id)) + } + + /// Implements the inherited `tuple.index(value[, start[, stop]])` method. + fn tuple_index(&self, args: ArgValues, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let pos_args = args.into_pos_only("tuple.index", vm.heap)?; + defer_drop!(pos_args, vm); + let len = self.get(vm.heap).items.len(); + let (value, start, end) = match pos_args.as_slice() { + [] => return Err(ExcType::type_error_at_least("tuple.index", 1, 0)), + [value] => (value, 0, len), + [value, start_arg] => (value, normalize_sequence_index(start_arg.as_int(vm)?, len), len), + [value, start_arg, end_arg] => { + let start = normalize_sequence_index(start_arg.as_int(vm)?, len); + let end = normalize_sequence_index(end_arg.as_int(vm)?, len).max(start); + (value, start, end) + } + other => return Err(ExcType::type_error_at_most("tuple.index", 3, other.len())), + }; + let iter = self.iter(vm)?; + defer_drop_vm_mut!(iter, vm); + while let Some((idx, item)) = iter.next_with_index(vm)? { + if idx >= end { + break; + } + if idx >= start && value.py_eq(item, vm)? { + return Ok(Value::Int(i64::try_from(idx).expect("index exceeds i64::MAX"))); + } + } + Err(ExcType::value_error_not_in_tuple()) + } + + /// Implements the inherited `tuple.count(value)` method. + fn tuple_count(&self, args: ArgValues, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let value = args.get_one_arg("tuple.count", vm.heap)?; + defer_drop!(value, vm); + let mut count = 0usize; + let iter = self.iter(vm)?; + defer_drop_vm_mut!(iter, vm); + while let Some(item) = iter.next(vm)? { + if value.py_eq(item, vm)? { + count += 1; + } + } + Ok(Value::Int(i64::try_from(count).expect("count exceeds i64::MAX"))) + } + + /// Implements `_make(iterable)`, building a new instance (with the same + /// name/fields) from an iterable of exactly `len(_fields)` items. + fn make(&self, args: ArgValues, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let iterable = args.get_one_arg("_make", vm.heap)?; + let items: Vec = MontyIter::new(iterable, vm)?.collect(vm)?; + let expected = self.get(vm.heap).field_names.len(); + if items.len() != expected { + let got = items.len(); + items.drop_with_heap(vm); + return Err(ExcType::type_error(format!("Expected {expected} arguments, got {got}"))); + } + let name = self.get(vm.heap).name.clone(); + let field_names = self.get(vm.heap).field_names.clone(); + let nt = NamedTuple::new(name, field_names, items); + let id = vm.heap.allocate(HeapData::NamedTuple(nt))?; + Ok(Value::Ref(id)) + } +} + +/// Renders a Python `repr()` of a string (single-quoted) for error messages. +fn py_repr_str(s: &str) -> String { + format!("'{s}'") } impl HeapItem for NamedTuple { diff --git a/crates/monty/src/types/namedtuple_class.rs b/crates/monty/src/types/namedtuple_class.rs new file mode 100644 index 000000000..7fccea25e --- /dev/null +++ b/crates/monty/src/types/namedtuple_class.rs @@ -0,0 +1,563 @@ +use std::{fmt::Write, mem}; + +use ruff_python_stdlib::keyword::is_keyword; + +use super::{Dict, LazyHeapSet, MontyIter, NamedTuple, PyTrait}; +use crate::{ + args::ArgValues, + bytecode::{CallResult, VM}, + exception_private::{ExcType, RunResult, SimpleException}, + heap::{DropWithHeap, HeapData, HeapId, HeapItem, HeapRead, HeapReadOutput}, + intern::StaticStrings, + resource::ResourceTracker, + types::{ + Type, allocate_tuple, + str::{allocate_string, str_isidentifier}, + }, + value::{EitherStr, Value}, +}; + +/// A class object produced by `collections.namedtuple(...)`. +/// +/// Calling it constructs a [`NamedTuple`] instance bound to `field_names` (with +/// trailing `defaults` applied). The class carries the class-level surface +/// (`__name__`, `_fields`, `_field_defaults`, `_make`); its own Python type is +/// `type` (like any class object). +/// +/// Field names and the type name are stored as [`EitherStr::Heap`] strings +/// because the factory runs at *runtime*, after the interner is frozen. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct NamedTupleClass { + /// The type name, e.g. `"Point"`. + typename: EitherStr, + /// Field names in definition order. + field_names: Vec, + /// Default values applied to the *last* `defaults.len()` fields. + defaults: Vec, +} + +impl NamedTupleClass { + /// Returns the type name. + pub fn typename<'a>(&'a self, vm: &'a VM<'_, impl ResourceTracker>) -> &'a str { + self.typename.as_str(vm.interns) + } + + /// Returns the default values (for GC traversal). + pub fn defaults(&self) -> &[Value] { + &self.defaults + } +} + +/// Implements the `namedtuple(typename, field_names, *, rename=False, +/// defaults=None, module=None)` factory, returning a new [`NamedTupleClass`]. +/// +/// Validates the type name and field names (identifiers, non-keywords, no +/// leading underscore, no duplicates) exactly as CPython does, honoring +/// `rename` to auto-fix invalid field names to `_0`, `_1`, …. +pub fn make_namedtuple(vm: &mut VM<'_, impl ResourceTracker>, args: ArgValues) -> RunResult { + let NamedtupleArgs { + typename, + field_names, + rename, + defaults, + } = parse_namedtuple_args(args, vm)?; + + // Extract the field-name strings, then release the `field_names` argument + // (it may be a heap list). `defaults` is dropped on every error path below. + let names_result = split_field_names(&field_names, vm); + field_names.drop_with_heap(vm); + let mut names = match names_result { + Ok(names) => names, + Err(e) => { + defaults.drop_with_heap(vm); + return Err(e); + } + }; + + // The typename and every field name must be a valid, non-keyword identifier. + if let Err(e) = validate_name(&typename) { + defaults.drop_with_heap(vm); + return Err(e); + } + if rename { + rename_invalid_fields(&mut names); + } + if let Err(e) = validate_field_names(&names, rename) { + defaults.drop_with_heap(vm); + return Err(e); + } + + // `defaults` longer than the field list is a CPython error. + if defaults.len() > names.len() { + defaults.drop_with_heap(vm); + return Err(SimpleException::new_msg(ExcType::TypeError, "Got more default values than field names").into()); + } + + let field_names: Vec = names.into_iter().map(EitherStr::Heap).collect(); + let class = NamedTupleClass { + typename: EitherStr::Heap(typename), + field_names, + defaults, + }; + let id = vm.heap.allocate(HeapData::NamedTupleClass(class))?; + Ok(Value::Ref(id)) +} + +/// Parsed and validated arguments for [`make_namedtuple`]. +struct NamedtupleArgs { + typename: String, + field_names: Value, + rename: bool, + defaults: Vec, +} + +/// Extracts the `namedtuple(...)` arguments, coercing `typename`/`rename` and +/// materializing `defaults` into an owned `Vec`. +fn parse_namedtuple_args(args: ArgValues, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult { + let (pos, kwargs) = args.into_parts(); + let mut pos: Vec = pos.collect(); + + let mut rename = false; + let mut defaults_val = Value::None; + // Bind keyword arguments (rename / defaults / module). `module` is accepted + // and ignored — Monty has no real module objects for namedtuples. + for (key, value) in kwargs { + let key_name = key.to_str(vm).map(str::to_owned).unwrap_or_default(); + match key_name.as_str() { + "rename" => rename = value.py_bool(vm), + "defaults" => defaults_val = value.clone_with_heap(vm.heap), + "module" => {} + _ => { + value.drop_with_heap(vm); + pos.drop_with_heap(vm); + defaults_val.drop_with_heap(vm); + return Err(ExcType::type_error_unexpected_keyword("namedtuple", &key_name)); + } + } + value.drop_with_heap(vm); + } + + if pos.len() < 2 || pos.len() > 2 { + // namedtuple(typename, field_names, ...) requires exactly these two + // positionals in Monty (rename/defaults/module are keyword-only here). + let n = pos.len(); + pos.drop_with_heap(vm); + defaults_val.drop_with_heap(vm); + return Err(ExcType::type_error_too_many_positional_range("namedtuple", 2, 2, n, 0)); + } + let field_names = pos.pop().expect("len == 2"); + let typename_val = pos.pop().expect("len == 2"); + + let Ok(typename) = typename_val.to_str(vm).map(str::to_owned) else { + let ty = typename_val.py_type_name(vm); + typename_val.drop_with_heap(vm); + field_names.drop_with_heap(vm); + defaults_val.drop_with_heap(vm); + return Err(ExcType::type_error(format!("expected str for typename, not {ty}"))); + }; + typename_val.drop_with_heap(vm); + + let defaults = if matches!(defaults_val, Value::None) { + Vec::new() + } else { + MontyIter::new(defaults_val, vm)?.collect(vm)? + }; + + Ok(NamedtupleArgs { + typename, + field_names, + rename, + defaults, + }) +} + +/// Splits the `field_names` argument into owned strings. A single string is +/// split on commas/whitespace (matching CPython); any other value is treated as +/// an iterable of strings. +fn split_field_names(field_names: &Value, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult> { + if let Ok(s) = field_names.to_str(vm) { + // CPython: field_names.replace(',', ' ').split() + Ok(s.replace(',', " ").split_whitespace().map(str::to_owned).collect()) + } else { + let items: Vec = MontyIter::new(field_names.clone_with_heap(vm.heap), vm)?.collect(vm)?; + let mut out = Vec::with_capacity(items.len()); + for item in items { + let result = item.to_str(vm).map(str::to_owned); + item.drop_with_heap(vm); + match result { + Ok(s) => out.push(s), + Err(_) => return Err(ExcType::type_error("field_names must be strings")), + } + } + Ok(out) + } +} + +/// Rewrites invalid field names to positional `_N` placeholders (the effect of +/// `rename=True`): a name that is not an identifier, is a keyword, starts with +/// an underscore, or duplicates an earlier field becomes `_{index}`. +fn rename_invalid_fields(names: &mut [String]) { + // `seen` tracks the *original* names (matching CPython), so a later + // duplicate of a valid name is itself renamed. + let mut seen: Vec = Vec::with_capacity(names.len()); + for (i, name) in names.iter_mut().enumerate() { + let original = name.clone(); + let invalid = !str_isidentifier(&original) + || is_keyword(&original) + || original.starts_with('_') + || seen.contains(&original); + if invalid { + *name = format!("_{i}"); + } + seen.push(original); + } +} + +/// Validates a single type/field name: must be a valid, non-keyword identifier. +/// +/// Uses `str.isidentifier` semantics (which accept keywords syntactically), so +/// the keyword check is applied separately — matching CPython's distinct error +/// messages. +fn validate_name(name: &str) -> RunResult<()> { + if !str_isidentifier(name) { + Err(SimpleException::new_msg( + ExcType::ValueError, + format!( + "Type names and field names must be valid identifiers: {}", + py_repr_str(name) + ), + ) + .into()) + } else if is_keyword(name) { + Err(SimpleException::new_msg( + ExcType::ValueError, + format!("Type names and field names cannot be a keyword: {}", py_repr_str(name)), + ) + .into()) + } else { + Ok(()) + } +} + +/// Validates every field name (identifier + keyword checks, then no leading +/// underscore and no duplicates), matching CPython's ordering and messages. +/// When `rename` is set, the leading-underscore check is skipped (renamed +/// fields legitimately start with `_`). +fn validate_field_names(names: &[String], rename: bool) -> RunResult<()> { + for name in names { + validate_name(name)?; + } + let mut seen: Vec<&str> = Vec::with_capacity(names.len()); + for name in names { + if name.starts_with('_') && !rename { + return Err(SimpleException::new_msg( + ExcType::ValueError, + format!("Field names cannot start with an underscore: {}", py_repr_str(name)), + ) + .into()); + } + if seen.contains(&name.as_str()) { + return Err(SimpleException::new_msg( + ExcType::ValueError, + format!("Encountered duplicate field name: {}", py_repr_str(name)), + ) + .into()); + } + seen.push(name); + } + Ok(()) +} + +/// Renders a Python `repr()` of a string (single-quoted) for error messages. +fn py_repr_str(s: &str) -> String { + format!("'{s}'") +} + +/// Constructs a [`NamedTuple`] instance from a call to a `NamedTupleClass`. +/// +/// Binds positional and keyword arguments to the class's field names (applying +/// trailing defaults), reproducing CPython's `{typename}.__new__` arity/keyword +/// errors. +pub(crate) fn instantiate( + vm: &mut VM<'_, impl ResourceTracker>, + class_id: HeapId, + args: ArgValues, +) -> RunResult { + // Snapshot the class's shape (names + defaults) so the borrow is released + // before we allocate the instance. + let (typename, field_names, defaults) = { + let HeapReadOutput::NamedTupleClass(class) = vm.heap.read(class_id) else { + unreachable!("namedtuple class") + }; + let c = class.get(vm.heap); + let typename = c.typename.clone(); + let field_names = c.field_names.clone(); + let defaults: Vec = c.defaults.iter().map(|v| v.clone_with_heap(vm.heap)).collect(); + (typename, field_names, defaults) + }; + + let new_name = format!("{}.__new__", typename.as_str(vm.interns)); + let items = bind_fields(&new_name, &field_names, defaults, args, vm)?; + let nt = NamedTuple::new(typename, field_names, items); + let id = vm.heap.allocate(HeapData::NamedTuple(nt))?; + Ok(CallResult::Value(Value::Ref(id))) +} + +/// Binds call arguments to field slots, applying `defaults` to trailing fields +/// and raising CPython-matching errors for arity/keyword mistakes. Consumes +/// `defaults` and all argument values. +fn bind_fields( + new_name: &str, + field_names: &[EitherStr], + defaults: Vec, + args: ArgValues, + vm: &mut VM<'_, impl ResourceTracker>, +) -> RunResult> { + let n = field_names.len(); + let required = n - defaults.len(); + let (pos, kwargs) = args.into_parts(); + let pos: Vec = pos.collect(); + + if pos.len() > n { + // "takes N positional arguments but M were given" (CPython counts self). + let given = pos.len(); + pos.drop_with_heap(vm); + kwargs.drop_with_heap(vm); + defaults.drop_with_heap(vm); + return Err(ExcType::type_error_too_many_positional_range( + new_name, + required + 1, + n + 1, + given + 1, + 0, + )); + } + + let mut slots: Vec> = Vec::with_capacity(n); + for v in pos { + slots.push(Some(v)); + } + while slots.len() < n { + slots.push(None); + } + + // Bind keyword arguments by field name. + for (key, value) in kwargs { + let key_str = key.to_str(vm).map(str::to_owned).unwrap_or_default(); + match field_names.iter().position(|f| f.as_str(vm.interns) == key_str) { + Some(idx) if slots[idx].is_none() => { + slots[idx] = Some(value); + } + Some(_) => { + // Already filled positionally → multiple values. + value.drop_with_heap(vm); + drop_slots(slots, vm); + defaults.drop_with_heap(vm); + return Err(ExcType::type_error_duplicate_arg(new_name, &key_str)); + } + None => { + value.drop_with_heap(vm); + drop_slots(slots, vm); + defaults.drop_with_heap(vm); + return Err(ExcType::type_error_unexpected_keyword(new_name, &key_str)); + } + } + } + + // Fill defaults for missing trailing fields; collect any still-missing names. + let default_start = required; + let mut result: Vec = Vec::with_capacity(n); + let mut missing: Vec<&str> = Vec::new(); + let mut defaults_iter = defaults.into_iter(); + let mut default_idx = 0; + for (i, slot) in slots.into_iter().enumerate() { + if let Some(v) = slot { + // Advance the defaults cursor so a later defaulted slot pairs up. + if i >= default_start { + if let Some(unused) = defaults_iter.next() { + unused.drop_with_heap(vm); + } + default_idx += 1; + } + result.push(v); + } else if i >= default_start { + // Use the matching default (defaults align to the last fields). + let target = i - default_start; + while default_idx < target { + if let Some(unused) = defaults_iter.next() { + unused.drop_with_heap(vm); + } + default_idx += 1; + } + match defaults_iter.next() { + Some(d) => result.push(d), + None => result.push(Value::None), + } + default_idx += 1; + } else { + missing.push(field_names[i].as_str(vm.interns)); + } + } + // Drop any defaults we never consumed. + for leftover in defaults_iter { + leftover.drop_with_heap(vm); + } + + if !missing.is_empty() { + result.drop_with_heap(vm); + return Err(ExcType::type_error_missing_positional_with_names(new_name, &missing)); + } + Ok(result) +} + +/// Drops all filled argument slots (used on error paths). +fn drop_slots(slots: Vec>, vm: &mut VM<'_, impl ResourceTracker>) { + for v in slots.into_iter().flatten() { + v.drop_with_heap(vm); + } +} + +impl<'h> HeapRead<'h, NamedTupleClass> { + /// Builds the `_fields` tuple (of field-name strings). + fn fields_tuple(&self, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let names: Vec = self + .get(vm.heap) + .field_names + .iter() + .map(|f| f.as_str(vm.interns).to_owned()) + .collect(); + let mut items = smallvec::SmallVec::<[Value; 3]>::new(); + for name in names { + items.push(allocate_string(name, vm.heap)?); + } + Ok(allocate_tuple(items, vm.heap)?) + } + + /// Builds the `_field_defaults` dict mapping each defaulted field to its + /// default value. + fn field_defaults_dict(&self, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult { + let (names, count) = { + let c = self.get(vm.heap); + let count = c.defaults.len(); + let start = c.field_names.len() - count; + let names: Vec = c.field_names[start..] + .iter() + .map(|f| f.as_str(vm.interns).to_owned()) + .collect(); + (names, count) + }; + let mut pairs = Vec::with_capacity(count); + for (i, name) in names.into_iter().enumerate() { + let key = allocate_string(name, vm.heap)?; + let value = self.get(vm.heap).defaults[i].clone_with_heap(vm.heap); + pairs.push((key, value)); + } + let dict = Dict::from_pairs(pairs, vm)?; + let id = vm.heap.allocate(HeapData::Dict(dict))?; + Ok(Value::Ref(id)) + } +} + +impl<'h> PyTrait<'h> for HeapRead<'h, NamedTupleClass> { + fn py_type(&self, _vm: &VM<'h, impl ResourceTracker>) -> Type { + // A class object's own type is `type` (matching `type(Point) is type`). + Type::Type + } + + fn py_len(&self, _vm: &VM<'h, impl ResourceTracker>) -> Option { + None + } + + fn py_bool(&self, _vm: &mut VM<'h, impl ResourceTracker>) -> bool { + true + } + + fn py_eq_impl(&self, _other: &Value, _vm: &mut VM<'h, impl ResourceTracker>) -> RunResult> { + // Class objects compare by identity (handled before the heap read). + Ok(None) + } + + fn py_repr_fmt( + &self, + f: &mut impl Write, + vm: &mut VM<'h, impl ResourceTracker>, + _heap_ids: &mut LazyHeapSet, + ) -> RunResult<()> { + // CPython renders ``. + Ok(write!(f, "", self.get(vm.heap).typename(vm))?) + } + + fn py_getattr(&self, attr: &EitherStr, vm: &mut VM<'h, impl ResourceTracker>) -> RunResult> { + let value = match attr.static_string() { + Some(StaticStrings::DunderName) => allocate_string(self.get(vm.heap).typename(vm).to_owned(), vm.heap)?, + Some(StaticStrings::Fields) => self.fields_tuple(vm)?, + Some(StaticStrings::FieldDefaults) => self.field_defaults_dict(vm)?, + _ => { + return Err(ExcType::attribute_error( + self.get(vm.heap).typename(vm), + attr.as_str(vm.interns), + )); + } + }; + Ok(Some(CallResult::Value(value))) + } + + fn py_call_attr( + &mut self, + self_id: HeapId, + vm: &mut VM<'h, impl ResourceTracker>, + attr: &EitherStr, + args: ArgValues, + ) -> RunResult { + if attr.static_string() == Some(StaticStrings::Make) { + namedtuple_make(self_id, args, vm) + } else { + args.drop_with_heap(vm); + Err(ExcType::attribute_error( + self.get(vm.heap).typename(vm), + attr.as_str(vm.interns), + )) + } + } +} + +/// Implements the `_make(iterable)` classmethod: builds an instance from an +/// iterable that must have exactly `len(_fields)` items. +fn namedtuple_make(class_id: HeapId, args: ArgValues, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult { + let iterable = args.get_one_arg("_make", vm.heap)?; + let items: Vec = MontyIter::new(iterable, vm)?.collect(vm)?; + + let (typename, field_names) = { + let HeapReadOutput::NamedTupleClass(class) = vm.heap.read(class_id) else { + unreachable!("namedtuple class") + }; + let c = class.get(vm.heap); + (c.typename.clone(), c.field_names.clone()) + }; + if items.len() != field_names.len() { + let got = items.len(); + let expected = field_names.len(); + items.drop_with_heap(vm); + return Err( + SimpleException::new_msg(ExcType::TypeError, format!("Expected {expected} arguments, got {got}")).into(), + ); + } + let nt = NamedTuple::new(typename, field_names, items); + let id = vm.heap.allocate(HeapData::NamedTuple(nt))?; + Ok(CallResult::Value(Value::Ref(id))) +} + +impl HeapItem for NamedTupleClass { + fn py_estimate_size(&self) -> usize { + mem::size_of::() + + self.typename.py_estimate_size() + + self.field_names.iter().map(EitherStr::py_estimate_size).sum::() + + self.defaults.len() * mem::size_of::() + } + + fn py_dec_ref_ids(&mut self, stack: &mut Vec) { + for d in &mut self.defaults { + d.py_dec_ref_ids(stack); + } + } +} diff --git a/crates/monty/src/types/str.rs b/crates/monty/src/types/str.rs index 56b9f0e08..386f75be0 100644 --- a/crates/monty/src/types/str.rs +++ b/crates/monty/src/types/str.rs @@ -1985,7 +1985,7 @@ struct EncodeArgs { /// Returns True if the string is a valid Python identifier according to /// the language definition (starts with letter or underscore, followed by /// letters, digits, or underscores). Empty strings return False. -fn str_isidentifier(s: &str) -> bool { +pub(crate) fn str_isidentifier(s: &str) -> bool { if s.is_empty() { return false; } diff --git a/crates/monty/src/types/type.rs b/crates/monty/src/types/type.rs index 6b6ce8383..205355d88 100644 --- a/crates/monty/src/types/type.rs +++ b/crates/monty/src/types/type.rs @@ -290,6 +290,9 @@ impl Type { } else if self == Self::DateTime && other == Self::Date { // datetime is a subtype of date in Python true + } else if self == Self::NamedTuple && other == Self::Tuple { + // a namedtuple is a subclass of tuple in Python + true } else { false } diff --git a/crates/monty/test_cases/collections__namedtuple.py b/crates/monty/test_cases/collections__namedtuple.py new file mode 100644 index 000000000..bdd933e55 --- /dev/null +++ b/crates/monty/test_cases/collections__namedtuple.py @@ -0,0 +1,148 @@ +# Tests for collections.namedtuple + +from collections import namedtuple + +# === Construction and field access === +Point = namedtuple('Point', ['x', 'y']) +p = Point(1, 2) +assert p.x == 1, 'attribute access x' +assert p.y == 2, 'attribute access y' +assert p[0] == 1, 'index access 0' +assert p[1] == 2, 'index access 1' +assert p[-1] == 2, 'negative index' +assert len(p) == 2, 'length' + +# === field_names as a space/comma string === +P = namedtuple('P', 'a b c') +assert P._fields == ('a', 'b', 'c'), 'space-separated fields' +Pc = namedtuple('Pc', 'a, b, c') +assert Pc._fields == ('a', 'b', 'c'), 'comma-separated fields' + +# === keyword construction === +assert Point(x=1, y=2) == Point(1, 2), 'keyword construction' +assert Point(1, y=2) == Point(1, 2), 'mixed positional and keyword' + +# === _fields (class and instance) === +assert Point._fields == ('x', 'y'), 'class _fields' +assert p._fields == ('x', 'y'), 'instance _fields' + +# === _make === +assert Point._make([3, 4]) == Point(3, 4), '_make from list' +assert Point._make((5, 6)) == Point(5, 6), '_make from tuple' + +# === _asdict === +assert p._asdict() == {'x': 1, 'y': 2}, '_asdict returns ordered dict' + +# === _replace === +assert p._replace(x=10) == Point(10, 2), '_replace one field' +assert p._replace(x=10, y=20) == Point(10, 20), '_replace both fields' +assert p == Point(1, 2), '_replace does not mutate original' + +# === defaults === +D = namedtuple('D', 'a b c', defaults=[10, 20]) +assert D(1) == D(1, 10, 20), 'trailing defaults applied' +assert D(1, 2) == D(1, 2, 20), 'partial defaults' +assert D(1, 2, 3) == D(1, 2, 3), 'all provided' +assert D._field_defaults == {'b': 10, 'c': 20}, '_field_defaults' +assert namedtuple('E', 'a b').__name__ == 'E', 'class __name__' + +# === Tuple compatibility === +assert p == (1, 2), 'equals plain tuple' +assert (1, 2) == p, 'plain tuple equals namedtuple' +assert isinstance(p, tuple), 'namedtuple is a tuple' +assert list(p) == [1, 2], 'iterable / list()' +a, b = p +assert (a, b) == (1, 2), 'sequence unpacking' +assert [v for v in p] == [1, 2], 'comprehension iteration' + +# === Inherited tuple methods === +T = namedtuple('T', 'a b c d') +t = T(1, 2, 2, 3) +assert t.count(2) == 2, 'inherited count' +assert t.index(2) == 1, 'inherited index' +assert t.index(3) == 3, 'inherited index later' + +# === Hashing (usable as dict key / set member) === +d = {Point(1, 2): 'a'} +assert d[Point(1, 2)] == 'a', 'namedtuple as dict key' +assert Point(1, 2) in {Point(1, 2)}, 'namedtuple as set member' + +# === Nested namedtuples === +Line = namedtuple('Line', 'start end') +line = Line(Point(0, 0), Point(1, 1)) +assert line.start.x == 0, 'nested field access' +assert line.end.y == 1, 'nested field access 2' +assert repr(line) == 'Line(start=Point(x=0, y=0), end=Point(x=1, y=1))', 'nested repr' + +# === repr === +assert repr(Point(1, 2)) == 'Point(x=1, y=2)', 'instance repr' +assert repr(namedtuple('Q', 'x')(5)) == 'Q(x=5)', 'single-field repr' + +# === rename === +R = namedtuple('R', 'x def 1y x', rename=True) +assert R._fields == ('x', '_1', '_2', '_3'), 'rename fixes invalid fields' + +# === bool === +assert bool(namedtuple('Z', 'a')(0)) is True, 'non-empty namedtuple is truthy' + +# === Construction errors === +try: + Point(1) + assert False, 'expected missing-argument error' +except TypeError as e: + assert str(e) == "Point.__new__() missing 1 required positional argument: 'y'", 'missing arg message' + +try: + Point(1, 2, 3) + assert False, 'expected too-many-arguments error' +except TypeError as e: + assert str(e) == 'Point.__new__() takes 3 positional arguments but 4 were given', 'too many args message' + +try: + Point(1, x=2) + assert False, 'expected multiple-values error' +except TypeError as e: + assert str(e) == "Point.__new__() got multiple values for argument 'x'", 'multiple values message' + +try: + Point(1, 2, z=3) + assert False, 'expected unexpected-keyword error' +except TypeError as e: + assert str(e) == "Point.__new__() got an unexpected keyword argument 'z'", 'unexpected keyword message' + +# === Factory errors === +try: + namedtuple('P', 'x x') + assert False, 'expected duplicate-field error' +except ValueError as e: + assert str(e) == "Encountered duplicate field name: 'x'", 'duplicate field message' + +try: + namedtuple('P', 'def') + assert False, 'expected keyword-field error' +except ValueError as e: + assert str(e) == "Type names and field names cannot be a keyword: 'def'", 'keyword field message' + +try: + namedtuple('1P', 'x') + assert False, 'expected invalid-identifier error' +except ValueError as e: + assert str(e) == "Type names and field names must be valid identifiers: '1P'", 'invalid identifier message' + +try: + namedtuple('P', '_x') + assert False, 'expected leading-underscore error' +except ValueError as e: + assert str(e) == "Field names cannot start with an underscore: '_x'", 'leading underscore message' + +try: + p._replace(z=9) + assert False, 'expected unexpected-field _replace error' +except ValueError as e: + assert str(e) == "Got unexpected field names: ['z']", '_replace unexpected field message' + +try: + Point._make([1]) + assert False, 'expected _make arity error' +except TypeError as e: + assert str(e) == 'Expected 2 arguments, got 1', '_make arity message' diff --git a/crates/monty/test_cases/import__collections.py b/crates/monty/test_cases/import__collections.py new file mode 100644 index 000000000..da6438e6e --- /dev/null +++ b/crates/monty/test_cases/import__collections.py @@ -0,0 +1,23 @@ +# Tests for importing the collections module and its members + +import collections + +# === Module attribute access === +d = collections.deque([1, 2, 3]) +assert list(d) == [1, 2, 3], 'collections.deque' +dd = collections.defaultdict(int) +dd['a'] += 1 +assert dd == {'a': 1}, 'collections.defaultdict' +c = collections.Counter('aab') +assert c == {'a': 2, 'b': 1}, 'collections.Counter' +Point = collections.namedtuple('Point', 'x y') +assert Point(1, 2).x == 1, 'collections.namedtuple' + +# === from collections import ... === +from collections import deque, defaultdict, Counter, namedtuple + +assert list(deque([4, 5])) == [4, 5], 'imported deque' +assert defaultdict(list)['k'] == [], 'imported defaultdict' +assert Counter([1, 1, 2]) == {1: 2, 2: 1}, 'imported Counter' +NT = namedtuple('NT', ['a', 'b']) +assert NT(a=1, b=2) == (1, 2), 'imported namedtuple' diff --git a/limitations/collections.md b/limitations/collections.md index 7f6924930..ce2fc0b95 100644 --- a/limitations/collections.md +++ b/limitations/collections.md @@ -9,6 +9,17 @@ following names are importable; everything else in CPython's `collections` - `deque` - `defaultdict` - `Counter` +- `namedtuple` + +The type checker uses a Monty-narrowed `collections` stub +(`crates/monty-typeshed/custom/collections/__init__.pyi`) exposing only these +members, so referencing an unimplemented member (e.g. `collections.OrderedDict`) +is a type error as well as a runtime `AttributeError`. + +**`collections.abc` is not importable at runtime.** `from collections.abc +import ...` type-checks (the ABCs are used only as annotations) but raises +`ModuleNotFoundError` if executed. Use the names only in annotations / under +`if TYPE_CHECKING:`. ### General @@ -91,3 +102,20 @@ Divergences from CPython: CPython too, but `total()` is missing here). - **Counts are integers.** Arithmetic assumes integer counts; non-integer values are treated as `0` for count purposes rather than raising. + +## `namedtuple` + +`collections.namedtuple(typename, field_names, *, rename=False, defaults=None, +module=None)` builds a callable class producing named-tuple instances. Field +names may be a whitespace/comma-separated string or an iterable of strings, and +are validated (identifiers, non-keywords, no leading underscore, no duplicates) +with CPython-matching `ValueError`s; `rename=True` auto-fixes invalid names to +`_0`, `_1`, …. + +See [namedtuple.md](namedtuple.md) for the full instance surface and +divergences. The headline divergence: **`type(instance) is TheClass` is +`False`** and `type(instance).__name__` is `'namedtuple'` — the class object +carries the correct name/`_fields`/repr, but the runtime type-identity link +between an instance and its specific class is not established. The `module` +argument is accepted and ignored (Monty has no per-namedtuple module objects, +so the class repr is always ``). diff --git a/limitations/namedtuple.md b/limitations/namedtuple.md index f98d92dad..66ae0eecf 100644 --- a/limitations/namedtuple.md +++ b/limitations/namedtuple.md @@ -1,38 +1,42 @@ # Named tuples -Named tuples exist as a heap type but cannot be **constructed** by -sandboxed Python code: +Named tuples can be created via `collections.namedtuple(...)` (see +[collections.md](collections.md)). They also enter the sandbox as +`sys.version_info` and as values passed in from the host via the `MontyObject` +API. -- `collections.namedtuple` — `collections` is not importable. -- `typing.NamedTuple` — exists as a marker only; subscripting / `class` - inheritance does not produce a type (no `class` statement; see - [language.md](language.md)). -- There is no builtin `namedtuple` factory. - -Named tuples enter the sandbox in two ways: as `sys.version_info`, and -as values passed in from the host via the `MontyObject` API. +`typing.NamedTuple` still exists as a marker only: subscripting / `class` +inheritance does not produce a type (no `class` statement; see +[language.md](language.md)). There is no bare builtin `namedtuple` factory — +it must be imported from `collections`. ## Supported operations +- Construction via a `collections.namedtuple` class (positional, keyword, and + trailing defaults). - Indexing by integer: `nt[0]`. `IndexError` on out-of-range. - Field access by name as an attribute: `nt.major`. `AttributeError` on unknown names. -- `len(nt)`, iteration (`for x in nt`). +- `len(nt)`, iteration (`for x in nt`), sequence unpacking (`a, b = nt`). - Equality: `nt == nt2` and `nt == (a, b, c)` — a named tuple equals a plain tuple with the same elements (matches CPython). +- `isinstance(nt, tuple)` is `True`. - Hashing: same hash as a plain tuple with the same elements; usable as a dict key or set element. - `repr(nt)` — `Name(field1=v1, field2=v2, ...)` matching CPython. - `bool(nt)` — `True` if non-empty, `False` if empty (tuple semantics). +- Named-tuple methods: `._replace(**kw)`, `._asdict()`, `._make(iterable)`, + `._fields`, and the inherited tuple methods `.count()` / `.index()`. The + class object also exposes `_fields`, `_field_defaults`, `_make`, `__name__`. -## NOT supported +## NOT supported / divergences +- **`type(nt)` identity.** `type(nt) is TheClass` is `False` and + `type(nt).__name__` is `'namedtuple'` (not the class name). The class object + itself reports the correct `__name__`/`repr`, and instances repr and behave + correctly — only the runtime type-identity link is missing. - Slicing: `nt[1:3]` raises — `__getitem__` only accepts integer keys. (CPython returns a plain tuple for slices.) -- Lookup by string key: `nt["major"]` raises `TypeError: ... indices must - be integers`. Use attribute access instead. -- Named-tuple methods from CPython: `._replace(**kw)`, `._asdict()`, - `._make(iterable)`, `._fields`, `._field_defaults`, `._source`. -- Concatenation / multiplication: `nt + nt2`, `nt * 3` are not - implemented (plain tuples support these; named tuples in Monty do not). -- Subclassing. +- Lookup by string key: `nt["major"]` raises `TypeError`. Use attribute access. +- Concatenation / multiplication: `nt + nt2`, `nt * 3` are not implemented. +- Subclassing, `._source`, and `._replace` with positional arguments.