diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 4330be922013f30..c5d24660fd8fe28 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -14,6 +14,14 @@ used to wrap these libraries in pure Python. .. include:: ../includes/optional-module.rst +.. warning:: + + :mod:`!ctypes` provides low-level access to native libraries and the + process's memory, bypassing Python's safety mechanisms and allowing + execution of arbitrary native code. + Incorrect use can corrupt data and objects, reveal sensitive information, + cause crashes, or otherwise compromise the running process. + .. _ctypes-ctypes-tutorial: @@ -198,10 +206,8 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`!ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). +The :mod:`faulthandler` module can help debug crashes, +such as segmentation faults produced by erroneous C library calls. ``None``, integers, bytes objects and (unicode) strings are the only native Python objects that can directly be used as parameters in these function calls. diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index f13bc2178b1e7eb..4c8ad11447aa591 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -1001,7 +1001,7 @@ struct _is { struct ast_state ast; struct types_state types; struct callable_cache callable_cache; - PyObject *common_consts[NUM_COMMON_CONSTANTS]; + _PyStackRef common_consts[NUM_COMMON_CONSTANTS]; bool jit; bool compiling; diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index ca4a7c216eda532..9495ccc8ac38896 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -263,6 +263,18 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) +static inline _PyStackRef +_PyStackRef_DupImmortal(_PyStackRef ref, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + PyObject *obj = _Py_stackref_get_object(ref); + assert(_Py_IsImmortal(obj)); + return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); +} +#define PyStackRef_DupImmortal(REF) _PyStackRef_DupImmortal((REF), __FILE__, __LINE__) + static inline void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber) { @@ -633,6 +645,15 @@ PyStackRef_DUP(_PyStackRef ref) } #endif +static inline _PyStackRef +PyStackRef_DupImmortal(_PyStackRef ref) +{ + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + assert(_Py_IsImmortal(BITS_TO_PTR_MASKED(ref))); + return ref; +} + static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { diff --git a/InternalDocs/qsbr.md b/InternalDocs/qsbr.md index 1c4a79a7b44436a..d511396fdab6452 100644 --- a/InternalDocs/qsbr.md +++ b/InternalDocs/qsbr.md @@ -101,15 +101,39 @@ Periodically, a polling mechanism processes this deferred-free list: To reduce memory contention from frequent updates to the global `wr_seq`, its advancement is sometimes deferred. Instead of incrementing `wr_seq` on every -reclamation request, each thread tracks its number of deferrals locally. Once -the deferral count reaches a limit (QSBR_DEFERRED_LIMIT, currently 10), the -thread advances the global `wr_seq` and resets its local count. - -When an object is added to the deferred-free list, its qsbr_goal is set to -`wr_seq` + 2. By setting the goal to the next sequence value, we ensure it's safe -to defer the global counter advancement. This optimization improves runtime -speed but may increase peak memory usage by slightly delaying when memory can -be reclaimed. +reclamation request, the object's qsbr_goal is set to `wr_seq` + 2 (the value +the counter *would* take on its next advance) without actually advancing the +global counter. This is safe because the goal still corresponds to a future +sequence value that no thread has yet observed as quiescent. + +Whether to actually advance `wr_seq` is decided per request, based on how +much memory and how many items the calling thread has already deferred since +its last advance: + +* For deferred object frees (`_PyMem_FreeDelayed`), the thread tracks both a + count (`deferred_count`) and an estimate of the held memory + (`deferred_memory`). The global `wr_seq` is advanced when the freed block + is larger than `QSBR_FREE_MEM_LIMIT` (1 MiB), when the accumulated deferred + memory exceeds that limit, or when the count exceeds `QSBR_DEFERRED_LIMIT` + (127, sized so a chunk of work items is processed before it overflows). + Crossing any of these thresholds also sets a per-thread `should_process` + flag, signalling that the deferred-free list should be drained. + +* For mimalloc pages held by QSBR, the thread tracks `deferred_page_memory` + and advances `wr_seq` when either the individual page or the accumulated + page memory exceeds `QSBR_PAGE_MEM_LIMIT` (4096 * 20 bytes). Advancing + promptly here matters because a held page cannot be reused for a different + size class or by a different thread. + +Processing of the deferred-free list normally happens from the eval breaker +(rather than from inside `_PyMem_FreeDelayed`), which gives the global +`rd_seq` a better chance to have advanced far enough that items can actually +be freed. `_PyMem_ProcessDelayed` is still called from the free path as a +safety valve when a work-item chunk fills up. + +This optimization improves runtime speed but may increase peak memory usage +by slightly delaying when memory can be reclaimed; the size-based thresholds +above bound that extra memory. ## Limitations diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index b463bb18b160564..11dfcc68eb2dacd 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -9314,7 +9314,7 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 300b7da753c2baf..993d231751409ba 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1974,7 +1974,7 @@ dummy_func( inst(LOAD_COMMON_CONSTANT, ( -- value)) { // Keep in sync with _common_constants in opcode.py assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); } inst(LOAD_BUILD_CLASS, ( -- bc)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 952860f01b8a682..9aaf9639b9b9015 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9396,7 +9396,7 @@ _PyStackRef value; oparg = CURRENT_OPARG(); assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -9410,7 +9410,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); @@ -9426,7 +9426,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); _tos_cache2 = value; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 6e3e5378cfbf155..eb0faf8cd183885 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -11,6 +11,7 @@ #include "pycore_opcode_utils.h" #include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() #include @@ -1330,7 +1331,8 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) } if (opcode == LOAD_COMMON_CONSTANT) { assert(oparg < NUM_COMMON_CONSTANTS); - return Py_NewRef(_PyInterpreterState_GET()->common_consts[oparg]); + return PyStackRef_AsPyObjectBorrow( + _PyInterpreterState_GET()->common_consts[oparg]); } if (constant == NULL) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 83051cf41cc043b..94384d5db3c107f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9313,7 +9313,7 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index c968185d77c3317..edb4c644bccbf6f 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2,6 +2,7 @@ #include "pycore_long.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" +#include "pycore_stackref.h" #include "pycore_typeobject.h" #include "pycore_uops.h" #include "pycore_uop_ids.h" @@ -870,15 +871,11 @@ dummy_func(void) { op(_LOAD_COMMON_CONSTANT, (-- value)) { assert(oparg < NUM_COMMON_CONSTANTS); - PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - if (_Py_IsImmortal(val)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(sym_new_const(ctx, val)); - } - else { - ADD_OP(_LOAD_CONST_INLINE, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } + PyObject *val = PyStackRef_AsPyObjectBorrow( + _PyInterpreterState_GET()->common_consts[oparg]); + assert(_Py_IsImmortal(val)); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index d52ebb9804197da..8895e02d47b1693 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1991,15 +1991,11 @@ case _LOAD_COMMON_CONSTANT: { JitOptRef value; assert(oparg < NUM_COMMON_CONSTANTS); - PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - if (_Py_IsImmortal(val)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(sym_new_const(ctx, val)); - } - else { - ADD_OP(_LOAD_CONST_INLINE, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } + PyObject *val = PyStackRef_AsPyObjectBorrow( + _PyInterpreterState_GET()->common_consts[oparg]); + assert(_Py_IsImmortal(val)); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a17c3baca3d006c..182233329498076 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -28,6 +28,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_stackref.h" // PyStackRef_FromPyObjectBorrow() #include "pycore_stats.h" // _PyStats_InterpInit() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_traceback.h" // PyUnstable_TracebackThreads() @@ -878,25 +879,28 @@ pycore_init_builtins(PyThreadState *tstate) goto error; } - interp->common_consts[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError; - interp->common_consts[CONSTANT_NOTIMPLEMENTEDERROR] = PyExc_NotImplementedError; - interp->common_consts[CONSTANT_BUILTIN_TUPLE] = (PyObject *)&PyTuple_Type; - interp->common_consts[CONSTANT_BUILTIN_ALL] = all; - interp->common_consts[CONSTANT_BUILTIN_ANY] = any; - interp->common_consts[CONSTANT_BUILTIN_LIST] = (PyObject *)&PyList_Type; - interp->common_consts[CONSTANT_BUILTIN_SET] = (PyObject *)&PySet_Type; - interp->common_consts[CONSTANT_NONE] = Py_None; - interp->common_consts[CONSTANT_EMPTY_STR] = + PyObject *common_objs[NUM_COMMON_CONSTANTS] = {NULL}; + common_objs[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError; + common_objs[CONSTANT_NOTIMPLEMENTEDERROR] = PyExc_NotImplementedError; + common_objs[CONSTANT_BUILTIN_TUPLE] = (PyObject *)&PyTuple_Type; + common_objs[CONSTANT_BUILTIN_ALL] = all; + common_objs[CONSTANT_BUILTIN_ANY] = any; + common_objs[CONSTANT_BUILTIN_LIST] = (PyObject *)&PyList_Type; + common_objs[CONSTANT_BUILTIN_SET] = (PyObject *)&PySet_Type; + common_objs[CONSTANT_NONE] = Py_None; + common_objs[CONSTANT_EMPTY_STR] = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_STR); - interp->common_consts[CONSTANT_TRUE] = Py_True; - interp->common_consts[CONSTANT_FALSE] = Py_False; - interp->common_consts[CONSTANT_MINUS_ONE] = + common_objs[CONSTANT_TRUE] = Py_True; + common_objs[CONSTANT_FALSE] = Py_False; + common_objs[CONSTANT_MINUS_ONE] = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS - 1]; - interp->common_consts[CONSTANT_BUILTIN_FROZENSET] = (PyObject *)&PyFrozenSet_Type; - interp->common_consts[CONSTANT_EMPTY_TUPLE] = + common_objs[CONSTANT_BUILTIN_FROZENSET] = (PyObject *)&PyFrozenSet_Type; + common_objs[CONSTANT_EMPTY_TUPLE] = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_TUPLE); for (int i = 0; i < NUM_COMMON_CONSTANTS; i++) { - assert(interp->common_consts[i] != NULL); + assert(common_objs[i] != NULL); + _Py_SetImmortal(common_objs[i]); + interp->common_consts[i] = PyStackRef_FromPyObjectBorrow(common_objs[i]); } PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); diff --git a/Python/pystate.c b/Python/pystate.c index 530bd567b770be3..fed1df0173bacf1 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -12,8 +12,9 @@ #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interpframe.h" // _PyThreadState_HasStackSpace() -#include "pycore_object.h" // _PyType_InitCache() +#include "pycore_object.h" // _PyType_InitCache(), _Py_ClearImmortal() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() +#include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS #include "pycore_optimizer.h" // JIT_CLEANUP_THRESHOLD #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() @@ -21,7 +22,7 @@ #include "pycore_pymem.h" // _PyMem_DebugEnabled() #include "pycore_runtime.h" // _PyRuntime #include "pycore_runtime_init.h" // _PyRuntimeState_INIT -#include "pycore_stackref.h" // Py_STACKREF_DEBUG +#include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() #include "pycore_stats.h" // FT_STAT_WORLD_STOP_INC() #include "pycore_time.h" // _PyTime_Init() #include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() @@ -778,6 +779,36 @@ extern void _Py_stackref_report_leaks(PyInterpreterState *interp); #endif +static int +common_const_is_initialized(_PyStackRef ref) +{ +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + return !PyStackRef_IsNull(ref); +#else + return ref.bits != 0 && !PyStackRef_IsNull(ref); +#endif +} + + +static void +common_constants_clear(PyInterpreterState *interp) +{ + for (int i = 0; i < NUM_COMMON_CONSTANTS; i++) { + _PyStackRef ref = interp->common_consts[i]; + if (!common_const_is_initialized(ref)) { + continue; + } + PyObject *obj = PyStackRef_AsPyObjectBorrow(ref); + PyStackRef_XCLOSE(ref); + interp->common_consts[i] = PyStackRef_NULL; + // Refcount reclamation skips heap immortals; release manually. + if (_Py_IsImmortal(obj) && !_Py_IsStaticImmortal(obj)) { + _Py_ClearImmortal(obj); + } + } +} + + static void interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) { @@ -904,6 +935,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) PyDict_Clear(interp->builtins); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + common_constants_clear(interp); #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) # ifdef Py_STACKREF_CLOSE_DEBUG diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 42459eedad6b1d7..22a321b4953de7d 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -609,6 +609,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_CLEAR", "PyStackRef_CLOSE_SPECIALIZED", "PyStackRef_DUP", + "PyStackRef_DupImmortal", "PyStackRef_False", "PyStackRef_FromPyObjectBorrow", "PyStackRef_FromPyObjectNew",