[fault_tolerance]
enable = true
group_size = 8
process_group = "nccl"
process_group_timeout_ms = 60000
Running on B200 nodes.
2026-03-12 07:14:52,572 INFO manager.py:906 -- job_id=03000000 replica_id=torchtitan_ft_5 rank=7 quorum_id=13 step=7101 commit_result=True worker_id=3e4273b0c3010f57c6918fcf2e1a57917f917306edbcd1832322b748 node_id=c99c19bddd447e25d5131f3e6393d7e94e45cbf67940897355e49d8d actor_id=5cb5af8742686029767e945f03000000 task_id=a7600a0d5834a75f5cb5af8742686029767e945f03000000 task_name=PretrainTrainingActor.start_training task_func_name=minted.ray.actors.PretrainTrainingActor.start_training actor_name=replica-5-rank-7 timestamp_ns=1773299692572719041
*** SIGSEGV received at time=1773299692 on cpu 56 ***
PC: @ 0x7fe555ebb7ba (unknown) uv_timer_stop
@ 0x7fe55a0a6520 (unknown) (unknown)
@ 0x7fe555f23ae0 (unknown) (unknown)
[2026-03-12 07:14:52,642 E 1221229 1226842] logging.cc:474: *** SIGSEGV received at time=1773299692 on cpu 56 ***
[2026-03-12 07:14:52,642 E 1221229 1226842] logging.cc:474: PC: @ 0x7fe555ebb7ba (unknown) uv_timer_stop
[2026-03-12 07:14:52,643 E 1221229 1226842] logging.cc:474: @ 0x7fe55a0a6520 (unknown) (unknown)
[2026-03-12 07:14:52,644 E 1221229 1226842] logging.cc:474: @ 0x7fe555f23ae0 (unknown) (unknown)
Fatal Python error: Segmentation fault
Stack (most recent call first):
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torchft/futures.py", line 46 in cancel
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torchft/futures.py", line 248 in context_timeout
File "/miniforge/envs/3.12/lib/python3.12/contextlib.py", line 144 in __exit__
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torchft/futures.py", line 353 in context_timeout
File "/miniforge/envs/3.12/lib/python3.12/contextlib.py", line 144 in __exit__
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torchft/process_group.py", line 759 in _stream_timeout
File "/miniforge/envs/3.12/lib/python3.12/contextlib.py", line 144 in __exit__
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torchft/process_group.py", line 770 in done_callback
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torchft/process_group.py", line 776 in get_future
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torchft/manager.py", line 1266 in _set_future_callback
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torchft/manager.py", line 1317 in wait
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py", line 3014 in all_reduce
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torch/distributed/c10d_logger.py", line 83 in wrapper
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torchtitan/components/ft/manager.py", line 92 in all_reduce_hook
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torch/distributed/fsdp/_fully_shard/_fsdp_collectives.py", line 597 in foreach_reduce
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 124 in decorate_context
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torch/distributed/fsdp/_fully_shard/_fsdp_param_group.py", line 566 in post_backward
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torch/distributed/fsdp/_fully_shard/_fsdp_param_group.py", line 900 in backward
File "/miniforge/envs/3.12/lib/python3.12/site-packages/torch/autograd/function.py", line 317 in apply
Extension modules: msgpack._cmsgpack, google._upb._message, psutil._psutil_linux, psutil._psutil_posix, yaml._yaml, _brotli, charset_normalizer.md, requests.packages.charset_normalizer.md, requests.packages.chardet.md, uvloop.loop, ray._raylet, numpy._core._multiarray_umath, numpy.linalg._umath_linalg, torch._C, torch._C._dynamo.autograd_compiler, torch._C._dynamo.eval_frame, torch._C._dynamo.guards, torch._C._dynamo.utils, torch._C._fft, torch._C._linalg, torch._C._nested, torch._C._nn, torch._C._sparse, torch._C._special, pyarrow.lib, pyarrow._json, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, xxhash._xxhash, markupsafe._speedups, zstandard.backend_c, _cffi_backend, multidict._multidict, yarl._quoting_c, propcache._helpers_c, aiohttp._http_writer, aiohttp._http_parser, aiohttp._websocket.mask, aiohttp._websocket.reader_c, frozenlist._frozenlist, grpc._cython.cygrpc, websockets.speedups, cuda_utils (total: 50)
Note that this happens intermittently, but with 64 GPUS (8 nodes) I am able to repro in first 30 min.
You can see the drops in GPU memory usage ^ is when a node goes down with this stack trace.
def _patch_torchft_timer_cancel() -> None:
"""Patch torchft _TimerHandle.cancel to be thread-safe.
torchft calls asyncio.TimerHandle.cancel() from the training thread,
but the event loop runs in a daemon thread. libuv's uv_timer_stop
is not thread-safe, causing intermittent SIGSEGV in uv_timer_stop.
This patch marshals the cancellation onto the event loop thread
via call_soon_threadsafe.
TODO(yurig): Remove once torchft fixes this upstream.
"""
from torchft.futures import _TimerHandle, _TIMEOUT_MANAGER
def _safe_cancel(self: _TimerHandle) -> None:
with self._lock:
assert not self._cancelled, "timer can only be cancelled once"
self._cancelled = True
if self._timer_handle is not None:
timer_handle = self._timer_handle
self._timer_handle = None
loop = _TIMEOUT_MANAGER._event_loop
if loop is not None and loop.is_running():
loop.call_soon_threadsafe(timer_handle.cancel)
else:
timer_handle.cancel()
_TimerHandle.cancel = _safe_cancel
Config:
Running on B200 nodes.
Stack Trace:
Note that this happens intermittently, but with 64 GPUS (8 nodes) I am able to repro in first 30 min.
You can see the drops in GPU memory usage ^ is when a node goes down with this stack trace.
It seems the issue is about the usage of
_TimerHandle.cancel()from the training thread in a non-thread-safe way.The following patch seems to address the issue for me: