Docket Redis Streams backend is generating cross slot exception #204
Description
I am getting this exception when I use the Docket task backend. The exception seem to originate in Docket ... it looks like its Redis Streams backend is generating this cross slot exception.
Traceback (most recent call last):
File "/app/.venv/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/fastapi/applications.py", line 1139, in __call__
await super().__call__(scope, receive, send)
File "/app/.venv/lib/python3.12/site-packages/starlette/applications.py", line 107, in __call__
await self.middleware_stack(scope, receive, send)
File "/app/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 186, in __call__
raise exc
File "/app/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 164, in __call__
await self.app(scope, receive, _send)
File "/app/.venv/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/app/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/app/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/app/.venv/lib/python3.12/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "/app/.venv/lib/python3.12/site-packages/starlette/routing.py", line 716, in __call__
await self.middleware_stack(scope, receive, send)
File "/app/.venv/lib/python3.12/site-packages/starlette/routing.py", line 736, in app
await route.handle(scope, receive, send)
File "/app/.venv/lib/python3.12/site-packages/starlette/routing.py", line 290, in handle
await self.app(scope, receive, send)
File "/app/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 119, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/app/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/app/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/app/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 106, in app
await response(scope, receive, send)
File "/app/.venv/lib/python3.12/site-packages/starlette/responses.py", line 167, in __call__
await self.background()
File "/app/.venv/lib/python3.12/site-packages/starlette/background.py", line 36, in __call__
await task()
File "/app/.venv/lib/python3.12/site-packages/starlette/background.py", line 21, in __call__
await self.func(*self.args, **self.kwargs)
File "/app/.venv/lib/python3.12/site-packages/agent_memory_server/long_term_memory.py", line 1051, in index_long_term_memories
background_tasks.add_task(extract_memory_structure, memory)
File "/app/.venv/lib/python3.12/site-packages/agent_memory_server/dependencies.py", line 63, in add_task
future.result() # Wait for completion
^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 456, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
raise self._exception
File "/usr/local/lib/python3.12/concurrent/futures/thread.py", line 59, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/agent_memory_server/dependencies.py", line 58, in run_in_thread
asyncio.run(schedule_task())
File "/usr/local/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/agent_memory_server/dependencies.py", line 55, in schedule_task
await docket.add(func)(*args, **kwargs)
File "/app/.venv/lib/python3.12/site-packages/docket/docket.py", line 390, in scheduler
await execution.schedule(replace=False)
File "/app/.venv/lib/python3.12/site-packages/docket/execution.py", line 647, in schedule
await schedule_script(
File "/app/.venv/lib/python3.12/site-packages/redis/commands/core.py", line 5773, in __call__
return await client.evalsha(self.sha, len(keys), *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/redis/asyncio/client.py", line 677, in execute_command
return await conn.retry.call_with_retry(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/redis/asyncio/retry.py", line 50, in call_with_retry
return await do()
^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/redis/asyncio/client.py", line 652, in _send_command_parse_response
return await self.parse_response(conn, command_name, **options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/redis/asyncio/client.py", line 698, in parse_response
response = await connection.read_response()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/redis/asyncio/connection.py", line 627, in read_response
raise response from None
redis.exceptions.ClusterCrossSlotError: Keys in request don't hash to the same slot (context='', command='evalsha', original-slot='7886', wrong-slot='8200', first-key='memory-server:stream', violating-key='memory-server:known:069b47d4-ea57-7e96-8000-e10dedea40ac')