Skip to content

Commit

Permalink
make sure this is actually doing something
Browse files Browse the repository at this point in the history
  • Loading branch information
AlexCheema committed Dec 17, 2024
1 parent 0a07223 commit 3a58576
Showing 1 changed file with 1 addition and 0 deletions.
1 change: 1 addition & 0 deletions exo/orchestration/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ async def process_inference_result(
if request_id not in self.buffered_token_output:
self.buffered_token_output[request_id] = ([], False)
is_finished = len(self.buffered_token_output[request_id][0]) >= self.max_generate_tokens
await asyncio.sleep(0.1)

if shard.is_last_layer() and not is_finished:
self.token_count += 1
Expand Down

0 comments on commit 3a58576

Please sign in to comment.