From 111d2bdd1774d19c3549f0a596b8d1ea63b1cf71 Mon Sep 17 00:00:00 2001 From: enochlev Date: Fri, 19 Apr 2024 15:16:04 -0600 Subject: [PATCH] Fixed Whitespace Error in Streaming mode --- .../inflight_batcher_llm/postprocessing/1/model.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/all_models/inflight_batcher_llm/postprocessing/1/model.py b/all_models/inflight_batcher_llm/postprocessing/1/model.py index 02aafad7..9af39cca 100644 --- a/all_models/inflight_batcher_llm/postprocessing/1/model.py +++ b/all_models/inflight_batcher_llm/postprocessing/1/model.py @@ -208,6 +208,12 @@ def _postprocessing(self, tokens_batch, sequence_lengths): seq_len = sequence_lengths[batch_idx][beam_idx] output = self.tokenizer.decode( tokens[:seq_len], - skip_special_tokens=self.skip_special_tokens) + skip_special_tokens=False) + + # for streamming mode, non-breaking if not streaming mode + token_id_string = self.tokenizer.convert_ids_to_tokens(tokens[:seq_len], skip_special_tokens=True) + if len(token_id_string) > 0 and len(token_id_string[0]) > 0 and token_id_string[0][0] == "▁": + output = " " + output + outputs.append(output.encode('utf8')) - return outputs + return output