diff --git a/src/open_clip/tokenizer.py b/src/open_clip/tokenizer.py index 5b4a238b9..08e6d6638 100644 --- a/src/open_clip/tokenizer.py +++ b/src/open_clip/tokenizer.py @@ -175,6 +175,7 @@ def tokenize(texts: Union[str, List[str]], context_length: int = 77) -> torch.Lo for i, tokens in enumerate(all_tokens): if len(tokens) > context_length: tokens = tokens[:context_length] # Truncate + tokens[-1] = eot_token result[i, :len(tokens)] = torch.tensor(tokens) return result