Merge pull request #132 from liyz15/eot_after_truncate

Change the last token to eot_token truncating
mlfoundations · Jul 26, 2022 · 46dc933 · 46dc933
2 parents 515d448 + 39d3a94
commit 46dc933
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/open_clip/tokenizer.py b/src/open_clip/tokenizer.py
@@ -175,6 +175,7 @@ def tokenize(texts: Union[str, List[str]], context_length: int = 77) -> torch.Lo
     for i, tokens in enumerate(all_tokens):
         if len(tokens) > context_length:
             tokens = tokens[:context_length]  # Truncate
+            tokens[-1] = eot_token
         result[i, :len(tokens)] = torch.tensor(tokens)
 
     return result