Update tokenizer: do the safety check before inserting EOL

brkygokcen · Mar 11, 2020 · 30579e0 · 30579e0
1 parent e0eaa1e
commit 30579e0
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/official/nlp/transformer/utils/tokenizer.py b/official/nlp/transformer/utils/tokenizer.py
@@ -140,6 +140,8 @@ def encode(self, raw_string, add_eos=False):
     for token in tokens:
       ret.extend(self._token_to_subtoken_ids(token))
     if add_eos:
+      assert EOS in self.subtoken_list, \
+          "Can't append 'EOS' because it is not in list of known subtokens."
       ret.append(EOS_ID)
     return ret