Add assert for old behavior

minimaxir · Apr 19, 2021 · 4a1c6dc · 4a1c6dc
1 parent e1740ef
commit 4a1c6dc
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/aitextgen/aitextgen.py b/aitextgen/aitextgen.py
@@ -16,6 +16,7 @@
 from transformers import (
     AutoConfig,
     AutoModelForCausalLM,
+    AutoTokenizer,
     GPT2Config,
     GPT2LMHeadModel,
     GPT2TokenizerFast,
@@ -97,6 +98,12 @@ def __init__(
         **kwargs,
     ) -> None:
 
+        if model:
+            assert not os.path.isfile(model), (
+                "As of aitextgen 0.5.0, you must "
+                + "use `model_folder` to load an existing model."
+            )
+
         if not verbose:
             for module in [
                 "transformers.file_utils",
@@ -189,7 +196,7 @@ def __init__(
             )
             if model and "gpt2" not in model:
                 logger.info(f"Using the tokenizer for {model}.")
-                self.tokenizer = GPT2TokenizerFast.from_pretrained(
+                self.tokenizer = AutoTokenizer.from_pretrained(
                     model,
                     cache_dir=cache_dir,
                 )