add individual .pypi

Author: ArthurZucker

bindings/python/py_src/tokenizers/__init__.pyi

@@ -0,0 +1,95 @@
+import tokenizers
+import tokenizers.decoders
+import typing
+
+class BPEDecoder:
+    def __new__(cls, /, suffix: str = ...) -> None: ...
+    @property
+    def suffix(self, /) -> str: ...
+    @suffix.setter
+    def suffix(self, /, suffix: str) -> None: ...
+
+class ByteFallback:
+    def __new__(cls, /) -> None: ...
+
+class ByteLevel:
+    def __new__(cls, /, **_kwargs) -> None: ...
+
+class CTC:
+    def __new__(cls, /, pad_token: str = ..., word_delimiter_token: str = ..., cleanup: bool = True) -> None: ...
+    @property
+    def cleanup(self, /) -> bool: ...
+    @cleanup.setter
+    def cleanup(self, /, cleanup: bool) -> None: ...
+    @property
+    def pad_token(self, /) -> str: ...
+    @pad_token.setter
+    def pad_token(self, /, pad_token: str) -> None: ...
+    @property
+    def word_delimiter_token(self, /) -> str: ...
+    @word_delimiter_token.setter
+    def word_delimiter_token(self, /, word_delimiter_token: str) -> None: ...
+
+class DecodeStream:
+    def __new__(cls, /, ids: typing.Any | None = None, skip_special_tokens: bool | None = False) -> None: ...
+    def step(self, /, tokenizer: tokenizers.Tokenizer, id: typing.Any) -> typing.Any: ...
+
+class Decoder:
+    def __getstate__(self, /) -> typing.Any: ...
+    def __repr__(self, /) -> str: ...
+    def __setstate__(self, /, state: typing.Any) -> typing.Any: ...
+    def __str__(self, /) -> str: ...
+    @staticmethod
+    def custom(decoder: typing.Any) -> tokenizers.decoders.Decoder: ...
+    def decode(self, /, tokens: typing.Any) -> str: ...
+
+class Fuse:
+    def __new__(cls, /) -> None: ...
+
+class Metaspace:
+    def __new__(cls, /, replacement: str = '▁', prepend_scheme: str = ..., split: bool = True) -> None: ...
+    @property
+    def prepend_scheme(self, /) -> str: ...
+    @prepend_scheme.setter
+    def prepend_scheme(self, /, prepend_scheme: str) -> typing.Any: ...
+    @property
+    def replacement(self, /) -> str: ...
+    @replacement.setter
+    def replacement(self, /, replacement: str) -> None: ...
+    @property
+    def split(self, /) -> bool: ...
+    @split.setter
+    def split(self, /, split: bool) -> None: ...
+
+class Replace:
+    def __new__(cls, /, pattern: str | tokenizers.Regex, content: str) -> None: ...
+
+class Sequence:
+    def __getnewargs__(self, /) -> typing.Any: ...
+    def __new__(cls, /, decoders_py: typing.Any) -> None: ...
+
+class Strip:
+    def __new__(cls, /, content: str = ' ', left: int = 0, right: int = 0) -> None: ...
+    @property
+    def content(self, /) -> str: ...
+    @content.setter
+    def content(self, /, content: str) -> None: ...
+    @property
+    def start(self, /) -> int: ...
+    @start.setter
+    def start(self, /, start: int) -> None: ...
+    @property
+    def stop(self, /) -> int: ...
+    @stop.setter
+    def stop(self, /, stop: int) -> None: ...
+
+class WordPiece:
+    def __new__(cls, /, prefix: str = ..., cleanup: bool = True) -> None: ...
+    @property
+    def cleanup(self, /) -> bool: ...
+    @cleanup.setter
+    def cleanup(self, /, cleanup: bool) -> None: ...
+    @property
+    def prefix(self, /) -> str: ...
+    @prefix.setter
+    def prefix(self, /, prefix: str) -> None: ...

bindings/python/py_src/tokenizers/decoders.pyi

@@ -0,0 +1,85 @@
+import typing
+
+class BPE:
+    def __new__(cls, /, vocab: typing.Any | str | None = None, merges: typing.Any | str | None = None, **kwargs) -> None: ...
+    def _clear_cache(self, /) -> typing.Any: ...
+    def _resize_cache(self, /, capacity: int) -> typing.Any: ...
+    @property
+    def byte_fallback(self, /) -> bool: ...
+    @byte_fallback.setter
+    def byte_fallback(self, /, byte_fallback: bool) -> None: ...
+    @property
+    def continuing_subword_prefix(self, /) -> typing.Any: ...
+    @continuing_subword_prefix.setter
+    def continuing_subword_prefix(self, /, continuing_subword_prefix: str | None) -> None: ...
+    @property
+    def dropout(self, /) -> typing.Any: ...
+    @dropout.setter
+    def dropout(self, /, dropout: float | None) -> None: ...
+    @property
+    def end_of_word_suffix(self, /) -> typing.Any: ...
+    @end_of_word_suffix.setter
+    def end_of_word_suffix(self, /, end_of_word_suffix: str | None) -> None: ...
+    @classmethod
+    def from_file(cls, /, vocab: str, merges: str, **kwargs) -> BPE: ...
+    @property
+    def fuse_unk(self, /) -> bool: ...
+    @fuse_unk.setter
+    def fuse_unk(self, /, fuse_unk: bool) -> None: ...
+    @property
+    def ignore_merges(self, /) -> bool: ...
+    @ignore_merges.setter
+    def ignore_merges(self, /, ignore_merges: bool) -> None: ...
+    @staticmethod
+    def read_file(vocab: str, merges: str) -> typing.Any: ...
+    @property
+    def unk_token(self, /) -> typing.Any: ...
+    @unk_token.setter
+    def unk_token(self, /, unk_token: str | None) -> None: ...
+
+class Model:
+    def __getstate__(self, /) -> typing.Any: ...
+    def __new__(cls, /) -> None: ...
+    def __repr__(self, /) -> str: ...
+    def __setstate__(self, /, state: typing.Any) -> typing.Any: ...
+    def __str__(self, /) -> str: ...
+    def get_trainer(self, /) -> typing.Any: ...
+    def id_to_token(self, /, id: int) -> typing.Any: ...
+    def save(self, /, folder: str, prefix: str | None = None, name: str | None = None) -> typing.Any: ...
+    def token_to_id(self, /, token: str) -> typing.Any: ...
+    def tokenize(self, /, sequence: str) -> typing.Any: ...
+
+class Unigram:
+    def __new__(cls, /, vocab: typing.Any | None = None, unk_id: int | None = None, byte_fallback: bool | None = None) -> None: ...
+    def _clear_cache(self, /) -> typing.Any: ...
+    def _resize_cache(self, /, capacity: int) -> typing.Any: ...
+
+class WordLevel:
+    def __new__(cls, /, vocab: typing.Any | str | None = None, unk_token: str | None = None) -> None: ...
+    @classmethod
+    def from_file(cls, /, vocab: str, unk_token: str | None = None) -> WordLevel: ...
+    @staticmethod
+    def read_file(vocab: str) -> typing.Any: ...
+    @property
+    def unk_token(self, /) -> str: ...
+    @unk_token.setter
+    def unk_token(self, /, unk_token: str) -> None: ...
+
+class WordPiece:
+    def __new__(cls, /, vocab: typing.Any | str | None = None, **kwargs) -> None: ...
+    @property
+    def continuing_subword_prefix(self, /) -> str: ...
+    @continuing_subword_prefix.setter
+    def continuing_subword_prefix(self, /, continuing_subword_prefix: str) -> None: ...
+    @classmethod
+    def from_file(cls, /, vocab: str, **kwargs) -> WordPiece: ...
+    @property
+    def max_input_chars_per_word(self, /) -> int: ...
+    @max_input_chars_per_word.setter
+    def max_input_chars_per_word(self, /, max: int) -> None: ...
+    @staticmethod
+    def read_file(vocab: str) -> typing.Any: ...
+    @property
+    def unk_token(self, /) -> str: ...
+    @unk_token.setter
+    def unk_token(self, /, unk_token: str) -> None: ...

bindings/python/py_src/tokenizers/models.pyi

@@ -0,0 +1,95 @@
+import tokenizers
+import tokenizers.normalizers
+import typing
+
+class BertNormalizer:
+    def __new__(cls, /, clean_text: bool = True, handle_chinese_chars: bool = True, strip_accents: bool | None = None, lowercase: bool = True) -> None: ...
+    @property
+    def clean_text(self, /) -> bool: ...
+    @clean_text.setter
+    def clean_text(self, /, clean_text: bool) -> None: ...
+    @property
+    def handle_chinese_chars(self, /) -> bool: ...
+    @handle_chinese_chars.setter
+    def handle_chinese_chars(self, /, handle_chinese_chars: bool) -> None: ...
+    @property
+    def lowercase(self, /) -> bool: ...
+    @lowercase.setter
+    def lowercase(self, /, lowercase: bool) -> None: ...
+    @property
+    def strip_accents(self, /) -> typing.Any: ...
+    @strip_accents.setter
+    def strip_accents(self, /, strip_accents: bool | None) -> None: ...
+
+class ByteLevel:
+    def __new__(cls, /) -> None: ...
+
+class Lowercase:
+    def __new__(cls, /) -> None: ...
+
+class NFC:
+    def __new__(cls, /) -> None: ...
+
+class NFD:
+    def __new__(cls, /) -> None: ...
+
+class NFKC:
+    def __new__(cls, /) -> None: ...
+
+class NFKD:
+    def __new__(cls, /) -> None: ...
+
+class Nmt:
+    def __new__(cls, /) -> None: ...
+
+class Normalizer:
+    def __getstate__(self, /) -> typing.Any: ...
+    def __repr__(self, /) -> str: ...
+    def __setstate__(self, /, state: typing.Any) -> typing.Any: ...
+    def __str__(self, /) -> str: ...
+    @staticmethod
+    def custom(obj: typing.Any) -> tokenizers.normalizers.Normalizer: ...
+    def normalize(self, /, normalized: tokenizers.NormalizedString | tokenizers.NormalizedStringRefMut) -> typing.Any: ...
+    def normalize_str(self, /, sequence: str) -> str: ...
+
+class Precompiled:
+    def __new__(cls, /, precompiled_charsmap: typing.Any) -> None: ...
+
+class Prepend:
+    def __new__(cls, /, prepend: str = ...) -> None: ...
+    @property
+    def prepend(self, /) -> str: ...
+    @prepend.setter
+    def prepend(self, /, prepend: str) -> None: ...
+
+class Replace:
+    def __new__(cls, /, pattern: str | tokenizers.Regex, content: str) -> None: ...
+    @property
+    def content(self, /) -> str: ...
+    @content.setter
+    def content(self, /, content: str) -> None: ...
+    @property
+    def pattern(self, /) -> typing.Any: ...
+    @pattern.setter
+    def pattern(self, /, _pattern: str | tokenizers.Regex) -> typing.Any: ...
+
+class Sequence:
+    def __getitem__(self, /, index: int) -> typing.Any: ...
+    def __getnewargs__(self, /) -> typing.Any: ...
+    def __len__(self, /) -> int: ...
+    def __new__(cls, /, normalizers: typing.Any) -> None: ...
+    def __setitem__(self, /, index: int, value: typing.Any) -> typing.Any: ...
+
+class Strip:
+    def __new__(cls, /, left: bool = True, right: bool = True) -> None: ...
+    @property
+    def left(self, /) -> bool: ...
+    @left.setter
+    def left(self, /, left: bool) -> None: ...
+    @property
+    def right(self, /) -> bool: ...
+    @right.setter
+    def right(self, /, right: bool) -> None: ...
+
+class StripAccents:
+    def __new__(cls, /) -> None: ...

bindings/python/py_src/tokenizers/normalizers.pyi

@@ -0,0 +1,108 @@
+import tokenizers
+import tokenizers.pre_tokenizers
+import typing
+
+class BertPreTokenizer:
+    def __new__(cls, /) -> None: ...
+
+class ByteLevel:
+    def __new__(cls, /, add_prefix_space: bool = True, trim_offsets: bool = True, use_regex: bool = True, **_kwargs) -> None: ...
+    @property
+    def add_prefix_space(self, /) -> bool: ...
+    @add_prefix_space.setter
+    def add_prefix_space(self, /, add_prefix_space: bool) -> None: ...
+    @staticmethod
+    def alphabet() -> typing.Any: ...
+    @property
+    def trim_offsets(self, /) -> bool: ...
+    @trim_offsets.setter
+    def trim_offsets(self, /, trim_offsets: bool) -> None: ...
+    @property
+    def use_regex(self, /) -> bool: ...
+    @use_regex.setter
+    def use_regex(self, /, use_regex: bool) -> None: ...
+
+class CharDelimiterSplit:
+    def __getnewargs__(self, /) -> typing.Any: ...
+    def __new__(cls, /, delimiter: str) -> None: ...
+    @property
+    def delimiter(self, /) -> str: ...
+    @delimiter.setter
+    def delimiter(self, /, delimiter: str) -> None: ...
+
+class Digits:
+    def __new__(cls, /, individual_digits: bool = False) -> None: ...
+    @property
+    def individual_digits(self, /) -> bool: ...
+    @individual_digits.setter
+    def individual_digits(self, /, individual_digits: bool) -> None: ...
+
+class FixedLength:
+    def __new__(cls, /, length: int = 5) -> None: ...
+    @property
+    def length(self, /) -> int: ...
+    @length.setter
+    def length(self, /, length: int) -> None: ...
+
+class Metaspace:
+    def __new__(cls, /, replacement: str = '▁', prepend_scheme: str = ..., split: bool = True) -> None: ...
+    @property
+    def prepend_scheme(self, /) -> str: ...
+    @prepend_scheme.setter
+    def prepend_scheme(self, /, prepend_scheme: str) -> typing.Any: ...
+    @property
+    def replacement(self, /) -> str: ...
+    @replacement.setter
+    def replacement(self, /, replacement: str) -> None: ...
+    @property
+    def split(self, /) -> bool: ...
+    @split.setter
+    def split(self, /, split: bool) -> None: ...
+
+class PreTokenizer:
+    def __getstate__(self, /) -> typing.Any: ...
+    def __repr__(self, /) -> str: ...
+    def __setstate__(self, /, state: typing.Any) -> typing.Any: ...
+    def __str__(self, /) -> str: ...
+    @staticmethod
+    def custom(pretok: typing.Any) -> tokenizers.pre_tokenizers.PreTokenizer: ...
+    def pre_tokenize(self, /, pretok: tokenizers.PreTokenizedString) -> typing.Any: ...
+    def pre_tokenize_str(self, /, s: str) -> typing.Any: ...
+
+class Punctuation:
+    def __new__(cls, /, behavior: typing.Any = ...) -> None: ...
+    @property
+    def behavior(self, /) -> str: ...
+    @behavior.setter
+    def behavior(self, /, behavior: str) -> typing.Any: ...
+
+class Sequence:
+    def __getitem__(self, /, index: int) -> typing.Any: ...
+    def __getnewargs__(self, /) -> typing.Any: ...
+    def __new__(cls, /, pre_tokenizers: typing.Any) -> None: ...
+    def __setitem__(self, /, index: int, value: typing.Any) -> typing.Any: ...
+
+class Split:
+    def __getnewargs__(self, /) -> typing.Any: ...
+    def __new__(cls, /, pattern: str | tokenizers.Regex, behavior: typing.Any, invert: bool = False) -> None: ...
+    @property
+    def behavior(self, /) -> str: ...
+    @behavior.setter
+    def behavior(self, /, behavior: str) -> typing.Any: ...
+    @property
+    def invert(self, /) -> bool: ...
+    @invert.setter
+    def invert(self, /, invert: bool) -> None: ...
+    @property
+    def pattern(self, /) -> typing.Any: ...
+    @pattern.setter
+    def pattern(self, /, _pattern: str | tokenizers.Regex) -> typing.Any: ...
+
+class UnicodeScripts:
+    def __new__(cls, /) -> None: ...
+
+class Whitespace:
+    def __new__(cls, /) -> None: ...
+
+class WhitespaceSplit:
+    def __new__(cls, /) -> None: ...