Merge pull request #1024 from PyThaiNLP/remove-clause_tokenize

Remove clause_tokenize

Author: bact

docs/api/tokenize.rst

@@ -8,10 +8,6 @@ The :mod:`pythainlp.tokenize` module contains a comprehensive set of functions a
 Modules
 -------
 
-.. autofunction:: clause_tokenize
-    :noindex:
-    
-    Tokenizes text into clauses. This function allows you to split text into meaningful sections, making it useful for more advanced text processing tasks.
 
 .. autofunction:: sent_tokenize
     :noindex:

pythainlp/tokenize/__init__.py

@@ -9,7 +9,6 @@
     "THAI2FIT_TOKENIZER",
     "Tokenizer",
     "Trie",
-    "clause_tokenize",
     "paragraph_tokenize",
     "sent_tokenize",
     "subword_tokenize",
@@ -32,7 +31,6 @@
 
 from pythainlp.tokenize.core import (
     Tokenizer,
-    clause_tokenize,
     paragraph_tokenize,
     sent_tokenize,
     subword_tokenize,

pythainlp/tokenize/core.py

@@ -25,43 +25,6 @@
 from pythainlp.util.trie import Trie, dict_trie
 
 
-def clause_tokenize(doc: List[str]) -> List[List[str]]:
-    """
-    Clause tokenizer. (or Clause segmentation)
-    Tokenizes running word list into list of clauses (list of strings).
-    Split by CRF trained on Blackboard Treebank.
-
-    :param str doc: word list to be clause tokenized
-    :return: list of clauses
-    :rtype: list[list[str]]
-    :Example:
-    ::
-
-        from pythainlp.tokenize import clause_tokenize
-
-        clause_tokenize(
-            [
-                "ฉัน",
-                "นอน",
-                "และ",
-                "คุณ",
-                "เล่น",
-                "มือถือ",
-                "ส่วน",
-                "น้อง",
-                "เขียน",
-                "โปรแกรม",
-            ]
-        )
-        # [['ฉัน', 'นอน'],
-        # ['และ', 'คุณ', 'เล่น', 'มือถือ'],
-        # ['ส่วน', 'น้อง', 'เขียน', 'โปรแกรม']]
-    """
-    from pythainlp.tokenize.crfcls import segment
-
-    return segment(doc)
-
-
 def word_detokenize(
     segments: Union[List[List[str]], List[str]], output: str = "str"
 ) -> Union[List[str], str]:

pythainlp/tokenize/crfcls.py

@@ -20,7 +20,6 @@
     tltk,
     word_tokenize,
 )
-from pythainlp.tokenize import clause_tokenize as sent_clause_tokenize
 
 from ..core.test_tokenize import (
     SENT_1,
@@ -31,12 +30,6 @@
 )
 
 
-class ClauseTokenizeTestCase(unittest.TestCase):
-    def test_clause_tokenize(self):
-        self.assertIsNotNone(sent_clause_tokenize(["ฉัน", "ทดสอบ"]))
-        self.assertIsInstance(sent_clause_tokenize(["ฉัน", "ทดสอบ"]), list)
-
-
 class DetokenizeTestCase(unittest.TestCase):
     def test_numeric_data_format(self):
         engines = ["attacut", "deepcut", "sefr_cut"]