Cleanup python bindings.

Author: MeetThePatel

bindings/python/src/decoders.rs

@@ -424,12 +424,12 @@ pub struct PyBPEDecoder {}
 impl PyBPEDecoder {
     #[getter]
     fn get_suffix(self_: PyRef<Self>) -> String {
-        getter!(self_, BPE, suffix.clone())
+        getter!(self_, BPE, suffix.to_string())
     }
 
     #[setter]
     fn set_suffix(self_: PyRef<Self>, suffix: String) {
-        setter!(self_, BPE, suffix, suffix);
+        setter!(self_, BPE, suffix, suffix.into());
     }
 
     #[new]
@@ -455,22 +455,27 @@ pub struct PyCTCDecoder {}
 impl PyCTCDecoder {
     #[getter]
     fn get_pad_token(self_: PyRef<Self>) -> String {
-        getter!(self_, CTC, pad_token.clone())
+        getter!(self_, CTC, pad_token.to_string())
     }
 
     #[setter]
     fn set_pad_token(self_: PyRef<Self>, pad_token: String) {
-        setter!(self_, CTC, pad_token, pad_token);
+        setter!(self_, CTC, pad_token, pad_token.into());
     }
 
     #[getter]
     fn get_word_delimiter_token(self_: PyRef<Self>) -> String {
-        getter!(self_, CTC, word_delimiter_token.clone())
+        getter!(self_, CTC, word_delimiter_token.clone()).to_string()
     }
 
     #[setter]
     fn set_word_delimiter_token(self_: PyRef<Self>, word_delimiter_token: String) {
-        setter!(self_, CTC, word_delimiter_token, word_delimiter_token);
+        setter!(
+            self_,
+            CTC,
+            word_delimiter_token,
+            word_delimiter_token.into()
+        );
     }
 
     #[getter]

bindings/python/src/encoding.rs

@@ -129,7 +129,7 @@ impl PyEncoding {
     fn get_tokens(&self) -> Vec<String> {
         self.encoding
             .get_tokens()
-            .into_iter()
+            .iter()
             .map(|x| x.to_string())
             .collect()
     }

bindings/python/src/models.rs

@@ -590,7 +590,6 @@ impl PyBPE {
                     e
                 ))
             })
-            .into()
     }
 
     /// Instantiate a BPE model from the given files.

bindings/python/src/normalizers.rs

@@ -521,12 +521,12 @@ pub struct PyPrepend {}
 impl PyPrepend {
     #[getter]
     fn get_prepend(self_: PyRef<Self>) -> String {
-        getter!(self_, Prepend, prepend)
+        getter!(self_, Prepend, prepend).into()
     }
 
     #[setter]
     fn set_prepend(self_: PyRef<Self>, prepend: String) {
-        setter!(self_, Prepend, prepend, prepend)
+        setter!(self_, Prepend, prepend, prepend.into())
     }
 
     #[new]
@@ -624,12 +624,12 @@ impl PyReplace {
 
     #[getter]
     fn get_content(self_: PyRef<Self>) -> String {
-        getter!(self_, Replace, content)
+        getter!(self_, Replace, content).to_string()
     }
 
     #[setter]
     fn set_content(self_: PyRef<Self>, content: String) {
-        setter!(self_, Replace, content, content)
+        setter!(self_, Replace, content, content.into())
     }
 }
 

bindings/python/src/processors.rs

@@ -721,7 +721,7 @@ impl PyTemplateProcessing {
 
     #[getter]
     fn get_single(self_: PyRef<Self>) -> String {
-        getter!(self_, Template, get_single())
+        getter!(self_, Template, get_single()).into()
     }
 
     #[setter]
@@ -847,7 +847,7 @@ mod test {
     fn get_subtype() {
         Python::with_gil(|py| {
             let py_proc = PyPostProcessor::new(PyPostProcessorTypeWrapper::Single(Arc::new(
-                RwLock::new(BertProcessing::new(("SEP".into(), 0), ("CLS".into(), 1)).into()),
+                RwLock::new(BertProcessing::new(("SEP", 0), ("CLS", 1)).into()),
             )));
             let py_bert = py_proc.get_as_subtype(py).unwrap();
             assert_eq!(
@@ -859,7 +859,7 @@ mod test {
 
     #[test]
     fn serialize() {
-        let rs_processing = BertProcessing::new(("SEP".into(), 0), ("CLS".into(), 1));
+        let rs_processing = BertProcessing::new(("SEP", 0), ("CLS", 1));
         let rs_wrapper: PostProcessorWrapper = rs_processing.clone().into();
         let rs_processing_ser = serde_json::to_string(&rs_processing).unwrap();
         let rs_wrapper_ser = serde_json::to_string(&rs_wrapper).unwrap();

bindings/python/src/tokenizer.rs

@@ -871,7 +871,9 @@ impl PyTokenizer {
                     }
                     "pad_id" => params.pad_id = value.extract()?,
                     "pad_type_id" => params.pad_type_id = value.extract()?,
-                    "pad_token" => params.pad_token = value.extract()?,
+                    "pad_token" => {
+                        params.pad_token = value.extract::<String>()?.to_compact_string()
+                    }
                     "max_length" => {
                         println!(
                             "enable_padding(max_length=X) is deprecated, \
@@ -927,7 +929,7 @@ impl PyTokenizer {
             )?;
             dict.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?;
             dict.set_item("pad_id", params.pad_id)?;
-            dict.set_item("pad_token", &params.pad_token)?;
+            dict.set_item("pad_token", &*params.pad_token)?;
             dict.set_item("pad_type_id", params.pad_type_id)?;
             dict.set_item("direction", params.direction.as_ref())?;
 

bindings/python/src/utils/normalization.rs

@@ -1,6 +1,7 @@
 use super::regex::PyRegex;
 use super::{DestroyPtr, RefMutContainer, RefMutGuard};
 use crate::error::ToPyResult;
+use compact_str::ToCompactString;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
@@ -38,17 +39,21 @@ impl Pattern for PyPattern {
 impl From<PyPattern> for tk::normalizers::replace::ReplacePattern {
     fn from(pattern: PyPattern) -> Self {
         match pattern {
-            PyPattern::Str(s) => Self::String(s.to_owned()),
-            PyPattern::Regex(r) => Python::with_gil(|py| Self::Regex(r.borrow(py).pattern.clone())),
+            PyPattern::Str(s) => Self::String(s.into()),
+            PyPattern::Regex(r) => {
+                Python::with_gil(|py| Self::Regex(r.borrow(py).pattern.to_compact_string()))
+            }
         }
     }
 }
 
 impl From<PyPattern> for tk::pre_tokenizers::split::SplitPattern {
     fn from(pattern: PyPattern) -> Self {
         match pattern {
-            PyPattern::Str(s) => Self::String(s.to_owned()),
-            PyPattern::Regex(r) => Python::with_gil(|py| Self::Regex(r.borrow(py).pattern.clone())),
+            PyPattern::Str(s) => Self::String(s.into()),
+            PyPattern::Regex(r) => {
+                Python::with_gil(|py| Self::Regex(r.borrow(py).pattern.to_compact_string()))
+            }
         }
     }
 }