File tree Expand file tree Collapse file tree 1 file changed +7
-2
lines changed
cocoa-core/src/main/kotlin/cc/unitmesh/nlp/embedding Expand file tree Collapse file tree 1 file changed +7
-2
lines changed Original file line number Diff line number Diff line change @@ -4,15 +4,20 @@ import com.knuddels.jtokkit.Encodings
44import com.knuddels.jtokkit.api.Encoding
55import com.knuddels.jtokkit.api.EncodingRegistry
66import com.knuddels.jtokkit.api.EncodingType
7+ import com.knuddels.jtokkit.api.IntArrayList
78
89class OpenAiEncoding : EncodingTokenizer {
910 private val registry: EncodingRegistry = Encodings .newLazyEncodingRegistry()
1011 private val encoding: Encoding = registry.getEncoding(EncodingType .CL100K_BASE )
12+
1113 override fun encode (text : String ): List <Int > {
12- return encoding.encode(text)
14+ val encode: IntArrayList = encoding.encode(text)
15+ return encode.boxed()
1316 }
1417
1518 override fun decode (tokens : List <Int >): String {
16- return encoding.decode(tokens)
19+ val intArray: IntArrayList = IntArrayList (tokens.size)
20+ tokens.forEach { intArray.add(it) }
21+ return encoding.decode(intArray)
1722 }
1823}
You can’t perform that action at this time.
0 commit comments