diff --git a/app/src/main/java/helium314/keyboard/latin/dictionary/DictionaryFactory.kt b/app/src/main/java/helium314/keyboard/latin/dictionary/DictionaryFactory.kt index dcd3ba9bca..9bd97fe362 100644 --- a/app/src/main/java/helium314/keyboard/latin/dictionary/DictionaryFactory.kt +++ b/app/src/main/java/helium314/keyboard/latin/dictionary/DictionaryFactory.kt @@ -74,11 +74,7 @@ object DictionaryFactory { } @JvmStatic - fun getDictionary( - file: File, - locale: Locale - ): Dictionary? { - if (!file.isFile) return null + fun getDictionary(file: File, locale: Locale): Dictionary? { val header = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(file) if (header == null) { killDictionary(file) diff --git a/app/src/main/java/helium314/keyboard/latin/dictionary/ExpandableBinaryDictionary.java b/app/src/main/java/helium314/keyboard/latin/dictionary/ExpandableBinaryDictionary.java index 1603c9263b..5c5ac91e01 100644 --- a/app/src/main/java/helium314/keyboard/latin/dictionary/ExpandableBinaryDictionary.java +++ b/app/src/main/java/helium314/keyboard/latin/dictionary/ExpandableBinaryDictionary.java @@ -275,13 +275,15 @@ public void addUnigramEntry(final String word, final int frequency, shortcutFreq, isNotAWord, isPossiblyOffensive, timestamp)); } - protected void addUnigramLocked(final String word, final int frequency, + protected boolean addUnigramLocked(final String word, final int frequency, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final boolean isPossiblyOffensive, final int timestamp) { if (!mBinaryDictionary.addUnigramEntry(word, frequency, shortcutTarget, shortcutFreq, false /* isBeginningOfSentence */, isNotAWord, isPossiblyOffensive, timestamp)) { Log.e(TAG, "Cannot add unigram entry. word: " + word); + return false; } + return true; } /** diff --git a/app/src/main/java/helium314/keyboard/latin/dictionary/UserAddedDictionary.kt b/app/src/main/java/helium314/keyboard/latin/dictionary/UserAddedDictionary.kt new file mode 100644 index 0000000000..9a408ed165 --- /dev/null +++ b/app/src/main/java/helium314/keyboard/latin/dictionary/UserAddedDictionary.kt @@ -0,0 +1,84 @@ +package helium314.keyboard.latin.dictionary + +import android.content.Context +import com.android.inputmethod.latin.BinaryDictionary +import helium314.keyboard.latin.makedict.DictionaryHeader +import helium314.keyboard.latin.utils.DictionaryInfoUtils +import helium314.keyboard.latin.utils.Log +import java.io.File +import java.util.Locale + +// todo +// how to add to correct directory? +// is it enough to provide the file? +// use user suffix? +// parse "everything" +// bigram +// shortcut +// ui for adding, with feedback +// ui for managing +// should show up in dictionaries screen +// should have a full header +// backup/restore +class UserAddedDictionary(context: Context, locale: Locale, name: String) : ExpandableBinaryDictionary( + context, + name, + locale, + name, + File(DictionaryInfoUtils.getCacheDirectoryForLocale(locale, context)!!, name + "_" + DictionaryInfoUtils.USER_DICTIONARY_SUFFIX) +) { + var content: List? = null + private set + var added = 0 + var failed = 0 + + fun setContents(contents: List) { + Log.i(TAG, "setting new contents for ") + content = contents + setNeedsToRecreate() + reloadDictionaryIfRequired() + } + + // todo: after around 350k words this becomes slow, and fails to add more words a bit later + // split at 300k words? not nice for query performance for large dicts i guess + // just tell the user that it's not working? + // ideally we'd create an actual .dict file, here we also have some unknown but larger limit + override fun loadInitialContentsLocked() { + added = 0 + failed = 0 + content?.forEach { line -> + if (!line.trim().startsWith("word")) return@forEach + val split = line.split(",").map { it.trim() } + val success = addUnigramLocked( + split.first { it.startsWith("word=") }.substringAfter("word="), + split.first { it.startsWith("f=") }.substringAfter("f=").toInt(), + null, + 0, + split.contains("not_a_word=true"), + split.contains("possibly_offensive=true"), + BinaryDictionary.NOT_A_VALID_TIMESTAMP + ) + if (success) added++ else failed++ + runGCIfRequiredLocked(true) + } + content = null + Log.i(TAG, "added $added entries, could not add $failed entries") + } + + companion object { + private val TAG = UserAddedDictionary::class.java.simpleName + + fun tryParseHeader(file: File): DictionaryHeader? { + val lines = file.readLines() + if (lines.size < 2) return null + if (!lines[1].trim().startsWith("word=", true) || !lines[1].contains("f=", true)) { + Log.e(TAG, "tryParseHeader: second line not a dictionary line: ${lines[1]}") + return null // not the right format (should be extended though, why not just accept word lists, or word + frequency) + } + + return if (lines[0].trim().startsWith("dictionary", true)) + runCatching { DictionaryHeader.fromString(lines[0]) }.getOrNull() + else DictionaryHeader.createEmptyHeader() + } + } +} diff --git a/app/src/main/java/helium314/keyboard/latin/makedict/DictionaryHeader.kt b/app/src/main/java/helium314/keyboard/latin/makedict/DictionaryHeader.kt index 9ca80b975a..febac22dc1 100644 --- a/app/src/main/java/helium314/keyboard/latin/makedict/DictionaryHeader.kt +++ b/app/src/main/java/helium314/keyboard/latin/makedict/DictionaryHeader.kt @@ -11,6 +11,7 @@ import helium314.keyboard.latin.makedict.FormatSpec.DictionaryOptions import java.text.DateFormat import java.util.Date import java.util.Locale +import kotlin.jvm.Throws /** * Class representing dictionary header. @@ -58,5 +59,17 @@ class DictionaryHeader( const val MAX_TRIGRAM_COUNT_KEY = "MAX_TRIGRAM_ENTRY_COUNT" const val ATTRIBUTE_VALUE_TRUE = "1" const val CODE_POINT_TABLE_KEY = "codePointTable" + + @Throws(UnsupportedFormatException::class) + fun fromString(string: String): DictionaryHeader { + val split = string.split(",").map { it.trim() }.filter { "=" in it } + val map = split.associateTo(HashMap()) { it.split("=").let { it[0] to it[1] } } + return DictionaryHeader(DictionaryOptions(map)) + } + + fun createEmptyHeader(): DictionaryHeader { + val map = hashMapOf(DICTIONARY_LOCALE_KEY to "", DICTIONARY_VERSION_KEY to "", DICTIONARY_ID_KEY to "") + return DictionaryHeader(DictionaryOptions(map)) + } } } diff --git a/app/src/main/java/helium314/keyboard/settings/FilePicker.kt b/app/src/main/java/helium314/keyboard/settings/FilePicker.kt index 9fd5d0a5fe..df85192bdd 100644 --- a/app/src/main/java/helium314/keyboard/settings/FilePicker.kt +++ b/app/src/main/java/helium314/keyboard/settings/FilePicker.kt @@ -2,6 +2,7 @@ package helium314.keyboard.settings import android.app.Activity +import android.content.Context import android.content.Intent import android.net.Uri import android.provider.OpenableColumns @@ -13,11 +14,16 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember +import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import helium314.keyboard.latin.R import helium314.keyboard.latin.common.FileUtils +import helium314.keyboard.latin.dictionary.UserAddedDictionary +import helium314.keyboard.latin.makedict.DictionaryHeader +import helium314.keyboard.latin.makedict.FormatSpec +import helium314.keyboard.latin.utils.DictionaryInfoUtils import helium314.keyboard.latin.utils.LayoutUtilsCustom import helium314.keyboard.latin.utils.getActivity import helium314.keyboard.settings.dialogs.InfoDialog @@ -46,17 +52,11 @@ fun layoutFilePicker( var errorDialog by remember { mutableStateOf(false) } val loadFilePicker = filePicker { uri -> val cr = ctx.getActivity()?.contentResolver ?: return@filePicker - val name = cr.query(uri, null, null, null, null)?.use { c -> - if (!c.moveToFirst()) return@use null - val index = c.getColumnIndex(OpenableColumns.DISPLAY_NAME) - if (index < 0) null - else c.getString(index) - } cr.openInputStream(uri)?.use { val content = it.reader().readText() errorDialog = !LayoutUtilsCustom.checkLayout(content, ctx) if (!errorDialog) - onSuccess(content, name) + onSuccess(content, getNameFromUri(ctx, uri)) } } if (errorDialog) @@ -68,18 +68,46 @@ fun layoutFilePicker( fun dictionaryFilePicker(mainLocale: Locale?): ManagedActivityResultLauncher { val ctx = LocalContext.current val cachedDictionaryFile = File(ctx.cacheDir?.path + File.separator + "temp_dict") - var done by remember { mutableStateOf(false) } + var name by rememberSaveable { mutableStateOf(null) } val picker = filePicker { uri -> cachedDictionaryFile.delete() FileUtils.copyContentUriToNewFile(uri, ctx, cachedDictionaryFile) - done = true + name = getNameFromUri(ctx, uri) ?: "dict" + } + if (name != null) { + val header = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(cachedDictionaryFile, 0, cachedDictionaryFile.length()) + val textHeader by lazy { UserAddedDictionary.tryParseHeader(cachedDictionaryFile) } // todo: got a "No such file or directory" exception here + if (header != null) { + NewDictionaryDialog( + onDismissRequest = { name = null }, + cachedDictionaryFile, + mainLocale, + header, + false + ) + } else if (textHeader != null) { + if (textHeader!!.mIdString == "") // no header in file + textHeader!!.mDictionaryOptions.mAttributes[DictionaryHeader.DICTIONARY_ID_KEY] = name + NewDictionaryDialog( + onDismissRequest = { name = null }, + cachedDictionaryFile, + mainLocale, + DictionaryHeader(FormatSpec.DictionaryOptions(textHeader!!.mDictionaryOptions.mAttributes)), + true + ) + } else { + InfoDialog(stringResource(R.string.dictionary_file_error)) { name = null } + } } - if (done) - NewDictionaryDialog( - onDismissRequest = { done = false }, - cachedDictionaryFile, - mainLocale - ) return picker } + +private fun getNameFromUri(context: Context, uri: Uri): String? { + return context.getActivity()?.contentResolver?.query(uri, null, null, null, null)?.use { c -> + if (!c.moveToFirst()) return@use null + val index = c.getColumnIndex(OpenableColumns.DISPLAY_NAME) + if (index < 0) null + else c.getString(index) + } +} diff --git a/app/src/main/java/helium314/keyboard/settings/SettingsActivity.kt b/app/src/main/java/helium314/keyboard/settings/SettingsActivity.kt index f3a1394dd1..9d44215301 100644 --- a/app/src/main/java/helium314/keyboard/settings/SettingsActivity.kt +++ b/app/src/main/java/helium314/keyboard/settings/SettingsActivity.kt @@ -39,11 +39,13 @@ import helium314.keyboard.latin.common.FileUtils import helium314.keyboard.latin.define.DebugFlags import helium314.keyboard.latin.settings.Settings import helium314.keyboard.latin.utils.DeviceProtectedUtils +import helium314.keyboard.latin.utils.DictionaryInfoUtils import helium314.keyboard.latin.utils.ExecutorUtils import helium314.keyboard.latin.utils.UncachedInputMethodManagerUtils import helium314.keyboard.latin.utils.cleanUnusedMainDicts import helium314.keyboard.latin.utils.prefs import helium314.keyboard.settings.dialogs.ConfirmationDialog +import helium314.keyboard.settings.dialogs.InfoDialog import helium314.keyboard.settings.dialogs.NewDictionaryDialog import kotlinx.coroutines.flow.MutableStateFlow import java.io.BufferedOutputStream @@ -138,11 +140,17 @@ open class SettingsActivity : ComponentActivity(), SharedPreferences.OnSharedPre } } if (dictUri != null) { - NewDictionaryDialog( - onDismissRequest = { dictUriFlow.value = null }, - cachedFile = cachedDictionaryFile, - mainLocale = null - ) + val header = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(cachedDictionaryFile, 0, cachedDictionaryFile.length()) + if (header == null) + InfoDialog(stringResource(R.string.dictionary_file_error)) { dictUriFlow.value = null } + else + NewDictionaryDialog( + onDismissRequest = { dictUriFlow.value = null }, + cachedFile = cachedDictionaryFile, + mainLocale = null, + header = header, + isTextFile = false + ) } } } diff --git a/app/src/main/java/helium314/keyboard/settings/dialogs/DictionaryDialog.kt b/app/src/main/java/helium314/keyboard/settings/dialogs/DictionaryDialog.kt index 79b84914aa..caf435a9c0 100644 --- a/app/src/main/java/helium314/keyboard/settings/dialogs/DictionaryDialog.kt +++ b/app/src/main/java/helium314/keyboard/settings/dialogs/DictionaryDialog.kt @@ -100,7 +100,7 @@ fun DictionaryDialog( onNeutral = { val intent = Intent(Intent.ACTION_OPEN_DOCUMENT) .addCategory(Intent.CATEGORY_OPENABLE) - .setType("application/octet-stream") + .setType("*/*") picker.launch(intent) } ) @@ -135,7 +135,7 @@ private fun DictionaryDetails(dict: File) { ConfirmationDialog( onDismissRequest = { showDeleteDialog = false }, confirmButtonText = stringResource(R.string.remove), - onConfirmed = { dict.delete() }, + onConfirmed = { dict.deleteRecursively() }, content = { Text(stringResource(R.string.remove_dictionary_message, type))} ) } diff --git a/app/src/main/java/helium314/keyboard/settings/dialogs/NewDictionaryDialog.kt b/app/src/main/java/helium314/keyboard/settings/dialogs/NewDictionaryDialog.kt index 2380d667f3..333f095862 100644 --- a/app/src/main/java/helium314/keyboard/settings/dialogs/NewDictionaryDialog.kt +++ b/app/src/main/java/helium314/keyboard/settings/dialogs/NewDictionaryDialog.kt @@ -11,6 +11,7 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember +import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext @@ -33,18 +34,23 @@ import helium314.keyboard.settings.WithSmallTitle import java.io.File import java.util.Locale import androidx.compose.ui.platform.LocalConfiguration +import helium314.keyboard.latin.dictionary.UserAddedDictionary +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch @Composable fun NewDictionaryDialog( onDismissRequest: () -> Unit, cachedFile: File, - mainLocale: Locale? + mainLocale: Locale?, + header: DictionaryHeader, + isTextFile: Boolean ) { - val (error, header) = checkDict(cachedFile) - if (error != null) { - InfoDialog(stringResource(error), onDismissRequest) + if (!isTextFile && !canLoadDictionary(cachedFile)) { + InfoDialog(stringResource(R.string.dictionary_load_error), onDismissRequest) cachedFile.delete() - } else if (header != null) { + } else { val ctx = LocalContext.current val dictLocale = header.mLocaleString.constructLocale() val enabledLanguages = SubtypeSettings.getEnabledSubtypes().map { it.locale().language } @@ -57,20 +63,47 @@ fun NewDictionaryDialog( val dictFile = File(cacheDir, header.mIdString.substringBefore(":") + "_" + DictionaryInfoUtils.USER_DICTIONARY_SUFFIX) val type = header.mIdString.substringBefore(":") val info = header.info(LocalConfiguration.current.locale()) + var showWait by rememberSaveable { mutableStateOf(null) } + if (showWait != null) + InfoDialog(showWait.toString()) { } // no way to cancel ThreeButtonAlertDialog( onDismissRequest = { onDismissRequest(); cachedFile.delete() }, onConfirmed = { dictFile.parentFile?.mkdirs() dictFile.delete() - cachedFile.renameTo(dictFile) - if (type == Dictionary.TYPE_MAIN) { - // replaced main dict, remove the one created from internal data - val internalMainDictFile = File(cacheDir, DictionaryInfoUtils.MAIN_DICT_FILE_NAME) - internalMainDictFile.delete() + fun finish() { + if (type == Dictionary.TYPE_MAIN) { + // replaced main dict, remove the one created from internal data + val internalMainDictFile = File(cacheDir, DictionaryInfoUtils.MAIN_DICT_FILE_NAME) + internalMainDictFile.delete() + } + val newDictBroadcast = Intent(DictionaryPackConstants.NEW_DICTIONARY_INTENT_ACTION) + ctx.sendBroadcast(newDictBroadcast) + onDismissRequest() + } + if (isTextFile) { + showWait = "please wait" + val dict = UserAddedDictionary(ctx, locale, type) + dict.setContents(cachedFile.readLines()) + // maybe this shit is not necessary, but if we don't do it the user doesn't know about failed imports + GlobalScope.launch { + var wait = 0 + while (dict.content != null && wait < 3000) { + delay(100) + wait++ + showWait = "please wait, ${dict.added} words added, failed for ${dict.failed} words" + } + dict.onFinishInput() + showWait = null + cachedFile.delete() + finish() + } + } else { + cachedFile.renameTo(dictFile) + finish() } - val newDictBroadcast = Intent(DictionaryPackConstants.NEW_DICTIONARY_INTENT_ACTION) - ctx.sendBroadcast(newDictBroadcast) }, + confirmDismissesDialog = false, confirmButtonText = stringResource(if (dictFile.exists()) R.string.replace_dictionary else android.R.string.ok), title = { Text(stringResource(R.string.add_new_dictionary_title)) }, content = { @@ -107,15 +140,11 @@ fun NewDictionaryDialog( } } -private fun checkDict(file: File): Pair { - val newHeader = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(file, 0, file.length()) - ?: return R.string.dictionary_file_error to null - +private fun canLoadDictionary(file: File): Boolean { + val newHeader = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(file, 0, file.length()) ?: return false val locale = newHeader.mLocaleString.constructLocale() val dict = ReadOnlyBinaryDictionary(file.absolutePath, 0, file.length(), false, locale, "test") - if (!dict.isValidDictionary) { - dict.close() - return R.string.dictionary_load_error to null - } - return null to newHeader + val isValid = dict.isValidDictionary + dict.close() + return isValid } diff --git a/app/src/main/java/helium314/keyboard/settings/dialogs/ThreeButtonAlertDialog.kt b/app/src/main/java/helium314/keyboard/settings/dialogs/ThreeButtonAlertDialog.kt index cc20a4e631..850d4fc0f7 100644 --- a/app/src/main/java/helium314/keyboard/settings/dialogs/ThreeButtonAlertDialog.kt +++ b/app/src/main/java/helium314/keyboard/settings/dialogs/ThreeButtonAlertDialog.kt @@ -42,6 +42,7 @@ fun ThreeButtonAlertDialog( cancelButtonText: String = stringResource(android.R.string.cancel), neutralButtonText: String? = null, reducePadding: Boolean = false, + confirmDismissesDialog: Boolean = true, properties: DialogProperties = DialogProperties() ) { Dialog( @@ -87,7 +88,7 @@ fun ThreeButtonAlertDialog( if (confirmButtonText != null) TextButton( enabled = checkOk(), - onClick = { onConfirmed(); onDismissRequest() }, + onClick = { onConfirmed(); if (confirmDismissesDialog) onDismissRequest() }, ) { Text(confirmButtonText) } } } diff --git a/app/src/main/java/helium314/keyboard/settings/screens/DictionaryScreen.kt b/app/src/main/java/helium314/keyboard/settings/screens/DictionaryScreen.kt index d94adebd52..b99cf613af 100644 --- a/app/src/main/java/helium314/keyboard/settings/screens/DictionaryScreen.kt +++ b/app/src/main/java/helium314/keyboard/settings/screens/DictionaryScreen.kt @@ -113,7 +113,7 @@ fun DictionaryScreen( onConfirmed = { val intent = Intent(Intent.ACTION_OPEN_DOCUMENT) .addCategory(Intent.CATEGORY_OPENABLE) - .setType("application/octet-stream") + .setType("*/*") dictPicker.launch(intent) }, title = { Text(stringResource(R.string.add_new_dictionary_title)) },