Merge pull request #3674 from kiwix/Fix#3646

Improved the scanning of ZIM files.

app/src/main/java/org/kiwix/kiwixmobile/nav/destination/library/LocalLibraryFragment.kt

@@ -86,6 +86,7 @@ import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.adapter.BooksOnDis
 import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.adapter.BooksOnDiskListItem
 import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.adapter.BooksOnDiskListItem.BookOnDisk
 import org.kiwix.kiwixmobile.databinding.FragmentDestinationLibraryBinding
+import org.kiwix.kiwixmobile.zimManager.MAX_PROGRESS
 import org.kiwix.kiwixmobile.zimManager.ZimManageViewModel
 import org.kiwix.kiwixmobile.zimManager.ZimManageViewModel.FileSelectActions
 import org.kiwix.kiwixmobile.zimManager.ZimManageViewModel.FileSelectActions.RequestDeleteMultiSelection
@@ -235,8 +236,14 @@ class LocalLibraryFragment : BaseFragment() {
       }
     disposable.add(sideEffects())
     disposable.add(fileSelectActions())
-    zimManageViewModel.deviceListIsRefreshing.observe(viewLifecycleOwner) {
-      fragmentDestinationLibraryBinding?.zimSwiperefresh?.isRefreshing = it!!
+    zimManageViewModel.deviceListScanningProgress.observe(viewLifecycleOwner) {
+      fragmentDestinationLibraryBinding?.scanningProgressView?.apply {
+        progress = it
+        // hide this progress bar when scanning is complete.
+        visibility = if (it == MAX_PROGRESS) GONE else VISIBLE
+        // enable if the previous scanning is completes.
+        fragmentDestinationLibraryBinding?.zimSwiperefresh?.isEnabled = it == MAX_PROGRESS
+      }
     }
     if (savedInstanceState != null && savedInstanceState.getBoolean(WAS_IN_ACTION_MODE)) {
       zimManageViewModel.fileSelectActions.offer(FileSelectActions.RestartActionMode)
@@ -258,6 +265,7 @@ class LocalLibraryFragment : BaseFragment() {
           SCROLL_DOWN -> {
             setBottomMarginToSwipeRefreshLayout(0)
           }
+
           SCROLL_UP -> {
             getBottomNavigationView()?.let {
               setBottomMarginToSwipeRefreshLayout(it.measuredHeight)
@@ -280,6 +288,18 @@ class LocalLibraryFragment : BaseFragment() {
           // the loading icon remains visible infinitely.
           fragmentDestinationLibraryBinding?.zimSwiperefresh?.isRefreshing = false
         } else {
+          fragmentDestinationLibraryBinding?.zimSwiperefresh?.apply {
+            // hide the swipe refreshing because now we are showing the ContentLoadingProgressBar
+            // to show the progress of how many files are scanned.
+            isRefreshing = false
+            // disable the swipe refresh layout until the ongoing scanning will not complete
+            // to avoid multiple scanning.
+            isEnabled = false
+          }
+          fragmentDestinationLibraryBinding?.scanningProgressView?.apply {
+            visibility = VISIBLE
+            progress = 0
+          }
           requestFileSystemCheck()
         }
       }
@@ -512,24 +532,16 @@ class LocalLibraryFragment : BaseFragment() {
   }
 
   private fun checkManageExternalStoragePermission() {
-    if (sharedPreferenceUtil.isPlayStoreBuild) {
-      requestFileSystemCheck()
-    } else {
-      if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
-        if (Environment.isExternalStorageManager()) {
-          // We already have permission!!
-          requestFileSystemCheck()
-        } else {
-          if (sharedPreferenceUtil.manageExternalFilesPermissionDialog) {
-            // We should only ask for first time, If the users wants to revoke settings
-            // then they can directly toggle this feature from settings screen
-            sharedPreferenceUtil.manageExternalFilesPermissionDialog = false
-            // Show Dialog and  Go to settings to give permission
-            showManageExternalStoragePermissionDialog()
-          }
+    if (!sharedPreferenceUtil.isPlayStoreBuild && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+      if (!Environment.isExternalStorageManager()) {
+        // We do not have the permission!!
+        if (sharedPreferenceUtil.manageExternalFilesPermissionDialog) {
+          // We should only ask for first time, If the users wants to revoke settings
+          // then they can directly toggle this feature from settings screen
+          sharedPreferenceUtil.manageExternalFilesPermissionDialog = false
+          // Show Dialog and  Go to settings to give permission
+          showManageExternalStoragePermissionDialog()
         }
-      } else {
-        requestFileSystemCheck()
       }
     }
   }

app/src/main/java/org/kiwix/kiwixmobile/zimManager/ZimManageViewModel.kt

@@ -47,6 +47,7 @@ import org.kiwix.kiwixmobile.core.extensions.calculateSearchMatches
 import org.kiwix.kiwixmobile.core.extensions.registerReceiver
 import org.kiwix.kiwixmobile.core.utils.BookUtils
 import org.kiwix.kiwixmobile.core.utils.SharedPreferenceUtil
+import org.kiwix.kiwixmobile.core.utils.files.ScanningProgressListener
 import org.kiwix.kiwixmobile.core.zim_manager.Language
 import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.SelectionMode.MULTI
 import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.SelectionMode.NORMAL
@@ -78,6 +79,8 @@ import java.util.Locale
 import java.util.concurrent.TimeUnit.MILLISECONDS
 import javax.inject.Inject
 
+const val DEFAULT_PROGRESS = 0
+const val MAX_PROGRESS = 100
 class ZimManageViewModel @Inject constructor(
   private val downloadDao: FetchDownloadDao,
   private val bookDao: NewBookDao,
@@ -107,7 +110,7 @@ class ZimManageViewModel @Inject constructor(
   val sideEffects = PublishProcessor.create<SideEffect<Any?>>()
   val libraryItems: MutableLiveData<List<LibraryListItem>> = MutableLiveData()
   val fileSelectListStates: MutableLiveData<FileSelectListState> = MutableLiveData()
-  val deviceListIsRefreshing = MutableLiveData<Boolean>()
+  val deviceListScanningProgress = MutableLiveData<Int>()
   val libraryListIsRefreshing = MutableLiveData<Boolean>()
   val shouldShowWifiOnlyDialog = MutableLiveData<Boolean>()
   val networkStates = MutableLiveData<NetworkState>()
@@ -430,15 +433,32 @@ class ZimManageViewModel @Inject constructor(
       .subscribeOn(Schedulers.io())
       .observeOn(Schedulers.io())
       .onBackpressureDrop()
-      .doOnNext { deviceListIsRefreshing.postValue(true) }
+      .doOnNext { deviceListScanningProgress.postValue(DEFAULT_PROGRESS) }
       .switchMap(
         {
-          booksFromStorageNotIn(booksFromDao)
+          booksFromStorageNotIn(
+            booksFromDao,
+            object : ScanningProgressListener {
+              override fun onProgressUpdate(scannedDirectory: Int, totalDirectory: Int) {
+                // Calculate the overall progress based on the number of processed directories
+                val overallProgress =
+                  (scannedDirectory.toDouble() / totalDirectory.toDouble() * MAX_PROGRESS).toInt()
+                if (overallProgress != MAX_PROGRESS) {
+                  // Send the progress if it is not 100% because after scanning the entire storage,
+                  // it takes a bit of time to organize the ZIM files, filter them,
+                  // and remove any duplicate ZIM files. We send the 100% progress
+                  // in the doOnNext method to hide the progressBar from the UI
+                  // and display all the filtered ZIM files.
+                  deviceListScanningProgress.postValue(overallProgress)
+                }
+              }
+            }
+          )
         },
         1
       )
       .onBackpressureDrop()
-      .doOnNext { deviceListIsRefreshing.postValue(false) }
+      .doOnNext { deviceListScanningProgress.postValue(MAX_PROGRESS) }
       .filter(List<BookOnDisk>::isNotEmpty)
       .map { it.distinctBy { bookOnDisk -> bookOnDisk.book.id } }
       .subscribe(
@@ -450,8 +470,11 @@ class ZimManageViewModel @Inject constructor(
     .subscribeOn(Schedulers.io())
     .map { it.sortedBy { book -> book.book.title } }
 
-  private fun booksFromStorageNotIn(booksFromDao: Flowable<List<BookOnDisk>>) =
-    storageObserver.booksOnFileSystem
+  private fun booksFromStorageNotIn(
+    booksFromDao: Flowable<List<BookOnDisk>>,
+    scanningProgressListener: ScanningProgressListener
+  ) =
+    storageObserver.getBooksOnFileSystem(scanningProgressListener)
       .withLatestFrom(
         booksFromDao.map { it.map { bookOnDisk -> bookOnDisk.book.id } },
         BiFunction(::removeBooksAlreadyInDao)

app/src/main/res/layout/fragment_destination_library.xml

@@ -37,6 +37,7 @@
       android:layout_height="wrap_content">
 
       <include layout="@layout/layout_scrolling_toolbar" />
+
     </com.google.android.material.appbar.AppBarLayout>
 
 
@@ -46,12 +47,32 @@
       android:layout_height="match_parent"
       app:layout_behavior="@string/appbar_scrolling_view_behavior">
 
-      <androidx.recyclerview.widget.RecyclerView
-        android:id="@+id/zimfilelist"
+      <androidx.constraintlayout.widget.ConstraintLayout
         android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        tools:listitem="@layout/item_book" />
+        android:layout_height="match_parent">
+
+        <androidx.recyclerview.widget.RecyclerView
+          android:id="@+id/zimfilelist"
+          android:layout_width="0dp"
+          android:layout_height="match_parent"
+          app:layout_constraintBottom_toBottomOf="parent"
+          app:layout_constraintEnd_toEndOf="parent"
+          app:layout_constraintStart_toStartOf="parent"
+          app:layout_constraintTop_toBottomOf="@+id/scanning_progress_view"
+          tools:listitem="@layout/item_book" />
 
+        <androidx.core.widget.ContentLoadingProgressBar
+          android:id="@+id/scanning_progress_view"
+          style="?android:attr/progressBarStyleHorizontal"
+          android:layout_width="match_parent"
+          android:layout_height="4dp"
+          android:indeterminate="false"
+          android:max="100"
+          android:theme="@style/ThemeOverlay.KiwixTheme.ProgressBar"
+          android:visibility="gone"
+          app:layout_constraintTop_toTopOf="parent"
+          tools:progress="0" />
+      </androidx.constraintlayout.widget.ConstraintLayout>
     </androidx.swiperefreshlayout.widget.SwipeRefreshLayout>
 
   </org.kiwix.kiwixmobile.core.utils.NestedCoordinatorLayout>
@@ -64,8 +85,8 @@
     app:layout_constraintEnd_toEndOf="parent"
     app:layout_constraintStart_toStartOf="parent"
     app:layout_constraintTop_toTopOf="parent"
-    tools:ignore="RequiredSize"
-    app:layout_constraintVertical_bias="0.45" />
+    app:layout_constraintVertical_bias="0.45"
+    tools:ignore="RequiredSize" />
 
   <Button
     android:id="@+id/go_to_downloads_button_no_files"

app/src/test/java/org/kiwix/kiwixmobile/zimManager/ZimManageViewModelTest.kt

@@ -47,6 +47,7 @@ import org.kiwix.kiwixmobile.core.downloader.model.DownloadModel
 import org.kiwix.kiwixmobile.core.entity.LibraryNetworkEntity.Book
 import org.kiwix.kiwixmobile.core.utils.BookUtils
 import org.kiwix.kiwixmobile.core.utils.SharedPreferenceUtil
+import org.kiwix.kiwixmobile.core.utils.files.ScanningProgressListener
 import org.kiwix.kiwixmobile.core.zim_manager.Language
 import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.SelectionMode.MULTI
 import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.SelectionMode.NORMAL
@@ -128,7 +129,11 @@ class ZimManageViewModelTest {
     every { connectivityBroadcastReceiver.action } returns "test"
     every { downloadDao.downloads() } returns downloads
     every { newBookDao.books() } returns books
-    every { storageObserver.booksOnFileSystem } returns booksOnFileSystem
+    every {
+      storageObserver.getBooksOnFileSystem(
+        any<ScanningProgressListener>()
+      )
+    } returns booksOnFileSystem
     every { newLanguagesDao.languages() } returns languages
     every { fat32Checker.fileSystemStates } returns fileSystemStates
     every { connectivityBroadcastReceiver.networkStates } returns networkStates

core/src/main/java/org/kiwix/kiwixmobile/core/StorageObserver.kt

@@ -25,6 +25,7 @@ import org.kiwix.kiwixmobile.core.dao.FetchDownloadDao
 import org.kiwix.kiwixmobile.core.downloader.model.DownloadModel
 import org.kiwix.kiwixmobile.core.reader.ZimFileReader
 import org.kiwix.kiwixmobile.core.utils.files.FileSearch
+import org.kiwix.kiwixmobile.core.utils.files.ScanningProgressListener
 import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.adapter.BooksOnDiskListItem.BookOnDisk
 import java.io.File
 import javax.inject.Inject
@@ -35,12 +36,16 @@ class StorageObserver @Inject constructor(
   private val zimReaderFactory: ZimFileReader.Factory
 ) {
 
-  val booksOnFileSystem: Flowable<List<BookOnDisk>>
-    get() = scanFiles()
+  fun getBooksOnFileSystem(
+    scanningProgressListener: ScanningProgressListener
+  ): Flowable<List<BookOnDisk>> {
+    return scanFiles(scanningProgressListener)
       .withLatestFrom(downloadDao.downloads(), BiFunction(::toFilesThatAreNotDownloading))
       .map { it.mapNotNull(::convertToBookOnDisk) }
+  }
 
-  private fun scanFiles() = fileSearch.scan().subscribeOn(Schedulers.io())
+  private fun scanFiles(scanningProgressListener: ScanningProgressListener) =
+    fileSearch.scan(scanningProgressListener).subscribeOn(Schedulers.io())
 
   private fun toFilesThatAreNotDownloading(files: List<File>, downloads: List<DownloadModel>) =
     files.filter { fileHasNoMatchingDownload(downloads, it) }

core/src/main/java/org/kiwix/kiwixmobile/core/utils/files/FileSearch.kt

@@ -35,9 +35,10 @@ class FileSearch @Inject constructor(private val context: Context) {
 
   private val zimFileExtensions = arrayOf("zim", "zimaa")
 
-  fun scan(): Flowable<List<File>> =
+  fun scan(scanningProgressListener: ScanningProgressListener): Flowable<List<File>> =
     Flowable.combineLatest(
-      Flowable.fromCallable(::scanFileSystem).subscribeOn(Schedulers.io()),
+      Flowable.fromCallable { scanFileSystem(scanningProgressListener) }
+        .subscribeOn(Schedulers.io()),
       Flowable.fromCallable(::scanMediaStore).subscribeOn(Schedulers.io()),
       BiFunction<List<File>, List<File>, List<File>> { filesSystemFiles, mediaStoreFiles ->
         filesSystemFiles + mediaStoreFiles
@@ -61,18 +62,46 @@ class FileSearch @Inject constructor(private val context: Context) {
       null
     )
 
-  private fun scanFileSystem() =
-    directoryRoots()
-      .fold(mutableListOf<File>(), { acc, root ->
-        acc.apply { addAll(scanDirectory(root)) }
-      })
-      .distinctBy { it.canonicalPath }
+  private fun scanFileSystem(scanningProgressListener: ScanningProgressListener): List<File> {
+    val directoryRoots = directoryRoots()
+    val totalDirectories = directoryRoots.size
+    var processedDirectories = 0
+
+    return directoryRoots.fold(mutableListOf<File>()) { acc, root ->
+      acc.apply {
+        addAll(
+          scanDirectory(root).also {
+            // Increment the count of processed directories and notify the progress
+            processedDirectories++
+            scanningProgressListener.onProgressUpdate(processedDirectories, totalDirectories)
+          }
+        )
+      }
+    }.distinctBy { it.canonicalPath }
+  }
 
   private fun directoryRoots() =
     StorageDeviceUtils.getReadableStorage(context).map(StorageDevice::name)
 
-  private fun scanDirectory(directory: String): List<File> =
-    File(directory).walk().filter { it.extension.isAny(*zimFileExtensions) }.toList()
+  private fun scanDirectory(directory: String): List<File> {
+    return File(directory).walk()
+      .onEnter { dir ->
+        // Excluding the "data," "obb," and "Trash" folders from scanning is justified for
+        // several reasons. The "Trash" folder contains deleted files,
+        // making it unnecessary for scanning. Additionally,
+        // the "data" and "obb" folders are specifically designed for the
+        // app's private directory, and users usually do not store ZIM files there.
+        // Most file managers prohibit direct copying of files into these directories.
+        // Therefore, scanning these folders is not essential. Moreover,
+        // such scans consume time, given the presence of numerous files written by other apps,
+        // which are irrelevant to our application.
+        !dir.name.equals(".Trash", ignoreCase = true) &&
+          !dir.name.equals("data", ignoreCase = true) &&
+          !dir.name.equals("obb", ignoreCase = true)
+      }.filter {
+        it.extension.isAny(*zimFileExtensions)
+      }.toList()
+  }
 }
 
 internal fun String.isAny(vararg suffixes: String) =

core/src/main/java/org/kiwix/kiwixmobile/core/utils/files/ScanningProgressListener.kt

@@ -0,0 +1,23 @@
+/*
+ * Kiwix Android
+ * Copyright (c) 2024 Kiwix <android.kiwix.org>
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package org.kiwix.kiwixmobile.core.utils.files
+
+interface ScanningProgressListener {
+  fun onProgressUpdate(scannedDirectory: Int, totalDirectory: Int)
+}

core/src/test/java/org/kiwix/kiwixmobile/core/StorageObserverTest.kt

@@ -33,6 +33,7 @@ import org.kiwix.kiwixmobile.core.reader.ZimFileReader
 import org.kiwix.kiwixmobile.core.reader.ZimFileReader.Factory
 import org.kiwix.kiwixmobile.core.utils.SharedPreferenceUtil
 import org.kiwix.kiwixmobile.core.utils.files.FileSearch
+import org.kiwix.kiwixmobile.core.utils.files.ScanningProgressListener
 import org.kiwix.sharedFunctions.book
 import org.kiwix.sharedFunctions.bookOnDisk
 import org.kiwix.sharedFunctions.resetSchedulers
@@ -48,6 +49,7 @@ class StorageObserverTest {
   private val file: File = mockk()
   private val readerFactory: Factory = mockk()
   private val zimFileReader: ZimFileReader = mockk()
+  private val scanningProgressListener: ScanningProgressListener = mockk()
 
   private val files: PublishProcessor<List<File>> = PublishProcessor.create()
   private val downloads: PublishProcessor<List<DownloadModel>> = PublishProcessor.create()
@@ -66,7 +68,7 @@ class StorageObserverTest {
   @BeforeEach fun init() {
     clearAllMocks()
     every { sharedPreferenceUtil.prefStorage } returns "a"
-    every { fileSearch.scan() } returns files
+    every { fileSearch.scan(scanningProgressListener) } returns files
     every { downloadDao.downloads() } returns downloads
     every { readerFactory.create(file) } returns zimFileReader
     storageObserver = StorageObserver(downloadDao, fileSearch, readerFactory)
@@ -92,7 +94,7 @@ class StorageObserverTest {
     verify { zimFileReader.dispose() }
   }
 
-  private fun booksOnFileSystem() = storageObserver.booksOnFileSystem
+  private fun booksOnFileSystem() = storageObserver.getBooksOnFileSystem(scanningProgressListener)
     .test()
     .also {
       downloads.offer(listOf(downloadModel))