[PRD-550] [PRD-558] Improve search

src/dataproxy/worker/shared-worker.ts

@@ -85,13 +85,12 @@ const updateState = (): void => {
 
   if (client && searchEngine && searchEngine.searchIndexes) {
     state.status = 'Ready'
-    state.indexLength = Object.keys(searchEngine.searchIndexes).map(
-      (indexKey: string) => ({
-        doctype: indexKey,
+    state.indexLength = Object.entries(searchEngine.searchIndexes).map(
+      ([doctype, searchIndex]) => ({
+        doctype,
         // @ts-expect-error index.store is not TS typed
         // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-        count: Object.keys(searchEngine.searchIndexes[indexKey].index.store)
-          .length
+        count: Object.keys(searchIndex.index.store).length
       })
     )
     broadcastChannel.postMessage(state)

src/search/SearchEngine.spec.js

@@ -0,0 +1,140 @@
+import { createMockClient } from 'cozy-client'
+
+import { APPS_DOCTYPE, CONTACTS_DOCTYPE, FILES_DOCTYPE } from '@/search/consts'
+
+import SearchEngine from './SearchEngine'
+
+jest.mock('cozy-client')
+jest.mock('flexsearch')
+jest.mock('flexsearch/dist/module/lang/latin/balance')
+
+jest.mock('@/search/helpers/client', () => ({
+  getPouchLink: jest.fn()
+}))
+jest.mock('@/search/helpers/getSearchEncoder', () => ({
+  getSearchEncoder: jest.fn()
+}))
+
+describe('sortSearchResults', () => {
+  let searchEngine
+
+  beforeEach(() => {
+    const client = createMockClient()
+    searchEngine = new SearchEngine(client)
+  })
+
+  afterEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it('should sort results by doctype order', () => {
+    const searchResults = [
+      { doctype: FILES_DOCTYPE, doc: { _type: FILES_DOCTYPE } },
+      { doctype: APPS_DOCTYPE, doc: { _type: APPS_DOCTYPE } },
+      { doctype: CONTACTS_DOCTYPE, doc: { _type: CONTACTS_DOCTYPE } }
+    ]
+
+    const sortedResults = searchEngine.sortSearchResults(searchResults)
+
+    expect(sortedResults[0].doctype).toBe(APPS_DOCTYPE)
+    expect(sortedResults[1].doctype).toBe(CONTACTS_DOCTYPE)
+    expect(sortedResults[2].doctype).toBe(FILES_DOCTYPE)
+  })
+
+  it('should sort apps by slug', () => {
+    const searchResults = [
+      { doctype: APPS_DOCTYPE, doc: { slug: 'appB', _type: APPS_DOCTYPE } },
+      { doctype: APPS_DOCTYPE, doc: { slug: 'appA', _type: APPS_DOCTYPE } }
+    ]
+
+    const sortedResults = searchEngine.sortSearchResults(searchResults)
+
+    expect(sortedResults[0].doc.slug).toBe('appA')
+    expect(sortedResults[1].doc.slug).toBe('appB')
+  })
+
+  it('should sort contacts by displayName', () => {
+    const searchResults = [
+      {
+        doctype: CONTACTS_DOCTYPE,
+        doc: { displayName: 'June', _type: CONTACTS_DOCTYPE }
+      },
+      {
+        doctype: CONTACTS_DOCTYPE,
+        doc: { displayName: 'Alice', _type: CONTACTS_DOCTYPE }
+      }
+    ]
+
+    const sortedResults = searchEngine.sortSearchResults(searchResults)
+
+    expect(sortedResults[0].doc.displayName).toBe('Alice')
+    expect(sortedResults[1].doc.displayName).toBe('June')
+  })
+
+  it('should sort files by type and name', () => {
+    const searchResults = [
+      {
+        doctype: FILES_DOCTYPE,
+        doc: { name: 'fileB', type: 'file', _type: FILES_DOCTYPE },
+        fields: ['name']
+      },
+      {
+        doctype: FILES_DOCTYPE,
+        doc: { name: 'fileA', type: 'file', _type: FILES_DOCTYPE },
+        fields: ['name']
+      },
+      {
+        doctype: FILES_DOCTYPE,
+        doc: { name: 'folderA', type: 'directory', _type: FILES_DOCTYPE },
+        fields: ['name']
+      }
+    ]
+
+    const sortedResults = searchEngine.sortSearchResults(searchResults)
+
+    expect(sortedResults[0].doc.type).toBe('directory') // Folders should come first
+    expect(sortedResults[1].doc.name).toBe('fileA')
+    expect(sortedResults[2].doc.name).toBe('fileB')
+  })
+
+  it('should sort files first if they match on name, then path', () => {
+    const searchResults = [
+      {
+        doctype: FILES_DOCTYPE,
+        doc: {
+          name: 'test11',
+          path: 'test/test11',
+          type: 'file',
+          _type: FILES_DOCTYPE
+        },
+        fields: ['name']
+      },
+      {
+        doctype: FILES_DOCTYPE,
+        doc: {
+          name: 'test1',
+          path: 'test/test1',
+          type: 'file',
+          _type: FILES_DOCTYPE
+        },
+        fields: ['name']
+      },
+      {
+        doctype: FILES_DOCTYPE,
+        doc: {
+          name: 'DirName',
+          path: 'test1/path',
+          type: 'directory',
+          _type: FILES_DOCTYPE
+        },
+        fields: ['path']
+      }
+    ]
+
+    const sortedResults = searchEngine.sortSearchResults(searchResults)
+
+    expect(sortedResults[0].doc.name).toBe('test1') // File match on name
+    expect(sortedResults[1].doc.name).toBe('test11') // File match on name
+    expect(sortedResults[2].doc.name).toBe('DirName') // Directory
+  })
+})

src/search/SearchEngine.ts

@@ -11,7 +11,9 @@
   CONTACTS_DOCTYPE,
   DOCTYPE_ORDER,
   LIMIT_DOCTYPE_SEARCH,
-  REPLICATION_DEBOUNCE
+  REPLICATION_DEBOUNCE,
+  ROOT_DIR_ID,
+  SHARED_DRIVES_DIR_ID
 } from '@/search/consts'
 import { getPouchLink } from '@/search/helpers/client'
 import { getSearchEncoder } from '@/search/helpers/getSearchEncoder'
@@ -66,7 +68,7 @@
     })
     this.client.on('login', () => {
       // Ensure login is done before plugin register
       this.client.registerPlugin(RealtimePlugin, {})
       this.handleUpdatedOrCreatedDoc = this.handleUpdatedOrCreatedDoc.bind(this)
       this.handleDeletedDoc = this.handleDeletedDoc.bind(this)
 
@@ -77,8 +79,8 @@
   }
 
   subscribeDoctype(client: CozyClient, doctype: string): void {
     const realtime = this.client.plugins.realtime
     realtime.subscribe('created', doctype, this.handleUpdatedOrCreatedDoc)
     realtime.subscribe('updated', doctype, this.handleUpdatedOrCreatedDoc)
     realtime.subscribe('deleted', doctype, this.handleDeletedDoc)
   }
@@ -94,7 +96,7 @@
       return
     }
     log.debug('[REALTIME] index doc after update : ', doc)
-    searchIndex.index.add(doc)
+    this.addDocToIndex(searchIndex.index, doc)
 
     this.debouncedReplication()
   }
@@ -133,12 +135,34 @@
     })
 
     for (const doc of docs) {
-      flexsearchIndex.add(doc)
+      this.addDocToIndex(flexsearchIndex, doc)
     }
 
     return flexsearchIndex
   }
 
+  addDocToIndex(
+    flexsearchIndex: FlexSearch.Document<CozyDoc, true>,
+    doc: CozyDoc
+  ): void {
+    if (this.shouldIndexDoc(doc)) {
+      flexsearchIndex.add(doc)
+    }
+  }
+
+  shouldIndexDoc(doc: CozyDoc): boolean {
+    if (isIOCozyFile(doc)) {
+      const notInTrash = !doc.trashed && !/^\/\.cozy_trash/.test(doc.path)
+      const notRootDir = doc._id !== ROOT_DIR_ID
+      // Shared drives folder to be hidden in search.
+      // The files inside it though must appear. Thus only the file with the folder ID is filtered out.
+      const notSharedDrivesDir = doc._id !== SHARED_DRIVES_DIR_ID
+
+      return notInTrash && notRootDir && notSharedDrivesDir
+    }
+    return true
+  }
+
   async indexDocsForSearch(doctype: string): Promise<SearchIndex | null> {
     const searchIndex = this.searchIndexes[doctype]
     const pouchLink = getPouchLink(this.client)
@@ -155,7 +179,8 @@
 
       this.searchIndexes[doctype] = {
         index,
-        lastSeq: info?.update_seq
+        lastSeq: info?.update_seq,
+        lastUpdated: new Date().toISOString()
       }
       return this.searchIndexes[doctype]
     }
@@ -175,11 +200,12 @@
         searchIndex.index.remove(change.id)
       } else {
         const normalizedDoc = { ...change.doc, _type: doctype }
-        searchIndex.index.add(normalizedDoc)
+        this.addDocToIndex(searchIndex.index, normalizedDoc)
       }
     }
 
     searchIndex.lastSeq = changes.last_seq
+    searchIndex.lastUpdated = new Date().toISOString()
     return searchIndex
   }
 
@@ -215,7 +241,7 @@
 
     const allResults = this.searchOnIndexes(query)
     const results = this.deduplicateAndFlatten(allResults)
-    const sortedResults = this.sortAndLimitSearchResults(results)
+    const sortedResults = this.sortSearchResults(results)
 
     return sortedResults
       .map(res => normalizeSearchResult(this.client, res, query))
@@ -232,7 +258,8 @@
       }
       // TODO: do not use flexsearch store and rely on pouch storage?
       // It's better for memory, but might slow down search queries
-      const indexResults = index.index.search(query, LIMIT_DOCTYPE_SEARCH, {
+      const indexResults = index.index.search(query, {
+        limit: LIMIT_DOCTYPE_SEARCH,
         enrich: true
       })
       const newResults = indexResults.map(res => ({
@@ -264,65 +291,50 @@
     return [...resultMap.values()]
   }
 
-  sortAndLimitSearchResults(
-    searchResults: RawSearchResult[]
-  ): RawSearchResult[] {
-    const sortedResults = this.sortSearchResults(searchResults)
-    return this.limitSearchResults(sortedResults)
+  compareStrings(str1: string, str2: string): number {
+    return str1.localeCompare(str2, undefined, { numeric: true })
   }
 
   sortSearchResults(searchResults: RawSearchResult[]): RawSearchResult[] {
     return searchResults.sort((a, b) => {
       const doctypeComparison =
         DOCTYPE_ORDER[a.doctype] - DOCTYPE_ORDER[b.doctype]
       if (doctypeComparison !== 0) return doctypeComparison
-
       if (
         a.doctype === APPS_DOCTYPE &&
         isIOCozyApp(a.doc) &&
         isIOCozyApp(b.doc)
       ) {
-        return a.doc.slug.localeCompare(b.doc.slug)
+        return this.compareStrings(a.doc.slug, b.doc.slug)
       } else if (
         a.doctype === CONTACTS_DOCTYPE &&
         isIOCozyContact(a.doc) &&
         isIOCozyContact(b.doc)
       ) {
-        return a.doc.displayName.localeCompare(b.doc.displayName)
+        return this.compareStrings(a.doc.displayName, b.doc.displayName)
       } else if (
         a.doctype === FILES_DOCTYPE &&
         isIOCozyFile(a.doc) &&
         isIOCozyFile(b.doc)
       ) {
-        if (a.doc.type !== b.doc.type) {
-          return a.doc.type === 'directory' ? -1 : 1
-        }
-        return a.doc.name.localeCompare(b.doc.name)
+        return this.sortFiles(a, b)
       }
 
       return 0
     })
   }
 
-  limitSearchResults(searchResults: RawSearchResult[]): RawSearchResult[] {
-    const limitedResults = {
-      [APPS_DOCTYPE]: [],
-      [CONTACTS_DOCTYPE]: [],
-      [FILES_DOCTYPE]: []
+  sortFiles(aRes: RawSearchResult, bRes: RawSearchResult): number {
+    if (!isIOCozyFile(aRes.doc) || !isIOCozyFile(bRes.doc)) {
+      return 0
     }
-
-    searchResults.forEach(item => {
-      const type = item.doctype as SearchedDoctype
-      if (limitedResults[type].length < LIMIT_DOCTYPE_SEARCH) {
-        limitedResults[type].push(item)
-      }
-    })
-
-    return [
-      ...limitedResults[APPS_DOCTYPE],
-      ...limitedResults[CONTACTS_DOCTYPE],
-      ...limitedResults[FILES_DOCTYPE]
-    ]
+    if (!aRes.fields.includes('name') || !bRes.fields.includes('name')) {
+      return aRes.fields.includes('name') ? -1 : 1
+    }
+    if (aRes.doc.type !== bRes.doc.type) {
+      return aRes.doc.type === 'directory' ? -1 : 1
+    }
+    return this.compareStrings(aRes.doc.name, bRes.doc.name)
   }
 }
 

src/search/types.ts

@@ -47,7 +47,8 @@ export interface SearchResult {
 
 export interface SearchIndex {
   index: FlexSearch.Document<CozyDoc, true>
-  lastSeq: number
+  lastSeq: number | null
+  lastUpdated: string
 }
 
 export type SearchIndexes = {