Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 121 additions & 52 deletions docs/api/cozy-pouch-link/classes/PouchLink.md

Large diffs are not rendered by default.

67 changes: 64 additions & 3 deletions packages/cozy-pouch-link/src/CozyPouchLink.js
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,22 @@ const addBasicAuth = (url, basicAuth) => {
return url.replace('//', `//${basicAuth}`)
}

export const getReplicationURL = (uri, token, doctype) => {
/**
* Constructs the replication URL for a given doctype and replication options.
*
* @param {string} uri - The base URI of the Cozy instance.
* @param {Object} token - The authentication token object, must have a toBasicAuth() method.
* @param {string} doctype - The doctype for which to construct the replication URL.
* @param {Object} [replicationOptions] - Additional replication options.
* @param {string} [replicationOptions.driveId] - If present, indicates replication is for a shared drive and which one.
* @returns {string} The fully constructed replication URL.
*/
export const getReplicationURL = (uri, token, doctype, replicationOptions) => {
const basicAuth = token.toBasicAuth()
const authenticatedURL = addBasicAuth(uri, basicAuth)
if (replicationOptions?.driveId) {
return `${authenticatedURL}/sharings/drives/${replicationOptions?.driveId}`
}
return `${authenticatedURL}/data/${doctype}`
}

Expand Down Expand Up @@ -140,7 +153,15 @@ class PouchLink extends CozyLink {
return storage.getAdapterName()
}

getReplicationURL(doctype) {
/**
* Get the authenticated replication URL for a specific doctype
*
* @param {string} doctype - The document type to replicate (e.g., 'io.cozy.files')
* @param {object} [replicationOptions={}] - Replication options
* @param {string} [replicationOptions.driveId] - The ID of the shared drive to replicate (for shared drives)
* @returns {string} The authenticated replication URL
*/
getReplicationURL(doctype, replicationOptions) {
const url = this.client && this.client.stackClient.uri
const token = this.client && this.client.stackClient.token

Expand All @@ -156,7 +177,7 @@ class PouchLink extends CozyLink {
)
}

return getReplicationURL(url, token, doctype)
return getReplicationURL(url, token, doctype, replicationOptions)
}

async registerClient(client) {
Expand Down Expand Up @@ -789,6 +810,46 @@ class PouchLink extends CozyLink {
}
this.pouches.syncImmediately()
}

/**
* Adds a new doctype to the list of managed doctypes, sets its replication options,
* adds it to the pouches, and starts replication.
*
* @param {string} doctype - The name of the doctype to add.
* @param {Object} replicationOptions - The replication options for the doctype.
* @param {Object} options - The replication options for the doctype.
* @param {boolean} [options.shouldStartReplication=true] - Whether the replication should be started.
*/
async addDoctype(doctype, replicationOptions, options) {
this.doctypes.push(doctype)
if (!this.doctypesReplicationOptions) {
this.doctypesReplicationOptions = {}
}
this.doctypesReplicationOptions[doctype] = replicationOptions
this.pouches.doctypes.push(doctype)
await this.pouches.addDoctype(doctype, replicationOptions)
if (options?.shouldStartReplication === true) {
this.startReplicationWithDebounce()
}
}

/**
* Removes a doctype from the list of managed doctypes, deletes its replication options,
* and removes it from the pouches.
*
* @param {string} doctype - The name of the doctype to remove.
*/
async removeDoctype(doctype) {
this.doctypes = this.doctypes.filter(d => d !== doctype)
delete this.doctypesReplicationOptions[doctype]
await this.pouches.removeDoctype(doctype)
}

getSharedDriveDoctypes() {
return this.doctypes.filter(
doctype => this.doctypesReplicationOptions[doctype]?.driveId
)
}
}

export default PouchLink
5 changes: 5 additions & 0 deletions packages/cozy-pouch-link/src/CozyPouchLink.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ jest.mock('./helpers', () => ({
withoutDesignDocuments: jest.fn(),
isAdapterBugged: jest.fn()
}))
jest.mock('./remote', () => ({
fetchRemoteInstance: jest.fn()
}))
import { fetchRemoteInstance } from './remote'

import CozyPouchLink from '.'
import { SCHEMA, TODO_1, TODO_2, TODO_3, TODO_4 } from './__tests__/fixtures'
Expand Down Expand Up @@ -53,6 +57,7 @@ async function setup(linkOpts = {}) {
await link.onLogin()

client.setData = jest.fn()
fetchRemoteInstance.mockResolvedValue({ rows: [] })
}

async function clean() {
Expand Down
97 changes: 68 additions & 29 deletions packages/cozy-pouch-link/src/PouchManager.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
import fromPairs from 'lodash/fromPairs'
import forEach from 'lodash/forEach'
import get from 'lodash/get'
import { isMobileApp } from 'cozy-device-helper'

import { PouchLocalStorage } from './localStorage'
Expand Down Expand Up @@ -55,44 +53,26 @@ class PouchManager {
}

async init() {
const pouchPlugins = get(this.options, 'pouch.plugins', [])
const pouchOptions = get(this.options, 'pouch.options', {})
const pouchPlugins = this.options?.pouch?.plugins ?? []
const pouchOptions = this.options?.pouch?.options ?? {}
if (!pouchOptions.view_update_changes_batch_size) {
pouchOptions.view_update_changes_batch_size = DEFAULT_VIEW_UPDATE_BATCH
}

forEach(pouchPlugins, plugin => this.PouchDB.plugin(plugin))
this.pouches = fromPairs(
this.doctypes.map(doctype => {
const dbName = getDatabaseName(this.options.prefix, doctype)
const pouch = new this.PouchDB(
getDatabaseName(this.options.prefix, doctype),
pouchOptions
)

return [dbName, pouch]
})
)

const dbNames = Object.keys(this.pouches)
dbNames.forEach(dbName => {
// Set query engine for all databases
const doctype = getDoctypeFromDatabaseName(dbName)
this.setQueryEngine(dbName, doctype)
})
this.pouches = {}
this.doctypesReplicationOptions =
this.options.doctypesReplicationOptions || {}
for (const doctype of this.doctypes) {
this.addDoctype(doctype, this.doctypesReplicationOptions[doctype])
}

// Persist db names for old browsers not supporting indexeddb.databases()
// This is useful for cleanup.
// Note PouchDB adds itself the _pouch_ prefix
const pouchDbNames = dbNames.map(dbName => `_pouch_${dbName}`)
await this.storage.persistDatabasesNames(pouchDbNames)
await this.persistDatabasesNames()

/** @type {Record<string, import('./types').SyncInfo>} - Stores synchronization info per doctype */
this.syncedDoctypes = await this.storage.getPersistedSyncedDoctypes()
this.warmedUpQueries = await this.storage.getPersistedWarmedUpQueries()
this.getReplicationURL = this.options.getReplicationURL
this.doctypesReplicationOptions =
this.options.doctypesReplicationOptions || {}
this.listenerLaunched = false

// We must ensure databases exist on the remote before
Expand Down Expand Up @@ -358,6 +338,65 @@ class PouchManager {
this.warmedUpQueries = {}
await this.storage.destroyWarmedUpQueries()
}

/**
* Adds a new doctype to the list of managed doctypes, sets its replication options,
* creates a new PouchDB instance for it, and sets up the query engine.
*
* @param {string} doctype - The name of the doctype to add.
* @param {Object} replicationOptions - The replication options for the doctype.
*/
async addDoctype(doctype, replicationOptions) {
if (!this.options?.doctypesReplicationOptions) {
this.options.doctypesReplicationOptions = {}
}
this.options.doctypesReplicationOptions[doctype] = replicationOptions
const pouchOptions = this.options?.pouch?.options ?? {}
if (!pouchOptions.view_update_changes_batch_size) {
pouchOptions.view_update_changes_batch_size = DEFAULT_VIEW_UPDATE_BATCH
}

const dbName = getDatabaseName(this.options.prefix, doctype)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@paultranvan will it works with #1633 ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, good catch.
@doubleface I think we could do 2 things here:

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • doctype is already added in the doctypes array by CozyPouchLink
  • database names are now persisted after addDoctype or removeDoctype 0468550

this.pouches[dbName] = new this.PouchDB(
getDatabaseName(this.options.prefix, doctype),
pouchOptions
)
await this.persistDatabasesNames()

this.setQueryEngine(dbName, getDoctypeFromDatabaseName(dbName))
}

/**
* Removes a doctype from the list of managed doctypes, deletes its replication options,
* destroys its PouchDB instance, and removes it from the pouches.
*
* @param {string} doctype - The name of the doctype to remove.
*/
async removeDoctype(doctype) {
this.doctypes = this.doctypes.filter(d => d !== doctype)
delete this.options?.doctypesReplicationOptions?.[doctype]

const dbName = getDatabaseName(this.options.prefix, doctype)
this.pouches[dbName].destroy()
delete this.pouches[dbName]
await this.persistDatabasesNames()
}

/**
* Persists the names of the PouchDB databases.
*
* This method is primarily used to ensure that database names are saved for
* old browsers that do not support `indexeddb.databases()`. This persistence
* facilitates cleanup processes. Note that PouchDB automatically adds the
* `_pouch_` prefix to database names.
*
* @returns {Promise<void>}
*/
async persistDatabasesNames() {
const dbNames = Object.keys(this.pouches)
const pouchDbNames = dbNames.map(dbName => `_pouch_${dbName}`)
await this.storage.persistDatabasesNames(pouchDbNames)
}
}

export default PouchManager
2 changes: 1 addition & 1 deletion packages/cozy-pouch-link/src/PouchManager.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import PouchDB from 'pouchdb-browser'
import { LOCALSTORAGE_STORAGE_KEYS, PouchLocalStorage } from './localStorage'
import { platformWeb } from './platformWeb'

import { fetchRemoteLastSequence, fetchRemoteInstance } from './remote'
import { fetchRemoteInstance, fetchRemoteLastSequence } from './remote'

const ls = new PouchLocalStorage(platformWeb.storage)

Expand Down
2 changes: 0 additions & 2 deletions packages/cozy-pouch-link/src/remote.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,6 @@ describe('remote', () => {
)
})
})

describe('fetchRemoteLastSequence', () => {
it('Should return data when found', async () => {
const remoteUrl =
Expand Down Expand Up @@ -165,7 +164,6 @@ const mockDatabaseNotFoundOn = url => {
status: 404
})
}

const mockDatabaseReservedDoctypeOn = url => {
fetch.mockOnceIf(
url,
Expand Down
26 changes: 13 additions & 13 deletions packages/cozy-pouch-link/src/replicateOnce.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import fromPairs from 'lodash/fromPairs'
import get from 'lodash/get'
import map from 'lodash/map'
import startsWith from 'lodash/startsWith'
import zip from 'lodash/zip'
Expand Down Expand Up @@ -33,30 +32,30 @@ export const replicateOnce = async pouchManager => {
pouchManager.pouches,
async (pouch, dbName) => {
const doctype = getDoctypeFromDatabaseName(dbName)
// Use optional chaining and nullish coalescing instead of get
const replicationOptions =
pouchManager.doctypesReplicationOptions?.[doctype] ?? {}
logger.info('PouchManager: Starting replication for ' + doctype)

const getReplicationURL = () => pouchManager.getReplicationURL(doctype)
const getReplicationURL = () =>
pouchManager.getReplicationURL(doctype, replicationOptions)

const initialReplication =
pouchManager.getSyncStatus(doctype) !== 'synced'
const replicationFilter = doc => {
return !startsWith(doc._id, '_design')
}
const isSharedDrive = Boolean(replicationOptions.driveId)
let seq = ''
if (initialReplication) {
if (initialReplication && !isSharedDrive) {
// Before the first replication, get the last remote sequence,
// which will be used as a checkpoint for the next replication
const lastSeq = await fetchRemoteLastSequence(getReplicationURL())
await pouchManager.storage.persistDoctypeLastSequence(doctype, lastSeq)
} else {
seq = await pouchManager.storage.getDoctypeLastSequence(doctype)
seq = (await pouchManager.storage.getDoctypeLastSequence(doctype)) || ''
}

const replicationOptions = get(
pouchManager.doctypesReplicationOptions,
doctype,
{}
)
replicationOptions.initialReplication = initialReplication
replicationOptions.filter = replicationFilter
replicationOptions.since = seq
Expand All @@ -69,11 +68,12 @@ export const replicateOnce = async pouchManager => {
pouch,
replicationOptions,
getReplicationURL,
pouchManager.storage
pouchManager.storage,
pouchManager.client
)
if (seq) {
// We only need the sequence for the second replication, as PouchDB
// will use a local checkpoint for the next runs.
if (seq && !isSharedDrive) {
// For shared drives, we always need to keep the last sequence since replication is handled manually, not by PouchDB.
// For standard PouchDB replications, the last sequence is only needed for the initial sync; subsequent runs use PouchDB's internal checkpointing.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But, for "standard replication", we now save the lastSeq at the end of each completed replication process. So there is probably no point to destroy it here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But I did not change that. the lastSeq was already saved on each completed replication.
It looks like the previous comment was wrong and the PouchDb replication already used the lastSeq saved in localStorage for every replication (the first one with alldocs expected).

Yes, it means that there is no point at destroying it here but what I don't know is if there is a need to avoid to use the lastSeq in localStorage.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the lastSeq was already saved on each completed replication.

Well, no, that's you who added the save on each completed replication 😄 ac5fb47#diff-6f9271f35894bccd13c05f69ba8e460d9cae55ed63666aa0b2038a5b4bc5a328

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes sorry. At this point, I think I just should revert ac5fb47 (#1632) since it has no point anymore

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This time, with 0468550, we have exactly the same behaviour as before for non shared drive doctypes : fetch last sequence from client only on initial replication and removing it after.

await pouchManager.storage.destroyDoctypeLastSequence(doctype)
}

Expand Down
Loading
Loading