From 94374761f6d446bdac38c84489b9f41d0be28e60 Mon Sep 17 00:00:00 2001 From: mathieuRA Date: Mon, 9 Sep 2024 16:09:05 +0200 Subject: [PATCH 1/6] feat(xo-server/rest-api/dashboard): add S3 backup repositories information --- @xen-orchestra/backups/RemoteAdapter.mjs | 12 +++++ packages/xo-server/src/xo-mixins/remotes.mjs | 7 +++ packages/xo-server/src/xo-mixins/rest-api.mjs | 51 +++++++++++-------- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/@xen-orchestra/backups/RemoteAdapter.mjs b/@xen-orchestra/backups/RemoteAdapter.mjs index 8ab8bc1e894..46eaee7a67c 100644 --- a/@xen-orchestra/backups/RemoteAdapter.mjs +++ b/@xen-orchestra/backups/RemoteAdapter.mjs @@ -18,6 +18,7 @@ import fromEvent from 'promise-toolbox/fromEvent' import groupBy from 'lodash/groupBy.js' import pDefer from 'promise-toolbox/defer' import pickBy from 'lodash/pickBy.js' +import reduce from 'lodash/reduce.js' import tar from 'tar' import zlib from 'zlib' @@ -826,6 +827,17 @@ export class RemoteAdapter { } return metadata } + + async getTotalVmBackupSize() { + const vmBackups = await this.listAllVmBackups() + return reduce(vmBackups, (sum, backups) => sum + backups.reduce((sum, backup) => sum + backup.size, 0), 0) + } + + // @TODO: add `getTotalXoBackupSize` and `getTotalPoolBackupSize` once `size` is implemented + async getTotalBackupSize() { + const backupsSize = await Promise.all([this.getTotalVmBackupSize()]) + return backupsSize.reduce((sum, backupSize) => sum + backupSize) + } } Object.assign(RemoteAdapter.prototype, { diff --git a/packages/xo-server/src/xo-mixins/remotes.mjs b/packages/xo-server/src/xo-mixins/remotes.mjs index 02988e53fa8..67ceb117994 100644 --- a/packages/xo-server/src/xo-mixins/remotes.mjs +++ b/packages/xo-server/src/xo-mixins/remotes.mjs @@ -1,4 +1,5 @@ import asyncMapSettled from '@xen-orchestra/async-map/legacy.js' +import Disposable from 'promise-toolbox/Disposable' import Obfuscate from '@vates/obfuscate' import { basename } from 'path' import { createLogger } from '@xen-orchestra/log' @@ -9,6 +10,7 @@ import { invalidParameters, noSuchObject } from 'xo-common/api-errors.js' import { synchronized } from 'decorator-synchronized' import patch from '../patch.mjs' +import { noop } from '../utils.mjs' import { Remotes } from '../models/remote.mjs' // =================================================================== @@ -173,10 +175,15 @@ export default class { : this.getRemoteHandler(remote.id).then(handler => handler.getInfo()) try { + const sizeUsedForBackup = await Disposable.use(this._app.getBackupsRemoteAdapter(remote), adapter => + adapter.getTotalBackupSize() + ).catch(noop) + await timeout.call( promise.then(info => { remotesInfo[remote.id] = { ...info, + sizeUsedForBackup, encryption, } }), diff --git a/packages/xo-server/src/xo-mixins/rest-api.mjs b/packages/xo-server/src/xo-mixins/rest-api.mjs index 9daa928a543..a2ca4f370e8 100644 --- a/packages/xo-server/src/xo-mixins/rest-api.mjs +++ b/packages/xo-server/src/xo-mixins/rest-api.mjs @@ -226,34 +226,43 @@ async function _getDashboardStats(app) { } try { - const remotes = await app.getAllRemotes() - const remotesInfo = await app.getAllRemotesInfo() - - const backupRepositoriesSize = remotes.reduce( - (prev, remote) => { - const { type } = parse(remote.url) - const remoteInfo = remotesInfo[remote.id] - - if (!remote.enabled || type === 's3' || remoteInfo === undefined) { - return prev - } - - return { - available: prev.available + remoteInfo.available, - backups: 0, // @TODO: compute the space used by backups - other: 0, // @TODO: compute the space used by everything that is not a backup - total: prev.total + remoteInfo.size, - used: prev.used + remoteInfo.used, - } + const backupRepositoriesSize = { + s3: { + backups: 0, }, - { + other: { available: 0, backups: 0, other: 0, total: 0, used: 0, + }, + } + + const remotes = await app.getAllRemotes() + const remotesInfo = await app.getAllRemotesInfo() + + for (const remote of remotes) { + const { type } = parse(remote.url) + const remoteInfo = remotesInfo[remote.id] + + if (!remote.enabled || remoteInfo === undefined) { + continue } - ) + + const sizeUsedForBackup = remoteInfo.sizeUsedForBackup ?? 0 + const isS3 = type === 's3' + const target = isS3 ? backupRepositoriesSize.s3 : backupRepositoriesSize.other + + target.backups += sizeUsedForBackup + if (!isS3) { + target.available += remoteInfo.available + target.other += remoteInfo.used - sizeUsedForBackup + target.total += remoteInfo.size + target.used += remoteInfo.used + } + } + dashboard.backupRepositories = { size: backupRepositoriesSize } } catch (error) { console.error(error) From 4a00286acd8bfdaeb0700f1d44956e19532f3e0d Mon Sep 17 00:00:00 2001 From: mathieuRA Date: Mon, 9 Sep 2024 16:18:20 +0200 Subject: [PATCH 2/6] changelog --- CHANGELOG.unreleased.md | 1 + packages/xo-server/src/xo-mixins/remotes.mjs | 10 +++++----- packages/xo-server/src/xo-mixins/rest-api.mjs | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 28f167ebf88..75981509962 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -19,6 +19,7 @@ - [V2V] Fix computation of `memory_static_max` - **XO 6**: - [Dashboard] Display backup issues data (PR [#7974](https://github.com/vatesfr/xen-orchestra/pull/7974)) +- [REST API] Add S3 backup repository information in the `/rest/v0/dashboard` endpoint (PR [#7978](https://github.com/vatesfr/xen-orchestra/pull/7978)) ### Bug fixes diff --git a/packages/xo-server/src/xo-mixins/remotes.mjs b/packages/xo-server/src/xo-mixins/remotes.mjs index 67ceb117994..0f47006b70f 100644 --- a/packages/xo-server/src/xo-mixins/remotes.mjs +++ b/packages/xo-server/src/xo-mixins/remotes.mjs @@ -174,16 +174,16 @@ export default class { }) : this.getRemoteHandler(remote.id).then(handler => handler.getInfo()) - try { - const sizeUsedForBackup = await Disposable.use(this._app.getBackupsRemoteAdapter(remote), adapter => - adapter.getTotalBackupSize() - ).catch(noop) + const sizeUsedForBackups = await Disposable.use(this._app.getBackupsRemoteAdapter(remote), adapter => + adapter.getTotalBackupSize() + ).catch(noop) + try { await timeout.call( promise.then(info => { remotesInfo[remote.id] = { ...info, - sizeUsedForBackup, + sizeUsedForBackups, encryption, } }), diff --git a/packages/xo-server/src/xo-mixins/rest-api.mjs b/packages/xo-server/src/xo-mixins/rest-api.mjs index a2ca4f370e8..8671adf58f9 100644 --- a/packages/xo-server/src/xo-mixins/rest-api.mjs +++ b/packages/xo-server/src/xo-mixins/rest-api.mjs @@ -250,14 +250,14 @@ async function _getDashboardStats(app) { continue } - const sizeUsedForBackup = remoteInfo.sizeUsedForBackup ?? 0 + const sizeUsedForBackups = remoteInfo.sizeUsedForBackups ?? 0 const isS3 = type === 's3' const target = isS3 ? backupRepositoriesSize.s3 : backupRepositoriesSize.other - target.backups += sizeUsedForBackup + target.backups += sizeUsedForBackups if (!isS3) { target.available += remoteInfo.available - target.other += remoteInfo.used - sizeUsedForBackup + target.other += remoteInfo.used - sizeUsedForBackups target.total += remoteInfo.size target.used += remoteInfo.used } From bcad7761ac63b4ad0ca61dc1c4cdf9c79beef642 Mon Sep 17 00:00:00 2001 From: mathieuRA Date: Mon, 9 Sep 2024 16:32:02 +0200 Subject: [PATCH 3/6] change 'backupRepositories' struct after discussion with Olivier.F --- packages/xo-server/src/xo-mixins/rest-api.mjs | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/packages/xo-server/src/xo-mixins/rest-api.mjs b/packages/xo-server/src/xo-mixins/rest-api.mjs index 8671adf58f9..c6e89f09443 100644 --- a/packages/xo-server/src/xo-mixins/rest-api.mjs +++ b/packages/xo-server/src/xo-mixins/rest-api.mjs @@ -226,44 +226,34 @@ async function _getDashboardStats(app) { } try { - const backupRepositoriesSize = { - s3: { - backups: 0, - }, - other: { - available: 0, - backups: 0, - other: 0, - total: 0, - used: 0, - }, - } + const s3Brsize = { backups: 0 } + const otherBrSize = { available: 0, backups: 0, other: 0, total: 0, used: 0 } - const remotes = await app.getAllRemotes() - const remotesInfo = await app.getAllRemotesInfo() + const backupRepositories = await app.getAllRemotes() + const backupRepositoriesInfo = await app.getAllRemotesInfo() - for (const remote of remotes) { - const { type } = parse(remote.url) - const remoteInfo = remotesInfo[remote.id] + for (const backupRepository of backupRepositories) { + const { type } = parse(backupRepository.url) + const backupRepositoryInfo = backupRepositoriesInfo[backupRepository.id] - if (!remote.enabled || remoteInfo === undefined) { + if (!backupRepository.enabled || backupRepositoryInfo === undefined) { continue } - const sizeUsedForBackups = remoteInfo.sizeUsedForBackups ?? 0 + const { available, size, sizeUsedForBackups = 0, used } = backupRepositoryInfo const isS3 = type === 's3' - const target = isS3 ? backupRepositoriesSize.s3 : backupRepositoriesSize.other + const target = isS3 ? s3Brsize : otherBrSize target.backups += sizeUsedForBackups if (!isS3) { - target.available += remoteInfo.available - target.other += remoteInfo.used - sizeUsedForBackups - target.total += remoteInfo.size - target.used += remoteInfo.used + target.available += available + target.other += used - sizeUsedForBackups + target.total += size + target.used += used } } - dashboard.backupRepositories = { size: backupRepositoriesSize } + dashboard.backupRepositories = { s3: { size: s3Brsize }, other: { size: otherBrSize } } } catch (error) { console.error(error) } From 25fe0c765f4290237bedb4e130fbeba9995e0b24 Mon Sep 17 00:00:00 2001 From: mathieuRA Date: Thu, 12 Sep 2024 17:25:05 +0200 Subject: [PATCH 4/6] add recusrive fn to compute backup instead code duplication --- @xen-orchestra/backups/RemoteAdapter.mjs | 19 +++++++++++++++---- packages/xo-server/src/xo-mixins/remotes.mjs | 4 ++-- packages/xo-server/src/xo-mixins/rest-api.mjs | 7 ++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/@xen-orchestra/backups/RemoteAdapter.mjs b/@xen-orchestra/backups/RemoteAdapter.mjs index 46eaee7a67c..3c4d9cae9af 100644 --- a/@xen-orchestra/backups/RemoteAdapter.mjs +++ b/@xen-orchestra/backups/RemoteAdapter.mjs @@ -828,15 +828,26 @@ export class RemoteAdapter { return metadata } + #computeTotalBackupSizeRecursively(backups) { + return reduce( + backups, + (prev, backup) => { + const _backup = Array.isArray(backup) ? this.#computeTotalBackupSizeRecursively(backup) : backup + return { + onDisk: prev.onDisk + (_backup.onDisk ?? _backup.size), + } + }, + { onDisk: 0 } + ) + } + async getTotalVmBackupSize() { - const vmBackups = await this.listAllVmBackups() - return reduce(vmBackups, (sum, backups) => sum + backups.reduce((sum, backup) => sum + backup.size, 0), 0) + return this.#computeTotalBackupSizeRecursively(await this.listAllVmBackups()) } // @TODO: add `getTotalXoBackupSize` and `getTotalPoolBackupSize` once `size` is implemented async getTotalBackupSize() { - const backupsSize = await Promise.all([this.getTotalVmBackupSize()]) - return backupsSize.reduce((sum, backupSize) => sum + backupSize) + return this.#computeTotalBackupSizeRecursively(await Promise.all([this.getTotalVmBackupSize()])) } } diff --git a/packages/xo-server/src/xo-mixins/remotes.mjs b/packages/xo-server/src/xo-mixins/remotes.mjs index 0f47006b70f..a9ff02225b1 100644 --- a/packages/xo-server/src/xo-mixins/remotes.mjs +++ b/packages/xo-server/src/xo-mixins/remotes.mjs @@ -174,7 +174,7 @@ export default class { }) : this.getRemoteHandler(remote.id).then(handler => handler.getInfo()) - const sizeUsedForBackups = await Disposable.use(this._app.getBackupsRemoteAdapter(remote), adapter => + const totalBackupSize = await Disposable.use(this._app.getBackupsRemoteAdapter(remote), adapter => adapter.getTotalBackupSize() ).catch(noop) @@ -183,7 +183,7 @@ export default class { promise.then(info => { remotesInfo[remote.id] = { ...info, - sizeUsedForBackups, + totalBackupSize, encryption, } }), diff --git a/packages/xo-server/src/xo-mixins/rest-api.mjs b/packages/xo-server/src/xo-mixins/rest-api.mjs index c6e89f09443..72f80ff59a1 100644 --- a/packages/xo-server/src/xo-mixins/rest-api.mjs +++ b/packages/xo-server/src/xo-mixins/rest-api.mjs @@ -240,14 +240,15 @@ async function _getDashboardStats(app) { continue } - const { available, size, sizeUsedForBackups = 0, used } = backupRepositoryInfo + const { available, size, totalBackupSize, used } = backupRepositoryInfo + const isS3 = type === 's3' const target = isS3 ? s3Brsize : otherBrSize - target.backups += sizeUsedForBackups + target.backups += totalBackupSize?.onDisk ?? 0 if (!isS3) { target.available += available - target.other += used - sizeUsedForBackups + target.other += used - (totalBackupSize?.onDisk ?? 0) target.total += size target.used += used } From d082f7902e5e5904859e03bf377cb442c6465c37 Mon Sep 17 00:00:00 2001 From: mathieuRA Date: Wed, 18 Sep 2024 10:46:44 +0200 Subject: [PATCH 5/6] do not inline code --- @xen-orchestra/backups/RemoteAdapter.mjs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/@xen-orchestra/backups/RemoteAdapter.mjs b/@xen-orchestra/backups/RemoteAdapter.mjs index 3c4d9cae9af..1905a4e6a50 100644 --- a/@xen-orchestra/backups/RemoteAdapter.mjs +++ b/@xen-orchestra/backups/RemoteAdapter.mjs @@ -845,9 +845,10 @@ export class RemoteAdapter { return this.#computeTotalBackupSizeRecursively(await this.listAllVmBackups()) } - // @TODO: add `getTotalXoBackupSize` and `getTotalPoolBackupSize` once `size` is implemented async getTotalBackupSize() { - return this.#computeTotalBackupSizeRecursively(await Promise.all([this.getTotalVmBackupSize()])) + const vmBackupSize = await this.getTotalVmBackupSize() + // @TODO: add `getTotalXoBackupSize` and `getTotalPoolBackupSize` once `size` is implemented by fs + return vmBackupSize } } From a3948100e44fd0e68d1c2688ca98b218f3bd1832 Mon Sep 17 00:00:00 2001 From: mathieuRA Date: Mon, 23 Sep 2024 14:54:27 +0200 Subject: [PATCH 6/6] remove getTotalBackupsize of remoteInfo --- packages/xo-server/src/xo-mixins/remotes.mjs | 7 ------- packages/xo-server/src/xo-mixins/rest-api.mjs | 10 +++++++--- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/xo-server/src/xo-mixins/remotes.mjs b/packages/xo-server/src/xo-mixins/remotes.mjs index a9ff02225b1..02988e53fa8 100644 --- a/packages/xo-server/src/xo-mixins/remotes.mjs +++ b/packages/xo-server/src/xo-mixins/remotes.mjs @@ -1,5 +1,4 @@ import asyncMapSettled from '@xen-orchestra/async-map/legacy.js' -import Disposable from 'promise-toolbox/Disposable' import Obfuscate from '@vates/obfuscate' import { basename } from 'path' import { createLogger } from '@xen-orchestra/log' @@ -10,7 +9,6 @@ import { invalidParameters, noSuchObject } from 'xo-common/api-errors.js' import { synchronized } from 'decorator-synchronized' import patch from '../patch.mjs' -import { noop } from '../utils.mjs' import { Remotes } from '../models/remote.mjs' // =================================================================== @@ -174,16 +172,11 @@ export default class { }) : this.getRemoteHandler(remote.id).then(handler => handler.getInfo()) - const totalBackupSize = await Disposable.use(this._app.getBackupsRemoteAdapter(remote), adapter => - adapter.getTotalBackupSize() - ).catch(noop) - try { await timeout.call( promise.then(info => { remotesInfo[remote.id] = { ...info, - totalBackupSize, encryption, } }), diff --git a/packages/xo-server/src/xo-mixins/rest-api.mjs b/packages/xo-server/src/xo-mixins/rest-api.mjs index 72f80ff59a1..b4998938468 100644 --- a/packages/xo-server/src/xo-mixins/rest-api.mjs +++ b/packages/xo-server/src/xo-mixins/rest-api.mjs @@ -8,6 +8,7 @@ import { pipeline } from 'node:stream/promises' import { json, Router } from 'express' import { Readable } from 'node:stream' import cloneDeep from 'lodash/cloneDeep.js' +import Disposable from 'promise-toolbox/Disposable' import groupBy from 'lodash/groupBy.js' import path from 'node:path' import pDefer from 'promise-toolbox/defer' @@ -240,15 +241,18 @@ async function _getDashboardStats(app) { continue } - const { available, size, totalBackupSize, used } = backupRepositoryInfo + const totalBackupSize = await Disposable.use(app.getBackupsRemoteAdapter(backupRepository), adapter => + adapter.getTotalBackupSize() + ) + const { available, size, used } = backupRepositoryInfo const isS3 = type === 's3' const target = isS3 ? s3Brsize : otherBrSize - target.backups += totalBackupSize?.onDisk ?? 0 + target.backups += totalBackupSize.onDisk if (!isS3) { target.available += available - target.other += used - (totalBackupSize?.onDisk ?? 0) + target.other += used - totalBackupSize.onDisk target.total += size target.used += used }