diff --git a/.gitignore b/.gitignore index 9a4546fc..2a2a4a37 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,5 @@ pids report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json /prisma/generated/ /prismaDmob/generated/ + +start_dmob_tunnel.sh diff --git a/src/service/allocator/allocator.service.ts b/src/service/allocator/allocator.service.ts index d9c18555..291b58e4 100644 --- a/src/service/allocator/allocator.service.ts +++ b/src/service/allocator/allocator.service.ts @@ -110,10 +110,15 @@ export class AllocatorService { ): Promise { const weeks = await this.storageProviderService.getWeeksTracked(); + const lastWeekAverageProviderRetrievability = + await this.storageProviderService.getLastWeekAverageProviderRetrievability( + isAccumulative, + ); + const results: AllocatorSpsComplianceWeek[] = []; for (const week of weeks) { - const weekAverageRetrievability = + const weekAverageProvidersRetrievability = await this.storageProviderService.getWeekAverageProviderRetrievability( week, isAccumulative, @@ -125,12 +130,13 @@ export class AllocatorService { ); const weekProvidersCompliance: { + // TODO refactor to a type complianceScore: number; provider: string; }[] = weekProviders.map((provider) => { return this.storageProviderService.getWeekProviderComplianceScore( provider, - weekAverageRetrievability, + weekAverageProvidersRetrievability, ); }); @@ -144,10 +150,11 @@ export class AllocatorService { (a) => a.allocator, ); - const weekResult: AllocatorSpsComplianceWeekSingle[] = await Promise.all( - Object.entries(clientsByAllocator).map( - // prettier-ignore - async ([allocator, clients]): Promise => { + const weekAllocators: AllocatorSpsComplianceWeekSingle[] = + await Promise.all( + Object.entries(clientsByAllocator).map( + // prettier-ignore + async ([allocator, clients]): Promise => { const weekProvidersForAllocator = await this.storageProviderService.getWeekProvidersForClients( week, @@ -157,7 +164,7 @@ export class AllocatorService { return { id: allocator, - totalDatacap: await this.getWeekAllocatorDatacap( + totalDatacap: await this.getWeekAllocatorTotalDatacap( week, isAccumulative, allocator, @@ -166,26 +173,29 @@ export class AllocatorService { weekProvidersCompliance, weekProvidersForAllocator.map((p) => p.provider), ), + totalSps: weekProvidersForAllocator.length, }; }, - ), - ); + ), + ); results.push({ week: week, - allocators: weekResult, - total: weekResult.length, + averageSuccessRate: weekAverageProvidersRetrievability * 100, + total: weekAllocators.length, + allocators: weekAllocators, }); } return new AllocatorSpsComplianceWeekResponse( + lastWeekAverageProviderRetrievability * 100, this.histogramHelper.withoutCurrentWeek( this.histogramHelper.sorted(results), ), ); } - public async getWeekAllocatorDatacap( + public async getWeekAllocatorTotalDatacap( week: Date, isAccumulative: boolean, allocatorId: string, diff --git a/src/service/allocator/types.allocator.ts b/src/service/allocator/types.allocator.ts index 1f5c100e..88b9f417 100644 --- a/src/service/allocator/types.allocator.ts +++ b/src/service/allocator/types.allocator.ts @@ -2,11 +2,19 @@ import { ApiProperty } from '@nestjs/swagger'; import { StorageProviderComplianceWeekPercentage } from '../storage-provider/types.storage-provider'; export class AllocatorSpsComplianceWeekSingle extends StorageProviderComplianceWeekPercentage { - @ApiProperty({ type: String }) + @ApiProperty({ type: String, description: 'Allocator ID' }) id: string; - @ApiProperty() + @ApiProperty({ + description: 'Total datacap of the allocator in the week', + }) totalDatacap: number; + + @ApiProperty({ + description: + 'Total number of storage providers for the allocator in the week', + }) + totalSps: number; } export class AllocatorSpsComplianceWeek { @@ -17,18 +25,34 @@ export class AllocatorSpsComplianceWeek { }) week: Date; - @ApiProperty({ type: AllocatorSpsComplianceWeekSingle, isArray: true }) - allocators: AllocatorSpsComplianceWeekSingle[]; - @ApiProperty() total: number; + + @ApiProperty({ + description: + 'Average storage providers retrievability success rate in the week', + }) + averageSuccessRate: number; + + @ApiProperty({ type: AllocatorSpsComplianceWeekSingle, isArray: true }) + allocators: AllocatorSpsComplianceWeekSingle[]; } export class AllocatorSpsComplianceWeekResponse { + @ApiProperty({ + description: + 'Last full week average storage providers retrievability success rate', + }) + averageSuccessRate: number; + @ApiProperty({ type: AllocatorSpsComplianceWeek, isArray: true }) results: AllocatorSpsComplianceWeek[]; - constructor(results: AllocatorSpsComplianceWeek[]) { + constructor( + averageSuccessRate: number, + results: AllocatorSpsComplianceWeek[], + ) { + this.averageSuccessRate = averageSuccessRate; this.results = results; } } diff --git a/src/service/histogram-helper/types.histogram-helper.ts b/src/service/histogram-helper/types.histogram-helper.ts index 9b1d9795..e1e84193 100644 --- a/src/service/histogram-helper/types.histogram-helper.ts +++ b/src/service/histogram-helper/types.histogram-helper.ts @@ -134,7 +134,7 @@ export class RetrievabilityHistogramWeekResponse { export class RetrievabilityWeekResponse { @ApiProperty({ - description: 'Last week average retrievability success rate', + description: 'Last full week average retrievability success rate', }) averageSuccessRate: number; diff --git a/src/service/storage-provider/storage-provider.service.ts b/src/service/storage-provider/storage-provider.service.ts index bacba9ca..729f9b07 100644 --- a/src/service/storage-provider/storage-provider.service.ts +++ b/src/service/storage-provider/storage-provider.service.ts @@ -17,6 +17,7 @@ import { StorageProviderComplianceWeekCount, StorageProviderComplianceWeekPercentage, StorageProviderComplianceWeekResponse, + StorageProviderComplianceWeekTotalDatacap, } from './types.storage-provider'; import { HistogramHelperService } from '../histogram-helper/histogram-helper.service'; import { @@ -83,14 +84,8 @@ export class StorageProviderService { public async getProviderRetrievabilityWeekly( isAccumulative: boolean, ): Promise { - const lastWeek = DateTime.now() - .toUTC() - .minus({ week: 1 }) - .startOf('week') - .toJSDate(); - const lastWeekAverageRetrievability = - await this.getWeekAverageProviderRetrievability(lastWeek, isAccumulative); + await this.getLastWeekAverageProviderRetrievability(isAccumulative); const providerCount = await this.getProviderCount(); @@ -129,11 +124,26 @@ export class StorageProviderService { ); } + public getLastWeekAverageProviderRetrievability( + isAccumulative: boolean, + ): Promise { + const lastWeek = DateTime.now() + .toUTC() + .minus({ week: 1 }) + .startOf('week') + .toJSDate(); + + return this.getWeekAverageProviderRetrievability(lastWeek, isAccumulative); + } + public async getProviderComplianceWeekly( isAccumulative: boolean, ): Promise { const weeks = await this.getWeeksTracked(); + const lastWeekAverageRetrievability = + await this.getLastWeekAverageProviderRetrievability(isAccumulative); + const result: StorageProviderComplianceWeek[] = await Promise.all( weeks.map(async (week) => { const weekAverageRetrievability = @@ -150,21 +160,48 @@ export class StorageProviderService { return { week: week, + averageSuccessRate: weekAverageRetrievability * 100, + totalSps: weekProviders.length, ...this.getProviderComplianceWeekCount( weekProvidersCompliance, weekProviders.map((provider) => provider.provider), ), + ...this.getProviderComplianceWeekTotalDatacap( + weekProvidersCompliance, + weekProviders.map((provider) => provider.provider), + await this.getWeekProvidersTotalDatacap(week, isAccumulative), + ), }; }), ); return new StorageProviderComplianceWeekResponse( + lastWeekAverageRetrievability * 100, this.histogramHelper.withoutCurrentWeek( this.histogramHelper.sorted(result), ), ); } + public async getWeekProvidersTotalDatacap( + week: Date, + isAccumulative: boolean, + ): Promise<{ total_deal_size: bigint; provider: string }[]> { + return ( + (isAccumulative + ? this.prismaService.providers_weekly_acc + : this.prismaService.providers_weekly) as any + ).findMany({ + where: { + week: week, + }, + select: { + provider: true, + total_deal_size: true, + }, + }); + } + public async getWeekProvidersForClients( week: Date, isAccumulative: boolean, @@ -243,7 +280,7 @@ export class StorageProviderService { providerWeekly: { avg_retrievability_success_rate: number; num_of_clients: number; - biggest_client_total_deal_size: bigint; + biggest_client_total_deal_size: bigint | null; total_deal_size: bigint | null; provider: string; }, @@ -281,25 +318,58 @@ export class StorageProviderService { complianceScore: number; provider: string; }[], - allProviders: string[], + validProviders: string[], ): StorageProviderComplianceWeekCount { return { compliantSps: this._getProviderComplianceWeekCount( weekProvidersCompliance, - allProviders, + validProviders, ProviderComplianceScoreRange.Compliant, ), partiallyCompliantSps: this._getProviderComplianceWeekCount( weekProvidersCompliance, - allProviders, + validProviders, ProviderComplianceScoreRange.PartiallyCompliant, ), nonCompliantSps: this._getProviderComplianceWeekCount( weekProvidersCompliance, - allProviders, + validProviders, ProviderComplianceScoreRange.NonCompliant, ), - totalSps: allProviders.length, + }; + } + + public getProviderComplianceWeekTotalDatacap( + weekProvidersCompliance: { + complianceScore: number; + provider: string; + }[], + validProviders: string[], + weekProvidersTotalDatacap: { + total_deal_size: bigint | null; + provider: string; + }[], + ): StorageProviderComplianceWeekTotalDatacap { + return { + compliantSpsTotalDatacap: this._getProviderComplianceWeekTotalDatacap( + weekProvidersCompliance, + validProviders, + ProviderComplianceScoreRange.Compliant, + weekProvidersTotalDatacap, + ), + partiallyCompliantSpsTotalDatacap: + this._getProviderComplianceWeekTotalDatacap( + weekProvidersCompliance, + validProviders, + ProviderComplianceScoreRange.PartiallyCompliant, + weekProvidersTotalDatacap, + ), + nonCompliantSpsTotalDatacap: this._getProviderComplianceWeekTotalDatacap( + weekProvidersCompliance, + validProviders, + ProviderComplianceScoreRange.NonCompliant, + weekProvidersTotalDatacap, + ), }; } @@ -308,37 +378,37 @@ export class StorageProviderService { complianceScore: number; provider: string; }[], - allProviders: string[], + validProviders: string[], ): StorageProviderComplianceWeekPercentage { return { compliantSpsPercentage: this._getProviderComplianceWeekPercentage( weekProvidersCompliance, - allProviders, + validProviders, ProviderComplianceScoreRange.Compliant, ), partiallyCompliantSpsPercentage: this._getProviderComplianceWeekPercentage( weekProvidersCompliance, - allProviders, + validProviders, ProviderComplianceScoreRange.PartiallyCompliant, ), nonCompliantSpsPercentage: this._getProviderComplianceWeekPercentage( weekProvidersCompliance, - allProviders, + validProviders, ProviderComplianceScoreRange.NonCompliant, ), - totalSps: allProviders.length, }; } - private _getProviderComplianceWeekCount( + // returns list of storage providers in validProviders with validComplianceScore compliance score + private getProviderComplianceWeekProviders( weekProvidersCompliance: { complianceScore: number; provider: string; }[], - allProviders: string[], + validProviders: string[], validComplianceScore: ProviderComplianceScoreRange, - ): number { + ): string[] { const validComplianceScores: number[] = []; switch (validComplianceScore) { @@ -353,28 +423,73 @@ export class StorageProviderService { break; } - return weekProvidersCompliance.filter( - (p) => - allProviders.includes(p.provider) && - validComplianceScores.includes(p.complianceScore), + return ( + weekProvidersCompliance + .filter( + (p) => + validProviders.includes(p.provider) && + validComplianceScores.includes(p.complianceScore), + ) + .map((p) => p.provider) ?? [] + ); + } + + private _getProviderComplianceWeekCount( + weekProvidersCompliance: { + complianceScore: number; + provider: string; + }[], + validProviders: string[], + validComplianceScore: ProviderComplianceScoreRange, + ): number { + return this.getProviderComplianceWeekProviders( + weekProvidersCompliance, + validProviders, + validComplianceScore, ).length; } + private _getProviderComplianceWeekTotalDatacap( + weekProvidersCompliance: { + complianceScore: number; + provider: string; + }[], + validProviders: string[], + validComplianceScore: ProviderComplianceScoreRange, + weekProvidersTotalDatacap: { + total_deal_size: bigint | null; + provider: string; + }[], + ): number { + // TODO calling this 2 times + const validWeekProviders = this.getProviderComplianceWeekProviders( + weekProvidersCompliance, + validProviders, + validComplianceScore, + ); + + return Number( + weekProvidersTotalDatacap + .filter((p) => validWeekProviders.includes(p.provider)) + .reduce((acc, p) => acc + p.total_deal_size, 0n) ?? 0, + ); + } + private _getProviderComplianceWeekPercentage( weekProvidersCompliance: { complianceScore: number; provider: string; }[], - allProviders: string[], + validProviders: string[], validComplianceScore: ProviderComplianceScoreRange, ): number { - return allProviders.length + return validProviders.length ? (this._getProviderComplianceWeekCount( weekProvidersCompliance, - allProviders, + validProviders, validComplianceScore, ) / - allProviders.length) * + validProviders.length) * 100 : 0; } diff --git a/src/service/storage-provider/types.storage-provider.ts b/src/service/storage-provider/types.storage-provider.ts index e5f4e327..b2f8e5ca 100644 --- a/src/service/storage-provider/types.storage-provider.ts +++ b/src/service/storage-provider/types.storage-provider.ts @@ -1,53 +1,101 @@ import { ApiProperty } from '@nestjs/swagger'; +import { IntersectionType } from '@nestjs/swagger'; export enum ProviderComplianceScoreRange { - NonCompliant, // 0-0 - PartiallyCompliant, // 1-2 - Compliant, // 3-3 + NonCompliant, // compliance score: 0 - 0 + PartiallyCompliant, // compliance score: 1 - 2 + Compliant, // compliance score: 3 - 3 } export class StorageProviderComplianceWeekPercentage { - @ApiProperty({ type: Number }) + @ApiProperty({ + description: 'Percentage of compliant storage providers', + }) compliantSpsPercentage: number; - @ApiProperty({ type: Number }) + @ApiProperty({ + description: 'Percentage of partially compliant storage providers', + }) partiallyCompliantSpsPercentage: number; - @ApiProperty({ type: Number }) + @ApiProperty({ + description: 'Percentage of non-compliant storage providers', + }) nonCompliantSpsPercentage: number; +} - @ApiProperty({ type: Number }) - totalSps: number; +export class StorageProviderComplianceWeekTotalDatacap { + @ApiProperty({ + description: 'Total datacap of compliant storage providers', + }) + compliantSpsTotalDatacap: number; + + @ApiProperty({ + description: 'Total datacap of partially compliant storage providers', + }) + partiallyCompliantSpsTotalDatacap: number; + + @ApiProperty({ + description: 'Total datacap of non-compliant storage providers', + }) + nonCompliantSpsTotalDatacap: number; } export class StorageProviderComplianceWeekCount { - @ApiProperty({ type: Number }) + @ApiProperty({ + description: 'Number of compliant storage providers', + }) compliantSps: number; - @ApiProperty({ type: Number }) + @ApiProperty({ + description: 'Number of partially compliant storage providers', + }) partiallyCompliantSps: number; - @ApiProperty({ type: Number }) + @ApiProperty({ + description: 'Number of non-compliant storage providers', + }) nonCompliantSps: number; - - @ApiProperty({ type: Number }) - totalSps: number; } -export class StorageProviderComplianceWeek extends StorageProviderComplianceWeekCount { +export class StorageProviderComplianceWeek extends IntersectionType( + StorageProviderComplianceWeekCount, + StorageProviderComplianceWeekTotalDatacap, +) { @ApiProperty({ type: String, format: 'date', example: '2024-04-22T00:00:00.000Z', }) week: Date; + + @ApiProperty({ + description: 'Total number of storage providers in the week', + }) + totalSps: number; + + @ApiProperty({ + description: + 'Average storage providers retrievability success rate in the week', + }) + averageSuccessRate: number; } export class StorageProviderComplianceWeekResponse { + @ApiProperty({ + description: + 'Last full week average storage providers retrievability success rate', + }) + averageSuccessRate: number; + @ApiProperty({ type: StorageProviderComplianceWeek, isArray: true }) results: StorageProviderComplianceWeek[]; - constructor(results: StorageProviderComplianceWeek[]) { + constructor( + averageSuccessRate: number, + results: StorageProviderComplianceWeek[], + ) { + this.averageSuccessRate = averageSuccessRate; this.results = results; } }