From 31a5f9b670c4c777eacb41c8d94d4fa042a050a9 Mon Sep 17 00:00:00 2001 From: godwill charles Date: Mon, 10 Mar 2025 09:46:49 +0100 Subject: [PATCH 01/11] fastify-postgres--implementation --- db/package.json | 4 +- stats/bin/migrate.js | 37 +++++++++++-- stats/bin/spark-stats.js | 38 ++++++++++++- stats/lib/db.js | 113 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 stats/lib/db.js diff --git a/db/package.json b/db/package.json index 11f504e0..ce5d47a6 100644 --- a/db/package.json +++ b/db/package.json @@ -14,7 +14,9 @@ "dependencies": { "@filecoin-station/spark-evaluate": "^1.2.0", "pg": "^8.13.3", - "postgrator": "^8.0.0" + "postgrator": "^8.0.0", + "@fastify/postgres": "^5.2.0", + "@fastify/url-data": "^6.0.3" }, "standard": { "env": [ diff --git a/stats/bin/migrate.js b/stats/bin/migrate.js index 846f4c21..0f58f70d 100644 --- a/stats/bin/migrate.js +++ b/stats/bin/migrate.js @@ -1,9 +1,38 @@ +import Fastify from 'fastify' +import fastifyPostgres from '@fastify/postgres' import { - getPgPools, migrateEvaluateDB, migrateStatsDB } from '@filecoin-station/spark-stats-db' -const pgPools = await getPgPools() -await migrateStatsDB(pgPools.stats) -await migrateEvaluateDB(pgPools.evaluate) +const { + DATABASE_URL, + EVALUATE_DB_URL +} = process.env + +const app = Fastify({ logger: false }) + +await app.register(fastifyPostgres, { + connectionString: DATABASE_URL, + name: 'stats' +}) + +await app.register(fastifyPostgres, { + connectionString: EVALUATE_DB_URL, + name: 'evaluate' +}) + +try { + console.log('Running migrations for stats database...') + await migrateStatsDB(app.pg.stats) + + console.log('Running migrations for evaluate database...') + await migrateEvaluateDB(app.pg.evaluate) + + console.log('All migrations completed successfully') +} catch (error) { + console.error('Migration failed:', error) + process.exit(1) +} finally { + await app.close() +} \ No newline at end of file diff --git a/stats/bin/spark-stats.js b/stats/bin/spark-stats.js index 9771a3a2..01674695 100644 --- a/stats/bin/spark-stats.js +++ b/stats/bin/spark-stats.js @@ -1,16 +1,50 @@ import '../lib/instrument.js' +import Fastify from 'fastify' +import fastifyPostgres from '@fastify/postgres' import { createApp } from '../lib/app.js' -import { getPgPools } from '@filecoin-station/spark-stats-db' const { PORT = '8080', HOST = '127.0.0.1', SPARK_API_BASE_URL = 'https://api.filspark.com/', - REQUEST_LOGGING = 'true' + REQUEST_LOGGING = 'true', + DATABASE_URL, + } = process.env const pgPools = await getPgPools() +const dbFastify = Fastify({ logger: false }) + +await dbFastify.register(fastifyPostgres, { + connectionString: DATABASE_URL, + name: 'stats', + pool: { + min: 0, + max: 100, + idleTimeoutMillis: 1000, + maxLifetimeSeconds: 60 + } +}) + +const pgPools = { + stats: dbFastify.pg.stats, + evaluate: dbFastify.pg.evaluate, + async end() { + await dbFastify.close() + } +} + + +export const withDb = async (poolName, queryFn) => { + const client = await dbFastify.pg[poolName].connect() + try { + return await queryFn(client) + } finally { + client.release() + } +} + const app = await createApp({ SPARK_API_BASE_URL, pgPools, diff --git a/stats/lib/db.js b/stats/lib/db.js new file mode 100644 index 00000000..36edabca --- /dev/null +++ b/stats/lib/db.js @@ -0,0 +1,113 @@ +// stats/lib/db.js +import Fastify from 'fastify' +import fastifyPostgres from '@fastify/postgres' + +let fastifyApp = null +let isInitialized = false + +/** + * Initialize database connections + * @param {Object} options + * @param {string} options.statsConnectionString - Stats DB connection string + * @param {string} options.evaluateConnectionString - Evaluate DB connection string + * @returns {Promise} + */ +export async function initializeDb({ statsConnectionString, evaluateConnectionString }) { + if (isInitialized) return + + // Create a minimal Fastify instance for database connections + fastifyApp = Fastify({ logger: false }) + + // Register the Postgres plugin for stats DB + await fastifyApp.register(fastifyPostgres, { + connectionString: statsConnectionString, + name: 'stats', + pool: { + min: 0, + max: 100, + idleTimeoutMillis: 1000, + maxLifetimeSeconds: 60 + } + }) + + // Register the Postgres plugin for evaluate DB + await fastifyApp.register(fastifyPostgres, { + connectionString: evaluateConnectionString, + name: 'evaluate', + pool: { + min: 0, + max: 100, + idleTimeoutMillis: 1000, + maxLifetimeSeconds: 60 + } + }) + + isInitialized = true + console.log('Database connections initialized') +} + +/** + * Execute a query on the stats database + * @param {Function} queryFn - Function that takes a client and executes a query + * @returns {Promise<*>} The result of the query function + */ +export async function withStatsDb(queryFn) { + if (!isInitialized) { + throw new Error('Database connections not initialized') + } + + const client = await fastifyApp.pg.stats.connect() + try { + return await queryFn(client) + } finally { + client.release() + } +} + +/** + * Execute a query on the evaluate database + * @param {Function} queryFn - Function that takes a client and executes a query + * @returns {Promise<*>} The result of the query function + */ +export async function withEvaluateDb(queryFn) { + if (!isInitialized) { + throw new Error('Database connections not initialized') + } + + const client = await fastifyApp.pg.evaluate.connect() + try { + return await queryFn(client) + } finally { + client.release() + } +} + +/** + * Get PgPools object compatible with the existing API + * @returns {Object} PgPools object + */ +export function getPgPools() { + if (!isInitialized) { + throw new Error('Database connections not initialized') + } + + return { + stats: fastifyApp.pg.stats, + evaluate: fastifyApp.pg.evaluate, + async end() { + await closeDb() + } + } +} + +/** + * Close all database connections + * @returns {Promise} + */ +export async function closeDb() { + if (isInitialized && fastifyApp) { + await fastifyApp.close() + isInitialized = false + console.log('Database connections closed') + } +} \ No newline at end of file From 62065731445e3885e8a507b977f14cc1119b7a80 Mon Sep 17 00:00:00 2001 From: godwill charles Date: Thu, 13 Mar 2025 19:58:28 +0100 Subject: [PATCH 02/11] fastify/postgress-implementation --- db/package.json | 8 ++- package-lock.json | 44 +++++++++------ stats/bin/migrate.js | 30 ++++------- stats/bin/spark-stats.js | 45 +++------------- stats/lib/app.js | 33 +++++++++--- stats/lib/db.js | 113 --------------------------------------- stats/package.json | 1 + 7 files changed, 77 insertions(+), 197 deletions(-) delete mode 100644 stats/lib/db.js diff --git a/db/package.json b/db/package.json index ce5d47a6..7c571123 100644 --- a/db/package.json +++ b/db/package.json @@ -13,14 +13,12 @@ }, "dependencies": { "@filecoin-station/spark-evaluate": "^1.2.0", - "pg": "^8.13.3", - "postgrator": "^8.0.0", - "@fastify/postgres": "^5.2.0", - "@fastify/url-data": "^6.0.3" + "pg": "^8.14.0", + "postgrator": "^8.0.0" }, "standard": { "env": [ "mocha" ] } -} +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index cd8bd3e7..69973a0e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "version": "1.0.0", "dependencies": { "@filecoin-station/spark-evaluate": "^1.2.0", - "pg": "^8.13.3", + "pg": "^8.14.0", "postgrator": "^8.0.0" }, "devDependencies": { @@ -301,6 +301,22 @@ "dequal": "^2.0.3" } }, + "node_modules/@fastify/postgres": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/@fastify/postgres/-/postgres-5.2.2.tgz", + "integrity": "sha512-8TWRqDSiXJp0SZjbHrqwyhl0f55eV4fpYAd9m7G0hGUpyEZJFwcxIDQYjnlRAXcVTq5NloUjFH6DxgmxZ3apbQ==", + "dependencies": { + "fastify-plugin": "^4.0.0" + }, + "peerDependencies": { + "pg": ">=6.0.0" + } + }, + "node_modules/@fastify/postgres/node_modules/fastify-plugin": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-4.5.1.tgz", + "integrity": "sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==" + }, "node_modules/@fastify/proxy-addr": { "version": "5.0.0", "license": "MIT", @@ -5720,14 +5736,13 @@ } }, "node_modules/pg": { - "version": "8.13.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.13.3.tgz", - "integrity": "sha512-P6tPt9jXbL9HVu/SSRERNYaYG++MjnscnegFh9pPHihfoBSujsrka0hyuymMzeJKFWrcG8wvCKy8rCe8e5nDUQ==", - "license": "MIT", + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.14.0.tgz", + "integrity": "sha512-nXbVpyoaXVmdqlKEzToFf37qzyeeh7mbiXsnoWvstSqohj88yaa/I/Rq/HEVn2QPSZEuLIJa/jSpRDyzjEx4FQ==", "dependencies": { "pg-connection-string": "^2.7.0", - "pg-pool": "^3.7.1", - "pg-protocol": "^1.7.1", + "pg-pool": "^3.8.0", + "pg-protocol": "^1.8.0", "pg-types": "^2.1.0", "pgpass": "1.x" }, @@ -5770,19 +5785,17 @@ } }, "node_modules/pg-pool": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.7.1.tgz", - "integrity": "sha512-xIOsFoh7Vdhojas6q3596mXFsR8nwBQBXX5JiV7p9buEVAGqYL4yFzclON5P9vFrpu1u7Zwl2oriyDa89n0wbw==", - "license": "MIT", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.8.0.tgz", + "integrity": "sha512-VBw3jiVm6ZOdLBTIcXLNdSotb6Iy3uOCwDGFAksZCXmi10nyRvnP2v3jl4d+IsLYRyXf6o9hIm/ZtUzlByNUdw==", "peerDependencies": { "pg": ">=8.0" } }, "node_modules/pg-protocol": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.7.1.tgz", - "integrity": "sha512-gjTHWGYWsEgy9MsY0Gp6ZJxV24IjDqdpTW7Eh0x+WfJLFsm/TJx1MzL6T0D88mBvkpxotCQ6TwW6N+Kko7lhgQ==", - "license": "MIT" + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.8.0.tgz", + "integrity": "sha512-jvuYlEkL03NRvOoyoRktBK7+qU5kOvlAwvmrH8sr3wbLrOdVWsRxQfz8mMy9sZFsqJ1hEWNfdWKI4SAmoL+j7g==" }, "node_modules/pg-types": { "version": "4.0.2", @@ -7414,6 +7427,7 @@ "name": "@filecoin-station/spark-stats", "dependencies": { "@fastify/cors": "^11.0.0", + "@fastify/postgres": "^5.2.0", "@fastify/url-data": "^6.0.3", "@filecoin-station/spark-stats-db": "^1.0.0", "@sentry/node": "^9.5.0", diff --git a/stats/bin/migrate.js b/stats/bin/migrate.js index 0f58f70d..0aea4967 100644 --- a/stats/bin/migrate.js +++ b/stats/bin/migrate.js @@ -1,8 +1,7 @@ -import Fastify from 'fastify' -import fastifyPostgres from '@fastify/postgres' import { migrateEvaluateDB, - migrateStatsDB + migrateStatsDB, + getPgPools } from '@filecoin-station/spark-stats-db' const { @@ -10,29 +9,20 @@ const { EVALUATE_DB_URL } = process.env -const app = Fastify({ logger: false }) - -await app.register(fastifyPostgres, { - connectionString: DATABASE_URL, - name: 'stats' -}) - -await app.register(fastifyPostgres, { - connectionString: EVALUATE_DB_URL, - name: 'evaluate' -}) - try { console.log('Running migrations for stats database...') - await migrateStatsDB(app.pg.stats) + const pgPools = await getPgPools() + + // @ts-ignore - PgPoolStats actually does have a query method at runtime + await migrateStatsDB(pgPools.stats) + + // @ts-ignore - Similarly for evaluate + await migrateEvaluateDB(pgPools.evaluate) - console.log('Running migrations for evaluate database...') - await migrateEvaluateDB(app.pg.evaluate) + await pgPools.end() console.log('All migrations completed successfully') } catch (error) { console.error('Migration failed:', error) process.exit(1) -} finally { - await app.close() } \ No newline at end of file diff --git a/stats/bin/spark-stats.js b/stats/bin/spark-stats.js index 01674695..8164b35a 100644 --- a/stats/bin/spark-stats.js +++ b/stats/bin/spark-stats.js @@ -1,6 +1,4 @@ import '../lib/instrument.js' -import Fastify from 'fastify' -import fastifyPostgres from '@fastify/postgres' import { createApp } from '../lib/app.js' const { @@ -9,49 +7,20 @@ const { SPARK_API_BASE_URL = 'https://api.filspark.com/', REQUEST_LOGGING = 'true', DATABASE_URL, - + EVALUATE_DB_URL } = process.env -const pgPools = await getPgPools() - -const dbFastify = Fastify({ logger: false }) - -await dbFastify.register(fastifyPostgres, { - connectionString: DATABASE_URL, - name: 'stats', - pool: { - min: 0, - max: 100, - idleTimeoutMillis: 1000, - maxLifetimeSeconds: 60 - } -}) - -const pgPools = { - stats: dbFastify.pg.stats, - evaluate: dbFastify.pg.evaluate, - async end() { - await dbFastify.close() - } -} - - -export const withDb = async (poolName, queryFn) => { - const client = await dbFastify.pg[poolName].connect() - try { - return await queryFn(client) - } finally { - client.release() - } -} const app = await createApp({ SPARK_API_BASE_URL, - pgPools, + DATABASE_URL, + EVALUATE_DB_URL, logger: { level: ['1', 'true'].includes(REQUEST_LOGGING) ? 'info' : 'error' } }) + console.log('Starting the http server on host %j port %s', HOST, PORT) -const baseUrl = app.listen({ port: Number(PORT), host: HOST }) -console.log(baseUrl) +await app.listen({ port: Number(PORT), host: HOST }) +console.log(`Server listening at ${HOST}:${PORT}`) + diff --git a/stats/lib/app.js b/stats/lib/app.js index 99aeb689..7b04a6d9 100644 --- a/stats/lib/app.js +++ b/stats/lib/app.js @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/node' import Fastify from 'fastify' import cors from '@fastify/cors' import urlData from '@fastify/url-data' +import fastifyPostgres from '@fastify/postgres' import { addRoutes } from './routes.js' import { addPlatformRoutes } from './platform-routes.js' @@ -12,18 +13,38 @@ import { addPlatformRoutes } from './platform-routes.js' /** * @param {object} args * @param {string} args.SPARK_API_BASE_URL - * @param {import('@filecoin-station/spark-stats-db').PgPools} args.pgPools - * @param {Fastify.FastifyLoggerOptions} args.logger - * @returns + * @param {string} args.DATABASE_URL - Connection string for stats database + * @param {string} args.EVALUATE_DB_URL - Connection string for evaluate database + * @param {import('fastify').FastifyLoggerOptions} args.logger + * @returns {Promise} */ -export const createApp = ({ +export const createApp = async ({ SPARK_API_BASE_URL, - pgPools, + DATABASE_URL, + EVALUATE_DB_URL, logger }) => { const app = Fastify({ logger }) Sentry.setupFastifyErrorHandler(app) + await app.register(fastifyPostgres, { + connectionString: DATABASE_URL, + name: 'stats' + }) + + await app.register(fastifyPostgres, { + connectionString: EVALUATE_DB_URL, + name: 'evaluate', + }) + + const pgPools = { + stats: app.pg.stats, + evaluate: app.pg.evaluate, + async end() { + await app.close() + } + } + app.register(cors, { origin: [ 'http://localhost:3000', @@ -40,4 +61,4 @@ export const createApp = ({ }) return app -} +} \ No newline at end of file diff --git a/stats/lib/db.js b/stats/lib/db.js deleted file mode 100644 index 36edabca..00000000 --- a/stats/lib/db.js +++ /dev/null @@ -1,113 +0,0 @@ -// stats/lib/db.js -import Fastify from 'fastify' -import fastifyPostgres from '@fastify/postgres' - -let fastifyApp = null -let isInitialized = false - -/** - * Initialize database connections - * @param {Object} options - * @param {string} options.statsConnectionString - Stats DB connection string - * @param {string} options.evaluateConnectionString - Evaluate DB connection string - * @returns {Promise} - */ -export async function initializeDb({ statsConnectionString, evaluateConnectionString }) { - if (isInitialized) return - - // Create a minimal Fastify instance for database connections - fastifyApp = Fastify({ logger: false }) - - // Register the Postgres plugin for stats DB - await fastifyApp.register(fastifyPostgres, { - connectionString: statsConnectionString, - name: 'stats', - pool: { - min: 0, - max: 100, - idleTimeoutMillis: 1000, - maxLifetimeSeconds: 60 - } - }) - - // Register the Postgres plugin for evaluate DB - await fastifyApp.register(fastifyPostgres, { - connectionString: evaluateConnectionString, - name: 'evaluate', - pool: { - min: 0, - max: 100, - idleTimeoutMillis: 1000, - maxLifetimeSeconds: 60 - } - }) - - isInitialized = true - console.log('Database connections initialized') -} - -/** - * Execute a query on the stats database - * @param {Function} queryFn - Function that takes a client and executes a query - * @returns {Promise<*>} The result of the query function - */ -export async function withStatsDb(queryFn) { - if (!isInitialized) { - throw new Error('Database connections not initialized') - } - - const client = await fastifyApp.pg.stats.connect() - try { - return await queryFn(client) - } finally { - client.release() - } -} - -/** - * Execute a query on the evaluate database - * @param {Function} queryFn - Function that takes a client and executes a query - * @returns {Promise<*>} The result of the query function - */ -export async function withEvaluateDb(queryFn) { - if (!isInitialized) { - throw new Error('Database connections not initialized') - } - - const client = await fastifyApp.pg.evaluate.connect() - try { - return await queryFn(client) - } finally { - client.release() - } -} - -/** - * Get PgPools object compatible with the existing API - * @returns {Object} PgPools object - */ -export function getPgPools() { - if (!isInitialized) { - throw new Error('Database connections not initialized') - } - - return { - stats: fastifyApp.pg.stats, - evaluate: fastifyApp.pg.evaluate, - async end() { - await closeDb() - } - } -} - -/** - * Close all database connections - * @returns {Promise} - */ -export async function closeDb() { - if (isInitialized && fastifyApp) { - await fastifyApp.close() - isInitialized = false - console.log('Database connections closed') - } -} \ No newline at end of file diff --git a/stats/package.json b/stats/package.json index 6aef72df..d85582fc 100644 --- a/stats/package.json +++ b/stats/package.json @@ -16,6 +16,7 @@ "dependencies": { "@fastify/cors": "^11.0.0", "@fastify/url-data": "^6.0.3", + "@fastify/postgres": "^5.2.0", "@filecoin-station/spark-stats-db": "^1.0.0", "@sentry/node": "^9.5.0", "@sentry/profiling-node": "^9.5.0", From 51bfcad51679ad521e6cc828c2164f656c66ec84 Mon Sep 17 00:00:00 2001 From: godwill charles Date: Wed, 19 Mar 2025 08:40:41 +0100 Subject: [PATCH 03/11] FIX:integrated fastify/postgress implementation into routes --- stats/bin/migrate.js | 29 +++++----------------------- stats/bin/spark-stats.js | 4 ++-- stats/lib/app.js | 17 +++++------------ stats/lib/platform-routes.js | 37 ++++++++++++++++++++++++++---------- stats/lib/routes.js | 34 ++++++++++++++++++++++++++++++++- 5 files changed, 72 insertions(+), 49 deletions(-) diff --git a/stats/bin/migrate.js b/stats/bin/migrate.js index 0aea4967..ae58af09 100644 --- a/stats/bin/migrate.js +++ b/stats/bin/migrate.js @@ -1,28 +1,9 @@ import { + getPgPools, migrateEvaluateDB, - migrateStatsDB, - getPgPools + migrateStatsDB } from '@filecoin-station/spark-stats-db' -const { - DATABASE_URL, - EVALUATE_DB_URL -} = process.env - -try { - console.log('Running migrations for stats database...') - const pgPools = await getPgPools() - - // @ts-ignore - PgPoolStats actually does have a query method at runtime - await migrateStatsDB(pgPools.stats) - - // @ts-ignore - Similarly for evaluate - await migrateEvaluateDB(pgPools.evaluate) - - await pgPools.end() - - console.log('All migrations completed successfully') -} catch (error) { - console.error('Migration failed:', error) - process.exit(1) -} \ No newline at end of file +const pgPools = await getPgPools() +await migrateStatsDB(pgPools.stats) +await migrateEvaluateDB(pgPools.evaluate) \ No newline at end of file diff --git a/stats/bin/spark-stats.js b/stats/bin/spark-stats.js index 8164b35a..529e0bd7 100644 --- a/stats/bin/spark-stats.js +++ b/stats/bin/spark-stats.js @@ -21,6 +21,6 @@ const app = await createApp({ }) console.log('Starting the http server on host %j port %s', HOST, PORT) -await app.listen({ port: Number(PORT), host: HOST }) -console.log(`Server listening at ${HOST}:${PORT}`) +const baseUrl = app.listen({ port: Number(PORT), host: HOST }) +console.log(baseUrl) diff --git a/stats/lib/app.js b/stats/lib/app.js index 7b04a6d9..d1572ced 100644 --- a/stats/lib/app.js +++ b/stats/lib/app.js @@ -7,7 +7,6 @@ import fastifyPostgres from '@fastify/postgres' import { addRoutes } from './routes.js' import { addPlatformRoutes } from './platform-routes.js' -/** @typedef {import('@filecoin-station/spark-stats-db').PgPools} PgPools */ /** @typedef {import('./typings.js').DateRangeFilter} DateRangeFilter */ /** @@ -18,6 +17,7 @@ import { addPlatformRoutes } from './platform-routes.js' * @param {import('fastify').FastifyLoggerOptions} args.logger * @returns {Promise} */ + export const createApp = async ({ SPARK_API_BASE_URL, DATABASE_URL, @@ -37,14 +37,6 @@ export const createApp = async ({ name: 'evaluate', }) - const pgPools = { - stats: app.pg.stats, - evaluate: app.pg.evaluate, - async end() { - await app.close() - } - } - app.register(cors, { origin: [ 'http://localhost:3000', @@ -54,11 +46,12 @@ export const createApp = async ({ ] }) app.register(urlData) - addRoutes(app, pgPools, SPARK_API_BASE_URL) - addPlatformRoutes(app, pgPools) + addRoutes(app, SPARK_API_BASE_URL) + addPlatformRoutes(app) app.get('/', (request, reply) => { reply.send('OK') }) return app -} \ No newline at end of file +} + diff --git a/stats/lib/platform-routes.js b/stats/lib/platform-routes.js index 7d1836d8..145280c8 100644 --- a/stats/lib/platform-routes.js +++ b/stats/lib/platform-routes.js @@ -14,39 +14,56 @@ import { filterPreHandlerHook, filterOnSendHook } from './request-helpers.js' /** @typedef {import('./typings.js').RequestWithFilter} RequestWithFilter */ -export const addPlatformRoutes = (app, pgPools) => { +/** + * Create an adapter to convert Fastify pg to the expected pgPools format + * @param {any} pg Fastify pg object + * @returns {object} pgPools compatible object + */ + +function adaptPgPools(pg) { + return { + stats: pg.stats, + evaluate: pg.evaluate, + end: async () => {} // Empty implementation + }; +} + + +export const addPlatformRoutes = (app) => { app.register(async app => { app.addHook('preHandler', filterPreHandlerHook) app.addHook('onSend', filterOnSendHook) app.get('/stations/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchDailyStationCount(pgPools.evaluate, request.filter)) + reply.send(await fetchDailyStationCount(request.server.pg.evaluate, request.filter)) }) app.get('/stations/monthly', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchMonthlyStationCount(pgPools.evaluate, request.filter)) + reply.send(await fetchMonthlyStationCount(request.server.pg.evaluate, request.filter)) }) app.get('/stations/desktop/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchDailyDesktopUsers(pgPools.stats, request.filter)) + reply.send(await fetchDailyDesktopUsers(request.server.pg.stats, request.filter)) }) app.get('/measurements/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchDailyStationMeasurementCounts(pgPools.evaluate, request.filter)) + reply.send(await fetchDailyStationMeasurementCounts(request.server.pg.evaluate, request.filter)) }) app.get('/participants/top-measurements', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchParticipantsWithTopMeasurements(pgPools.evaluate, request.filter)) + reply.send(await fetchParticipantsWithTopMeasurements(request.server.pg.evaluate, request.filter)) }) app.get('/participants/top-earning', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchTopEarningParticipants(pgPools.stats, request.filter)) - }) + const pgPools = adaptPgPools(request.server.pg); + reply.send(await fetchTopEarningParticipants(pgPools.stats, request.filter)) }) + app.get('/participants/accumulative/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchAccumulativeDailyParticipantCount(pgPools.evaluate, request.filter)) + reply.send(await fetchAccumulativeDailyParticipantCount(request.server.pg.evaluate, request.filter)) }) app.get('/transfers/daily', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchDailyRewardTransfers(pgPools.stats, request.filter)) }) }) app.get('/participants/summary', async (request, reply) => { reply.header('cache-control', `public, max-age=${24 * 3600 /* one day */}`) - reply.send(await fetchParticipantsSummary(pgPools.evaluate)) + reply.send(await fetchParticipantsSummary(request.server.pg.evaluate)) }) } diff --git a/stats/lib/routes.js b/stats/lib/routes.js index 01e45367..ddf3f323 100644 --- a/stats/lib/routes.js +++ b/stats/lib/routes.js @@ -23,57 +23,89 @@ import { /** @typedef {import('./typings.js').RequestWithFilterAndMinerId} RequestWithFilterAndMinerId */ /** @typedef {import('./typings.js').RequestWithFilterAndClientId} RequestWithFilterAndClientId */ -export const addRoutes = (app, pgPools, SPARK_API_BASE_URL) => { +/** + * Create an adapter to convert Fastify pg to the expected pgPools format + * @param {any} pg Fastify pg object + * @returns {object} pgPools compatible object + */ + +function adaptPgPools(pg) { + return { + stats: pg.stats, + evaluate: pg.evaluate, + end: async () => {} + }; +} + +export const addRoutes = (app, SPARK_API_BASE_URL) => { app.register(async app => { app.addHook('preHandler', filterPreHandlerHook) app.addHook('onSend', filterOnSendHook) app.get('/deals/daily', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchDailyDealStats(pgPools, request.filter)) }) + + app.get('/deals/summary', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg) reply.send(await fetchDealSummary(pgPools, request.filter)) }) app.get('/retrieval-success-rate', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchRetrievalSuccessRate(pgPools, request.filter)) }) app.get('/participants/daily', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg) reply.send(await fetchDailyParticipants(pgPools, request.filter)) }) app.get('/participants/monthly', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchMonthlyParticipants(pgPools, request.filter)) }) app.get('/participants/change-rates', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchParticipantChangeRates(pgPools, request.filter)) }) app.get('/participant/:address/scheduled-rewards', async (/** @type {RequestWithFilterAndAddress} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchParticipantScheduledRewards(pgPools, request.filter, request.params.address)) }) app.get('/participant/:address/reward-transfers', async (/** @type {RequestWithFilterAndAddress} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchParticipantRewardTransfers(pgPools, request.filter, request.params.address)) }) app.get('/miners/retrieval-success-rate/summary', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchMinersRSRSummary(pgPools, request.filter)) }) app.get('/miners/retrieval-timings/summary', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchMinersTimingsSummary(pgPools, request.filter)) }) app.get('/retrieval-result-codes/daily', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchDailyRetrievalResultCodes(pgPools, request.filter)) }) app.get('/retrieval-timings/daily', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchDailyRetrievalTimings(pgPools, request.filter)) }) app.get('/miner/:minerId/retrieval-timings/summary', async (/** @type {RequestWithFilterAndMinerId} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchDailyMinerRetrievalTimings(pgPools, request.filter, request.params.minerId)) }) app.get('/miner/:minerId/retrieval-success-rate/summary', async (/** @type {RequestWithFilterAndMinerId} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchDailyMinerRSRSummary(pgPools, request.filter, request.params.minerId)) }) app.get('/clients/retrieval-success-rate/summary', async (/** @type {RequestWithFilter} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchClientsRSRSummary(pgPools, request.filter)) }) app.get('/client/:clientId/retrieval-success-rate/summary', async (/** @type {RequestWithFilterAndClientId} */ request, reply) => { + const pgPools = adaptPgPools(request.server.pg); reply.send(await fetchDailyClientRSRSummary(pgPools, request.filter, request.params.clientId)) }) }) From 3717210e6185cf3cdbf797bdb4dad9cf89f2e8d7 Mon Sep 17 00:00:00 2001 From: godwill charles Date: Wed, 19 Mar 2025 11:37:24 +0100 Subject: [PATCH 04/11] FIX: modified fetcher functiond signatures to accept Fastify's pg object directly --- stats/lib/app.js | 2 + stats/lib/platform-routes.js | 20 +----- stats/lib/platform-stats-fetchers.js | 58 +++++++++--------- stats/lib/routes.js | 61 ++++++------------- stats/lib/stats-fetchers.js | 91 ++++++++++++++-------------- 5 files changed, 95 insertions(+), 137 deletions(-) diff --git a/stats/lib/app.js b/stats/lib/app.js index d1572ced..8821b0f3 100644 --- a/stats/lib/app.js +++ b/stats/lib/app.js @@ -35,6 +35,8 @@ export const createApp = async ({ await app.register(fastifyPostgres, { connectionString: EVALUATE_DB_URL, name: 'evaluate', + + }) app.register(cors, { diff --git a/stats/lib/platform-routes.js b/stats/lib/platform-routes.js index 145280c8..4cc9741b 100644 --- a/stats/lib/platform-routes.js +++ b/stats/lib/platform-routes.js @@ -14,20 +14,6 @@ import { filterPreHandlerHook, filterOnSendHook } from './request-helpers.js' /** @typedef {import('./typings.js').RequestWithFilter} RequestWithFilter */ -/** - * Create an adapter to convert Fastify pg to the expected pgPools format - * @param {any} pg Fastify pg object - * @returns {object} pgPools compatible object - */ - -function adaptPgPools(pg) { - return { - stats: pg.stats, - evaluate: pg.evaluate, - end: async () => {} // Empty implementation - }; -} - export const addPlatformRoutes = (app) => { app.register(async app => { @@ -50,15 +36,13 @@ export const addPlatformRoutes = (app) => { reply.send(await fetchParticipantsWithTopMeasurements(request.server.pg.evaluate, request.filter)) }) app.get('/participants/top-earning', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchTopEarningParticipants(pgPools.stats, request.filter)) }) + reply.send(await fetchTopEarningParticipants(request.server.pg.stats, request.filter)) }) app.get('/participants/accumulative/daily', async (/** @type {RequestWithFilter} */ request, reply) => { reply.send(await fetchAccumulativeDailyParticipantCount(request.server.pg.evaluate, request.filter)) }) app.get('/transfers/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchDailyRewardTransfers(pgPools.stats, request.filter)) + reply.send(await fetchDailyRewardTransfers(request.server.pg.stats, request.filter)) }) }) diff --git a/stats/lib/platform-stats-fetchers.js b/stats/lib/platform-stats-fetchers.js index d4533f90..cfccee46 100644 --- a/stats/lib/platform-stats-fetchers.js +++ b/stats/lib/platform-stats-fetchers.js @@ -1,16 +1,16 @@ import assert from 'http-assert' import { today, yesterday } from './request-helpers.js' -/** @typedef {import('@filecoin-station/spark-stats-db').Queryable} Queryable */ +/** @typedef {import('./typings.js').DateRangeFilter} DateRangeFilter */ const ONE_DAY = 24 * 60 * 60 * 1000 /** - * @param {Queryable} pgPool + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyStationCount = async (pgPool, filter) => { - const { rows } = await pgPool.query(` +export const fetchDailyStationCount = async (pg, filter) => { + const { rows } = await pg.query(` SELECT day::TEXT, station_count FROM daily_platform_stats WHERE day >= $1 AND day <= $2 @@ -20,11 +20,11 @@ export const fetchDailyStationCount = async (pgPool, filter) => { } /** - * @param {Queryable} pgPool + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchMonthlyStationCount = async (pgPool, filter) => { - const { rows } = await pgPool.query(` +export const fetchMonthlyStationCount = async (pg, filter) => { + const { rows } = await pg.query(` SELECT month::TEXT, station_count FROM monthly_active_station_count WHERE @@ -36,11 +36,11 @@ export const fetchMonthlyStationCount = async (pgPool, filter) => { } /** - * @param {Queryable} pgPool + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyStationMeasurementCounts = async (pgPool, filter) => { - const { rows } = await pgPool.query(` +export const fetchDailyStationMeasurementCounts = async (pg, filter) => { + const { rows } = await pg.query(` SELECT day::TEXT, accepted_measurement_count, total_measurement_count FROM daily_platform_stats WHERE day >= $1 AND day <= $2 @@ -50,31 +50,31 @@ export const fetchDailyStationMeasurementCounts = async (pgPool, filter) => { } /** - * @param {Queryable} pgPool + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchParticipantsWithTopMeasurements = async (pgPool, filter) => { +export const fetchParticipantsWithTopMeasurements = async (pg, filter) => { assert(filter.to === filter.from, 400, 'Multi-day queries are not supported for this endpoint') assert(filter.to === yesterday(), 400, 'filter.to must be set to yesterday, other values are not supported yet') // Ignore the filter for this query // Get the top measurement stations from the Materialized View - return (await pgPool.query(` + return (await pg.query(` SELECT day::TEXT, participant_address, station_count, accepted_measurement_count, inet_group_count FROM top_measurement_participants_yesterday_mv `)).rows } /** - * @param {Queryable} pgPool + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyRewardTransfers = async (pgPool, filter) => { +export const fetchDailyRewardTransfers = async (pg, filter) => { assert( new Date(filter.to).getTime() - new Date(filter.from).getTime() <= 31 * ONE_DAY, 400, 'Date range must be 31 days max' ) - const { rows } = await pgPool.query(` + const { rows } = await pg.query(` SELECT day::TEXT, to_address, amount FROM daily_reward_transfers WHERE day >= $1 AND day <= $2 @@ -99,11 +99,11 @@ export const fetchDailyRewardTransfers = async (pgPool, filter) => { } /** - * @param {Queryable} pgPool + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchAccumulativeDailyParticipantCount = async (pgPool, filter) => { - const { rows } = await pgPool.query(` +export const fetchAccumulativeDailyParticipantCount = async (pg, filter) => { + const { rows } = await pg.query(` WITH first_appearance AS ( SELECT participant_id, MIN(day) as day FROM daily_participants @@ -126,15 +126,15 @@ export const fetchAccumulativeDailyParticipantCount = async (pgPool, filter) => } /** - * @param {Queryable} pgPool + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchTopEarningParticipants = async (pgPool, filter) => { +export const fetchTopEarningParticipants = async (pg, filter) => { // The query combines "transfers until filter.to" with "latest scheduled rewards as of today". // As a result, it produces incorrect result if `to` is different from `now()`. // See https://github.com/filecoin-station/spark-stats/pull/170#discussion_r1664080395 assert(filter.to === today(), 400, 'filter.to must be today, other values are not supported') - const { rows } = await pgPool.query(` + const { rows } = await pg.query(` WITH latest_scheduled_rewards AS ( SELECT DISTINCT ON (participant_address) participant_address, scheduled_rewards FROM daily_scheduled_rewards @@ -154,10 +154,10 @@ export const fetchTopEarningParticipants = async (pgPool, filter) => { } /** - * @param {Queryable} pgPool - */ -export const fetchParticipantsSummary = async (pgPool) => { - const { rows } = await pgPool.query(` + * @param {Object} pg +*/ +export const fetchParticipantsSummary = async (pg) => { + const { rows } = await pg.query(` SELECT COUNT(DISTINCT participant_id) FROM daily_participants `) return { @@ -166,11 +166,11 @@ export const fetchParticipantsSummary = async (pgPool) => { } /** - * @param {Queryable} pgPool + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyDesktopUsers = async (pgPool, filter) => { - const { rows } = await pgPool.query(` +export const fetchDailyDesktopUsers = async (pg, filter) => { + const { rows } = await pg.query(` SELECT day::TEXT, user_count diff --git a/stats/lib/routes.js b/stats/lib/routes.js index ddf3f323..5299865c 100644 --- a/stats/lib/routes.js +++ b/stats/lib/routes.js @@ -23,19 +23,7 @@ import { /** @typedef {import('./typings.js').RequestWithFilterAndMinerId} RequestWithFilterAndMinerId */ /** @typedef {import('./typings.js').RequestWithFilterAndClientId} RequestWithFilterAndClientId */ -/** - * Create an adapter to convert Fastify pg to the expected pgPools format - * @param {any} pg Fastify pg object - * @returns {object} pgPools compatible object - */ -function adaptPgPools(pg) { - return { - stats: pg.stats, - evaluate: pg.evaluate, - end: async () => {} - }; -} export const addRoutes = (app, SPARK_API_BASE_URL) => { app.register(async app => { @@ -43,70 +31,54 @@ export const addRoutes = (app, SPARK_API_BASE_URL) => { app.addHook('onSend', filterOnSendHook) app.get('/deals/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchDailyDealStats(pgPools, request.filter)) + reply.send(await fetchDailyDealStats(request.server.pg, request.filter)) }) app.get('/deals/summary', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg) - reply.send(await fetchDealSummary(pgPools, request.filter)) + reply.send(await fetchDealSummary(request.server.pg, request.filter)) }) app.get('/retrieval-success-rate', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchRetrievalSuccessRate(pgPools, request.filter)) + reply.send(await fetchRetrievalSuccessRate(request.server.pg, request.filter)) }) app.get('/participants/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg) - reply.send(await fetchDailyParticipants(pgPools, request.filter)) + reply.send(await fetchDailyParticipants(request.server.pg, request.filter)) }) app.get('/participants/monthly', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchMonthlyParticipants(pgPools, request.filter)) + reply.send(await fetchMonthlyParticipants(request.server.pg, request.filter)) }) app.get('/participants/change-rates', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchParticipantChangeRates(pgPools, request.filter)) + reply.send(await fetchParticipantChangeRates(request.server.pg, request.filter)) }) app.get('/participant/:address/scheduled-rewards', async (/** @type {RequestWithFilterAndAddress} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchParticipantScheduledRewards(pgPools, request.filter, request.params.address)) + reply.send(await fetchParticipantScheduledRewards(request.server.pg, request.filter, request.params.address)) }) app.get('/participant/:address/reward-transfers', async (/** @type {RequestWithFilterAndAddress} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchParticipantRewardTransfers(pgPools, request.filter, request.params.address)) + reply.send(await fetchParticipantRewardTransfers(request.server.pg, request.filter, request.params.address)) }) app.get('/miners/retrieval-success-rate/summary', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchMinersRSRSummary(pgPools, request.filter)) + reply.send(await fetchMinersRSRSummary(request.server.pg, request.filter)) }) app.get('/miners/retrieval-timings/summary', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchMinersTimingsSummary(pgPools, request.filter)) + reply.send(await fetchMinersTimingsSummary(request.server.pg, request.filter)) }) app.get('/retrieval-result-codes/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchDailyRetrievalResultCodes(pgPools, request.filter)) + reply.send(await fetchDailyRetrievalResultCodes(request.server.pg, request.filter)) }) app.get('/retrieval-timings/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchDailyRetrievalTimings(pgPools, request.filter)) + reply.send(await fetchDailyRetrievalTimings(request.server.pg, request.filter)) }) app.get('/miner/:minerId/retrieval-timings/summary', async (/** @type {RequestWithFilterAndMinerId} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchDailyMinerRetrievalTimings(pgPools, request.filter, request.params.minerId)) + reply.send(await fetchDailyMinerRetrievalTimings(request.server.pg, request.filter, request.params.minerId)) }) app.get('/miner/:minerId/retrieval-success-rate/summary', async (/** @type {RequestWithFilterAndMinerId} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchDailyMinerRSRSummary(pgPools, request.filter, request.params.minerId)) + reply.send(await fetchDailyMinerRSRSummary(request.server.pg, request.filter, request.params.minerId)) }) app.get('/clients/retrieval-success-rate/summary', async (/** @type {RequestWithFilter} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchClientsRSRSummary(pgPools, request.filter)) + reply.send(await fetchClientsRSRSummary(request.server.pg, request.filter)) }) app.get('/client/:clientId/retrieval-success-rate/summary', async (/** @type {RequestWithFilterAndClientId} */ request, reply) => { - const pgPools = adaptPgPools(request.server.pg); - reply.send(await fetchDailyClientRSRSummary(pgPools, request.filter, request.params.clientId)) + reply.send(await fetchDailyClientRSRSummary(request.server.pg, request.filter, request.params.clientId)) }) }) @@ -133,3 +105,4 @@ const redirectToSparkApi = (request, reply, SPARK_API_BASE_URL) => { const location = new URL(request.url, SPARK_API_BASE_URL).toString() reply.redirect(location, 302) } + diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js index cc99db88..65a92b02 100644 --- a/stats/lib/stats-fetchers.js +++ b/stats/lib/stats-fetchers.js @@ -1,12 +1,11 @@ -/** @typedef {import('@filecoin-station/spark-stats-db').PgPools} PgPools */ /** - * @param {PgPools} pgPools + * @param {Object} pg - Fastify pg object with database connections * @param {import('./typings.js').DateRangeFilter & {nonZero?: 'true'}} filter */ -export const fetchRetrievalSuccessRate = async (pgPools, filter) => { +export const fetchRetrievalSuccessRate = async (pg, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres for converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pgPools.evaluate.query(` + const { rows } = await pg.evaluate.query(` SELECT day::text, SUM(total) as total, @@ -36,13 +35,13 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => { } /** - * @param {import('@filecoin-station/spark-stats-db').PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyDealStats = async (pgPools, filter) => { +export const fetchDailyDealStats = async (pg, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pgPools.evaluate.query(` + const { rows } = await pg.evaluate.query(` SELECT day::text, SUM(tested) AS tested, @@ -63,11 +62,11 @@ export const fetchDailyDealStats = async (pgPools, filter) => { } /** - * @param {import('@filecoin-station/spark-stats-db').PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDealSummary = async (pgPools, filter) => { - const { rows: [summary] } = await pgPools.evaluate.query(` +export const fetchDealSummary = async (pg, filter) => { + const { rows: [summary] } = await pg.evaluate.query(` SELECT SUM(tested) AS tested, SUM(index_majority_found) AS "indexMajorityFound", @@ -83,10 +82,10 @@ export const fetchDealSummary = async (pgPools, filter) => { return summary } -export const fetchDailyParticipants = async (pgPools, filter) => { +export const fetchDailyParticipants = async (pg, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pgPools.evaluate.query(` + const { rows } = await pg.evaluate.query(` SELECT day::TEXT, COUNT(DISTINCT participant_id)::INT as participants FROM daily_participants WHERE day >= $1 AND day <= $2 @@ -96,10 +95,10 @@ export const fetchDailyParticipants = async (pgPools, filter) => { return rows } -export const fetchMonthlyParticipants = async (pgPools, filter) => { +export const fetchMonthlyParticipants = async (pg, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pgPools.evaluate.query(` + const { rows } = await pg.evaluate.query(` SELECT date_trunc('month', day)::DATE::TEXT as month, COUNT(DISTINCT participant_id)::INT as participants @@ -115,13 +114,13 @@ export const fetchMonthlyParticipants = async (pgPools, filter) => { } /** - * @param {PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchParticipantChangeRates = async (pgPools, filter) => { +export const fetchParticipantChangeRates = async (pg, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pgPools.evaluate.query(` + const { rows } = await pg.evaluate.query(` SELECT date_trunc('month', day)::DATE::TEXT as month, participant_id @@ -183,12 +182,12 @@ export const fetchParticipantChangeRates = async (pgPools, filter) => { } /** - * @param {PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter * @param {string} address */ -export const fetchParticipantScheduledRewards = async (pgPools, { from, to }, address) => { - const { rows } = await pgPools.stats.query(` +export const fetchParticipantScheduledRewards = async (pg, { from, to }, address) => { + const { rows } = await pg.stats.query(` SELECT day::text, scheduled_rewards FROM daily_scheduled_rewards WHERE participant_address = $1 AND day >= $2 AND day <= $3 @@ -197,12 +196,12 @@ export const fetchParticipantScheduledRewards = async (pgPools, { from, to }, ad } /** - * @param {PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter * @param {string} address */ -export const fetchParticipantRewardTransfers = async (pgPools, { from, to }, address) => { - const { rows } = await pgPools.stats.query(` +export const fetchParticipantRewardTransfers = async (pg, { from, to }, address) => { + const { rows } = await pg.stats.query(` SELECT day::TEXT, amount FROM daily_reward_transfers WHERE to_address = $1 AND day >= $2 AND day <= $3 @@ -212,11 +211,11 @@ export const fetchParticipantRewardTransfers = async (pgPools, { from, to }, add /** * Fetches the retrieval stats summary for all miners for given date range. - * @param {PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchMinersRSRSummary = async (pgPools, filter) => { - const { rows } = await pgPools.evaluate.query(` +export const fetchMinersRSRSummary = async (pg, filter) => { + const { rows } = await pg.evaluate.query(` SELECT miner_id, SUM(total) as total, @@ -246,12 +245,12 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => { /** * Fetches the retrieval stats summary for a single miner for given date range. - * @param {PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter * @param {string} minerId */ -export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId) => { - const { rows } = await pgPools.evaluate.query(` +export const fetchDailyMinerRSRSummary = async (pg, { from, to }, minerId) => { + const { rows } = await pg.evaluate.query(` SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful, @@ -280,8 +279,8 @@ export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId) return stats } -export const fetchDailyRetrievalResultCodes = async (pgPools, filter) => { - const { rows } = await pgPools.stats.query(` +export const fetchDailyRetrievalResultCodes = async (pg, filter) => { + const { rows } = await pg.stats.query(` SELECT day::TEXT, code, rate FROM daily_retrieval_result_codes WHERE day >= $1 AND day <= $2 @@ -302,11 +301,11 @@ export const fetchDailyRetrievalResultCodes = async (pgPools, filter) => { /** * Fetches daily global retrieval time statistics - * @param {import('@filecoin-station/spark-stats-db').PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyRetrievalTimings = async (pgPools, filter) => { - const { rows } = await pgPools.evaluate.query(` +export const fetchDailyRetrievalTimings = async (pg, filter) => { + const { rows } = await pg.evaluate.query(` SELECT day::text, CEIL(percentile_cont(0.5) WITHIN GROUP (ORDER BY ttfb_p50_values)) AS ttfb_ms @@ -323,12 +322,12 @@ export const fetchDailyRetrievalTimings = async (pgPools, filter) => { /** * Fetches per miner daily retrieval time statistics - * @param {import('@filecoin-station/spark-stats-db').PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter * @param {string} minerId */ -export const fetchDailyMinerRetrievalTimings = async (pgPools, { from, to }, minerId) => { - const { rows } = await pgPools.evaluate.query(` +export const fetchDailyMinerRetrievalTimings = async (pg, { from, to }, minerId) => { + const { rows } = await pg.evaluate.query(` SELECT day::text, miner_id, @@ -347,11 +346,11 @@ export const fetchDailyMinerRetrievalTimings = async (pgPools, { from, to }, min /** * Fetches retrieval time statistics summary for all miners for given date range. - * @param {import('@filecoin-station/spark-stats-db').PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchMinersTimingsSummary = async (pgPools, { from, to }) => { - const { rows } = await pgPools.evaluate.query(` +export const fetchMinersTimingsSummary = async (pg, { from, to }) => { + const { rows } = await pg.evaluate.query(` SELECT miner_id, CEIL(percentile_cont(0.5) WITHIN GROUP (ORDER BY ttfb_p50_values)) AS ttfb_ms @@ -368,11 +367,11 @@ export const fetchMinersTimingsSummary = async (pgPools, { from, to }) => { /** * Fetches the retrieval stats summary for all clients for given date range. - * @param {PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchClientsRSRSummary = async (pgPools, filter) => { - const { rows } = await pgPools.evaluate.query(` +export const fetchClientsRSRSummary = async (pg, filter) => { + const { rows } = await pg.evaluate.query(` SELECT client_id, SUM(total) as total, @@ -400,12 +399,12 @@ export const fetchClientsRSRSummary = async (pgPools, filter) => { /** * Fetches the retrieval stats summary for a single client for given date range. - * @param {PgPools} pgPools + * @param {Object} pg * @param {import('./typings.js').DateRangeFilter} filter * @param {string} clientId */ -export const fetchDailyClientRSRSummary = async (pgPools, { from, to }, clientId) => { - const { rows } = await pgPools.evaluate.query(` +export const fetchDailyClientRSRSummary = async (pg, { from, to }, clientId) => { + const { rows } = await pg.evaluate.query(` SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful, From 973b0dafa3cb0cdc312c2c640b157c1643fe4f02 Mon Sep 17 00:00:00 2001 From: godwill charles Date: Thu, 20 Mar 2025 11:23:48 +0100 Subject: [PATCH 05/11] FIX: introduced FastifyPy type param --- db/package.json | 2 +- stats/lib/app.js | 2 -- stats/lib/platform-routes.js | 15 ++++----- stats/lib/platform-stats-fetchers.js | 50 ++++++++++++++++------------ stats/lib/routes.js | 2 ++ stats/lib/stats-fetchers.js | 41 +++++++++++++---------- stats/lib/typings.d.ts | 6 ++++ 7 files changed, 69 insertions(+), 49 deletions(-) diff --git a/db/package.json b/db/package.json index 7c571123..77cdb27a 100644 --- a/db/package.json +++ b/db/package.json @@ -13,7 +13,7 @@ }, "dependencies": { "@filecoin-station/spark-evaluate": "^1.2.0", - "pg": "^8.14.0", + "pg": "^8.13.3", "postgrator": "^8.0.0" }, "standard": { diff --git a/stats/lib/app.js b/stats/lib/app.js index 8821b0f3..d1572ced 100644 --- a/stats/lib/app.js +++ b/stats/lib/app.js @@ -35,8 +35,6 @@ export const createApp = async ({ await app.register(fastifyPostgres, { connectionString: EVALUATE_DB_URL, name: 'evaluate', - - }) app.register(cors, { diff --git a/stats/lib/platform-routes.js b/stats/lib/platform-routes.js index 4cc9741b..4739b9d1 100644 --- a/stats/lib/platform-routes.js +++ b/stats/lib/platform-routes.js @@ -21,28 +21,27 @@ export const addPlatformRoutes = (app) => { app.addHook('onSend', filterOnSendHook) app.get('/stations/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchDailyStationCount(request.server.pg.evaluate, request.filter)) }) app.get('/stations/monthly', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchMonthlyStationCount(request.server.pg.evaluate, request.filter)) + reply.send(await fetchMonthlyStationCount(request.server.pg, request.filter)) }) app.get('/stations/desktop/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchDailyDesktopUsers(request.server.pg.stats, request.filter)) + reply.send(await fetchDailyDesktopUsers(request.server.pg, request.filter)) }) app.get('/measurements/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchDailyStationMeasurementCounts(request.server.pg.evaluate, request.filter)) + reply.send(await fetchDailyStationMeasurementCounts(request.server.pg, request.filter)) }) app.get('/participants/top-measurements', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchParticipantsWithTopMeasurements(request.server.pg.evaluate, request.filter)) + reply.send(await fetchParticipantsWithTopMeasurements(request.server.pg, request.filter)) }) app.get('/participants/top-earning', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchTopEarningParticipants(request.server.pg.stats, request.filter)) }) + reply.send(await fetchTopEarningParticipants(request.server.pg, request.filter)) }) app.get('/participants/accumulative/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchAccumulativeDailyParticipantCount(request.server.pg.evaluate, request.filter)) + reply.send(await fetchAccumulativeDailyParticipantCount(request.server.pg, request.filter)) }) app.get('/transfers/daily', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchDailyRewardTransfers(request.server.pg.stats, request.filter)) + reply.send(await fetchDailyRewardTransfers(request.server.pg, request.filter)) }) }) diff --git a/stats/lib/platform-stats-fetchers.js b/stats/lib/platform-stats-fetchers.js index cfccee46..fea90c3d 100644 --- a/stats/lib/platform-stats-fetchers.js +++ b/stats/lib/platform-stats-fetchers.js @@ -1,16 +1,24 @@ import assert from 'http-assert' import { today, yesterday } from './request-helpers.js' -/** @typedef {import('./typings.js').DateRangeFilter} DateRangeFilter */ +/** + @typedef {import('./typings.js').DateRangeFilter} DateRangeFilter + @typedef {import('@filecoin-station/spark-stats-db').Queryable} Queryable + @typedef {import('./typings.js').FastifyPg} FastifyPg +*/ + +/** + * @param {FastifyPg} pg - Fastify pg object with database connections + */ const ONE_DAY = 24 * 60 * 60 * 1000 /** - * @param {Object} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyStationCount = async (pg, filter) => { - const { rows } = await pg.query(` + const { rows } = await pg.evaluate.query(` SELECT day::TEXT, station_count FROM daily_platform_stats WHERE day >= $1 AND day <= $2 @@ -20,11 +28,11 @@ export const fetchDailyStationCount = async (pg, filter) => { } /** - * @param {Object} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchMonthlyStationCount = async (pg, filter) => { - const { rows } = await pg.query(` + const { rows } = await pg.evaluate.query(` SELECT month::TEXT, station_count FROM monthly_active_station_count WHERE @@ -36,11 +44,11 @@ export const fetchMonthlyStationCount = async (pg, filter) => { } /** - * @param {Object} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyStationMeasurementCounts = async (pg, filter) => { - const { rows } = await pg.query(` + const { rows } = await pg.evaluate.query(` SELECT day::TEXT, accepted_measurement_count, total_measurement_count FROM daily_platform_stats WHERE day >= $1 AND day <= $2 @@ -50,7 +58,7 @@ export const fetchDailyStationMeasurementCounts = async (pg, filter) => { } /** - * @param {Object} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchParticipantsWithTopMeasurements = async (pg, filter) => { @@ -58,14 +66,14 @@ export const fetchParticipantsWithTopMeasurements = async (pg, filter) => { assert(filter.to === yesterday(), 400, 'filter.to must be set to yesterday, other values are not supported yet') // Ignore the filter for this query // Get the top measurement stations from the Materialized View - return (await pg.query(` + return (await pg.evaluate.query(` SELECT day::TEXT, participant_address, station_count, accepted_measurement_count, inet_group_count FROM top_measurement_participants_yesterday_mv `)).rows } /** - * @param {Object} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyRewardTransfers = async (pg, filter) => { @@ -74,7 +82,7 @@ export const fetchDailyRewardTransfers = async (pg, filter) => { 400, 'Date range must be 31 days max' ) - const { rows } = await pg.query(` + const { rows } = await pg.stats.query(` SELECT day::TEXT, to_address, amount FROM daily_reward_transfers WHERE day >= $1 AND day <= $2 @@ -99,11 +107,11 @@ export const fetchDailyRewardTransfers = async (pg, filter) => { } /** - * @param {Object} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchAccumulativeDailyParticipantCount = async (pg, filter) => { - const { rows } = await pg.query(` + const { rows } = await pg.evaluate.query(` WITH first_appearance AS ( SELECT participant_id, MIN(day) as day FROM daily_participants @@ -126,7 +134,7 @@ export const fetchAccumulativeDailyParticipantCount = async (pg, filter) => { } /** - * @param {Object} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchTopEarningParticipants = async (pg, filter) => { @@ -134,7 +142,7 @@ export const fetchTopEarningParticipants = async (pg, filter) => { // As a result, it produces incorrect result if `to` is different from `now()`. // See https://github.com/filecoin-station/spark-stats/pull/170#discussion_r1664080395 assert(filter.to === today(), 400, 'filter.to must be today, other values are not supported') - const { rows } = await pg.query(` + const { rows } = await pg.stats.query(` WITH latest_scheduled_rewards AS ( SELECT DISTINCT ON (participant_address) participant_address, scheduled_rewards FROM daily_scheduled_rewards @@ -154,10 +162,10 @@ export const fetchTopEarningParticipants = async (pg, filter) => { } /** - * @param {Object} pg -*/ + * @param {FastifyPg} pg + */ export const fetchParticipantsSummary = async (pg) => { - const { rows } = await pg.query(` + const { rows } = await pg.evaluate.query(` SELECT COUNT(DISTINCT participant_id) FROM daily_participants `) return { @@ -166,11 +174,11 @@ export const fetchParticipantsSummary = async (pg) => { } /** - * @param {Object} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyDesktopUsers = async (pg, filter) => { - const { rows } = await pg.query(` + const { rows } = await pg.stats.query(` SELECT day::TEXT, user_count @@ -180,4 +188,4 @@ export const fetchDailyDesktopUsers = async (pg, filter) => { [filter.from, filter.to]) return rows -} +} \ No newline at end of file diff --git a/stats/lib/routes.js b/stats/lib/routes.js index 5299865c..892f115d 100644 --- a/stats/lib/routes.js +++ b/stats/lib/routes.js @@ -25,6 +25,8 @@ import { + + export const addRoutes = (app, SPARK_API_BASE_URL) => { app.register(async app => { app.addHook('preHandler', filterPreHandlerHook) diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js index 65a92b02..95ad2cca 100644 --- a/stats/lib/stats-fetchers.js +++ b/stats/lib/stats-fetchers.js @@ -1,7 +1,13 @@ +/** + @typedef {import('@filecoin-station/spark-stats-db').Queryable} Queryable + @typedef {import('./typings.js').FastifyPg} FastifyPg + */ + /** - * @param {Object} pg - Fastify pg object with database connections * @param {import('./typings.js').DateRangeFilter & {nonZero?: 'true'}} filter + * @param {FastifyPg} pg - Fastify pg object with database connections */ + export const fetchRetrievalSuccessRate = async (pg, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres for converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. @@ -35,7 +41,7 @@ export const fetchRetrievalSuccessRate = async (pg, filter) => { } /** - * @param {Object} pg + * @param {any} pg - Fastify pg object with database connections * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyDealStats = async (pg, filter) => { @@ -62,7 +68,7 @@ export const fetchDailyDealStats = async (pg, filter) => { } /** - * @param {Object} pg + * @param {FastifyPg} pg - Fastify pg object with database connections * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDealSummary = async (pg, filter) => { @@ -114,7 +120,7 @@ export const fetchMonthlyParticipants = async (pg, filter) => { } /** - * @param {Object} pg + * @param {FastifyPg} pg - Fastify pg object with database connections * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchParticipantChangeRates = async (pg, filter) => { @@ -182,7 +188,7 @@ export const fetchParticipantChangeRates = async (pg, filter) => { } /** - * @param {Object} pg + * @param {FastifyPg} pg - Fastify pg object with database connections * @param {import('./typings.js').DateRangeFilter} filter * @param {string} address */ @@ -196,7 +202,7 @@ export const fetchParticipantScheduledRewards = async (pg, { from, to }, address } /** - * @param {Object} pg + * @param {FastifyPg} pg - Fastify pg object with database connections * @param {import('./typings.js').DateRangeFilter} filter * @param {string} address */ @@ -211,7 +217,7 @@ export const fetchParticipantRewardTransfers = async (pg, { from, to }, address) /** * Fetches the retrieval stats summary for all miners for given date range. - * @param {Object} pg + * @param {FastifyPg} pg - Fastify pg object with database connections * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchMinersRSRSummary = async (pg, filter) => { @@ -245,7 +251,7 @@ export const fetchMinersRSRSummary = async (pg, filter) => { /** * Fetches the retrieval stats summary for a single miner for given date range. - * @param {Object} pg + * @param {FastifyPg} pg - Fastify pg object with database connections * @param {import('./typings.js').DateRangeFilter} filter * @param {string} minerId */ @@ -301,7 +307,7 @@ export const fetchDailyRetrievalResultCodes = async (pg, filter) => { /** * Fetches daily global retrieval time statistics - * @param {Object} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyRetrievalTimings = async (pg, filter) => { @@ -322,7 +328,7 @@ export const fetchDailyRetrievalTimings = async (pg, filter) => { /** * Fetches per miner daily retrieval time statistics - * @param {Object} pg + * @param {{stats: Queryable, evaluate: Queryable}} pg * @param {import('./typings.js').DateRangeFilter} filter * @param {string} minerId */ @@ -346,9 +352,10 @@ export const fetchDailyMinerRetrievalTimings = async (pg, { from, to }, minerId) /** * Fetches retrieval time statistics summary for all miners for given date range. - * @param {Object} pg + /** + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter - */ +*/ export const fetchMinersTimingsSummary = async (pg, { from, to }) => { const { rows } = await pg.evaluate.query(` SELECT @@ -366,10 +373,10 @@ export const fetchMinersTimingsSummary = async (pg, { from, to }) => { } /** - * Fetches the retrieval stats summary for all clients for given date range. - * @param {Object} pg - * @param {import('./typings.js').DateRangeFilter} filter - */ +* Fetches the retrieval stats summary for all clients for given date range. +* @param {FastifyPg} pg +* @param {import('./typings.js').DateRangeFilter} filter +*/ export const fetchClientsRSRSummary = async (pg, filter) => { const { rows } = await pg.evaluate.query(` SELECT @@ -399,7 +406,7 @@ export const fetchClientsRSRSummary = async (pg, filter) => { /** * Fetches the retrieval stats summary for a single client for given date range. - * @param {Object} pg + * @param {{stats: Queryable, evaluate: Queryable}} pg * @param {import('./typings.js').DateRangeFilter} filter * @param {string} clientId */ diff --git a/stats/lib/typings.d.ts b/stats/lib/typings.d.ts index 377374ea..72379608 100644 --- a/stats/lib/typings.d.ts +++ b/stats/lib/typings.d.ts @@ -1,10 +1,16 @@ import { FastifyRequest } from 'fastify' +import { PostgresDb } from '@fastify/postgres' +import { Queryable } from '@filecoin-station/spark-stats-db' + + +export type FastifyPg = PostgresDb & Record export interface DateRangeFilter { from: string; to: string; } + export type RequestWithFilter = FastifyRequest<{ Querystring: { from: string, to: string } }> From e89a63c893c95af6d9a638413c69cb28ec11eb33 Mon Sep 17 00:00:00 2001 From: godwill charles Date: Thu, 20 Mar 2025 12:27:08 +0100 Subject: [PATCH 06/11] FIX: ci issue --- package-lock.json | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index d2f0dc0c..e298db74 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "version": "1.0.0", "dependencies": { "@filecoin-station/spark-evaluate": "^1.2.0", - "pg": "^8.13.3", + "pg": "^8.14.1", "postgrator": "^8.0.0" }, "devDependencies": { @@ -5736,10 +5736,9 @@ } }, "node_modules/pg": { - "version": "8.13.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.13.3.tgz", - "integrity": "sha512-P6tPt9jXbL9HVu/SSRERNYaYG++MjnscnegFh9pPHihfoBSujsrka0hyuymMzeJKFWrcG8wvCKy8rCe8e5nDUQ==", - "license": "MIT", + "version": "8.14.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.14.1.tgz", + "integrity": "sha512-0TdbqfjwIun9Fm/r89oB7RFQ0bLgduAhiIqIXOsyKoiC/L54DbuAAzIEN/9Op0f1Po9X7iCPXGoa/Ah+2aI8Xw==", "dependencies": { "pg-connection-string": "^2.7.0", "pg-pool": "^3.8.0", @@ -5786,19 +5785,17 @@ } }, "node_modules/pg-pool": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.7.1.tgz", - "integrity": "sha512-xIOsFoh7Vdhojas6q3596mXFsR8nwBQBXX5JiV7p9buEVAGqYL4yFzclON5P9vFrpu1u7Zwl2oriyDa89n0wbw==", - "license": "MIT", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.8.0.tgz", + "integrity": "sha512-VBw3jiVm6ZOdLBTIcXLNdSotb6Iy3uOCwDGFAksZCXmi10nyRvnP2v3jl4d+IsLYRyXf6o9hIm/ZtUzlByNUdw==", "peerDependencies": { "pg": ">=8.0" } }, "node_modules/pg-protocol": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.7.1.tgz", - "integrity": "sha512-gjTHWGYWsEgy9MsY0Gp6ZJxV24IjDqdpTW7Eh0x+WfJLFsm/TJx1MzL6T0D88mBvkpxotCQ6TwW6N+Kko7lhgQ==", - "license": "MIT" + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.8.0.tgz", + "integrity": "sha512-jvuYlEkL03NRvOoyoRktBK7+qU5kOvlAwvmrH8sr3wbLrOdVWsRxQfz8mMy9sZFsqJ1hEWNfdWKI4SAmoL+j7g==" }, "node_modules/pg-types": { "version": "4.0.2", From 26d73c783586688814e40c63f4cdfe0390495a7e Mon Sep 17 00:00:00 2001 From: godwill charles Date: Thu, 20 Mar 2025 15:49:47 +0100 Subject: [PATCH 07/11] FIX: added missing line of code --- stats/bin/migrate.js | 2 +- stats/bin/spark-stats.js | 2 -- stats/lib/app.js | 3 +-- stats/lib/platform-routes.js | 5 +++-- stats/lib/platform-stats-fetchers.js | 24 ++++++++++++------------ stats/lib/routes.js | 8 +------- stats/lib/stats-fetchers.js | 8 ++++---- 7 files changed, 22 insertions(+), 30 deletions(-) diff --git a/stats/bin/migrate.js b/stats/bin/migrate.js index ae58af09..846f4c21 100644 --- a/stats/bin/migrate.js +++ b/stats/bin/migrate.js @@ -6,4 +6,4 @@ import { const pgPools = await getPgPools() await migrateStatsDB(pgPools.stats) -await migrateEvaluateDB(pgPools.evaluate) \ No newline at end of file +await migrateEvaluateDB(pgPools.evaluate) diff --git a/stats/bin/spark-stats.js b/stats/bin/spark-stats.js index 529e0bd7..da122ac0 100644 --- a/stats/bin/spark-stats.js +++ b/stats/bin/spark-stats.js @@ -10,7 +10,6 @@ const { EVALUATE_DB_URL } = process.env - const app = await createApp({ SPARK_API_BASE_URL, DATABASE_URL, @@ -23,4 +22,3 @@ const app = await createApp({ console.log('Starting the http server on host %j port %s', HOST, PORT) const baseUrl = app.listen({ port: Number(PORT), host: HOST }) console.log(baseUrl) - diff --git a/stats/lib/app.js b/stats/lib/app.js index 9577ff5d..942e4285 100644 --- a/stats/lib/app.js +++ b/stats/lib/app.js @@ -34,7 +34,7 @@ export const createApp = async ({ await app.register(fastifyPostgres, { connectionString: EVALUATE_DB_URL, - name: 'evaluate', + name: 'evaluate' }) app.register(cors, { @@ -59,4 +59,3 @@ export const createApp = async ({ return app } - diff --git a/stats/lib/platform-routes.js b/stats/lib/platform-routes.js index 4739b9d1..86dc3057 100644 --- a/stats/lib/platform-routes.js +++ b/stats/lib/platform-routes.js @@ -14,13 +14,13 @@ import { filterPreHandlerHook, filterOnSendHook } from './request-helpers.js' /** @typedef {import('./typings.js').RequestWithFilter} RequestWithFilter */ - export const addPlatformRoutes = (app) => { app.register(async app => { app.addHook('preHandler', filterPreHandlerHook) app.addHook('onSend', filterOnSendHook) app.get('/stations/daily', async (/** @type {RequestWithFilter} */ request, reply) => { + reply.send(await fetchDailyStationCount(request.server.pg, request.filter)) }) app.get('/stations/monthly', async (/** @type {RequestWithFilter} */ request, reply) => { reply.send(await fetchMonthlyStationCount(request.server.pg, request.filter)) @@ -35,7 +35,8 @@ export const addPlatformRoutes = (app) => { reply.send(await fetchParticipantsWithTopMeasurements(request.server.pg, request.filter)) }) app.get('/participants/top-earning', async (/** @type {RequestWithFilter} */ request, reply) => { - reply.send(await fetchTopEarningParticipants(request.server.pg, request.filter)) }) + reply.send(await fetchTopEarningParticipants(request.server.pg, request.filter)) + }) app.get('/participants/accumulative/daily', async (/** @type {RequestWithFilter} */ request, reply) => { reply.send(await fetchAccumulativeDailyParticipantCount(request.server.pg, request.filter)) diff --git a/stats/lib/platform-stats-fetchers.js b/stats/lib/platform-stats-fetchers.js index fea90c3d..6b6e125b 100644 --- a/stats/lib/platform-stats-fetchers.js +++ b/stats/lib/platform-stats-fetchers.js @@ -1,10 +1,10 @@ import assert from 'http-assert' import { today, yesterday } from './request-helpers.js' -/** - @typedef {import('./typings.js').DateRangeFilter} DateRangeFilter - @typedef {import('@filecoin-station/spark-stats-db').Queryable} Queryable - @typedef {import('./typings.js').FastifyPg} FastifyPg +/** + @typedef {import('./typings.js').DateRangeFilter} DateRangeFilter + @typedef {import('@filecoin-station/spark-stats-db').Queryable} Queryable + @typedef {import('./typings.js').FastifyPg} FastifyPg */ /** @@ -14,7 +14,7 @@ import { today, yesterday } from './request-helpers.js' const ONE_DAY = 24 * 60 * 60 * 1000 /** - * @param {FastifyPg} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyStationCount = async (pg, filter) => { @@ -58,7 +58,7 @@ export const fetchDailyStationMeasurementCounts = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchParticipantsWithTopMeasurements = async (pg, filter) => { @@ -73,7 +73,7 @@ export const fetchParticipantsWithTopMeasurements = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyRewardTransfers = async (pg, filter) => { @@ -107,7 +107,7 @@ export const fetchDailyRewardTransfers = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchAccumulativeDailyParticipantCount = async (pg, filter) => { @@ -134,7 +134,7 @@ export const fetchAccumulativeDailyParticipantCount = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchTopEarningParticipants = async (pg, filter) => { @@ -162,7 +162,7 @@ export const fetchTopEarningParticipants = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pg */ export const fetchParticipantsSummary = async (pg) => { const { rows } = await pg.evaluate.query(` @@ -174,7 +174,7 @@ export const fetchParticipantsSummary = async (pg) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyDesktopUsers = async (pg, filter) => { @@ -188,4 +188,4 @@ export const fetchDailyDesktopUsers = async (pg, filter) => { [filter.from, filter.to]) return rows -} \ No newline at end of file +} diff --git a/stats/lib/routes.js b/stats/lib/routes.js index 892f115d..3443d778 100644 --- a/stats/lib/routes.js +++ b/stats/lib/routes.js @@ -23,10 +23,6 @@ import { /** @typedef {import('./typings.js').RequestWithFilterAndMinerId} RequestWithFilterAndMinerId */ /** @typedef {import('./typings.js').RequestWithFilterAndClientId} RequestWithFilterAndClientId */ - - - - export const addRoutes = (app, SPARK_API_BASE_URL) => { app.register(async app => { app.addHook('preHandler', filterPreHandlerHook) @@ -35,8 +31,7 @@ export const addRoutes = (app, SPARK_API_BASE_URL) => { app.get('/deals/daily', async (/** @type {RequestWithFilter} */ request, reply) => { reply.send(await fetchDailyDealStats(request.server.pg, request.filter)) }) - - + app.get('/deals/summary', async (/** @type {RequestWithFilter} */ request, reply) => { reply.send(await fetchDealSummary(request.server.pg, request.filter)) }) @@ -107,4 +102,3 @@ const redirectToSparkApi = (request, reply, SPARK_API_BASE_URL) => { const location = new URL(request.url, SPARK_API_BASE_URL).toString() reply.redirect(location, 302) } - diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js index 95ad2cca..fc278702 100644 --- a/stats/lib/stats-fetchers.js +++ b/stats/lib/stats-fetchers.js @@ -1,6 +1,6 @@ -/** - @typedef {import('@filecoin-station/spark-stats-db').Queryable} Queryable - @typedef {import('./typings.js').FastifyPg} FastifyPg +/** + @typedef {import('@filecoin-station/spark-stats-db').Queryable} Queryable + @typedef {import('./typings.js').FastifyPg} FastifyPg */ /** @@ -307,7 +307,7 @@ export const fetchDailyRetrievalResultCodes = async (pg, filter) => { /** * Fetches daily global retrieval time statistics - * @param {FastifyPg} pg + * @param {FastifyPg} pg * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyRetrievalTimings = async (pg, filter) => { From 2608fdc3bef74f5922aab7db00ab40d7b564b935 Mon Sep 17 00:00:00 2001 From: godwill charles Date: Fri, 21 Mar 2025 13:55:55 +0100 Subject: [PATCH 08/11] FIX: CI test issues --- stats/bin/spark-stats.js | 4 ++-- stats/test/app.test.js | 10 ++++++++-- stats/test/platform-routes.test.js | 12 +++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/stats/bin/spark-stats.js b/stats/bin/spark-stats.js index da122ac0..626e4c96 100644 --- a/stats/bin/spark-stats.js +++ b/stats/bin/spark-stats.js @@ -6,8 +6,8 @@ const { HOST = '127.0.0.1', SPARK_API_BASE_URL = 'https://api.filspark.com/', REQUEST_LOGGING = 'true', - DATABASE_URL, - EVALUATE_DB_URL + DATABASE_URL = 'postgres://localhost:5432/spark_stats', + EVALUATE_DB_URL = 'postgres://localhost:5432/spark_evaluate' } = process.env const app = await createApp({ diff --git a/stats/test/app.test.js b/stats/test/app.test.js index 408c6573..604f8877 100644 --- a/stats/test/app.test.js +++ b/stats/test/app.test.js @@ -16,10 +16,16 @@ describe('HTTP request handler', () => { before(async () => { pgPools = await getPgPools() + + const { + DATABASE_URL = 'postgres://localhost:5432/spark_stats', + EVALUATE_DB_URL = 'postgres://localhost:5432/spark_evaluate' + } = process.env - app = createApp({ + app = await createApp({ SPARK_API_BASE_URL: 'https://api.filspark.com/', - pgPools, + DATABASE_URL, + EVALUATE_DB_URL, logger: { level: process.env.DEBUG === '*' || process.env.DEBUG?.includes('test') ? 'debug' diff --git a/stats/test/platform-routes.test.js b/stats/test/platform-routes.test.js index d1542dde..c7b80622 100644 --- a/stats/test/platform-routes.test.js +++ b/stats/test/platform-routes.test.js @@ -15,10 +15,16 @@ describe('Platform Routes HTTP request handler', () => { before(async () => { pgPools = await getPgPools() + + const { + DATABASE_URL = 'postgres://localhost:5432/spark_stats', + EVALUATE_DB_URL = 'postgres://localhost:5432/spark_evaluate' + } = process.env - app = createApp({ + app = await createApp({ SPARK_API_BASE_URL: 'https://api.filspark.com/', - pgPools, + DATABASE_URL, + EVALUATE_DB_URL, logger: { level: process.env.DEBUG === '*' || process.env.DEBUG?.includes('test') ? 'debug' @@ -28,7 +34,7 @@ describe('Platform Routes HTTP request handler', () => { baseUrl = await app.listen() }) - + after(async () => { await app.close() await pgPools.end() From f6a0319ec44498d10c77ca73e551faa604526f2d Mon Sep 17 00:00:00 2001 From: godwill charles Date: Mon, 31 Mar 2025 08:47:54 +0100 Subject: [PATCH 09/11] FIX: test implementation for fastify/postgres --- stats/bin/migrate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/bin/migrate.js b/stats/bin/migrate.js index 825961f9..c161cb5c 100644 --- a/stats/bin/migrate.js +++ b/stats/bin/migrate.js @@ -5,4 +5,4 @@ import { } from '@filecoin-station/spark-stats-db' const pgPools = await getPgPools() await migrateStatsDB(pgPools.stats) -await migrateEvaluateDB(pgPools.evaluate) \ No newline at end of file +await migrateEvaluateDB(pgPools.evaluate) From 59448e414b7e456bb23861d93d9f24d5e032db70 Mon Sep 17 00:00:00 2001 From: godwill charles Date: Wed, 2 Apr 2025 05:19:27 +0100 Subject: [PATCH 10/11] FIX: changed naming convention to pgPOols and CI test --- stats/bin/migrate.js | 3 +- stats/lib/platform-stats-fetchers.js | 56 +++--- stats/lib/stats-fetchers.js | 102 +++++----- stats/test/app.test.js | 272 ++++++++++++++------------- stats/test/platform-routes.test.js | 51 ++--- 5 files changed, 244 insertions(+), 240 deletions(-) diff --git a/stats/bin/migrate.js b/stats/bin/migrate.js index c161cb5c..ae58af09 100644 --- a/stats/bin/migrate.js +++ b/stats/bin/migrate.js @@ -3,6 +3,7 @@ import { migrateEvaluateDB, migrateStatsDB } from '@filecoin-station/spark-stats-db' + const pgPools = await getPgPools() await migrateStatsDB(pgPools.stats) -await migrateEvaluateDB(pgPools.evaluate) +await migrateEvaluateDB(pgPools.evaluate) \ No newline at end of file diff --git a/stats/lib/platform-stats-fetchers.js b/stats/lib/platform-stats-fetchers.js index 6b6e125b..6482e080 100644 --- a/stats/lib/platform-stats-fetchers.js +++ b/stats/lib/platform-stats-fetchers.js @@ -8,17 +8,17 @@ import { today, yesterday } from './request-helpers.js' */ /** - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools */ const ONE_DAY = 24 * 60 * 60 * 1000 /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyStationCount = async (pg, filter) => { - const { rows } = await pg.evaluate.query(` +export const fetchDailyStationCount = async (pgPools, filter) => { + const { rows } = await pgPools.evaluate.query(` SELECT day::TEXT, station_count FROM daily_platform_stats WHERE day >= $1 AND day <= $2 @@ -28,11 +28,11 @@ export const fetchDailyStationCount = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchMonthlyStationCount = async (pg, filter) => { - const { rows } = await pg.evaluate.query(` +export const fetchMonthlyStationCount = async (pgPools, filter) => { + const { rows } = await pgPools.evaluate.query(` SELECT month::TEXT, station_count FROM monthly_active_station_count WHERE @@ -44,11 +44,11 @@ export const fetchMonthlyStationCount = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyStationMeasurementCounts = async (pg, filter) => { - const { rows } = await pg.evaluate.query(` +export const fetchDailyStationMeasurementCounts = async (pgPools, filter) => { + const { rows } = await pgPools.evaluate.query(` SELECT day::TEXT, accepted_measurement_count, total_measurement_count FROM daily_platform_stats WHERE day >= $1 AND day <= $2 @@ -58,31 +58,31 @@ export const fetchDailyStationMeasurementCounts = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchParticipantsWithTopMeasurements = async (pg, filter) => { +export const fetchParticipantsWithTopMeasurements = async (pgPools, filter) => { assert(filter.to === filter.from, 400, 'Multi-day queries are not supported for this endpoint') assert(filter.to === yesterday(), 400, 'filter.to must be set to yesterday, other values are not supported yet') // Ignore the filter for this query // Get the top measurement stations from the Materialized View - return (await pg.evaluate.query(` + return (await pgPools.evaluate.query(` SELECT day::TEXT, participant_address, station_count, accepted_measurement_count, inet_group_count FROM top_measurement_participants_yesterday_mv `)).rows } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyRewardTransfers = async (pg, filter) => { +export const fetchDailyRewardTransfers = async (pgPools, filter) => { assert( new Date(filter.to).getTime() - new Date(filter.from).getTime() <= 31 * ONE_DAY, 400, 'Date range must be 31 days max' ) - const { rows } = await pg.stats.query(` + const { rows } = await pgPools.stats.query(` SELECT day::TEXT, to_address, amount FROM daily_reward_transfers WHERE day >= $1 AND day <= $2 @@ -107,11 +107,11 @@ export const fetchDailyRewardTransfers = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchAccumulativeDailyParticipantCount = async (pg, filter) => { - const { rows } = await pg.evaluate.query(` +export const fetchAccumulativeDailyParticipantCount = async (pgPools, filter) => { + const { rows } = await pgPools.evaluate.query(` WITH first_appearance AS ( SELECT participant_id, MIN(day) as day FROM daily_participants @@ -134,15 +134,15 @@ export const fetchAccumulativeDailyParticipantCount = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchTopEarningParticipants = async (pg, filter) => { +export const fetchTopEarningParticipants = async (pgPools, filter) => { // The query combines "transfers until filter.to" with "latest scheduled rewards as of today". // As a result, it produces incorrect result if `to` is different from `now()`. // See https://github.com/filecoin-station/spark-stats/pull/170#discussion_r1664080395 assert(filter.to === today(), 400, 'filter.to must be today, other values are not supported') - const { rows } = await pg.stats.query(` + const { rows } = await pgPools.stats.query(` WITH latest_scheduled_rewards AS ( SELECT DISTINCT ON (participant_address) participant_address, scheduled_rewards FROM daily_scheduled_rewards @@ -162,10 +162,10 @@ export const fetchTopEarningParticipants = async (pg, filter) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools */ -export const fetchParticipantsSummary = async (pg) => { - const { rows } = await pg.evaluate.query(` +export const fetchParticipantsSummary = async (pgPools) => { + const { rows } = await pgPools.evaluate.query(` SELECT COUNT(DISTINCT participant_id) FROM daily_participants `) return { @@ -174,11 +174,11 @@ export const fetchParticipantsSummary = async (pg) => { } /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyDesktopUsers = async (pg, filter) => { - const { rows } = await pg.stats.query(` +export const fetchDailyDesktopUsers = async (pgPools, filter) => { + const { rows } = await pgPools.stats.query(` SELECT day::TEXT, user_count diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js index 76316864..f0621a2b 100644 --- a/stats/lib/stats-fetchers.js +++ b/stats/lib/stats-fetchers.js @@ -5,13 +5,13 @@ /** * @param {import('./typings.js').DateRangeFilter & {nonZero?: 'true'}} filter - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools */ -export const fetchRetrievalSuccessRate = async (pg, filter) => { +export const fetchRetrievalSuccessRate = async (pgPools, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres for converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pg.evaluate.query(` + const { rows } = await pgPools.evaluate.query(` SELECT day::text, SUM(total) as total, @@ -41,13 +41,13 @@ export const fetchRetrievalSuccessRate = async (pg, filter) => { } /** - * @param {any} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyDealStats = async (pg, filter) => { +export const fetchDailyDealStats = async (pgPools, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pg.evaluate.query(` + const { rows } = await pgPools.evaluate.query(` SELECT day::text, SUM(tested) AS tested, @@ -68,11 +68,11 @@ export const fetchDailyDealStats = async (pg, filter) => { } /** - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDealSummary = async (pg, filter) => { - const { rows: [summary] } = await pg.evaluate.query(` +export const fetchDealSummary = async (pgPools, filter) => { + const { rows: [summary] } = await pgPools.evaluate.query(` SELECT SUM(tested) AS tested, SUM(index_majority_found) AS "indexMajorityFound", @@ -88,10 +88,10 @@ export const fetchDealSummary = async (pg, filter) => { return summary } -export const fetchDailyParticipants = async (pg, filter) => { +export const fetchDailyParticipants = async (pgPools, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pg.evaluate.query(` + const { rows } = await pgPools.evaluate.query(` SELECT day::TEXT, COUNT(DISTINCT participant_id)::INT as participants FROM daily_participants WHERE day >= $1 AND day <= $2 @@ -101,10 +101,10 @@ export const fetchDailyParticipants = async (pg, filter) => { return rows } -export const fetchMonthlyParticipants = async (pg, filter) => { +export const fetchMonthlyParticipants = async (pgPools, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pg.evaluate.query(` + const { rows } = await pgPools.evaluate.query(` SELECT date_trunc('month', day)::DATE::TEXT as month, COUNT(DISTINCT participant_id)::INT as participants @@ -120,13 +120,13 @@ export const fetchMonthlyParticipants = async (pg, filter) => { } /** - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchParticipantChangeRates = async (pg, filter) => { +export const fetchParticipantChangeRates = async (pgPools, filter) => { // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into // a JavaScript Date with a timezone, as that could change the date one day forward or back. - const { rows } = await pg.evaluate.query(` + const { rows } = await pgPools.evaluate.query(` SELECT date_trunc('month', day)::DATE::TEXT as month, participant_id @@ -188,12 +188,12 @@ export const fetchParticipantChangeRates = async (pg, filter) => { } /** - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter * @param {string} address */ -export const fetchParticipantScheduledRewards = async (pg, { from, to }, address) => { - const { rows } = await pg.stats.query(` +export const fetchParticipantScheduledRewards = async (pgPools, { from, to }, address) => { + const { rows } = await pgPools.stats.query(` SELECT day::text, scheduled_rewards FROM daily_scheduled_rewards WHERE participant_address = $1 AND day >= $2 AND day <= $3 @@ -202,12 +202,12 @@ export const fetchParticipantScheduledRewards = async (pg, { from, to }, address } /** - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter * @param {string} address */ -export const fetchParticipantRewardTransfers = async (pg, { from, to }, address) => { - const { rows } = await pg.stats.query(` +export const fetchParticipantRewardTransfers = async (pgPools, { from, to }, address) => { + const { rows } = await pgPools.stats.query(` SELECT day::TEXT, amount FROM daily_reward_transfers WHERE to_address = $1 AND day >= $2 AND day <= $3 @@ -217,11 +217,11 @@ export const fetchParticipantRewardTransfers = async (pg, { from, to }, address) /** * Fetches the retrieval stats summary for all miners for given date range. - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchMinersRSRSummary = async (pg, filter) => { - const { rows } = await pg.evaluate.query(` +export const fetchMinersRSRSummary = async (pgPools, filter) => { + const { rows } = await pgPools.evaluate.query(` SELECT miner_id, SUM(total) as total, @@ -251,12 +251,12 @@ export const fetchMinersRSRSummary = async (pg, filter) => { /** * Fetches the retrieval stats summary for a single miner for given date range. - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter * @param {string} minerId */ -export const fetchDailyMinerRSRSummary = async (pg, { from, to }, minerId) => { - const { rows } = await pg.evaluate.query(` +export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId) => { + const { rows } = await pgPools.evaluate.query(` SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful, @@ -285,8 +285,8 @@ export const fetchDailyMinerRSRSummary = async (pg, { from, to }, minerId) => { return stats } -export const fetchDailyRetrievalResultCodes = async (pg, filter) => { - const { rows } = await pg.stats.query(` +export const fetchDailyRetrievalResultCodes = async (pgPools, filter) => { + const { rows } = await pgPools.stats.query(` SELECT day::TEXT, code, rate FROM daily_retrieval_result_codes WHERE day >= $1 AND day <= $2 @@ -307,11 +307,11 @@ export const fetchDailyRetrievalResultCodes = async (pg, filter) => { /** * Fetches daily global retrieval time statistics - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchDailyRetrievalTimings = async (pg, filter) => { - const { rows } = await pg.evaluate.query(` +export const fetchDailyRetrievalTimings = async (pgPools, filter) => { + const { rows } = await pgPools.evaluate.query(` SELECT day::text, CEIL(percentile_cont(0.5) WITHIN GROUP (ORDER BY ttfb_p50_values)) AS ttfb_ms @@ -328,12 +328,12 @@ export const fetchDailyRetrievalTimings = async (pg, filter) => { /** * Fetches per miner daily retrieval time statistics - * @param {{stats: Queryable, evaluate: Queryable}} pg + * @param {{stats: Queryable, evaluate: Queryable}} pgPools * @param {import('./typings.js').DateRangeFilter} filter * @param {string} minerId */ -export const fetchDailyMinerRetrievalTimings = async (pg, { from, to }, minerId) => { - const { rows } = await pg.evaluate.query(` +export const fetchDailyMinerRetrievalTimings = async (pgPools, { from, to }, minerId) => { + const { rows } = await pgPools.evaluate.query(` SELECT day::text, miner_id, @@ -353,11 +353,11 @@ export const fetchDailyMinerRetrievalTimings = async (pg, { from, to }, minerId) /** * Fetches retrieval time statistics summary for all miners for given date range. /** - * @param {FastifyPg} pg + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchMinersTimingsSummary = async (pg, { from, to }) => { - const { rows } = await pg.evaluate.query(` +export const fetchMinersTimingsSummary = async (pgPools, { from, to }) => { + const { rows } = await pgPools.evaluate.query(` SELECT miner_id, CEIL(percentile_cont(0.5) WITHIN GROUP (ORDER BY ttfb_p50_values)) AS ttfb_ms @@ -374,11 +374,11 @@ export const fetchMinersTimingsSummary = async (pg, { from, to }) => { /** * Fetches the retrieval stats summary for all clients for given date range. -* @param {FastifyPg} pg +* @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchClientsRSRSummary = async (pg, filter) => { - const { rows } = await pg.evaluate.query(` +export const fetchClientsRSRSummary = async (pgPools, filter) => { + const { rows } = await pgPools.evaluate.query(` SELECT client_id, SUM(total) as total, @@ -406,12 +406,12 @@ export const fetchClientsRSRSummary = async (pg, filter) => { /** * Fetches the retrieval stats summary for a single client for given date range. - * @param {{stats: Queryable, evaluate: Queryable}} pg + * @param {{stats: Queryable, evaluate: Queryable}} pgPools * @param {import('./typings.js').DateRangeFilter} filter * @param {string} clientId */ -export const fetchDailyClientRSRSummary = async (pg, { from, to }, clientId) => { - const { rows } = await pg.evaluate.query(` +export const fetchDailyClientRSRSummary = async (pgPools, { from, to }, clientId) => { + const { rows } = await pgPools.evaluate.query(` SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful, @@ -440,11 +440,11 @@ export const fetchDailyClientRSRSummary = async (pg, { from, to }, clientId) => /** * Fetches the retrieval stats summary for all allocators for given date range. - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ -export const fetchAllocatorsRSRSummary = async (pg, filter) => { - const { rows } = await pg.evaluate.query(` +export const fetchAllocatorsRSRSummary = async (pgPools, filter) => { + const { rows } = await pgPools.evaluate.query(` SELECT allocator_id, SUM(total) as total, @@ -472,12 +472,12 @@ export const fetchAllocatorsRSRSummary = async (pg, filter) => { /** * Fetches the retrieval stats summary for a single allocator for given date range. - * @param {FastifyPg} pg - Fastify pg object with database connections + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter * @param {string} allocatorId */ -export const fetchDailyAllocatorRSRSummary = async (pg, { from, to }, allocatorId) => { - const { rows } = await pg.evaluate.query(` +export const fetchDailyAllocatorRSRSummary = async (pgPools, { from, to }, allocatorId) => { + const { rows } = await pgPools.evaluate.query(` SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful, diff --git a/stats/test/app.test.js b/stats/test/app.test.js index bbc8421b..e0c5a12e 100644 --- a/stats/test/app.test.js +++ b/stats/test/app.test.js @@ -10,11 +10,12 @@ describe('HTTP request handler', () => { let app /** @type {string} */ let baseUrl + let pgPools before(async () => { // Use test database connection strings - const DATABASE_URL = 'postgres://postgres:postgres@localhost:5433/spark_stats' - const EVALUATE_DB_URL = 'postgres://postgres:postgres@localhost:5433/spark_evaluate' + const DATABASE_URL = 'postgres://postgres:postgres@localhost:5432/spark_stats' + const EVALUATE_DB_URL = 'postgres://postgres:postgres@localhost:5432/spark_evaluate' app = await createApp({ SPARK_API_BASE_URL: 'https://api.filspark.com/', @@ -26,6 +27,7 @@ describe('HTTP request handler', () => { : 'error' } }) + pgPools = app.pg baseUrl = await app.listen() }) @@ -35,14 +37,14 @@ describe('HTTP request handler', () => { }) beforeEach(async () => { - await app.pg.evaluate.query('DELETE FROM retrieval_stats') - await app.pg.evaluate.query('DELETE FROM daily_participants') - await app.pg.evaluate.query('DELETE FROM daily_deals') - await app.pg.evaluate.query('DELETE FROM retrieval_timings') - await app.pg.stats.query('DELETE FROM daily_scheduled_rewards') - await app.pg.stats.query('DELETE FROM daily_reward_transfers') - await app.pg.stats.query('DELETE FROM daily_retrieval_result_codes') - await app.pg.evaluate.query('DELETE FROM daily_client_retrieval_stats') + await pgPools.evaluate.query('DELETE FROM retrieval_stats') + await pgPools.evaluate.query('DELETE FROM daily_participants') + await pgPools.evaluate.query('DELETE FROM daily_deals') + await pgPools.evaluate.query('DELETE FROM retrieval_timings') + await pgPools.stats.query('DELETE FROM daily_scheduled_rewards') + await pgPools.stats.query('DELETE FROM daily_reward_transfers') + await pgPools.stats.query('DELETE FROM daily_retrieval_result_codes') + await pgPools.evaluate.query('DELETE FROM daily_client_retrieval_stats') }) it('returns 200 for GET /', async () => { @@ -62,12 +64,12 @@ describe('HTTP request handler', () => { describe('GET /retrieval-success-rate', () => { beforeEach(async () => { - await app.pg.evaluate.query('DELETE FROM retrieval_stats') + await pgPools.evaluate.query('DELETE FROM retrieval_stats') }) it('returns today stats for no query string', async () => { const day = today() - await givenRetrievalStats(app.pg.evaluate, { day, total: 10, successful: 1, successfulHttp: 0, successfulHttpHead: 0 }) + await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, successfulHttp: 0, successfulHttpHead: 0 }) const res = await fetch(new URL('/retrieval-success-rate', baseUrl), { redirect: 'follow' }) await assertResponseStatus(res, 200) const stats = await res.json() @@ -77,10 +79,10 @@ describe('HTTP request handler', () => { }) it('applies from & to in YYYY-MM-DD format', async () => { - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-10', total: 10, successful: 1, successfulHttp: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-11', total: 20, successful: 1, successfulHttp: 0 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-12', total: 30, successful: 3, successfulHttp: 3 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-13', total: 40, successful: 1, successfulHttp: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 1, successfulHttp: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', total: 20, successful: 1, successfulHttp: 0 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', total: 30, successful: 3, successfulHttp: 3 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-13', total: 40, successful: 1, successfulHttp: 1 }) const res = await fetch( new URL( @@ -137,10 +139,10 @@ describe('HTTP request handler', () => { }) it('sums daily retrievals from all miners', async () => { - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50, successfulHttp: 35, successfulHttpHead: 35 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0, successfulHttpHead: 0 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 60, successfulHttp: 50, successfulHttpHead: 50 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50, successfulHttp: 35, successfulHttpHead: 35 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0, successfulHttpHead: 0 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 60, successfulHttp: 50, successfulHttpHead: 50 }) const res = await fetch( new URL( @@ -162,8 +164,8 @@ describe('HTTP request handler', () => { }) it('sorts items by date ascending', async () => { - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-10', total: 10, successful: 5, successfulHttp: 3, successfulHttpHead: 3 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 5, successfulHttp: 3, successfulHttpHead: 3 }) const res = await fetch( new URL( @@ -184,8 +186,8 @@ describe('HTTP request handler', () => { }) it('filters out miners with zero RSR when asked', async () => { - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-20', total: 10, successful: 1, minerId: 'f1one', successfulHttp: 1, successfulHttpHead: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-20', total: 10, successful: 0, minerId: 'f1two', successfulHttp: 0, successfulHttpHead: 0 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, minerId: 'f1one', successfulHttp: 1, successfulHttpHead: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 0, minerId: 'f1two', successfulHttp: 0, successfulHttpHead: 0 }) const res = await fetch( new URL( @@ -206,8 +208,8 @@ describe('HTTP request handler', () => { it('preserves additional query string arguments when redirecting', async () => { const day = today() - await givenRetrievalStats(app.pg.evaluate, { day, total: 10, successful: 1, minerId: 'f1one', successfulHttp: 1, successfulHttpHead: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day, total: 10, successful: 0, minerId: 'f1two', successfulHttp: 0, successfulHttpHead: 0 }) + await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, minerId: 'f1one', successfulHttp: 1, successfulHttpHead: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 0, minerId: 'f1two', successfulHttp: 0, successfulHttpHead: 0 }) const res = await fetch(new URL('/retrieval-success-rate?nonZero=true', baseUrl), { redirect: 'follow' }) await assertResponseStatus(res, 200) const stats = await res.json() @@ -216,9 +218,9 @@ describe('HTTP request handler', () => { ]) }) it('handles successful_http values 0, null, undefined', async () => { - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 0 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-21', total: 10, successful: 1, successfulHttp: undefined }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-22', total: 10, successful: 1, successfulHttp: null }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 0 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', total: 10, successful: 1, successfulHttp: undefined }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', total: 10, successful: 1, successfulHttp: null }) const res = await fetch( new URL( @@ -240,10 +242,10 @@ describe('HTTP request handler', () => { describe('GET /participants/daily', () => { it('returns daily active participants for the given date range', async () => { - await givenDailyParticipants(app.pg.evaluate, '2024-01-10', ['0x10', '0x20']) - await givenDailyParticipants(app.pg.evaluate, '2024-01-11', ['0x10', '0x20', '0x30']) - await givenDailyParticipants(app.pg.evaluate, '2024-01-12', ['0x10', '0x20', '0x40', '0x50']) - await givenDailyParticipants(app.pg.evaluate, '2024-01-13', ['0x10']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-10', ['0x10', '0x20']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-11', ['0x10', '0x20', '0x30']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-12', ['0x10', '0x20', '0x40', '0x50']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-13', ['0x10']) const res = await fetch( new URL( @@ -265,14 +267,14 @@ describe('HTTP request handler', () => { describe('GET /participants/monthly', () => { it('returns monthly active participants for the given date range ignoring the day number', async () => { // before the range - await givenDailyParticipants(app.pg.evaluate, '2023-12-31', ['0x01', '0x02']) + await givenDailyParticipants(pgPools.evaluate, '2023-12-31', ['0x01', '0x02']) // in the range - await givenDailyParticipants(app.pg.evaluate, '2024-01-10', ['0x10', '0x20']) - await givenDailyParticipants(app.pg.evaluate, '2024-01-11', ['0x10', '0x20', '0x30']) - await givenDailyParticipants(app.pg.evaluate, '2024-01-12', ['0x10', '0x20', '0x40', '0x50']) - await givenDailyParticipants(app.pg.evaluate, '2024-02-13', ['0x10', '0x60']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-10', ['0x10', '0x20']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-11', ['0x10', '0x20', '0x30']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-12', ['0x10', '0x20', '0x40', '0x50']) + await givenDailyParticipants(pgPools.evaluate, '2024-02-13', ['0x10', '0x60']) // after the range - await givenDailyParticipants(app.pg.evaluate, '2024-03-01', ['0x99']) + await givenDailyParticipants(pgPools.evaluate, '2024-03-01', ['0x99']) const res = await fetch( new URL( @@ -294,19 +296,19 @@ describe('HTTP request handler', () => { describe('GET /participants/change-rates', () => { it('returns monthly change rates for the given date range ignoring the day number', async () => { // before the range - await givenDailyParticipants(app.pg.evaluate, '2023-12-31', ['0x01', '0x02']) + await givenDailyParticipants(pgPools.evaluate, '2023-12-31', ['0x01', '0x02']) // the last month before the range - await givenDailyParticipants(app.pg.evaluate, '2024-01-10', ['0x10', '0x20']) - await givenDailyParticipants(app.pg.evaluate, '2024-01-11', ['0x10', '0x20', '0x30']) - await givenDailyParticipants(app.pg.evaluate, '2024-01-12', ['0x10', '0x20', '0x40', '0x50']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-10', ['0x10', '0x20']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-11', ['0x10', '0x20', '0x30']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-12', ['0x10', '0x20', '0x40', '0x50']) // the first month in the range - 0x50 is gone - await givenDailyParticipants(app.pg.evaluate, '2024-02-11', ['0x10', '0x20']) - await givenDailyParticipants(app.pg.evaluate, '2024-02-13', ['0x20', '0x30', '0x40']) + await givenDailyParticipants(pgPools.evaluate, '2024-02-11', ['0x10', '0x20']) + await givenDailyParticipants(pgPools.evaluate, '2024-02-13', ['0x20', '0x30', '0x40']) // the second month in the range - 0x30 and 0x40 is gone, new participant 0x60 - await givenDailyParticipants(app.pg.evaluate, '2024-03-11', ['0x10', '0x20']) - await givenDailyParticipants(app.pg.evaluate, '2024-03-13', ['0x10', '0x60']) + await givenDailyParticipants(pgPools.evaluate, '2024-03-11', ['0x10', '0x20']) + await givenDailyParticipants(pgPools.evaluate, '2024-03-13', ['0x10', '0x60']) // after the range - await givenDailyParticipants(app.pg.evaluate, '2024-04-01', ['0x99']) + await givenDailyParticipants(pgPools.evaluate, '2024-04-01', ['0x99']) const res = await fetch( new URL( @@ -346,11 +348,11 @@ describe('HTTP request handler', () => { it('handles a single-month range', async () => { // the last month before the range - await givenDailyParticipants(app.pg.evaluate, '2024-01-10', ['0x10', '0x20']) + await givenDailyParticipants(pgPools.evaluate, '2024-01-10', ['0x10', '0x20']) // the only month in the range - 0x20 is gone - await givenDailyParticipants(app.pg.evaluate, '2024-02-11', ['0x10']) + await givenDailyParticipants(pgPools.evaluate, '2024-02-11', ['0x10']) // after the range - await givenDailyParticipants(app.pg.evaluate, '2024-03-01', ['0x99']) + await givenDailyParticipants(pgPools.evaluate, '2024-03-01', ['0x99']) const res = await fetch( new URL( @@ -373,7 +375,7 @@ describe('HTTP request handler', () => { describe('GET /participant/:address/scheduled-rewards', () => { it('returns daily scheduled rewards for the given date range', async () => { - await app.pg.stats.query( + await pgPools.stats.query( 'INSERT INTO daily_scheduled_rewards (day, participant_address, scheduled_rewards) VALUES ($1, $2, $3)', ['2024-01-11', '0x20', '1'] ) @@ -396,7 +398,7 @@ describe('HTTP request handler', () => { describe('GET /participant/:address/reward-transfers', () => { it('returns daily reward transfers for the given date range', async () => { - await app.pg.stats.query(` + await pgPools.stats.query(` INSERT INTO daily_reward_transfers (day, to_address, amount, last_checked_block) VALUES @@ -422,14 +424,14 @@ describe('HTTP request handler', () => { describe('GET /miners/retrieval-success-rate/summary', () => { it('returns a summary of miners RSR for the given date range', async () => { // before the range - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 20, successfulHttp: 10, successfulHttpHead: 10 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 20, successfulHttp: 10, successfulHttpHead: 10 }) // in the range - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0, successfulHttpHead: 0 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 150, successfulHttp: 100, successfulHttpHead: 100 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0, successfulHttpHead: 0 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 150, successfulHttp: 100, successfulHttpHead: 100 }) // after the range - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-12', minerId: 'f1one', total: 30, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-12', minerId: 'f1two', total: 300, successful: 60, successfulHttp: 60, successfulHttpHead: 60 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1one', total: 30, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1two', total: 300, successful: 60, successfulHttp: 60, successfulHttpHead: 60 }) const res = await fetch( new URL( @@ -447,11 +449,11 @@ describe('HTTP request handler', () => { ]) }) it('handles successful_http values 0, null, undefined', async () => { - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 0 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 10, successful: 1, successfulHttp: undefined }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-22', minerId: 'f1one', total: 10, successful: 1, successfulHttp: null }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-23', minerId: 'f2two', total: 10, successful: 1, successfulHttp: undefined }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-24', minerId: 'f3three', total: 20, successful: 2, successfulHttp: null }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 0 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 10, successful: 1, successfulHttp: undefined }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', minerId: 'f1one', total: 10, successful: 1, successfulHttp: null }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-23', minerId: 'f2two', total: 10, successful: 1, successfulHttp: undefined }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-24', minerId: 'f3three', total: 20, successful: 2, successfulHttp: null }) let res = await fetch( new URL( @@ -488,7 +490,7 @@ describe('HTTP request handler', () => { describe('GET /retrieval-result-codes/daily', () => { it('returns daily retrieval result codes for the given date range', async () => { - await app.pg.stats.query(` + await pgPools.stats.query(` INSERT INTO daily_retrieval_result_codes (day, code, rate) VALUES @@ -548,8 +550,8 @@ describe('HTTP request handler', () => { describe('GET /deals/daily', () => { it('returns daily deal stats for the given date range', async () => { - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-10', tested: 10, indexed: 5, retrievable: 1 }) - await givenDailyDealStats(app.pg.evaluate, { + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-10', tested: 10, indexed: 5, retrievable: 1 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-11', tested: 20, indexMajorityFound: 10, @@ -558,8 +560,8 @@ describe('HTTP request handler', () => { retrievalMajorityFound: 5, retrievable: 2 }) - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-12', tested: 30, indexed: 7, retrievable: 3 }) - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-13', tested: 40, indexed: 8, retrievable: 4 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-12', tested: 30, indexed: 7, retrievable: 3 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-13', tested: 40, indexed: 8, retrievable: 4 }) const res = await fetch( new URL( @@ -594,10 +596,10 @@ describe('HTTP request handler', () => { }) it('aggregates stats over miners', async () => { - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-11', minerId: 'f1aa', tested: 10 }) - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-11', minerId: 'f1bb', tested: 20 }) - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-12', minerId: 'f1aa', tested: 30 }) - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-12', minerId: 'f1bb', tested: 40 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1aa', tested: 10 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1bb', tested: 20 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1aa', tested: 30 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1bb', tested: 40 }) const res = await fetch( new URL( @@ -622,10 +624,10 @@ describe('HTTP request handler', () => { }) it('aggregates stats over clients', async () => { - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-11', clientId: 'f1aa', tested: 10 }) - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-11', clientId: 'f1bb', tested: 20 }) - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-12', clientId: 'f1aa', tested: 30 }) - await givenDailyDealStats(app.pg.evaluate, { day: '2024-01-12', minerId: 'f1bb', tested: 40 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-11', clientId: 'f1aa', tested: 10 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-11', clientId: 'f1bb', tested: 20 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-12', clientId: 'f1aa', tested: 30 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1bb', tested: 40 }) const res = await fetch( new URL( @@ -652,16 +654,16 @@ describe('HTTP request handler', () => { describe('GET /deals/summary', () => { it('returns deal summary for the given date range (including the end day)', async () => { - await givenDailyDealStats(app.pg.evaluate, { day: '2024-03-12', tested: 200, indexed: 52, retrievable: 2 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-12', tested: 200, indexed: 52, retrievable: 2 }) // filter.to - 7 days -> should be excluded - await givenDailyDealStats(app.pg.evaluate, { day: '2024-03-23', tested: 300, indexed: 53, retrievable: 3 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-23', tested: 300, indexed: 53, retrievable: 3 }) // last 7 days - await givenDailyDealStats(app.pg.evaluate, { day: '2024-03-24', tested: 400, indexed: 54, retrievable: 4 }) - await givenDailyDealStats(app.pg.evaluate, { day: '2024-03-29', tested: 500, indexed: 55, retrievable: 5 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-24', tested: 400, indexed: 54, retrievable: 4 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-29', tested: 500, indexed: 55, retrievable: 5 }) // `filter.to` (e.g. today) - should be included - await givenDailyDealStats(app.pg.evaluate, { day: '2024-03-30', tested: 6000, indexed: 600, retrievable: 60 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-30', tested: 6000, indexed: 600, retrievable: 60 }) // after the requested range - await givenDailyDealStats(app.pg.evaluate, { day: '2024-03-31', tested: 70000, indexed: 7000, retrievable: 700 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-31', tested: 70000, indexed: 7000, retrievable: 700 }) const res = await fetch( new URL( @@ -729,16 +731,16 @@ describe('HTTP request handler', () => { describe('GET /miner/{id}/retrieval-success-rate/summary', () => { it('lists daily retrieval stats summary for specified miner in given date range', async () => { // before the range - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-09', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-09', minerId: 'f1two', total: 100, successful: 20, successfulHttp: 10, successfulHttpHead: 10 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1two', total: 100, successful: 20, successfulHttp: 10, successfulHttpHead: 10 }) // in the range - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0, successfulHttpHead: 0 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-20', minerId: 'f1two', total: 200, successful: 60, successfulHttp: 50, successfulHttpHead: 50 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50, successfulHttp: 35, successfulHttpHead: 35 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0, successfulHttpHead: 0 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1two', total: 200, successful: 60, successfulHttp: 50, successfulHttpHead: 50 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50, successfulHttp: 35, successfulHttpHead: 35 }) // after the range - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 30, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) - await givenRetrievalStats(app.pg.evaluate, { day: '2024-01-21', minerId: 'f1two', total: 300, successful: 60, successfulHttp: 60, successfulHttpHead: 60 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 30, successful: 1, successfulHttp: 1, successfulHttpHead: 1 }) + await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1two', total: 300, successful: 60, successfulHttp: 60, successfulHttpHead: 60 }) const res = await fetch( new URL( @@ -763,17 +765,17 @@ describe('HTTP request handler', () => { describe('miner retrieval timing stats', () => { beforeEach(async () => { // before the range - await givenRetrievalTimings(app.pg.evaluate, { day: '2024-01-09', minerId: 'f1one', timeToFirstByteP50: [1000] }) - await givenRetrievalTimings(app.pg.evaluate, { day: '2024-01-09', minerId: 'f1two', timeToFirstByteP50: [1000] }) + await givenRetrievalTimings(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1one', timeToFirstByteP50: [1000] }) + await givenRetrievalTimings(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1two', timeToFirstByteP50: [1000] }) // in the range - await givenRetrievalTimings(app.pg.evaluate, { day: '2024-01-20', minerId: 'f1one', timeToFirstByteP50: [1000] }) - await givenRetrievalTimings(app.pg.evaluate, { day: '2024-01-20', minerId: 'f1two', timeToFirstByteP50: [1000] }) + await givenRetrievalTimings(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', timeToFirstByteP50: [1000] }) + await givenRetrievalTimings(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1two', timeToFirstByteP50: [1000] }) - await givenRetrievalTimings(app.pg.evaluate, { day: '2024-01-10', minerId: 'f1one', timeToFirstByteP50: [123, 345] }) - await givenRetrievalTimings(app.pg.evaluate, { day: '2024-01-10', minerId: 'f1two', timeToFirstByteP50: [654, 789] }) + await givenRetrievalTimings(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', timeToFirstByteP50: [123, 345] }) + await givenRetrievalTimings(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', timeToFirstByteP50: [654, 789] }) // after the range - await givenRetrievalTimings(app.pg.evaluate, { day: '2024-01-21', minerId: 'f1one', timeToFirstByteP50: [1000] }) - await givenRetrievalTimings(app.pg.evaluate, { day: '2024-01-21', minerId: 'f1two', timeToFirstByteP50: [1000] }) + await givenRetrievalTimings(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', timeToFirstByteP50: [1000] }) + await givenRetrievalTimings(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1two', timeToFirstByteP50: [1000] }) }) it('lists daily retrieval timings in given date range', async () => { @@ -837,18 +839,18 @@ describe('HTTP request handler', () => { }) describe('GET /clients/retrieval-success-rate/summary', () => { beforeEach(async () => { - await app.pg.evaluate.query('DELETE FROM daily_client_retrieval_stats') + await pgPools.evaluate.query('DELETE FROM daily_client_retrieval_stats') }) it('returns a summary of clients RSR for the given date range', async () => { // before the range - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-10', clientId: 'f1oneClient', total: 10, successful: 1, successfulHttp: 1 }) - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-10', clientId: 'f1twoClient', total: 100, successful: 20, successfulHttp: 10 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-10', clientId: 'f1oneClient', total: 10, successful: 1, successfulHttp: 1 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-10', clientId: 'f1twoClient', total: 100, successful: 20, successfulHttp: 10 }) // in the range - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-11', clientId: 'f1oneClient', total: 20, successful: 1, successfulHttp: 0 }) - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-11', clientId: 'f1twoClient', total: 200, successful: 150, successfulHttp: 100 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-11', clientId: 'f1oneClient', total: 20, successful: 1, successfulHttp: 0 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-11', clientId: 'f1twoClient', total: 200, successful: 150, successfulHttp: 100 }) // after the range - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-12', clientId: 'f1oneClient', total: 30, successful: 1, successfulHttp: 1 }) - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-12', clientId: 'f1twoClient', total: 300, successful: 60, successfulHttp: 60 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-12', clientId: 'f1oneClient', total: 30, successful: 1, successfulHttp: 1 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-12', clientId: 'f1twoClient', total: 300, successful: 60, successfulHttp: 60 }) const res = await fetch( new URL( @@ -866,8 +868,8 @@ describe('HTTP request handler', () => { ]) }) it('handles total value being smaller or equal to 0', async () => { - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-11', clientId: 'f1oneClient', total: 0, successful: 0, successfulHttp: 0 }) - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-11', clientId: 'f2twoClient', total: -1, successful: 0, successfulHttp: 0 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-11', clientId: 'f1oneClient', total: 0, successful: 0, successfulHttp: 0 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-11', clientId: 'f2twoClient', total: -1, successful: 0, successfulHttp: 0 }) const res = await fetch( new URL( @@ -887,20 +889,20 @@ describe('HTTP request handler', () => { }) describe('GET /client/{id}/retrieval-success-rate/summary', () => { beforeEach(async () => { - await app.pg.evaluate.query('DELETE FROM daily_client_retrieval_stats') + await pgPools.evaluate.query('DELETE FROM daily_client_retrieval_stats') }) it('lists daily retrieval stats summary for specified client in given date range', async () => { // before the range - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-09', clientId: 'f1oneClient', total: 10, successful: 1, successfulHttp: 1 }) - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-09', clientId: 'f1twoClient', total: 100, successful: 20, successfulHttp: 10 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-09', clientId: 'f1oneClient', total: 10, successful: 1, successfulHttp: 1 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-09', clientId: 'f1twoClient', total: 100, successful: 20, successfulHttp: 10 }) // in the range - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-20', clientId: 'f1oneClient', total: 20, successful: 1, successfulHttp: 0 }) - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-20', clientId: 'f1twoClient', total: 200, successful: 60, successfulHttp: 50 }) - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-10', clientId: 'f1oneClient', total: 10, successful: 1, successfulHttp: 1 }) - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-10', clientId: 'f1twoClient', total: 100, successful: 50, successfulHttp: 35 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-20', clientId: 'f1oneClient', total: 20, successful: 1, successfulHttp: 0 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-20', clientId: 'f1twoClient', total: 200, successful: 60, successfulHttp: 50 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-10', clientId: 'f1oneClient', total: 10, successful: 1, successfulHttp: 1 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-10', clientId: 'f1twoClient', total: 100, successful: 50, successfulHttp: 35 }) // after the range - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-21', clientId: 'f1oneClient', total: 30, successful: 1, successfulHttp: 1 }) - await givenClientRetrievalStats(app.pg.evaluate, { day: '2024-01-21', clientId: 'f1twoClient', total: 300, successful: 60, successfulHttp: 60 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-21', clientId: 'f1oneClient', total: 30, successful: 1, successfulHttp: 1 }) + await givenClientRetrievalStats(pgPools.evaluate, { day: '2024-01-21', clientId: 'f1twoClient', total: 300, successful: 60, successfulHttp: 60 }) const res = await fetch( new URL( @@ -923,18 +925,18 @@ describe('HTTP request handler', () => { }) describe('GET /allocators/retrieval-success-rate/summary', () => { beforeEach(async () => { - await app.pg.evaluate.query('DELETE FROM daily_allocator_retrieval_stats') + await pgPools.evaluate.query('DELETE FROM daily_allocator_retrieval_stats') }) it('returns a summary of allocators RSR for the given date range', async () => { // before the range - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-10', allocatorId: 'f1oneAllocator', total: 10, successful: 1, successfulHttp: 1 }) - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-10', allocatorId: 'f1twoAllocator', total: 100, successful: 20, successfulHttp: 10 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-10', allocatorId: 'f1oneAllocator', total: 10, successful: 1, successfulHttp: 1 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-10', allocatorId: 'f1twoAllocator', total: 100, successful: 20, successfulHttp: 10 }) // in the range - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-11', allocatorId: 'f1oneAllocator', total: 20, successful: 1, successfulHttp: 0 }) - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-11', allocatorId: 'f2twoAllocator', total: 200, successful: 150, successfulHttp: 100 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-11', allocatorId: 'f1oneAllocator', total: 20, successful: 1, successfulHttp: 0 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-11', allocatorId: 'f2twoAllocator', total: 200, successful: 150, successfulHttp: 100 }) // after the range - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-12', allocatorId: 'f1oneAllocator', total: 30, successful: 1, successfulHttp: 1 }) - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-12', allocatorId: 'f2twoAllocator', total: 300, successful: 60, successfulHttp: 60 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-12', allocatorId: 'f1oneAllocator', total: 30, successful: 1, successfulHttp: 1 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-12', allocatorId: 'f2twoAllocator', total: 300, successful: 60, successfulHttp: 60 }) const res = await fetch( new URL( @@ -952,8 +954,8 @@ describe('HTTP request handler', () => { ]) }) it('handles total value being smaller or equal to 0', async () => { - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-11', allocatorId: 'f1oneAllocator', total: 0, successful: 0, successfulHttp: 0 }) - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-11', allocatorId: 'f2twoAllocator', total: -1, successful: 0, successfulHttp: 0 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-11', allocatorId: 'f1oneAllocator', total: 0, successful: 0, successfulHttp: 0 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-11', allocatorId: 'f2twoAllocator', total: -1, successful: 0, successfulHttp: 0 }) const res = await fetch( new URL( @@ -973,20 +975,20 @@ describe('HTTP request handler', () => { }) describe('GET /allocator/{id}/retrieval-success-rate/summary', () => { beforeEach(async () => { - await app.pg.evaluate.query('DELETE FROM daily_allocator_retrieval_stats') + await pgPools.evaluate.query('DELETE FROM daily_allocator_retrieval_stats') }) it('lists daily retrieval stats summary for specified allocator in given date range', async () => { // before the range - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-09', allocatorId: 'f1oneAllocator', total: 10, successful: 1, successfulHttp: 1 }) - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-09', allocatorId: 'f2twoAllocator', total: 100, successful: 20, successfulHttp: 10 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-09', allocatorId: 'f1oneAllocator', total: 10, successful: 1, successfulHttp: 1 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-09', allocatorId: 'f2twoAllocator', total: 100, successful: 20, successfulHttp: 10 }) // in the range - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-20', allocatorId: 'f1oneAllocator', total: 20, successful: 1, successfulHttp: 0 }) - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-20', allocatorId: 'f2twoAllocator', total: 200, successful: 60, successfulHttp: 50 }) - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-10', allocatorId: 'f1oneAllocator', total: 10, successful: 1, successfulHttp: 1 }) - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-10', allocatorId: 'f2twoAllocator', total: 100, successful: 50, successfulHttp: 35 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-20', allocatorId: 'f1oneAllocator', total: 20, successful: 1, successfulHttp: 0 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-20', allocatorId: 'f2twoAllocator', total: 200, successful: 60, successfulHttp: 50 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-10', allocatorId: 'f1oneAllocator', total: 10, successful: 1, successfulHttp: 1 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-10', allocatorId: 'f2twoAllocator', total: 100, successful: 50, successfulHttp: 35 }) // after the range - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-21', allocatorId: 'f1oneAllocator', total: 30, successful: 1, successfulHttp: 1 }) - await givenAllocatorRetrievalStats(app.pg.evaluate, { day: '2024-01-21', allocatorId: 'f2twoAllocator', total: 300, successful: 60, successfulHttp: 60 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-21', allocatorId: 'f1oneAllocator', total: 30, successful: 1, successfulHttp: 1 }) + await givenAllocatorRetrievalStats(pgPools.evaluate, { day: '2024-01-21', allocatorId: 'f2twoAllocator', total: 300, successful: 60, successfulHttp: 60 }) const res = await fetch( new URL( diff --git a/stats/test/platform-routes.test.js b/stats/test/platform-routes.test.js index dd2c9ba3..644c19e9 100644 --- a/stats/test/platform-routes.test.js +++ b/stats/test/platform-routes.test.js @@ -10,11 +10,12 @@ describe('Platform Routes HTTP request handler', () => { let app /** @type {string} */ let baseUrl + let pgPools before(async () => { // Use test database connection strings - const DATABASE_URL = 'postgres://postgres:postgres@localhost:5433/spark_stats' - const EVALUATE_DB_URL = 'postgres://postgres:postgres@localhost:5433/spark_evaluate' + const DATABASE_URL = 'postgres://postgres:postgres@localhost:5432/spark_stats' + const EVALUATE_DB_URL = 'postgres://postgres:postgres@localhost:5432/spark_evaluate' // Await the app creation since it's async app = await createApp({ @@ -27,7 +28,7 @@ describe('Platform Routes HTTP request handler', () => { : 'error' } }) - + pgPools = app.pg baseUrl = await app.listen({ port: 0 }) // Use random port for tests }) @@ -36,18 +37,18 @@ describe('Platform Routes HTTP request handler', () => { }) beforeEach(async () => { - await app.pg.evaluate.query('DELETE FROM recent_station_details') - await app.pg.evaluate.query('DELETE FROM recent_participant_subnets') - await app.pg.evaluate.query('DELETE FROM daily_participants') - await app.pg.evaluate.query('DELETE FROM participants') - await app.pg.evaluate.query('DELETE FROM monthly_active_station_count') - await app.pg.evaluate.query('DELETE FROM daily_platform_stats') - - await app.pg.evaluate.query('REFRESH MATERIALIZED VIEW top_measurement_participants_yesterday_mv') - - await app.pg.stats.query('DELETE FROM daily_reward_transfers') - await app.pg.stats.query('DELETE FROM daily_scheduled_rewards') - await app.pg.stats.query('DELETE FROM daily_desktop_users') + await pgPools.evaluate.query('DELETE FROM recent_station_details') + await pgPools.evaluate.query('DELETE FROM recent_participant_subnets') + await pgPools.evaluate.query('DELETE FROM daily_participants') + await pgPools.evaluate.query('DELETE FROM participants') + await pgPools.evaluate.query('DELETE FROM monthly_active_station_count') + await pgPools.evaluate.query('DELETE FROM daily_platform_stats') + + await pgPools.evaluate.query('REFRESH MATERIALIZED VIEW top_measurement_participants_yesterday_mv') + + await pgPools.stats.query('DELETE FROM daily_reward_transfers') + await pgPools.stats.query('DELETE FROM daily_scheduled_rewards') + await pgPools.stats.query('DELETE FROM daily_desktop_users') }) // Helper functions updated to use app's database connections @@ -61,7 +62,7 @@ describe('Platform Routes HTTP request handler', () => { inet_group_count: row.inet_group_count ?? 8 })) - await app.pg.evaluate.query(` + await pgPools.evaluate.query(` INSERT INTO daily_platform_stats ( day, accepted_measurement_count, @@ -89,7 +90,7 @@ describe('Platform Routes HTTP request handler', () => { } const givenMonthlyActiveStationCount = async (month, stationCount) => { - await app.pg.evaluate.query(` + await pgPools.evaluate.query(` INSERT INTO monthly_active_station_count (month, station_count) VALUES ($1, $2) ON CONFLICT DO NOTHING @@ -100,7 +101,7 @@ describe('Platform Routes HTTP request handler', () => { } const givenDailyRewardTransferMetrics = async (day, transferStats) => { - await app.pg.stats.query(` + await pgPools.stats.query(` INSERT INTO daily_reward_transfers (day, to_address, amount, last_checked_block) SELECT $1 AS day, UNNEST($2::text[]) AS to_address, UNNEST($3::int[]) AS amount, UNNEST($4::int[]) AS last_checked_block ON CONFLICT DO NOTHING @@ -116,7 +117,7 @@ describe('Platform Routes HTTP request handler', () => { const givenDailyParticipants = async (day, participantAddresses) => { // This is a simplified implementation - you may need to adjust based on your actual schema for (const address of participantAddresses) { - await app.pg.evaluate.query(` + await pgPools.evaluate.query(` WITH participant AS ( INSERT INTO participants (participant_address) VALUES ($1) @@ -131,7 +132,7 @@ describe('Platform Routes HTTP request handler', () => { } const givenDailyDesktopUsers = async (day, count) => { - await app.pg.stats.query(` + await pgPools.stats.query(` INSERT INTO daily_desktop_users (day, user_count) VALUES ($1, $2) ON CONFLICT DO NOTHING @@ -221,14 +222,14 @@ describe('Platform Routes HTTP request handler', () => { it('returns top measurement stations for the given date', async () => { const day = yesterday() - await app.pg.evaluate.query(` + await pgPools.evaluate.query(` INSERT INTO participants (id, participant_address) VALUES (1, 'f1abcdef'), (2, 'f1ghijkl'), (3, 'f1mnopqr') `) - await app.pg.evaluate.query(` + await pgPools.evaluate.query(` INSERT INTO recent_station_details (day, participant_id, station_id, accepted_measurement_count, total_measurement_count) VALUES ($1, 1, 'station1', 20, 25), ($1, 1, 'station2', 20, 25), @@ -238,7 +239,7 @@ describe('Platform Routes HTTP request handler', () => { ($1, 3, 'station6', 10, 15) `, [day]) - await app.pg.evaluate.query(` + await pgPools.evaluate.query(` INSERT INTO recent_participant_subnets (day, participant_id, subnet) VALUES ($1, 1, 'subnet1'), ($1, 1, 'subnet2'), @@ -249,7 +250,7 @@ describe('Platform Routes HTTP request handler', () => { `, [day]) // Refresh the materialized view - await app.pg.evaluate.query('REFRESH MATERIALIZED VIEW top_measurement_participants_yesterday_mv') + await pgPools.evaluate.query('REFRESH MATERIALIZED VIEW top_measurement_participants_yesterday_mv') const res = await fetch( new URL( @@ -367,7 +368,7 @@ describe('Platform Routes HTTP request handler', () => { const oneWeekAgo = getLocalDayAsISOString(new Date(Date.now() - 7 * 24 * 60 * 60 * 1000)) const setupScheduledRewardsData = async () => { - await app.pg.stats.query(` + await pgPools.stats.query(` INSERT INTO daily_scheduled_rewards (day, participant_address, scheduled_rewards) VALUES ('${yesterday()}', 'address1', 10), From 97ac2bd5aa2b349e2573cb37499a7422478e0ab7 Mon Sep 17 00:00:00 2001 From: godwill charles Date: Wed, 2 Apr 2025 16:25:04 +0100 Subject: [PATCH 11/11] FIX: linting issues --- stats/bin/migrate.js | 2 +- stats/lib/platform-stats-fetchers.js | 2 +- stats/lib/stats-fetchers.js | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/stats/bin/migrate.js b/stats/bin/migrate.js index ae58af09..846f4c21 100644 --- a/stats/bin/migrate.js +++ b/stats/bin/migrate.js @@ -6,4 +6,4 @@ import { const pgPools = await getPgPools() await migrateStatsDB(pgPools.stats) -await migrateEvaluateDB(pgPools.evaluate) \ No newline at end of file +await migrateEvaluateDB(pgPools.evaluate) diff --git a/stats/lib/platform-stats-fetchers.js b/stats/lib/platform-stats-fetchers.js index 6482e080..1a3e2926 100644 --- a/stats/lib/platform-stats-fetchers.js +++ b/stats/lib/platform-stats-fetchers.js @@ -8,7 +8,7 @@ import { today, yesterday } from './request-helpers.js' */ /** - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools */ const ONE_DAY = 24 * 60 * 60 * 1000 diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js index f0621a2b..d79f1fb1 100644 --- a/stats/lib/stats-fetchers.js +++ b/stats/lib/stats-fetchers.js @@ -5,7 +5,7 @@ /** * @param {import('./typings.js').DateRangeFilter & {nonZero?: 'true'}} filter - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools */ export const fetchRetrievalSuccessRate = async (pgPools, filter) => { @@ -41,7 +41,7 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => { } /** - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDailyDealStats = async (pgPools, filter) => { @@ -68,7 +68,7 @@ export const fetchDailyDealStats = async (pgPools, filter) => { } /** - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDealSummary = async (pgPools, filter) => { @@ -120,7 +120,7 @@ export const fetchMonthlyParticipants = async (pgPools, filter) => { } /** - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchParticipantChangeRates = async (pgPools, filter) => { @@ -188,7 +188,7 @@ export const fetchParticipantChangeRates = async (pgPools, filter) => { } /** - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter * @param {string} address */ @@ -202,7 +202,7 @@ export const fetchParticipantScheduledRewards = async (pgPools, { from, to }, ad } /** - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter * @param {string} address */ @@ -217,7 +217,7 @@ export const fetchParticipantRewardTransfers = async (pgPools, { from, to }, add /** * Fetches the retrieval stats summary for all miners for given date range. - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchMinersRSRSummary = async (pgPools, filter) => { @@ -251,7 +251,7 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => { /** * Fetches the retrieval stats summary for a single miner for given date range. - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter * @param {string} minerId */ @@ -440,7 +440,7 @@ export const fetchDailyClientRSRSummary = async (pgPools, { from, to }, clientId /** * Fetches the retrieval stats summary for all allocators for given date range. - * @param {FastifyPg} pgPools + * @param {FastifyPg} pgPools * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchAllocatorsRSRSummary = async (pgPools, filter) => {