diff --git a/defi/package.json b/defi/package.json index b4f1dadf7c..d47eb0ac8e 100644 --- a/defi/package.json +++ b/defi/package.json @@ -28,6 +28,7 @@ "backfill-dex": "npm run update-submodules && npm run backfill-dex-script", "backfill-dex-local": "npm run update-submodules && npm run backfill-dex-local-script", "check-adapters": "npm run update-submodules && npx ts-node src/dexVolumes/cli/checkNonEnabledDexs.ts", + "checkDataDimensions": "npx ts-node src/api2/scripts/checkDataDimensions.ts", "update-submodules": "git submodule update --init --recursive --remote --merge && npm run prebuild ", "update-metadata-file": "bash src/api2/scripts/updateMetadataScript.sh", "get-snapshot-ids": "npx ts-node src/governance/getSnapshotIds.ts", diff --git a/defi/src/adaptors/cli/backfillUtilities/backfillFunction.ts b/defi/src/adaptors/cli/backfillUtilities/backfillFunction.ts index 160c8dc088..d8f968bc4b 100644 --- a/defi/src/adaptors/cli/backfillUtilities/backfillFunction.ts +++ b/defi/src/adaptors/cli/backfillUtilities/backfillFunction.ts @@ -1,103 +1,146 @@ -import { formatTimestampAsDate } from "../../../utils/date" -import executeAsyncBackfill from "./executeAsyncBackfill" -import getBackfillEvent from "./getBackfillEvent" -import readline from 'readline'; import { AdapterType } from "@defillama/dimension-adapters/adapters/types"; -import { IJSON } from "../../data/types"; +import fs from "fs"; +import path from "path"; +import readline from "readline"; import { getStringArrUnique } from "../../utils/getAllChainsFromAdaptors"; -import sleep from "../../../utils/shared/sleep"; +import executeAsyncBackfill from "./executeAsyncBackfill"; const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout + input: process.stdin, + output: process.stdout, }); +const formatTimestampAsDate = (timestamp: number) => { + const date = new Date(timestamp * 1000); + return `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String( + date.getUTCDate() + ).padStart(2, "0")}`; +}; + export interface ICliArgs { - onlyMissing: boolean, - chain?: string, - version?: string, - timestamp?: number, - endTimestamp?: number, - recordTypes?: string[] + onlyMissing: boolean; + chain?: string; + version?: string; + timestamp?: number; + timestamps?: number[]; + endTimestamp?: number; + recordTypes?: string[]; } export const autoBackfill = async (argv: string[]) => { - if (argv.length < 3) { - console.error(`Not enough args! Please, use\nnpm run backfill \nor\nnpm run backfill onlyMissing\nor\nnpm run backfill-local`) - process.exit(1) - } - const type = argv[2] as AdapterType - const adapterName = argv[3].split(',') as string[] - const cliArguments = argv.filter(arg => arg.includes("=")).reduce((acc, cliArg) => { - const [rawArgumentName, value] = cliArg.split("=") - if (rawArgumentName === 'onlyMissing') { - if (value === 'true' || !value) acc.onlyMissing = true - else acc['onlyMissing'] === false - } - else if (rawArgumentName === 'chain') { - if (value) acc.chain = value - else throw new Error("Please provide a value for chain=[chain]") - } - else if (rawArgumentName === 'version') { - if (value) acc.version = value - else throw new Error("Please provide a value for version=[version]") - } - else if (rawArgumentName === 'timestamp') { - if (!isNaN(+value)) acc.timestamp = +value - else throw new Error("Please provide a proper value for timestamp=[timestamp]") - } - else if (rawArgumentName === 'endTimestamp') { - if (!isNaN(+value)) acc.endTimestamp = +value - else throw new Error("Please provide a proper value for endTimestamp=[timestamp]") - } - else if (rawArgumentName === 'recordTypes') { - const recordTypes = value?.split(",") - if (recordTypes && recordTypes.length > 0) acc.recordTypes = recordTypes - else throw new Error("Please provide a value for recordType=[recordType]") + if (argv.length < 3) { + console.error( + `Not enough args! Please, use\nnpm run backfill \nor\nnpm run backfill onlyMissing\nor\nnpm run backfill-local` + ); + process.exit(1); + } + + const type = argv[2] as AdapterType; + const adapterName = argv[3].split(",") as string[]; + + const cliArguments = argv + .filter((arg) => arg.includes("=")) + .reduce( + (acc, cliArg) => { + const [rawArgumentName, value] = cliArg.split("="); + + if (rawArgumentName === "onlyMissing") { + acc.onlyMissing = value === "true" || !value ? true : false; + } else if (rawArgumentName === "chain") { + if (value) acc.chain = value; + else throw new Error("Please provide a value for chain=[chain]"); + } else if (rawArgumentName === "version") { + if (value) acc.version = value; + else throw new Error("Please provide a value for version=[version]"); + } else if (rawArgumentName === "timestamps") { + const timestampsArray = value.split(",").map((ts) => parseInt(ts, 10)); + if (timestampsArray.length > 0) acc.timestamps = timestampsArray; + else throw new Error("Please provide valid timestamps as a comma-separated list"); + } else if (rawArgumentName === "timestamp") { + if (!isNaN(+value)) acc.timestamp = +value; + else throw new Error("Please provide a proper value for timestamp=[timestamp]"); + } else if (rawArgumentName === "endTimestamp") { + if (!isNaN(+value)) acc.endTimestamp = +value; + else throw new Error("Please provide a proper value for endTimestamp=[timestamp]"); + } else if (rawArgumentName === "recordTypes") { + const recordTypes = value?.split(","); + if (recordTypes && recordTypes.length > 0) acc.recordTypes = recordTypes; + else throw new Error("Please provide a value for recordType=[recordType]"); } - return acc - }, { - onlyMissing: false - } as ICliArgs) - console.info(`Started backfilling for ${adapterName} adapter`) - console.info(`Generating backfill event...`) - const backfillEvent = await getBackfillEvent(adapterName, type, cliArguments) - console.info(`Backfill event generated!`) - if (!backfillEvent.backfill || backfillEvent.backfill.length <= 0) { - console.info("Has been generated an empty event, nothing to backfill...") - rl.close(); - return + + return acc; + }, + { + onlyMissing: false, + } as ICliArgs + ); + + if (cliArguments.timestamp && !cliArguments.timestamps) { + const todayTimestamp = Math.floor(Date.now() / 1000); + cliArguments.endTimestamp = cliArguments.endTimestamp || todayTimestamp; + + cliArguments.timestamps = []; + for (let ts = cliArguments.timestamp; ts <= cliArguments.endTimestamp; ts += 86400) { + cliArguments.timestamps.push(ts); } - const uniqueModules2Backfill = getStringArrUnique([...backfillEvent.backfill.map(n => n.dexNames).flat(1)]) - console.info(uniqueModules2Backfill.length, "protocols will be backfilled") - console.info(`${uniqueModules2Backfill} will be backfilled starting from ${formatTimestampAsDate(String(backfillEvent.backfill[0].timestamp!))}`) - console.info(`${backfillEvent.backfill.length} days will be filled. If a chain is already available will be refilled.`) - console.info(`With the following parameters:\nChain: ${backfillEvent.backfill[0].chain}\nRecord types: ${backfillEvent.backfill[0].adaptorRecordTypes}\nVersion: ${backfillEvent.backfill[0].protocolVersion}`) - if (argv[0] !== '') - rl.question('Do you wish to continue? y/n\n', async function (yn) { - if (yn.toLowerCase() === 'y') { - backfillEvent.backfill = backfillEvent.backfill ? backfillEvent.backfill?.reverse() as any[] : [] - await executeAsyncBackfill(backfillEvent) - /* for (let i = 70; i < backfillEvent.backfill.length; i += 50) { - const smallbackfillEvent = { - ...backfillEvent, - backfill: backfillEvent.backfill.slice(i, i + 50), - }; - console.info(`${smallbackfillEvent.backfill[0].dexNames[0].toUpperCase()} will be backfilled starting from ${formatTimestampAsDate(String(smallbackfillEvent.backfill[0].timestamp!))}`) - console.info(`${smallbackfillEvent.backfill.length} days will be filled. If a chain is already available will be refilled.`) - await executeAsyncBackfill(smallbackfillEvent) - await sleep(1000 * 60 * 2) - } */ - console.info(`Don't forget to enable the adapter to src/adaptors/data/${type}/config.ts, bye llama🦙`) - rl.close(); - } - else { - console.info("Backfill cancelled... bye llama🦙") - rl.close(); - } - }); - else { - rl.close(); - await executeAsyncBackfill(backfillEvent) + } + + if (!cliArguments.timestamps && !cliArguments.timestamp && !cliArguments.endTimestamp) { + console.warn("No specific timestamp provided. Using last 30 days by default."); + + const today = Math.floor(Date.now() / 1000); + const thirtyDaysAgo = today - 86400 * 30; + cliArguments.timestamps = []; + + for (let ts = thirtyDaysAgo; ts <= today; ts += 86400) { + cliArguments.timestamps.push(ts); } -} + } + + console.info(`Started backfilling for ${adapterName} adapter`); + + if (cliArguments.timestamps && cliArguments.timestamps.length > 0) { + const backfillEvent = { + type: type, + backfill: cliArguments.timestamps.map((timestamp) => ({ + dexNames: adapterName, + timestamp, + })), + }; + + const backfillFilePath = path.join(__dirname, "output", "backfill_event.json"); + fs.writeFileSync(backfillFilePath, JSON.stringify(backfillEvent, null, 2)); + console.info(`Backfill event saved to ${backfillFilePath}`); + + const uniqueModules2Backfill = getStringArrUnique([...backfillEvent.backfill.map((n) => n.dexNames).flat(1)]); + console.info(`${uniqueModules2Backfill.length} protocols will be backfilled`); + console.info( + `${uniqueModules2Backfill} will be backfilled starting from ${formatTimestampAsDate( + backfillEvent.backfill[0].timestamp! + )}` + ); + console.info( + `${backfillEvent.backfill.length} days will be filled. If a chain is already available, it will be refilled.` + ); + + rl.question("Do you wish to continue? y/n\n", async function (yn) { + if (yn.toLowerCase() === "y") { + for (const event of backfillEvent.backfill) { + console.info(`Backfilling for ${formatTimestampAsDate(event.timestamp)}...`); + await executeAsyncBackfill({ + type: backfillEvent.type, + backfill: [event], + }); + console.info(`Backfill completed for ${formatTimestampAsDate(event.timestamp)}.`); + } + console.info("Backfill completed successfully for all timestamps."); + } else { + console.info("Backfill cancelled."); + } + rl.close(); + }); + } else { + console.error("No timestamps provided!"); + rl.close(); + } +}; diff --git a/defi/src/api2/scripts/checkDataDimensions.ts b/defi/src/api2/scripts/checkDataDimensions.ts new file mode 100644 index 0000000000..ec7b33d592 --- /dev/null +++ b/defi/src/api2/scripts/checkDataDimensions.ts @@ -0,0 +1,216 @@ +import { AdapterType, ProtocolType } from "@defillama/dimension-adapters/adapters/types"; +import fs from "fs"; +import path from "path"; +import loadAdaptorsData from "../../adaptors/data"; +import { AdaptorData } from "../../adaptors/data/types"; +import { getAllItemsAfter } from "../../adaptors/db-utils/db2"; +import { DEFAULT_CHART_BY_ADAPTOR_TYPE } from "../../adaptors/handlers/getOverviewProcess"; +import { ADAPTER_TYPES } from "../../adaptors/handlers/triggerStoreAdaptorData"; + +const OUTPUT_DIR = path.join(__dirname, "reports"); +const JAN_1ST_2023_TIMESTAMP = Date.UTC(2023, 0, 1) / 1000; + +if (!fs.existsSync(OUTPUT_DIR)) { + fs.mkdirSync(OUTPUT_DIR); +} + +const CSV_HEADER = "Protocol Name,Timestamps,Adapter Type,Backfill Command\n"; +const filePaths: Record = {}; + +for (const adapterType of ADAPTER_TYPES) { + filePaths[adapterType] = { + missing: initCsvFile(adapterType, "missing"), + negative: initCsvFile(adapterType, "negative"), + }; +} + +function initCsvFile(adapterType: string, dayType: "missing" | "negative"): string { + const filePath = path.join(OUTPUT_DIR, `${adapterType}_${dayType}_days_report.csv`); + fs.writeFileSync(filePath, CSV_HEADER); + return filePath; +} + +async function run() { + const allCache: Record }> = {}; + + for (const adapterType of ADAPTER_TYPES) { + await fetchData(adapterType, allCache); + await generateSummaries(adapterType, allCache); + } + + await cleanupEmptyCsvFiles(); + process.exit(0); +} + +interface ProtocolData { + records: Record; + firstDate: string; +} + +async function fetchData( + adapterType: AdapterType, + allCache: Record }> +) { + if (!allCache[adapterType]) { + allCache[adapterType] = { protocols: {} }; + } + const adapterData = allCache[adapterType]; + + const results = await getAllItemsAfter({ adapterType, timestamp: 0 }); + const recordType = DEFAULT_CHART_BY_ADAPTOR_TYPE[adapterType]; + if (!recordType) return; + + for (const result of results) { + const { id, data, timeS } = result; + const aggData = data?.aggregated?.[recordType]?.value; + + if (!adapterData.protocols[id]) { + adapterData.protocols[id] = { records: {}, firstDate: timeS }; + } + + adapterData.protocols[id].records[timeS] = aggData; + } +} + +async function generateSummaries( + adapterType: AdapterType, + allCache: Record }> +) { + const recordType = DEFAULT_CHART_BY_ADAPTOR_TYPE[adapterType]; + if (!recordType) return; + + const dataModule = loadAdaptorsData(adapterType); + const { protocolMap } = dataModule; + const adapterData = allCache[adapterType]; + + const protocols = Object.values(protocolMap) as any[]; + + const processedModules = new Set(); + + for (const protocolInfo of protocols) { + if (protocolInfo.enabled === false || protocolInfo.disabled) { + console.log(`Excluding disabled adapter: ${protocolInfo.name}`); + continue; + } + + if (processedModules.has(protocolInfo.module)) continue; + + if (await shouldExcludeAdapter(dataModule, protocolInfo.module)) { + console.log(`Excluding adapter with runAtCurrTime set to true: ${protocolInfo.name}`); + continue; + } + + processedModules.add(protocolInfo.module); + + const protocolId = protocolInfo.protocolType === ProtocolType.CHAIN ? protocolInfo.id2 : protocolInfo.id; + const protocolData = adapterData.protocols[protocolId]; + if (!protocolData) continue; + + const protocolRecords = protocolData.records; + const firstDBDate = protocolData.firstDate; + const earliestStart = await getEarliestStartFromModule(dataModule, protocolInfo.module); + + let startTimestamp: number | null = null; + if (earliestStart) { + // Adjust earliestStart to the next day's UTC midnight + startTimestamp = alignToNextUTCMidnight(earliestStart); + } else if (firstDBDate) { + startTimestamp = alignToUTCMidnight(Date.parse(firstDBDate) / 1000); + } + + if (!startTimestamp || startTimestamp < JAN_1ST_2023_TIMESTAMP) continue; + + const todayTimestamp = Math.floor(Date.now() / 1000); + const currentDayTimestamp = Math.floor(new Date().setUTCHours(0, 0, 0, 0) / 1000); + + const missingDays: number[] = []; + const negativeDays: number[] = []; + + for (let i = startTimestamp; i < todayTimestamp; i += 86400) { + if (i >= currentDayTimestamp) break; + + const timeS = new Date(i * 1000).toISOString().slice(0, 10); + const aggValue = protocolRecords[timeS]; + + if (aggValue === undefined) { + missingDays.push(i); + } else if (aggValue < 0) { + negativeDays.push(i); + } + } + + if (missingDays.length > 730) continue; + + addCsvRow(filePaths[adapterType].missing, missingDays, adapterType, protocolInfo.name); + addCsvRow(filePaths[adapterType].negative, negativeDays, adapterType, protocolInfo.name, false); + } +} + +function alignToUTCMidnight(timestamp: number): number { + const date = new Date(timestamp * 1000); + return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()) / 1000; +} + +function alignToNextUTCMidnight(timestamp: number): number { + const date = new Date((timestamp + (86400 *2)) * 1000); + return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()) / 1000; +} + +function addCsvRow(filePath: string, days: number[], adapterType: string, protocolName: string, onlyMissing = true) { + if (days.length === 0) return; + + const backfillCommand = `npm run backfill-local ${adapterType.toLowerCase()} "${protocolName}"${ + onlyMissing ? " onlyMissing=true" : "" + } timestamps=${days.join(",")}`; + const csvRow = `${protocolName},"${days.join(",")}",${adapterType},"${backfillCommand}"\n`; + fs.appendFileSync(filePath, csvRow); +} + +async function cleanupEmptyCsvFiles() { + for (const { missing, negative } of Object.values(filePaths)) { + for (const filePath of [missing, negative]) { + if (fs.existsSync(filePath)) { + const fileSize = fs.statSync(filePath).size; + if (fileSize <= CSV_HEADER.length) { + fs.unlinkSync(filePath); + } + } + } + } +} + +async function getEarliestStartFromModule(dataModule: AdaptorData, moduleName: string): Promise { + try { + const module = await dataModule.importModule(moduleName); + const adapterEntries = Object.values(module.default?.adapter ?? {}); + + const chainStarts = adapterEntries + .map((chainAdapter: any) => { + const startTimestamp = chainAdapter?.start; + return typeof startTimestamp === "number" ? startTimestamp : null; + }) + .filter((start): start is number => start !== null); + + return chainStarts.length > 0 ? Math.min(...chainStarts) : null; + } catch { + return null; + } +} + +async function shouldExcludeAdapter(dataModule: AdaptorData, moduleName: string): Promise { + try { + const module = await dataModule.importModule(moduleName); + const adapter = module.default?.adapter; + + if (!adapter) return false; + + return Object.values(adapter).some((chainAdapter: any) => chainAdapter?.runAtCurrTime === true); + } catch { + return false; + } +} + +run().catch((error) => { + console.error('An error occurred:', error); + process.exit(1); +});