ft: adding cron jobs to increase cache hit ratio for oracle data (#157)

* ft: adding cron jobs to increase cache hit ratio for oracle data

* update version

* changes to make native oracle calls only for chainlink

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "arka",
-  "version": "1.7.2",
+  "version": "1.7.3",
   "description": "ARKA - (Albanian for Cashier's case) is the first open source Paymaster as a service software",
   "type": "module",
   "directories": {

backend/src/paymaster/index.ts

@@ -273,14 +273,16 @@ export class Paymaster {
     return provider.send('eth_estimateUserOperationGas', [userOp, entryPoint]);
   }
 
-  private async getLatestAnswerAndDecimals(
+  async getLatestAnswerAndDecimals(
     provider: providers.JsonRpcProvider,
     nativeOracleAddress: string,
-    chainId: number
+    chainId: number,
+    useCache = true
   ) {
     const cacheKey = `${chainId}-${nativeOracleAddress}`;
     const cache = this.nativeCurrencyPrice.get(cacheKey);
-    if(cache && cache.expiry > Date.now()) {
+
+    if(useCache && cache && cache.expiry > Date.now()) {
       return {
         latestAnswer: cache.data,
         decimals: NativeOracleDecimals
@@ -336,15 +338,16 @@ export class Paymaster {
     })
   }
 
-  private async getPriceFromOrochi(
+  async getPriceFromOrochi(
     oracleAddress: string,
     provider: providers.JsonRpcProvider,
     gasToken: string,
-    chainId: number
+    chainId: number,
+    useCache = true
   ) {
     const cacheKey = `${chainId}-${oracleAddress}-${gasToken}`;
     const cache = this.priceAndMetadata.get(cacheKey);
-    if(cache && cache.expiry > Date.now()) {
+    if(useCache && cache && cache.expiry > Date.now()) {
       return cache.data;
     }
 
@@ -385,17 +388,18 @@ export class Paymaster {
     });
   }
 
-  private async getPriceFromChainlink(
+  async getPriceFromChainlink(
     oracleAddress: string,
     provider: providers.JsonRpcProvider,
     gasToken: string,
     ethUsdPrice: any,
     ethUsdPriceDecimal: any,
-    chainId: number
+    chainId: number,
+    useCache = true
   ) {
     const cacheKey = `${chainId}-${oracleAddress}-${gasToken}`;
     const cache = this.priceAndMetadata.get(cacheKey);
-    if(cache && cache.expiry > Date.now()) {
+    if(useCache && cache && cache.expiry > Date.now()) {
       return cache.data;
     }
 
@@ -444,15 +448,16 @@ export class Paymaster {
     });
   }
 
-  private async getPriceFromEtherspotChainlink(
+  async getPriceFromEtherspotChainlink(
     oracleAddress: string,
     provider: providers.JsonRpcProvider,
     gasToken: string,
-    chainId: number
+    chainId: number,
+    useCache = true
   ) {
     const cacheKey = `${chainId}-${oracleAddress}-${gasToken}`;
     const cache = this.priceAndMetadata.get(cacheKey);
-    if(cache && cache.expiry > Date.now()) {
+    if(useCache && cache && cache.expiry > Date.now()) {
       return cache.data;
     }
     const ecContract = new ethers.Contract(oracleAddress, EtherspotChainlinkOracleAbi, provider);

backend/src/plugins/config.ts

@@ -32,7 +32,9 @@ const ConfigSchema = Type.Strict(
     DELETE_KEY_RECOVER_WINDOW: Type.Number(),
     KMS_KEY_ID: Type.String() || undefined,
     USE_KMS: Type.Boolean() || false,
-    DEFAULT_BUNDLER_API_KEY: Type.String()
+    DEFAULT_BUNDLER_API_KEY: Type.String(),
+    MULTI_TOKEN_PAYMASTERS: Type.String(),
+    MULTI_TOKEN_ORACLES: Type.String(),
   })
 );
 
@@ -69,7 +71,9 @@ const configPlugin: FastifyPluginAsync = async (server) => {
     DELETE_KEY_RECOVER_WINDOW: process.env.DELETE_KEY_RECOVER_WINDOW,
     KMS_KEY_ID: process.env.KMS_KEY_ID,
     USE_KMS: process.env.USE_KMS,
-    DEFAULT_BUNDLER_API_KEY: process.env.DEFAULT_BUNDLER_API_KEY
+    DEFAULT_BUNDLER_API_KEY: process.env.DEFAULT_BUNDLER_API_KEY,
+    MULTI_TOKEN_PAYMASTERS: process.env.MULTI_TOKEN_PAYMASTERS,
+    MULTI_TOKEN_ORACLES: process.env.MULTI_TOKEN_ORACLES
   }
 
   const valid = validate(envVar);
@@ -102,7 +106,9 @@ const configPlugin: FastifyPluginAsync = async (server) => {
     DELETE_KEY_RECOVER_WINDOW: parseInt(process.env.DELETE_KEY_RECOVER_WINDOW || '7'),
     KMS_KEY_ID: process.env.KMS_KEY_ID ?? '',
     USE_KMS: process.env.USE_KMS === 'true',
-    DEFAULT_BUNDLER_API_KEY: process.env.DEFAULT_BUNDLER_API_KEY ?? ''
+    DEFAULT_BUNDLER_API_KEY: process.env.DEFAULT_BUNDLER_API_KEY ?? '',
+    MULTI_TOKEN_PAYMASTERS: process.env.MULTI_TOKEN_PAYMASTERS ?? '',
+    MULTI_TOKEN_ORACLES: process.env.MULTI_TOKEN_ORACLES ?? '',
   }
 
   server.log.info(config, "config:");

backend/src/routes/paymaster-routes.ts

@@ -3,7 +3,6 @@ import { FastifyPluginAsync } from "fastify";
 import { BigNumber, Wallet, ethers, providers } from "ethers";
 import { gql, request as GLRequest } from "graphql-request";
 import { GetSecretValueCommand, SecretsManagerClient } from "@aws-sdk/client-secrets-manager";
-import { Paymaster } from "../paymaster/index.js";
 import SupportedNetworks from "../../config.json" assert { type: "json" };
 import { PAYMASTER_ADDRESS } from "../constants/Pimlico.js";
 import ErrorMessage, { generateErrorMessage } from "../constants/ErrorMessage.js";
@@ -13,9 +12,11 @@ import { printRequest, getNetworkConfig } from "../utils/common.js";
 import { SponsorshipPolicy } from "../models/sponsorship-policy.js";
 import { DEFAULT_EP_VERSION, EPVersions, getEPVersion } from "../types/sponsorship-policy-dto.js";
 import { NativeOracles } from "../constants/ChainlinkOracles.js";
+import { PaymasterRoutesOpts } from "../types/arka-config-dto.js";
 
-const paymasterRoutes: FastifyPluginAsync = async (server) => {
-  const paymaster = new Paymaster(server.config.FEE_MARKUP, server.config.MULTI_TOKEN_MARKUP, server.config.EP7_TOKEN_VGL, server.config.EP7_TOKEN_PGL);
+const paymasterRoutes: FastifyPluginAsync<PaymasterRoutesOpts> = async (server, options: PaymasterRoutesOpts) => {
+
+  const {paymaster} = options;
 
   const SUPPORTED_ENTRYPOINTS = {
     EPV_06: server.config.EPV_06,

backend/src/server.ts

@@ -15,7 +15,6 @@ import { getNetworkConfig } from './utils/common.js';
 import { checkDeposit } from './utils/monitorTokenPaymaster.js';
 import { APIKey } from './models/api-key.js';
 import { APIKeyRepository } from './repository/api-key-repository.js';
-import { ArkaConfig } from './models/arka-config.js';
 import { ArkaConfigRepository } from './repository/arka-config-repository.js';
 import adminRoutes from './routes/admin-routes.js';
 import depositRoutes from './routes/deposit-route.js';
@@ -25,6 +24,8 @@ import pimlicoRoutes from './routes/pimlico-routes.js';
 import whitelistRoutes from './routes/whitelist-routes.js';
 import sponsorshipPolicyRoutes from './routes/sponsorship-policy-routes.js';
 import SupportedNetworks from "../config.json" assert { type: "json" };
+import { Paymaster } from './paymaster/index.js';
+import { NativeOracles } from './constants/ChainlinkOracles.js';
 
 let server: FastifyInstance;
 
@@ -55,7 +56,9 @@ const initializeServer = async (): Promise<void> => {
     logLevel: "warn"
   });
 
-  await server.register(paymasterRoutes);
+  const paymaster = new Paymaster(server.config.FEE_MARKUP, server.config.MULTI_TOKEN_MARKUP, server.config.EP7_TOKEN_VGL, server.config.EP7_TOKEN_PGL);
+
+  await server.register(paymasterRoutes, {paymaster});
 
   await server.register(adminRoutes);
 
@@ -317,6 +320,109 @@ const initializeServer = async (): Promise<void> => {
             }
           }
         }
+      },
+      {
+        name: 'updateNativeTokenOracleData',
+        cronTime: process.env.NATIVE_ORACLE_UPDATE_CRON_EXP || "*/2 * * * *", // every 2 mins.
+        onTick: async () => {
+          const chainIds = Object.keys(NativeOracles);
+          for(const chainId of chainIds) {
+            const networkConfig = getNetworkConfig(chainId, '', server.config.EPV_06);
+            if(!networkConfig) {
+              continue;
+            }
+            const provider = new ethers.providers.JsonRpcProvider(networkConfig.bundler);
+            paymaster.getLatestAnswerAndDecimals(provider, NativeOracles[+chainId], +chainId, false)
+              .catch((error) => {
+                server.log.error(`Failed to update native oracle price for chain id: ${chainId}, ${error}`);
+              });
+          }
+        }
+      },
+      {
+        name: 'updateTokenOracleData',
+        cronTime: process.env.TOKEN_ORACLE_UPDATE_CRON_EXP || '*/5 * * * *', // every 5 mins.
+        onTick: async () => {
+          let buffer = Buffer.from(server.config.MULTI_TOKEN_PAYMASTERS, 'base64');
+          const multiTokenPaymasters = JSON.parse(buffer.toString());
+
+          buffer = Buffer.from(server.config.MULTI_TOKEN_ORACLES, 'base64');
+          const multiTokenOracles = JSON.parse(buffer.toString());
+
+          const chainIds = Object.keys(multiTokenPaymasters);
+
+          for(const chainId of chainIds) {
+            const networkConfig = getNetworkConfig(chainId, '', server.config.EPV_06);
+            if(!networkConfig) {
+              continue;
+            }
+            const provider = new ethers.providers.JsonRpcProvider(networkConfig.bundler);
+
+            if(networkConfig.MultiTokenPaymasterOracleUsed === 'chainlink') {
+              paymaster.getLatestAnswerAndDecimals(
+                provider,
+                NativeOracles[+chainId],
+                +chainId
+              ).then((data) => {
+                  const {latestAnswer, decimals} = data;
+                  if(!latestAnswer || !decimals) {
+                    return;
+                  }
+                  const tokens = Object.keys(multiTokenPaymasters[chainId]);
+                  for(const token of tokens) {
+                    paymaster.getPriceFromChainlink(
+                      multiTokenOracles[chainId][token],
+                      provider,
+                      token,
+                      latestAnswer,
+                      decimals,
+                      +chainId,
+                      false
+                    ).catch((error) => {
+                      server.log.error(
+                        `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}`
+                      );
+                    });
+                  }
+                })
+                .catch((error) => {
+                server.log.error(
+                  `Failed to get native prices while updating oracle token data: chain id ${chainId}, ${error}`
+                );
+              });
+            } else if (networkConfig.MultiTokenPaymasterOracleUsed === 'orochi') {
+              const tokens = Object.keys(multiTokenPaymasters[chainId]);
+              for(const token of tokens) {
+                paymaster.getPriceFromOrochi(
+                  multiTokenOracles[chainId][token],
+                  provider,
+                  token,
+                  +chainId,
+                  false
+                ).catch((error) => {
+                  server.log.error(
+                    `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}`
+                  );
+                });
+              }
+            } else {
+              const tokens = Object.keys(multiTokenPaymasters[chainId]);
+              for(const token of tokens) {
+                paymaster.getPriceFromEtherspotChainlink(
+                  multiTokenOracles[chainId][token],
+                  provider,
+                  token,
+                  +chainId,
+                  false
+                ).catch((error) => {
+                  server.log.error(
+                    `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}`
+                  );
+                });
+              }
+            }
+          }
+        }
       }
     ]
   });

backend/src/types/arka-config-dto.ts

@@ -1,3 +1,4 @@
+import { Paymaster } from "../paymaster/index.js";
 
 export interface ArkaConfigUpdateData {
     deployedErc20Paymasters: string;
@@ -11,3 +12,6 @@ export interface ArkaConfigUpdateData {
     coingeckoApiUrl: string;
 }
 
+export interface PaymasterRoutesOpts {
+    paymaster: Paymaster;
+}