From 7bc672f04e8d96841264ef40e5d23447909d8fe1 Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Mon, 3 Jun 2024 20:15:36 +0700 Subject: [PATCH 01/15] feat: use external config --- IPFS.json | 10 ++- config/external-config/index.ts | 7 +++ config/external-config/types.ts | 20 ++++++ .../use-external-config-context.ts | 51 +++++++++++++++ config/external-config/utils.ts | 63 +++++++++++++++++++ config/get-config.ts | 3 + config/groups/cache.ts | 4 ++ config/groups/revalidation.ts | 15 +++++ config/provider.tsx | 23 ++++++- consts/external-links.ts | 4 ++ .../use-remote-version.ts | 59 +++++++---------- .../request/form/options/options-picker.tsx | 3 +- .../withdrawal-rates/use-withdrawal-rates.ts | 15 ++--- .../withdrawals-constants/index.ts | 9 --- pages/_app.tsx | 10 +-- pages/index.tsx | 7 +-- pages/referral.tsx | 3 + pages/rewards.tsx | 3 + pages/settings.tsx | 6 +- pages/withdrawals/[mode].tsx | 11 ++-- pages/wrap/[[...mode]].tsx | 9 +-- providers/index.tsx | 11 +++- utilsApi/fetch-external-manifest.ts | 60 ++++++++++++++++++ utilsApi/get-default-static-props.ts | 30 +++++++++ 24 files changed, 353 insertions(+), 83 deletions(-) create mode 100644 config/external-config/index.ts create mode 100644 config/external-config/types.ts create mode 100644 config/external-config/use-external-config-context.ts create mode 100644 config/external-config/utils.ts create mode 100644 config/groups/revalidation.ts create mode 100644 utilsApi/fetch-external-manifest.ts create mode 100644 utilsApi/get-default-static-props.ts diff --git a/IPFS.json b/IPFS.json index 64256ff70..df2d553d1 100644 --- a/IPFS.json +++ b/IPFS.json @@ -1,7 +1,10 @@ { "1": { "cid": "bafybeib3zmyqlmantvdd6i5q4ehmo4larvorgquyanne3uoqdbedwgh3aq", - "leastSafeVersion": "0.36.1" + "leastSafeVersion": "0.36.1", + "config": { + "enabledWithdrawalDexes": ["one-inch", "paraswap", "bebop"] + } }, "5": { "__warning__": "For testing purposes only", @@ -11,6 +14,9 @@ "17000": { "__warning__": "For testing purposes only", "cid": "bafybeibbsoqlofslw273b4ih2pdxfaz2zbjmred2ijog725tcmfoewix7y", - "leastSafeVersion": "0.36.1" + "leastSafeVersion": "0.36.1", + "config": { + "enabledWithdrawalDexes": ["one-inch", "paraswap", "bebop"] + } } } diff --git a/config/external-config/index.ts b/config/external-config/index.ts new file mode 100644 index 000000000..c7d70a8b7 --- /dev/null +++ b/config/external-config/index.ts @@ -0,0 +1,7 @@ +export { useExternalConfigContext } from './use-external-config-context'; +export type { + ManifestConfig, + Manifest, + ManifestEntry, + ExternalConfig, +} from './types'; diff --git a/config/external-config/types.ts b/config/external-config/types.ts new file mode 100644 index 000000000..92ea1ad04 --- /dev/null +++ b/config/external-config/types.ts @@ -0,0 +1,20 @@ +import type { DexWithdrawalApi } from 'features/withdrawals/request/withdrawal-rates'; +import { SWRResponse } from 'swr'; + +export type Manifest = Record; + +export type ManifestEntry = { + cid?: string; + ens?: string; + leastSafeVersion?: string; + config: ManifestConfig; +}; + +export type ManifestConfig = { + enabledWithdrawalDexes: DexWithdrawalApi[]; +}; + +export type ExternalConfig = Omit & + ManifestConfig & { + fetchMeta: SWRResponse; + }; diff --git a/config/external-config/use-external-config-context.ts b/config/external-config/use-external-config-context.ts new file mode 100644 index 000000000..3ca8410b6 --- /dev/null +++ b/config/external-config/use-external-config-context.ts @@ -0,0 +1,51 @@ +import { STRATEGY_LAZY } from 'consts/swr-strategies'; +import { getConfig } from '../get-config'; +import { standardFetcher } from 'utils/standardFetcher'; +import { IPFS_MANIFEST_PATH } from 'consts/external-links'; +import { isManifestEntryValid, useFallbackManifestEntry } from './utils'; +import { ExternalConfig, ManifestEntry } from './types'; +import useSWR from 'swr'; +import { useMemo } from 'react'; + +const onFetchError = (e: any) => { + console.warn('[useExternalConfigContext] while fetching external config:', e); +}; + +export const useExternalConfigContext = ( + prefetchedManifest?: unknown, +): ExternalConfig => { + const { defaultChain } = getConfig(); + const fallbackData = useFallbackManifestEntry( + prefetchedManifest, + defaultChain, + ); + + const swr = useSWR( + ['swr:external-config', defaultChain], + async () => { + const result = await standardFetcher>( + IPFS_MANIFEST_PATH, + { + headers: { Accept: 'application/json' }, + }, + ); + const entry = result[defaultChain.toString()]; + if (isManifestEntryValid(entry)) return entry; + throw new Error( + '[useExternalConfigContext] received invalid manifest', + result, + ); + }, + { + ...STRATEGY_LAZY, + fallbackData: fallbackData, + onError: onFetchError, + }, + ); + + return useMemo(() => { + const { config, ...rest } = swr.data ?? fallbackData; + + return { ...config, ...rest, fetchMeta: swr }; + }, [swr, fallbackData]); +}; diff --git a/config/external-config/utils.ts b/config/external-config/utils.ts new file mode 100644 index 000000000..482f3e57f --- /dev/null +++ b/config/external-config/utils.ts @@ -0,0 +1,63 @@ +import { useMemo } from 'react'; +import { getConfig } from '../get-config'; +import type { Manifest, ManifestEntry } from './types'; +import { + type DexWithdrawalApi, + getDexConfig, +} from 'features/withdrawals/request/withdrawal-rates'; + +const config = getConfig(); + +// TODO: refactor on config expansion +export const isManifestEntryValid = ( + entry?: unknown, +): entry is ManifestEntry => { + if ( + entry && + typeof entry === 'object' && + entry && + typeof entry === 'object' && + // entry.config = {} + 'config' in entry && + typeof entry.config === 'object' && + entry.config + ) { + const config = entry.config; + if ( + 'enabledWithdrawalDexes' in config && + Array.isArray(config.enabledWithdrawalDexes) + ) { + const enabledWithdrawalDexes = config.enabledWithdrawalDexes; + return ( + (enabledWithdrawalDexes as string[]).every( + (dex) => !!getDexConfig(dex as DexWithdrawalApi), + ) && + new Set(enabledWithdrawalDexes).size === enabledWithdrawalDexes.length + ); + } + return false; + } + return false; +}; + +export const isManifestValid = ( + manifest: unknown, + chain: number, +): manifest is Manifest => { + const stringChain = chain.toString(); + if (manifest && typeof manifest === 'object' && stringChain in manifest) + return isManifestEntryValid( + (manifest as Record)[stringChain], + ); + return false; +}; + +export const useFallbackManifestEntry = ( + prefetchedManifest: unknown, + chain: number, +): ManifestEntry => { + return useMemo(() => { + const isValid = isManifestValid(prefetchedManifest, chain); + return isValid ? prefetchedManifest[chain] : config.FALLBACK_MANIFEST_ENTRY; + }, [prefetchedManifest, chain]); +}; diff --git a/config/get-config.ts b/config/get-config.ts index 31cc81fa2..71a8c3c5e 100644 --- a/config/get-config.ts +++ b/config/get-config.ts @@ -4,6 +4,7 @@ import * as estimate from './groups/estimate'; import * as ipfs from './groups/ipfs'; import * as locale from './groups/locale'; import * as stake from './groups/stake'; +import * as revalidation from './groups/revalidation'; import * as withdrawalQueueEstimate from './groups/withdrawal-queue-estimate'; export type ConfigType = { @@ -14,6 +15,7 @@ export type ConfigType = { typeof ipfs & typeof locale & typeof stake & + typeof revalidation & typeof withdrawalQueueEstimate & PreConfigType; @@ -27,6 +29,7 @@ export const getConfig = (): ConfigType => { ...ipfs, ...locale, ...stake, + ...revalidation, ...withdrawalQueueEstimate, // highest priority diff --git a/config/groups/cache.ts b/config/groups/cache.ts index 9d0a050e5..6b6246107 100644 --- a/config/groups/cache.ts +++ b/config/groups/cache.ts @@ -32,6 +32,10 @@ export const CACHE_ONE_INCH_RATE_TTL = ms('5m'); export const CACHE_TOTAL_SUPPLY_KEY = 'cache-total-supply'; export const CACHE_TOTAL_SUPPLY_TTL = ms('1m'); + +export const CACHE_EXTERNAL_CONFIG_KEY = 'cache-external-config'; +export const CACHE_EXTERNAL_CONFIG_TTL = ms('10m'); + export const CACHE_TOTAL_SUPPLY_HEADERS = 'public, max-age=60, stale-if-error=1200, stale-while-revalidate=30'; diff --git a/config/groups/revalidation.ts b/config/groups/revalidation.ts new file mode 100644 index 000000000..4e8f850ae --- /dev/null +++ b/config/groups/revalidation.ts @@ -0,0 +1,15 @@ +import type { ManifestConfig, ManifestEntry } from 'config/external-config'; + +export const DEFAULT_REVALIDATION = 60 * 15; // 15 minutes +export const ERROR_REVALIDATION = 60; // 1 minute + +export const FALLBACK_CONFIG: ManifestConfig = { + enabledWithdrawalDexes: [], +}; + +export const FALLBACK_MANIFEST_ENTRY: ManifestEntry = { + cid: undefined, + ens: undefined, + leastSafeVersion: undefined, + config: FALLBACK_CONFIG, +}; diff --git a/config/provider.tsx b/config/provider.tsx index 3c07a9784..fe42b1d49 100644 --- a/config/provider.tsx +++ b/config/provider.tsx @@ -6,26 +6,45 @@ import { useFeatureFlagsContext, FeatureFlagsContextType, } from './feature-flags'; +import { + type ExternalConfig, + useExternalConfigContext, +} from './external-config'; type ConfigProviderType = { config: ConfigType; userConfig: UserConfigContextType; featureFlags: FeatureFlagsContextType; + externalConfig: ExternalConfig; }; export const ConfigContext = createContext(null); -export const ConfigProvider = ({ children }: PropsWithChildren) => { +type ConfigProviderProps = { + prefetchedManifest?: unknown; +}; + +export const ConfigProvider = ({ + children, + prefetchedManifest, +}: PropsWithChildren) => { const userConfigContextValue = useUserConfigContext(); const featureFlagsContextValue = useFeatureFlagsContext(); + const externalConfigContextValue = + useExternalConfigContext(prefetchedManifest); const contextValue = useMemo( () => ({ config: getConfig(), userConfig: userConfigContextValue, featureFlags: featureFlagsContextValue, + externalConfig: externalConfigContextValue, }), - [userConfigContextValue, featureFlagsContextValue], + [ + userConfigContextValue, + featureFlagsContextValue, + externalConfigContextValue, + ], ); return ( diff --git a/consts/external-links.ts b/consts/external-links.ts index 8b186d87f..649f00ee1 100644 --- a/consts/external-links.ts +++ b/consts/external-links.ts @@ -3,3 +3,7 @@ export const LINK_ADD_NFT_GUIDE = export const OPEN_OCEAN_REFERRAL_ADDRESS = '0xbb1263222b2c020f155d409dba05c4a3861f18f8'; + +// for dev and local testing you can set to '/runtime/IPFS.json' and have file at /public/runtime/IPFS.json +export const IPFS_MANIFEST_PATH = + 'https://raw.githubusercontent.com/lidofinance/ethereum-staking-widget/main/IPFS.json'; diff --git a/features/ipfs/security-status-banner/use-remote-version.ts b/features/ipfs/security-status-banner/use-remote-version.ts index 9d27b74c4..52e727c15 100644 --- a/features/ipfs/security-status-banner/use-remote-version.ts +++ b/features/ipfs/security-status-banner/use-remote-version.ts @@ -1,66 +1,55 @@ import { useLidoSWR } from '@lido-sdk/react'; -import { config } from 'config'; +import { useConfig } from 'config'; import { STRATEGY_LAZY } from 'consts/swr-strategies'; import { useMainnetStaticRpcProvider } from 'shared/hooks/use-mainnet-static-rpc-provider'; -import { standardFetcher } from 'utils/standardFetcher'; type EnsHashCheckReturn = { cid: string; ens?: string; leastSafeVersion?: string; link: string; -} | null; - -type ReleaseInfoData = Record; - -type ReleaseInfo = { - cid?: string; - ens?: string; - leastSafeVersion?: string; }; -// for dev and local testing you can set to '/runtime/IPFS.json' and have file at /public/runtime/ -const IPFS_RELEASE_URL = - 'https://raw.githubusercontent.com/lidofinance/ethereum-staking-widget/main/IPFS.json'; - export const useRemoteVersion = () => { const provider = useMainnetStaticRpcProvider(); - // ens cid extraction + + // we use directly non-optimistic manifest data + // can't trust static props(in IPFS esp) to generate warnings/disconnect wallet + const { data, error } = useConfig().externalConfig.fetchMeta; + + // ens&cid extraction return useLidoSWR( - ['swr:use-remote-version'], + ['swr:use-remote-version', data], async (): Promise => { - const releaseInfoData = await standardFetcher( - IPFS_RELEASE_URL, - { - headers: { Accept: 'application/json' }, - }, - ); - - const releaseInfo = releaseInfoData[config.defaultChain.toString()]; - if (releaseInfo?.ens) { - const resolver = await provider.getResolver(releaseInfo.ens); + if (data?.ens) { + const resolver = await provider.getResolver(data.ens); if (resolver) { const contentHash = await resolver.getContentHash(); if (contentHash) { return { cid: contentHash, - ens: releaseInfo.ens, - link: `https://${releaseInfo.ens}.limo`, - leastSafeVersion: releaseInfo.leastSafeVersion, + ens: data.ens, + link: `https://${externalConfig.ens}.limo`, + leastSafeVersion: data.leastSafeVersion, }; } } } - if (releaseInfo?.cid) { + if (data?.cid) { return { - cid: releaseInfo.cid, - link: `https://${releaseInfo.cid}.ipfs.cf-ipfs.com`, - leastSafeVersion: releaseInfo.leastSafeVersion, + cid: data.cid, + link: `https://${data.cid}.ipfs.cf-ipfs.com`, + leastSafeVersion: data.leastSafeVersion, }; } - throw new Error('invalid IPFS manifest content'); + throw new Error('[useRemoteVersion] invalid IPFS manifest content'); + }, + { + ...STRATEGY_LAZY, + // we postpone fetch if we don't have external data and don't have error + // empty data will force fetcher to produce correct error + isPaused: () => !(data || error), }, - { ...STRATEGY_LAZY }, ); }; diff --git a/features/withdrawals/request/form/options/options-picker.tsx b/features/withdrawals/request/form/options/options-picker.tsx index abc8c359c..6ea5cad15 100644 --- a/features/withdrawals/request/form/options/options-picker.tsx +++ b/features/withdrawals/request/form/options/options-picker.tsx @@ -7,7 +7,6 @@ import { MATOMO_CLICK_EVENTS_TYPES } from 'consts/matomo-click-events'; import { DATA_UNAVAILABLE } from 'consts/text'; import { useWaitingTime } from 'features/withdrawals/hooks/useWaitingTime'; import { RequestFormInputType } from 'features/withdrawals/request/request-form-context'; -import { ENABLED_WITHDRAWAL_DEXES } from 'features/withdrawals/withdrawals-constants'; import { getDexConfig, useWithdrawalRates, @@ -99,7 +98,7 @@ const DexButton: React.FC = ({ isActive, onClick }) => { Use aggregators - {ENABLED_WITHDRAWAL_DEXES.map((dexKey) => { + {enabledDexes.map((dexKey) => { const Icon = getDexConfig(dexKey).icon; return ; })} diff --git a/features/withdrawals/request/withdrawal-rates/use-withdrawal-rates.ts b/features/withdrawals/request/withdrawal-rates/use-withdrawal-rates.ts index 9eb775b4c..1ccc15da9 100644 --- a/features/withdrawals/request/withdrawal-rates/use-withdrawal-rates.ts +++ b/features/withdrawals/request/withdrawal-rates/use-withdrawal-rates.ts @@ -11,13 +11,12 @@ import { useDebouncedValue } from 'shared/hooks/useDebouncedValue'; import type { RequestFormInputType } from '../request-form-context'; import { getDexConfig } from './integrations'; -import { ENABLED_WITHDRAWAL_DEXES } from 'features/withdrawals/withdrawals-constants'; - import type { DexWithdrawalApi, GetWithdrawalRateParams, GetWithdrawalRateResult, } from './types'; +import { useConfig } from 'config'; export type useWithdrawalRatesOptions = { fallbackValue?: BigNumber; @@ -57,15 +56,11 @@ export const useWithdrawalRates = ({ const [token, amount] = useWatch({ name: ['token', 'amount'], }); + const enabledDexes = useConfig().externalConfig.enabledWithdrawalDexes; const fallbackedAmount = amount ?? fallbackValue; const debouncedAmount = useDebouncedValue(fallbackedAmount, 1000); const swr = useLidoSWR( - [ - 'swr:withdrawal-rates', - debouncedAmount.toString(), - token, - ENABLED_WITHDRAWAL_DEXES, - ], + ['swr:withdrawal-rates', debouncedAmount.toString(), token, enabledDexes], // eslint-disable-next-line @typescript-eslint/no-explicit-any (_, amount, token, enabledDexes) => getWithdrawalRates({ @@ -78,7 +73,7 @@ export const useWithdrawalRates = ({ isPaused: () => !debouncedAmount || !debouncedAmount._isBigNumber || - ENABLED_WITHDRAWAL_DEXES.length === 0, + enabledDexes.length === 0, }, ); @@ -89,7 +84,7 @@ export const useWithdrawalRates = ({ return { amount: fallbackedAmount, bestRate, - enabledDexes: ENABLED_WITHDRAWAL_DEXES, + enabledDexes, selectedToken: token, data: swr.data, get initialLoading() { diff --git a/features/withdrawals/withdrawals-constants/index.ts b/features/withdrawals/withdrawals-constants/index.ts index 1fcc9bbdc..72fd896a5 100644 --- a/features/withdrawals/withdrawals-constants/index.ts +++ b/features/withdrawals/withdrawals-constants/index.ts @@ -1,6 +1,3 @@ -import { overrideWithQAMockArray } from 'utils/qa'; -import type { DexWithdrawalApi } from '../request/withdrawal-rates'; - // max requests count for one tx export const MAX_REQUESTS_COUNT = 256; export const MAX_REQUESTS_COUNT_LEDGER_LIMIT = 2; @@ -15,9 +12,3 @@ export const WHAT_IS_TURBO = '#whatIsTurboMode'; // time that validation function waits for context data to resolve // should be enough to load token balances/tvl/max&min amounts and other contract data export const VALIDATION_CONTEXT_TIMEOUT = 4000; - -export const ENABLED_WITHDRAWAL_DEXES: DexWithdrawalApi[] = - overrideWithQAMockArray( - ['one-inch', 'paraswap', 'bebop'], - 'mock-qa-helpers-enabled-withdrawal-dexes', - ); diff --git a/pages/_app.tsx b/pages/_app.tsx index 2ae16e58f..4566a8a37 100644 --- a/pages/_app.tsx +++ b/pages/_app.tsx @@ -34,11 +34,11 @@ const App = (props: AppProps) => { const MemoApp = memo(App); -const AppWrapper = (props: AppProps): JSX.Element => { - const { ...rest } = props; - +const AppWrapper = ( + props: AppProps<{ ___prefetch_manifest___?: object }>, +): JSX.Element => { return ( - + {/* see https://nextjs.org/docs/messages/no-document-viewport-meta */} { }} /> - + diff --git a/pages/index.tsx b/pages/index.tsx index 3cea362e5..8ac95a62b 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -2,10 +2,9 @@ import { config } from 'config'; import { StakePage } from 'features/stake'; import { HomePageIpfs } from 'features/ipfs'; -import { GetStaticProps } from 'next'; -export const getStaticProps: GetStaticProps = async () => { - return { props: {} }; -}; +import { getDefaultStaticProps } from 'utilsApi/get-default-static-props'; + +export const getStaticProps = getDefaultStaticProps(); export default config.ipfsMode ? HomePageIpfs : StakePage; diff --git a/pages/referral.tsx b/pages/referral.tsx index 40dff0604..3ab13965b 100644 --- a/pages/referral.tsx +++ b/pages/referral.tsx @@ -1,4 +1,6 @@ import { FC } from 'react'; + +import { getDefaultStaticProps } from 'utilsApi/get-default-static-props'; import { Banner } from 'features/referral'; import { Layout } from 'shared/components'; @@ -9,5 +11,6 @@ const Referral: FC = () => { ); }; +export const getStaticProps = getDefaultStaticProps(); export default Referral; diff --git a/pages/rewards.tsx b/pages/rewards.tsx index b197c291e..6bc2ae327 100644 --- a/pages/rewards.tsx +++ b/pages/rewards.tsx @@ -5,6 +5,7 @@ import { TopCard, RewardsList } from 'features/rewards/features'; import RewardsHistoryProvider from 'providers/rewardsHistory'; import { Fallback } from 'shared/wallet'; import { GoerliSunsetBanner } from 'shared/banners/goerli-sunset'; +import { getDefaultStaticProps } from 'utilsApi/get-default-static-props'; const Rewards: FC = () => { return ( @@ -31,4 +32,6 @@ const Rewards: FC = () => { ); }; +export const getStaticProps = getDefaultStaticProps(); + export default Rewards; diff --git a/pages/settings.tsx b/pages/settings.tsx index bd584333c..736b4db11 100644 --- a/pages/settings.tsx +++ b/pages/settings.tsx @@ -1,9 +1,9 @@ import { FC } from 'react'; -import { GetStaticProps } from 'next'; import { config } from 'config'; import { Layout } from 'shared/components'; import { SettingsForm } from 'features/settings/settings-form'; +import { getDefaultStaticProps } from 'utilsApi/get-default-static-props'; const Settings: FC = () => { return ( @@ -15,8 +15,8 @@ const Settings: FC = () => { export default Settings; -export const getStaticProps: GetStaticProps = async () => { +export const getStaticProps = getDefaultStaticProps(async () => { if (!config.ipfsMode) return { notFound: true }; return { props: {} }; -}; +}); diff --git a/pages/withdrawals/[mode].tsx b/pages/withdrawals/[mode].tsx index 7aada07a3..bb69d1825 100644 --- a/pages/withdrawals/[mode].tsx +++ b/pages/withdrawals/[mode].tsx @@ -1,5 +1,5 @@ -import { FC } from 'react'; -import { GetStaticPaths, GetStaticProps } from 'next'; +import type { FC } from 'react'; +import type { GetStaticPaths } from 'next'; import Head from 'next/head'; import { Layout } from 'shared/components'; @@ -8,6 +8,7 @@ import NoSSRWrapper from 'shared/components/no-ssr-wrapper'; import { WithdrawalsTabs } from 'features/withdrawals'; import { WithdrawalsProvider } from 'features/withdrawals/contexts/withdrawals-context'; import { useWeb3Key } from 'shared/hooks/useWeb3Key'; +import { getDefaultStaticProps } from 'utilsApi/get-default-static-props'; const Withdrawals: FC = ({ mode }) => { const key = useWeb3Key(); @@ -44,10 +45,10 @@ export const getStaticPaths: GetStaticPaths< }; }; -export const getStaticProps: GetStaticProps< +export const getStaticProps = getDefaultStaticProps< WithdrawalsModePageParams, WithdrawalsModePageParams -> = async ({ params }) => { +>(async ({ params }) => { if (!params?.mode) return { notFound: true }; return { props: { mode: params.mode } }; -}; +}); diff --git a/pages/wrap/[[...mode]].tsx b/pages/wrap/[[...mode]].tsx index 97a695c2d..651097bf9 100644 --- a/pages/wrap/[[...mode]].tsx +++ b/pages/wrap/[[...mode]].tsx @@ -1,9 +1,10 @@ import { FC } from 'react'; -import { GetStaticPaths, GetStaticProps } from 'next'; +import { GetStaticPaths } from 'next'; import Head from 'next/head'; import { Layout } from 'shared/components'; import { WrapUnwrapTabs } from 'features/wsteth/wrap-unwrap-tabs'; import { useWeb3Key } from 'shared/hooks/useWeb3Key'; +import { getDefaultStaticProps } from 'utilsApi/get-default-static-props'; const WrapPage: FC = ({ mode }) => { const key = useWeb3Key(); @@ -38,13 +39,13 @@ export const getStaticPaths: GetStaticPaths = async () => { }; // we need [[...]] pattern for / and /unwrap -export const getStaticProps: GetStaticProps< +export const getStaticProps = getDefaultStaticProps< WrapModePageProps, WrapModePageParams -> = async ({ params }) => { +>(async ({ params }) => { const mode = params?.mode; if (!mode) return { props: { mode: 'wrap' } }; if (mode[0] === 'unwrap') return { props: { mode: 'unwrap' } }; return { notFound: true }; -}; +}); diff --git a/providers/index.tsx b/providers/index.tsx index 5da5a1bdc..abc7d966d 100644 --- a/providers/index.tsx +++ b/providers/index.tsx @@ -10,8 +10,15 @@ import { InpageNavigationProvider } from './inpage-navigation'; import { ModalProvider } from './modal-provider'; import Web3Provider from './web3'; -export const Providers: FC = ({ children }) => ( - +type ProvidersProps = { + prefetchedManifest?: unknown; +}; + +export const Providers: FC> = ({ + children, + prefetchedManifest, +}) => ( + diff --git a/utilsApi/fetch-external-manifest.ts b/utilsApi/fetch-external-manifest.ts new file mode 100644 index 000000000..5a8f9b1a5 --- /dev/null +++ b/utilsApi/fetch-external-manifest.ts @@ -0,0 +1,60 @@ +import { Cache } from 'memory-cache'; +import { IPFS_MANIFEST_PATH } from 'consts/external-links'; +import { responseTimeExternalMetricWrapper } from './fetchApiWrapper'; +import { standardFetcher } from 'utils/standardFetcher'; +import { config } from 'config'; + +export type ExternalConfigResult = { + ___prefetch_manifest___: object | null; + revalidate: number; +}; + +const cache = new Cache< + typeof config.CACHE_EXTERNAL_CONFIG_KEY, + ExternalConfigResult +>(); + +export const fetchExternalManifest = async () => { + const cachedConfig = cache.get(config.CACHE_EXTERNAL_CONFIG_KEY); + if (cachedConfig) return cachedConfig; + + let retries = 3; + while (retries > 0) { + try { + const data = await responseTimeExternalMetricWrapper({ + payload: IPFS_MANIFEST_PATH, + request: () => + standardFetcher(IPFS_MANIFEST_PATH, { + headers: { Accept: 'application/json' }, + }), + }); + if (!data || typeof data !== 'object') + throw new Error(`invalid config received: ${data}`); + + const result = { + ___prefetch_manifest___: data, + revalidate: config.DEFAULT_REVALIDATION, + }; + + cache.put( + config.CACHE_EXTERNAL_CONFIG_KEY, + result, + config.CACHE_EXTERNAL_CONFIG_TTL, + ); + + console.debug(`[fetchExternalConfig] fetched external config`, result); + + return result; + } catch (e) { + console.error(`[fetchExternalConfig] failed to fetch external config`, e); + retries -= 1; + } + } + console.error( + `[fetchExternalConfig] failed to fetch external config after retries, revalidation is set to ${config.ERROR_REVALIDATION}`, + ); + return { + revalidate: config.ERROR_REVALIDATION, + ___prefetch_manifest___: null, + }; +}; diff --git a/utilsApi/get-default-static-props.ts b/utilsApi/get-default-static-props.ts new file mode 100644 index 000000000..a7f39df77 --- /dev/null +++ b/utilsApi/get-default-static-props.ts @@ -0,0 +1,30 @@ +import type { GetStaticProps, PreviewData } from 'next'; +import type { ParsedUrlQuery } from 'querystring'; +import { fetchExternalManifest } from './fetch-external-manifest'; + +export const getDefaultStaticProps = < + P extends { [key: string]: any } = { [key: string]: any }, + Q extends ParsedUrlQuery = ParsedUrlQuery, + D extends PreviewData = PreviewData, +>( + custom?: GetStaticProps, +): GetStaticProps

=> { + return async (context) => { + const { ___prefetch_manifest___, revalidate } = + await fetchExternalManifest(); + const props = ___prefetch_manifest___ ? { ___prefetch_manifest___ } : {}; + const base = { + props, + revalidate, + }; + if (custom) { + const { props: customProps, ...rest } = (await custom(context)) as any; + return { + ...base, + ...rest, + props: { ...base.props, ...customProps }, + }; + } + return base; + }; +}; From c0b9ef16ba446518eecd7265a762d388a0a5fa48 Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Mon, 3 Jun 2024 20:27:55 +0700 Subject: [PATCH 02/15] fix: start up revalidation --- config/external-config/utils.ts | 1 + features/ipfs/security-status-banner/use-remote-version.ts | 2 +- utilsApi/get-default-static-props.ts | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/external-config/utils.ts b/config/external-config/utils.ts index 482f3e57f..46cb88583 100644 --- a/config/external-config/utils.ts +++ b/config/external-config/utils.ts @@ -13,6 +13,7 @@ export const isManifestEntryValid = ( entry?: unknown, ): entry is ManifestEntry => { if ( + // entry = {} entry && typeof entry === 'object' && entry && diff --git a/features/ipfs/security-status-banner/use-remote-version.ts b/features/ipfs/security-status-banner/use-remote-version.ts index 52e727c15..38c41cd9c 100644 --- a/features/ipfs/security-status-banner/use-remote-version.ts +++ b/features/ipfs/security-status-banner/use-remote-version.ts @@ -29,7 +29,7 @@ export const useRemoteVersion = () => { return { cid: contentHash, ens: data.ens, - link: `https://${externalConfig.ens}.limo`, + link: `https://${data.ens}.limo`, leastSafeVersion: data.leastSafeVersion, }; } diff --git a/utilsApi/get-default-static-props.ts b/utilsApi/get-default-static-props.ts index a7f39df77..b35623ae9 100644 --- a/utilsApi/get-default-static-props.ts +++ b/utilsApi/get-default-static-props.ts @@ -9,13 +9,14 @@ export const getDefaultStaticProps = < >( custom?: GetStaticProps, ): GetStaticProps

=> { + let shouldZeroRevalidate = true; return async (context) => { const { ___prefetch_manifest___, revalidate } = await fetchExternalManifest(); const props = ___prefetch_manifest___ ? { ___prefetch_manifest___ } : {}; const base = { props, - revalidate, + revalidate: shouldZeroRevalidate ? 1 : revalidate, }; if (custom) { const { props: customProps, ...rest } = (await custom(context)) as any; @@ -25,6 +26,7 @@ export const getDefaultStaticProps = < props: { ...base.props, ...customProps }, }; } + shouldZeroRevalidate = false; return base; }; }; From db3e62c1a5eee6ec766a1d614d080364583e622e Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Tue, 4 Jun 2024 19:44:47 +0700 Subject: [PATCH 03/15] feat: add metrics --- consts/metrics.ts | 1 + utilsApi/fetch-external-manifest.ts | 12 +++++++++--- utilsApi/get-default-static-props.ts | 19 ++++++++++++++++--- utilsApi/metrics/request.ts | 11 +++++++++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/consts/metrics.ts b/consts/metrics.ts index 5e1ac3074..8414c7538 100644 --- a/consts/metrics.ts +++ b/consts/metrics.ts @@ -5,4 +5,5 @@ export const enum METRIC_NAMES { API_RESPONSE = 'api_response', SUBGRAPHS_RESPONSE = 'subgraphs_response', ETH_CALL_ADDRESS_TO = 'eth_call_address_to', + SSR_COUNT = 'ssr_count', } diff --git a/utilsApi/fetch-external-manifest.ts b/utilsApi/fetch-external-manifest.ts index 5a8f9b1a5..d7567306e 100644 --- a/utilsApi/fetch-external-manifest.ts +++ b/utilsApi/fetch-external-manifest.ts @@ -42,16 +42,22 @@ export const fetchExternalManifest = async () => { config.CACHE_EXTERNAL_CONFIG_TTL, ); - console.debug(`[fetchExternalConfig] fetched external config`, result); + console.debug( + `[fetchExternalManifest] fetched external manifest`, + result, + ); return result; } catch (e) { - console.error(`[fetchExternalConfig] failed to fetch external config`, e); + console.error( + `[fetchExternalManifest] failed to fetch external manifest`, + e, + ); retries -= 1; } } console.error( - `[fetchExternalConfig] failed to fetch external config after retries, revalidation is set to ${config.ERROR_REVALIDATION}`, + `[fetchExternalManifest] failed to fetch external manifest after retries, revalidation is set to ${config.ERROR_REVALIDATION}`, ); return { revalidate: config.ERROR_REVALIDATION, diff --git a/utilsApi/get-default-static-props.ts b/utilsApi/get-default-static-props.ts index b35623ae9..aefc98079 100644 --- a/utilsApi/get-default-static-props.ts +++ b/utilsApi/get-default-static-props.ts @@ -1,5 +1,7 @@ -import type { GetStaticProps, PreviewData } from 'next'; +import type { GetStaticProps, GetStaticPropsResult, PreviewData } from 'next'; import type { ParsedUrlQuery } from 'querystring'; + +import Metrics from 'utilsApi/metrics'; import { fetchExternalManifest } from './fetch-external-manifest'; export const getDefaultStaticProps = < @@ -11,6 +13,7 @@ export const getDefaultStaticProps = < ): GetStaticProps

=> { let shouldZeroRevalidate = true; return async (context) => { + /// common props const { ___prefetch_manifest___, revalidate } = await fetchExternalManifest(); const props = ___prefetch_manifest___ ? { ___prefetch_manifest___ } : {}; @@ -18,15 +21,25 @@ export const getDefaultStaticProps = < props, revalidate: shouldZeroRevalidate ? 1 : revalidate, }; + + /// custom getStaticProps + let result = base as GetStaticPropsResult

; if (custom) { const { props: customProps, ...rest } = (await custom(context)) as any; - return { + result = { ...base, ...rest, props: { ...base.props, ...customProps }, }; } + + /// metrics + console.debug( + `[getDefaultStaticProps] running revalidation, next revalidation in ${base.revalidate}`, + ); + Metrics.request.ssrCounter.labels({ revalidate: base.revalidate }).inc(1); + shouldZeroRevalidate = false; - return base; + return result; }; }; diff --git a/utilsApi/metrics/request.ts b/utilsApi/metrics/request.ts index 1553528e4..c966fa9fb 100644 --- a/utilsApi/metrics/request.ts +++ b/utilsApi/metrics/request.ts @@ -6,12 +6,14 @@ export class RequestMetrics { apiTimingsExternal: Histogram<'hostname' | 'route' | 'entity' | 'status'>; requestCounter: Counter<'route'>; ethCallToAddress: Counter<'address' | 'referrer'>; + ssrCounter: Counter<'revalidate'>; constructor(public registry: Registry) { this.apiTimings = this.apiTimingsInit('internal'); this.apiTimingsExternal = this.apiTimingsInit('external'); this.requestCounter = this.requestsCounterInit(); this.ethCallToAddress = this.ethCallToAddressInit(); + this.ssrCounter = this.ssrCounterInit(); } apiTimingsInit(postfix: string) { @@ -53,4 +55,13 @@ export class RequestMetrics { registers: [this.registry], }); } + + ssrCounterInit() { + return new Counter({ + name: METRICS_PREFIX + METRIC_NAMES.SSR_COUNT, + help: 'Counts of running getDefaultStaticProps with revalidation param', + labelNames: ['revalidate'], + registers: [this.registry], + }); + } } From 122a925850f141b94572076d8261ba05e9d836b0 Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Wed, 5 Jun 2024 20:37:15 +0700 Subject: [PATCH 04/15] fix: naming --- config/external-config/use-external-config-context.ts | 11 +++++++---- config/external-config/utils.ts | 2 -- consts/external-links.ts | 4 ++-- utilsApi/fetch-external-manifest.ts | 10 +++++----- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/config/external-config/use-external-config-context.ts b/config/external-config/use-external-config-context.ts index 3ca8410b6..900ed8073 100644 --- a/config/external-config/use-external-config-context.ts +++ b/config/external-config/use-external-config-context.ts @@ -1,14 +1,17 @@ import { STRATEGY_LAZY } from 'consts/swr-strategies'; import { getConfig } from '../get-config'; import { standardFetcher } from 'utils/standardFetcher'; -import { IPFS_MANIFEST_PATH } from 'consts/external-links'; +import { IPFS_MANIFEST_URL } from 'consts/external-links'; import { isManifestEntryValid, useFallbackManifestEntry } from './utils'; import { ExternalConfig, ManifestEntry } from './types'; import useSWR from 'swr'; import { useMemo } from 'react'; -const onFetchError = (e: any) => { - console.warn('[useExternalConfigContext] while fetching external config:', e); +const onFetchError = (error: unknown) => { + console.warn( + '[useExternalConfigContext] while fetching external config:', + error, + ); }; export const useExternalConfigContext = ( @@ -24,7 +27,7 @@ export const useExternalConfigContext = ( ['swr:external-config', defaultChain], async () => { const result = await standardFetcher>( - IPFS_MANIFEST_PATH, + IPFS_MANIFEST_URL, { headers: { Accept: 'application/json' }, }, diff --git a/config/external-config/utils.ts b/config/external-config/utils.ts index 46cb88583..a709f53e2 100644 --- a/config/external-config/utils.ts +++ b/config/external-config/utils.ts @@ -16,8 +16,6 @@ export const isManifestEntryValid = ( // entry = {} entry && typeof entry === 'object' && - entry && - typeof entry === 'object' && // entry.config = {} 'config' in entry && typeof entry.config === 'object' && diff --git a/consts/external-links.ts b/consts/external-links.ts index 649f00ee1..be791f8b9 100644 --- a/consts/external-links.ts +++ b/consts/external-links.ts @@ -4,6 +4,6 @@ export const LINK_ADD_NFT_GUIDE = export const OPEN_OCEAN_REFERRAL_ADDRESS = '0xbb1263222b2c020f155d409dba05c4a3861f18f8'; -// for dev and local testing you can set to '/runtime/IPFS.json' and have file at /public/runtime/IPFS.json -export const IPFS_MANIFEST_PATH = +// for dev and local testing you can set to 'http:/localhost:3000/runtime/IPFS.json' and have file at /public/runtime/IPFS.json +export const IPFS_MANIFEST_URL = 'https://raw.githubusercontent.com/lidofinance/ethereum-staking-widget/main/IPFS.json'; diff --git a/utilsApi/fetch-external-manifest.ts b/utilsApi/fetch-external-manifest.ts index d7567306e..b53cb6c9e 100644 --- a/utilsApi/fetch-external-manifest.ts +++ b/utilsApi/fetch-external-manifest.ts @@ -1,5 +1,5 @@ import { Cache } from 'memory-cache'; -import { IPFS_MANIFEST_PATH } from 'consts/external-links'; +import { IPFS_MANIFEST_URL } from 'consts/external-links'; import { responseTimeExternalMetricWrapper } from './fetchApiWrapper'; import { standardFetcher } from 'utils/standardFetcher'; import { config } from 'config'; @@ -22,9 +22,9 @@ export const fetchExternalManifest = async () => { while (retries > 0) { try { const data = await responseTimeExternalMetricWrapper({ - payload: IPFS_MANIFEST_PATH, + payload: IPFS_MANIFEST_URL, request: () => - standardFetcher(IPFS_MANIFEST_PATH, { + standardFetcher(IPFS_MANIFEST_URL, { headers: { Accept: 'application/json' }, }), }); @@ -48,10 +48,10 @@ export const fetchExternalManifest = async () => { ); return result; - } catch (e) { + } catch (error) { console.error( `[fetchExternalManifest] failed to fetch external manifest`, - e, + error, ); retries -= 1; } From a2f27f81ae4895f262442cf741dc26edc34372f3 Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Wed, 19 Jun 2024 13:44:54 +0700 Subject: [PATCH 05/15] feat: use local manifest as fallback --- config/external-config/utils.ts | 7 ++++--- utilsApi/fetch-external-manifest.ts | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/config/external-config/utils.ts b/config/external-config/utils.ts index a709f53e2..d2cfa8a78 100644 --- a/config/external-config/utils.ts +++ b/config/external-config/utils.ts @@ -1,12 +1,11 @@ import { useMemo } from 'react'; -import { getConfig } from '../get-config'; import type { Manifest, ManifestEntry } from './types'; import { type DexWithdrawalApi, getDexConfig, } from 'features/withdrawals/request/withdrawal-rates'; -const config = getConfig(); +import FallbackLocalManifest from 'IPFS.json' assert { type: 'json' }; // TODO: refactor on config expansion export const isManifestEntryValid = ( @@ -57,6 +56,8 @@ export const useFallbackManifestEntry = ( ): ManifestEntry => { return useMemo(() => { const isValid = isManifestValid(prefetchedManifest, chain); - return isValid ? prefetchedManifest[chain] : config.FALLBACK_MANIFEST_ENTRY; + return isValid + ? prefetchedManifest[chain] + : (FallbackLocalManifest as unknown as Manifest)[chain]; }, [prefetchedManifest, chain]); }; diff --git a/utilsApi/fetch-external-manifest.ts b/utilsApi/fetch-external-manifest.ts index b53cb6c9e..8c5b6373c 100644 --- a/utilsApi/fetch-external-manifest.ts +++ b/utilsApi/fetch-external-manifest.ts @@ -4,6 +4,8 @@ import { responseTimeExternalMetricWrapper } from './fetchApiWrapper'; import { standardFetcher } from 'utils/standardFetcher'; import { config } from 'config'; +import FallbackLocalManifest from 'IPFS.json' assert { type: 'json' }; + export type ExternalConfigResult = { ___prefetch_manifest___: object | null; revalidate: number; @@ -61,6 +63,6 @@ export const fetchExternalManifest = async () => { ); return { revalidate: config.ERROR_REVALIDATION, - ___prefetch_manifest___: null, + ___prefetch_manifest___: FallbackLocalManifest, }; }; From e43a6a69ee07d9e841582c42ff9ae2e1a26841eb Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Wed, 19 Jun 2024 17:07:20 +0700 Subject: [PATCH 06/15] fix: use local manifest for IPFS build --- utilsApi/fetch-external-manifest.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/utilsApi/fetch-external-manifest.ts b/utilsApi/fetch-external-manifest.ts index 8c5b6373c..46791e33e 100644 --- a/utilsApi/fetch-external-manifest.ts +++ b/utilsApi/fetch-external-manifest.ts @@ -20,6 +20,15 @@ export const fetchExternalManifest = async () => { const cachedConfig = cache.get(config.CACHE_EXTERNAL_CONFIG_KEY); if (cachedConfig) return cachedConfig; + // for IPFS build we use local manifest + // this allows local CID verification + if (config.ipfsMode) { + return { + ___prefetch_manifest___: FallbackLocalManifest, + revalidate: config.DEFAULT_REVALIDATION, + }; + } + let retries = 3; while (retries > 0) { try { From e7da3925c6065ad605f37ad5d6e697df78e50093 Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Wed, 19 Jun 2024 17:27:28 +0700 Subject: [PATCH 07/15] feat: add config backward compatibility --- .../use-external-config-context.ts | 10 +++++++--- config/external-config/utils.ts | 18 +++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/config/external-config/use-external-config-context.ts b/config/external-config/use-external-config-context.ts index 900ed8073..be96391fd 100644 --- a/config/external-config/use-external-config-context.ts +++ b/config/external-config/use-external-config-context.ts @@ -2,7 +2,11 @@ import { STRATEGY_LAZY } from 'consts/swr-strategies'; import { getConfig } from '../get-config'; import { standardFetcher } from 'utils/standardFetcher'; import { IPFS_MANIFEST_URL } from 'consts/external-links'; -import { isManifestEntryValid, useFallbackManifestEntry } from './utils'; +import { + getBackwardCompatibleConfig, + isManifestEntryValid, + useFallbackManifestEntry, +} from './utils'; import { ExternalConfig, ManifestEntry } from './types'; import useSWR from 'swr'; import { useMemo } from 'react'; @@ -48,7 +52,7 @@ export const useExternalConfigContext = ( return useMemo(() => { const { config, ...rest } = swr.data ?? fallbackData; - - return { ...config, ...rest, fetchMeta: swr }; + const cleanConfig = getBackwardCompatibleConfig(config); + return { ...cleanConfig, ...rest, fetchMeta: swr }; }, [swr, fallbackData]); }; diff --git a/config/external-config/utils.ts b/config/external-config/utils.ts index d2cfa8a78..9dc7e8d86 100644 --- a/config/external-config/utils.ts +++ b/config/external-config/utils.ts @@ -1,9 +1,6 @@ import { useMemo } from 'react'; import type { Manifest, ManifestEntry } from './types'; -import { - type DexWithdrawalApi, - getDexConfig, -} from 'features/withdrawals/request/withdrawal-rates'; +import { getDexConfig } from 'features/withdrawals/request/withdrawal-rates'; import FallbackLocalManifest from 'IPFS.json' assert { type: 'json' }; @@ -27,9 +24,6 @@ export const isManifestEntryValid = ( ) { const enabledWithdrawalDexes = config.enabledWithdrawalDexes; return ( - (enabledWithdrawalDexes as string[]).every( - (dex) => !!getDexConfig(dex as DexWithdrawalApi), - ) && new Set(enabledWithdrawalDexes).size === enabledWithdrawalDexes.length ); } @@ -38,6 +32,16 @@ export const isManifestEntryValid = ( return false; }; +export const getBackwardCompatibleConfig = ( + config: ManifestEntry['config'], +): ManifestEntry['config'] => { + return { + enabledWithdrawalDexes: config.enabledWithdrawalDexes.filter( + (dex) => !!getDexConfig(dex), + ), + }; +}; + export const isManifestValid = ( manifest: unknown, chain: number, From 3805e0e991f74ae8b5c1903fabdf4ebd080ea891 Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Mon, 19 Aug 2024 14:46:38 +0700 Subject: [PATCH 08/15] fix: correct swr error merge --- .../use-remote-version.ts | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/features/ipfs/security-status-banner/use-remote-version.ts b/features/ipfs/security-status-banner/use-remote-version.ts index 38c41cd9c..0c524f5ad 100644 --- a/features/ipfs/security-status-banner/use-remote-version.ts +++ b/features/ipfs/security-status-banner/use-remote-version.ts @@ -15,11 +15,11 @@ export const useRemoteVersion = () => { // we use directly non-optimistic manifest data // can't trust static props(in IPFS esp) to generate warnings/disconnect wallet - const { data, error } = useConfig().externalConfig.fetchMeta; + const externalConfigSwr = useConfig().externalConfig.fetchMeta; + const { data, error } = externalConfigSwr; - // ens&cid extraction - return useLidoSWR( - ['swr:use-remote-version', data], + const swr = useLidoSWR( + ['swr:use-remote-version', externalConfigSwr.data], async (): Promise => { if (data?.ens) { const resolver = await provider.getResolver(data.ens); @@ -52,4 +52,22 @@ export const useRemoteVersion = () => { isPaused: () => !(data || error), }, ); + + // merged externalConfigSwr && cidSwr + return { + data: swr.data, + get initialLoading() { + return ( + swr.initialLoading || + (externalConfigSwr.data == null && externalConfigSwr.isValidating) + ); + }, + get loading() { + return swr.loading || externalConfigSwr.isValidating; + }, + get error() { + return swr.error || error; + }, + update: swr.update, + }; }; From b1333f18202ed7e6e23e6576161d4fc1cec54d2c Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Mon, 19 Aug 2024 14:51:56 +0700 Subject: [PATCH 09/15] chore: comment --- features/ipfs/security-status-banner/use-remote-version.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/features/ipfs/security-status-banner/use-remote-version.ts b/features/ipfs/security-status-banner/use-remote-version.ts index 0c524f5ad..d970860bc 100644 --- a/features/ipfs/security-status-banner/use-remote-version.ts +++ b/features/ipfs/security-status-banner/use-remote-version.ts @@ -18,6 +18,8 @@ export const useRemoteVersion = () => { const externalConfigSwr = useConfig().externalConfig.fetchMeta; const { data, error } = externalConfigSwr; + // we only need this as swr because of possible future ENS support + // otherwise there is no fetch const swr = useLidoSWR( ['swr:use-remote-version', externalConfigSwr.data], async (): Promise => { From efd0df58abe70512ac072f9215b28da04bd72066 Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Mon, 19 Aug 2024 18:19:17 +0700 Subject: [PATCH 10/15] fix: swr data --- config/external-config/use-external-config-context.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/config/external-config/use-external-config-context.ts b/config/external-config/use-external-config-context.ts index be96391fd..c6a109714 100644 --- a/config/external-config/use-external-config-context.ts +++ b/config/external-config/use-external-config-context.ts @@ -45,7 +45,6 @@ export const useExternalConfigContext = ( }, { ...STRATEGY_LAZY, - fallbackData: fallbackData, onError: onFetchError, }, ); From 9699e1d4f9e799b88eb5747919834097f7716100 Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Mon, 26 Aug 2024 19:46:55 +0700 Subject: [PATCH 11/15] chore: imports --- config/external-config/use-external-config-context.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config/external-config/use-external-config-context.ts b/config/external-config/use-external-config-context.ts index c6a109714..dc3dceca0 100644 --- a/config/external-config/use-external-config-context.ts +++ b/config/external-config/use-external-config-context.ts @@ -1,3 +1,6 @@ +import { useMemo } from 'react'; +import useSWR from 'swr'; + import { STRATEGY_LAZY } from 'consts/swr-strategies'; import { getConfig } from '../get-config'; import { standardFetcher } from 'utils/standardFetcher'; @@ -7,9 +10,8 @@ import { isManifestEntryValid, useFallbackManifestEntry, } from './utils'; -import { ExternalConfig, ManifestEntry } from './types'; -import useSWR from 'swr'; -import { useMemo } from 'react'; + +import type { ExternalConfig, ManifestEntry } from './types'; const onFetchError = (error: unknown) => { console.warn( From 4817ece12d3fad09f2cfaa5cf5a9559b6cb652b9 Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Mon, 26 Aug 2024 20:13:59 +0700 Subject: [PATCH 12/15] chore: naming --- utilsApi/fetch-external-manifest.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utilsApi/fetch-external-manifest.ts b/utilsApi/fetch-external-manifest.ts index 46791e33e..a388c1efb 100644 --- a/utilsApi/fetch-external-manifest.ts +++ b/utilsApi/fetch-external-manifest.ts @@ -68,10 +68,10 @@ export const fetchExternalManifest = async () => { } } console.error( - `[fetchExternalManifest] failed to fetch external manifest after retries, revalidation is set to ${config.ERROR_REVALIDATION}`, + `[fetchExternalManifest] failed to fetch external manifest after retries, revalidation is set to ${config.ERROR_REVALIDATION_SECONDS}`, ); return { - revalidate: config.ERROR_REVALIDATION, + revalidate: config.ERROR_REVALIDATION_SECONDS, ___prefetch_manifest___: FallbackLocalManifest, }; }; From 9245e790b3ea93581cec3a55b9338f211947918b Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Mon, 26 Aug 2024 20:14:46 +0700 Subject: [PATCH 13/15] chore: naming --- config/groups/revalidation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/groups/revalidation.ts b/config/groups/revalidation.ts index 4e8f850ae..59c55f761 100644 --- a/config/groups/revalidation.ts +++ b/config/groups/revalidation.ts @@ -1,7 +1,7 @@ import type { ManifestConfig, ManifestEntry } from 'config/external-config'; export const DEFAULT_REVALIDATION = 60 * 15; // 15 minutes -export const ERROR_REVALIDATION = 60; // 1 minute +export const ERROR_REVALIDATION_SECONDS = 60; // 1 minute export const FALLBACK_CONFIG: ManifestConfig = { enabledWithdrawalDexes: [], From e5b8f80a79c74182c480c7f21cd62d70980c0bde Mon Sep 17 00:00:00 2001 From: Evgeny Taktarov Date: Mon, 2 Sep 2024 20:05:13 +0700 Subject: [PATCH 14/15] chore: holesky test ipfs config --- IPFS.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPFS.json b/IPFS.json index df2d553d1..3be584f63 100644 --- a/IPFS.json +++ b/IPFS.json @@ -16,7 +16,7 @@ "cid": "bafybeibbsoqlofslw273b4ih2pdxfaz2zbjmred2ijog725tcmfoewix7y", "leastSafeVersion": "0.36.1", "config": { - "enabledWithdrawalDexes": ["one-inch", "paraswap", "bebop"] + "enabledWithdrawalDexes": ["one-inch", "paraswap"] } } } From 18d09e6f82947924d1c98c8dbb8a4b4a02c79bd0 Mon Sep 17 00:00:00 2001 From: Anton Shalimov Date: Tue, 3 Sep 2024 09:10:01 +0300 Subject: [PATCH 15/15] feat: add env vars logging, rpc startup checks, new metrics * feat: add ENV logs * feat: push open ENV vars to prom * refactor: simplify Metrics class * refactor: use the openKeys in the startup metrics * feat: add startup check RPC * fix: log level * feat: check all rpc * refactor: dirs * feat: remove RUN_STARTUP_CHECKS from 'yarn dev' * feat: pass startup-rpc-checks from mjs to Metrics class (ts source code) * feat: rpc startup checks metrics * feat: viem instead of ethers * chore: remove debug * refactor: rpc timeout 10 seconds * feat: remove stopping the app --- consts/metrics.ts | 1 + next.config.mjs | 8 +++ package.json | 2 +- pages/api/metrics.ts | 4 +- scripts/log-environment-variables.mjs | 80 +++++++++++++++++++++++++++ scripts/startup-checks/rpc.mjs | 78 ++++++++++++++++++++++++++ utilsApi/metrics/metrics.ts | 21 +------ utilsApi/metrics/startup-checks.ts | 22 ++++++++ utilsApi/metrics/startup-metrics.ts | 54 ++++++++++++++++++ 9 files changed, 248 insertions(+), 22 deletions(-) create mode 100644 scripts/log-environment-variables.mjs create mode 100644 scripts/startup-checks/rpc.mjs create mode 100644 utilsApi/metrics/startup-checks.ts create mode 100644 utilsApi/metrics/startup-metrics.ts diff --git a/consts/metrics.ts b/consts/metrics.ts index 5b529fd4f..b1a9ba104 100644 --- a/consts/metrics.ts +++ b/consts/metrics.ts @@ -2,6 +2,7 @@ export const METRICS_PREFIX = 'eth_stake_widget_ui_'; export const enum METRIC_NAMES { REQUESTS_TOTAL = 'requests_total', + STARTUP_CHECKS_RPC = 'startup_checks_rpc', API_RESPONSE = 'api_response', ETH_CALL_ADDRESS_TO = 'eth_call_address_to', SSR_COUNT = 'ssr_count', diff --git a/next.config.mjs b/next.config.mjs index 9cd936eb3..ce2c62c2f 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -1,9 +1,17 @@ import NextBundleAnalyzer from '@next/bundle-analyzer'; import buildDynamics from './scripts/build-dynamics.mjs'; +import { logEnvironmentVariables } from './scripts/log-environment-variables.mjs'; import generateBuildId from './scripts/generate-build-id.mjs'; +import { startupCheckRPCs } from './scripts/startup-checks/rpc.mjs'; +logEnvironmentVariables(); buildDynamics(); +if (process.env.RUN_STARTUP_CHECKS === 'true' && typeof window === 'undefined') { + void startupCheckRPCs(); +} + + // https://nextjs.org/docs/pages/api-reference/next-config-js/basePath const basePath = process.env.BASE_PATH; diff --git a/package.json b/package.json index 1876485f2..5ca71f1a9 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "build": "NODE_OPTIONS='--no-warnings=ExperimentalWarning' next build", "build:analyze": "ANALYZE_BUNDLE=true yarn build", "build:ipfs": "IPFS_MODE=true yarn build && IPFS_MODE=true next export", - "start": "NODE_ENV=production node -r next-logger --no-warnings=ExperimentalWarning server.mjs", + "start": "NODE_ENV=production RUN_STARTUP_CHECKS=true node -r next-logger --no-warnings=ExperimentalWarning server.mjs", "lint": "eslint --ext ts,tsx,js,mjs .", "lint:fix": "yarn lint --fix", "types": "tsc --noEmit", diff --git a/pages/api/metrics.ts b/pages/api/metrics.ts index 44913d34a..02902da85 100644 --- a/pages/api/metrics.ts +++ b/pages/api/metrics.ts @@ -1,13 +1,13 @@ import { wrapRequest as wrapNextRequest } from '@lidofinance/next-api-wrapper'; +import { metricsFactory } from '@lidofinance/next-pages'; + import { API_ROUTES } from 'consts/api'; import { responseTimeMetric, errorAndCacheDefaultWrappers, rateLimit, } from 'utilsApi'; - import Metrics from 'utilsApi/metrics'; -import { metricsFactory } from '@lidofinance/next-pages'; const metrics = metricsFactory({ registry: Metrics.registry, diff --git a/scripts/log-environment-variables.mjs b/scripts/log-environment-variables.mjs new file mode 100644 index 000000000..e019db116 --- /dev/null +++ b/scripts/log-environment-variables.mjs @@ -0,0 +1,80 @@ +export const openKeys = [ + 'SELF_ORIGIN', + 'ROOT_ORIGIN', + 'DOCS_ORIGIN', + 'HELP_ORIGIN', + 'RESEARCH_ORIGIN', + + 'SUPPORTED_CHAINS', + 'DEFAULT_CHAIN', + + 'CSP_TRUSTED_HOSTS', + 'CSP_REPORT_ONLY', + 'CSP_REPORT_URI', + + 'ENABLE_QA_HELPERS', + + 'REWARDS_BACKEND', + + 'RATE_LIMIT', + 'RATE_LIMIT_TIME_FRAME', + + 'ETH_API_BASE_PATH', + 'WQ_API_BASE_PATH', + 'MATOMO_URL', + 'WALLETCONNECT_PROJECT_ID', + 'REWARDS_BACKEND_BASE_PATH', +]; + +export const secretKeys = [ + 'EL_RPC_URLS_1', + 'EL_RPC_URLS_5', + 'EL_RPC_URLS_17000', + 'EL_RPC_URLS_11155111', +] + + +export const logOpenEnvironmentVariables = () => { + console.log('---------------------------------------------'); + console.log('Log environment variables (without secrets):'); + console.log('---------------------------------------------'); + + for (const key of openKeys) { + if (!process.env.hasOwnProperty(key)) { + console.error(`${key} - ERROR (not exist in process.env)`); + continue; + } + + console.info(`${key} = ${process.env[key]}`); + } + + console.log('---------------------------------------------'); + console.log(''); +}; + +export const logSecretEnvironmentVariables = () => { + console.log('---------------------------------------------'); + console.log('Log secret environment variables:'); + console.log('---------------------------------------------'); + + // console.log('process.env:', process.env) + for (const key of secretKeys) { + if (!process.env.hasOwnProperty(key)) { + console.error(`Secret ${key} - ERROR (not exist in process.env)`); + continue; + } + + if (process.env[key].length > 0) { + console.info(`Secret ${key} - OK (exist and not empty)`); + } else { + console.warn(`Secret ${key} - WARN (exist but empty)`); + } + } + + console.log('---------------------------------------------'); +}; + +export const logEnvironmentVariables = () => { + logOpenEnvironmentVariables(); + logSecretEnvironmentVariables(); +}; diff --git a/scripts/startup-checks/rpc.mjs b/scripts/startup-checks/rpc.mjs new file mode 100644 index 000000000..7bb55cb4e --- /dev/null +++ b/scripts/startup-checks/rpc.mjs @@ -0,0 +1,78 @@ +import { createClient, http } from 'viem'; +import { getChainId } from 'viem/actions' + +// Safely initialize a global variable +let globalRPCCheckResults = globalThis.__rpcCheckResults || []; +globalThis.__rpcCheckResults = globalRPCCheckResults; + +export const BROKEN_URL = 'BROKEN_URL'; +export const RPC_TIMEOUT_SECONDS = 10_000; + +const pushRPCCheckResult = (domain, success) => { + globalRPCCheckResults.push({ domain, success }); +}; + +export const getRPCCheckResults = () => globalThis.__rpcCheckResults || []; + +const getRpcUrls = (chainId) => { + const rpcUrls = process.env[`EL_RPC_URLS_${chainId}`]?.split(','); + return rpcUrls?.filter((url) => url); +}; + +export const startupCheckRPCs = async () => { + console.info('[startupCheckRPCs] Starting...'); + + try { + const defaultChain = parseInt(process.env.DEFAULT_CHAIN, 10); + const rpcUrls = getRpcUrls(defaultChain); + + if (!rpcUrls || rpcUrls.length === 0) { + throw new Error('[startupCheckRPCs] No RPC URLs found!'); + } + + let errorCount = 0; + + for (const url of rpcUrls) { + let domain; + try { + domain = new URL(url).hostname; + } catch (err) { + errorCount += 1; + console.error('There is a broken URL.'); + pushRPCCheckResult(BROKEN_URL, false); + continue; + } + + try { + const client = createClient({ + transport: http(url, { retryCount: 0, timeout: RPC_TIMEOUT_SECONDS }) + }); + + const chainId = await getChainId(client); + + if (defaultChain === chainId) { + pushRPCCheckResult(domain, true); + console.info(`[startupCheckRPCs] RPC ${domain} works!`); + } else { + throw(`[startupCheckRPCs] RPC ${domain} does not work!`); + } + } catch (err) { + errorCount += 1; + pushRPCCheckResult(domain, false); + console.error(`[startupCheckRPCs] Error with RPC ${domain}:`); + console.error(String(err).replaceAll(rpcUrls, domain)); + console.error(`[startupCheckRPCs] Timeout: ${RPC_TIMEOUT_SECONDS} seconds`); + } + } + + if (errorCount > 0) { + console.info(`[startupCheckRPCs] Number of working RPCs: ${rpcUrls.length - errorCount}`); + console.info(`[startupCheckRPCs] Number of broken RPCs: ${errorCount}`); + } else { + console.info('[startupCheckRPCs] All RPC works!'); + } + } catch (err) { + console.error('[startupCheckRPCs] Error during startup check:'); + console.error(err); + } +}; diff --git a/utilsApi/metrics/metrics.ts b/utilsApi/metrics/metrics.ts index 3c5b2337e..d75d75c4d 100644 --- a/utilsApi/metrics/metrics.ts +++ b/utilsApi/metrics/metrics.ts @@ -1,10 +1,7 @@ import { collectDefaultMetrics, Registry } from 'prom-client'; -import { collectStartupMetrics } from '@lidofinance/api-metrics'; - -import { config } from 'config'; import { METRICS_PREFIX } from 'consts/metrics'; -import buildInfoJson from 'build-info.json'; +import { collectStartupMetrics } from './startup-metrics'; import { RequestMetrics } from './request'; class Metrics { @@ -14,23 +11,9 @@ class Metrics { request = new RequestMetrics(this.registry); constructor() { - this.collectStartupMetricsInit(); + collectStartupMetrics(this.registry); collectDefaultMetrics({ prefix: METRICS_PREFIX, register: this.registry }); } - - collectStartupMetricsInit() { - collectStartupMetrics({ - prefix: METRICS_PREFIX, - registry: this.registry, - defaultChain: `${config.defaultChain}`, - supportedChains: config.supportedChains.map( - (chain: number) => `${chain}`, - ), - version: buildInfoJson.version, - commit: buildInfoJson.commit, - branch: buildInfoJson.branch, - }); - } } export default new Metrics(); diff --git a/utilsApi/metrics/startup-checks.ts b/utilsApi/metrics/startup-checks.ts new file mode 100644 index 000000000..aa1e4f5dc --- /dev/null +++ b/utilsApi/metrics/startup-checks.ts @@ -0,0 +1,22 @@ +import { Counter, Registry } from 'prom-client'; +import { METRICS_PREFIX, METRIC_NAMES } from 'consts/metrics'; + +export class StartupChecksRPCMetrics { + requestCounter: Counter<'rpc_domain' | 'success'>; + + constructor(public registry: Registry) { + this.requestCounter = this.requestsCounterInit(); + } + + requestsCounterInit() { + const requestsCounterName = + METRICS_PREFIX + METRIC_NAMES.STARTUP_CHECKS_RPC; + + return new Counter({ + name: requestsCounterName, + help: 'The total number of RPC checks after the app started.', + labelNames: ['rpc_domain', 'success'], + registers: [this.registry], + }); + } +} diff --git a/utilsApi/metrics/startup-metrics.ts b/utilsApi/metrics/startup-metrics.ts new file mode 100644 index 000000000..9fd417851 --- /dev/null +++ b/utilsApi/metrics/startup-metrics.ts @@ -0,0 +1,54 @@ +import { Gauge, type Registry } from 'prom-client'; +import { collectStartupMetrics as collectBuildInfoMetrics } from '@lidofinance/api-metrics'; + +import buildInfoJson from 'build-info.json'; +import { openKeys } from 'scripts/log-environment-variables.mjs'; +import { getRPCCheckResults } from 'scripts/startup-checks/rpc.mjs'; + +import { config } from 'config'; +import { METRICS_PREFIX } from 'consts/metrics'; + +import { StartupChecksRPCMetrics } from './startup-checks'; + +const collectStartupChecksRPCMetrics = (registry: Registry): void => { + const rpcMetrics = new StartupChecksRPCMetrics(registry); + + getRPCCheckResults().forEach( + (_check: { domain: string; success: boolean }) => { + rpcMetrics.requestCounter + .labels(_check.domain, _check.success.toString()) + .inc(); + }, + ); +}; + +const collectEnvInfoMetrics = (registry: Registry): void => { + const labelPairs = openKeys.map((key) => ({ + name: key, + value: process.env[key] ?? '', + })); + + const envInfo = new Gauge({ + name: METRICS_PREFIX + 'env_info', + help: 'Environment variables of the current runtime', + labelNames: labelPairs.map((pair) => pair.name), + registers: [registry], + }); + envInfo.labels(...labelPairs.map((pair) => pair.value)).set(1); +}; + +export const collectStartupMetrics = (registry: Registry): void => { + collectEnvInfoMetrics(registry); + + collectBuildInfoMetrics({ + prefix: METRICS_PREFIX, + registry: registry, + defaultChain: `${config.defaultChain}`, + supportedChains: config.supportedChains.map((chain: number) => `${chain}`), + version: buildInfoJson.version, + commit: buildInfoJson.commit, + branch: buildInfoJson.branch, + }); + + collectStartupChecksRPCMetrics(registry); +};