feat: Add dynamic configuration refresh and remove panic on missing RPCs

cmd/relayer_exporter/relayer_exporter.go

@@ -6,6 +6,10 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -26,10 +30,44 @@ func getVersion() string {
 	return fmt.Sprintf("version: %s commit: %s date: %s", version, commit, date)
 }
 
+// refreshCollectors updates the collectors with new configuration
+func refreshCollectors(ctx context.Context, cfg *config.Config, registry *prometheus.Registry) error {
+	paths, err := cfg.IBCPaths(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get IBC paths: %w", err)
+	}
+
+	rpcs, err := cfg.GetRPCsMap()
+	if err != nil {
+		return fmt.Errorf("failed to get RPCs map: %w", err)
+	}
+
+	// Unregister existing collectors
+	registry.Unregister(collector.IBCCollector{})
+	registry.Unregister(collector.WalletBalanceCollector{})
+
+	// Create and register new collectors
+	ibcCollector := collector.IBCCollector{
+		RPCs:  rpcs,
+		Paths: paths,
+	}
+
+	balancesCollector := collector.WalletBalanceCollector{
+		RPCs:     rpcs,
+		Accounts: cfg.Accounts,
+	}
+
+	registry.MustRegister(ibcCollector)
+	registry.MustRegister(balancesCollector)
+
+	return nil
+}
+
 func main() {
 	port := flag.Int("p", 8008, "Server port")
 	version := flag.Bool("version", false, "Print version")
 	configPath := flag.String("config", "./config.yml", "path to config file")
+	refreshInterval := flag.Duration("refresh", 5*time.Minute, "Configuration refresh interval")
 	logLevel := log.LevelFlag()
 
 	flag.Parse()
@@ -55,34 +93,82 @@ func main() {
 		zap.String("Testnet Directory", cfg.GitHub.TestnetsIBCDir),
 	)
 
-	ctx := context.Background()
-	// TODO: Add a feature to refresh paths at configured interval
-	paths, err := cfg.IBCPaths(ctx)
-	if err != nil {
-		log.Fatal(err.Error())
-	}
+	// Create a context with cancel
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel() // Ensure context is cancelled when main exits
 
-	rpcs, err := cfg.GetRPCsMap(paths)
-	if err != nil {
+	// Setup signal handling for graceful shutdown
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	registry := prometheus.NewRegistry()
+
+	// Initial setup of collectors
+	if err := refreshCollectors(ctx, cfg, registry); err != nil {
 		log.Fatal(err.Error())
 	}
 
-	ibcCollector := collector.IBCCollector{
-		RPCs:  rpcs,
-		Paths: paths,
+	// Start periodic refresh in background
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+		ticker := time.NewTicker(*refreshInterval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ctx.Done():
+				log.Info("Stopping collector refresh routine")
+				return
+			case <-ticker.C:
+				log.Info("Refreshing configuration and collectors")
+				if err := refreshCollectors(ctx, cfg, registry); err != nil {
+					log.Error(fmt.Sprintf("Failed to refresh collectors: %v", err))
+					continue
+				}
+				log.Info("Successfully refreshed configuration and collectors")
+			}
+		}
+	}()
+
+	// Setup HTTP server
+	server := &http.Server{
+		Addr:    fmt.Sprintf(":%d", *port),
+		Handler: nil,
 	}
 
-	balancesCollector := collector.WalletBalanceCollector{
-		RPCs:     rpcs,
-		Accounts: cfg.Accounts,
+	// Setup HTTP handler with custom registry
+	handler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
+	http.Handle("/metrics", handler)
+
+	// Start server in a goroutine
+	go func() {
+		log.Info(fmt.Sprintf("Starting server on addr: %s", server.Addr))
+		log.Info(fmt.Sprintf("Configuration refresh interval: %s", refreshInterval.String()))
+		if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Fatal(fmt.Sprintf("Server error: %v", err))
+		}
+	}()
+
+	// Wait for shutdown signal
+	<-sigChan
+	log.Info("Received shutdown signal")
+
+	// Initiate graceful shutdown
+	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer shutdownCancel()
+
+	// Trigger context cancellation to stop the refresh routine
+	cancel()
+
+	// Shutdown the HTTP server
+	if err := server.Shutdown(shutdownCtx); err != nil {
+		log.Error(fmt.Sprintf("Server shutdown error: %v", err))
 	}
 
-	prometheus.MustRegister(ibcCollector)
-	prometheus.MustRegister(balancesCollector)
-
-	http.Handle("/metrics", promhttp.Handler())
-
-	addr := fmt.Sprintf(":%d", *port)
-	log.Info(fmt.Sprintf("Starting server on addr: %s", addr))
-	log.Fatal(http.ListenAndServe(addr, nil).Error())
+	// Wait for background tasks to complete
+	wg.Wait()
+	log.Info("Shutdown complete")
 }

config.yaml

@@ -19,13 +19,13 @@ rpc:
     url: https://rpc.bitcanna.io:443
   - chainName: cosmoshub
     chainId: cosmoshub-4
-    url: https://rpc.cosmoshub.strange.love:443
+    url: https://cosmos-rpc.quickapi.com:443
   - chainName: decentr
     chainId: mainnet-3
     url: https://rpc.decentr.chaintools.tech:443
   - chainName: jackal
     chainId: jackal-1
-    url: https://jackal-rpc.polkachu.com:443
+    url: https://jackal.rpc.kjnodes.com:443
   - chainName: juno
     chainId: juno-1
     url: https://juno-rpc.publicnode.com:443
@@ -37,25 +37,25 @@ rpc:
     url: https://noble-rpc.polkachu.com:443
   - chainName: nois
     chainId: nois-1
-    url: https://nois.rpc.kjnodes.com:443
+    url: https://rpc.nois.silentvalidator.com:443
   - chainName: omniflixhub
     chainId: omniflixhub-1
-    url: https://rpc.omniflix.stakeup.tech:443
+    url: https://omniflix-rpc.dakshavalidator.in:443
   - chainName: osmosis
     chainId: osmosis-1
-    url: https://rpc-osmosis.whispernode.com:443
+    url: https://rpc-osmosis.ecostake.com:443
   - chainName: quicksilver
     chainId: quicksilver-2
     url: https://rpc.quicksilver.zone:443
   - chainName: umee
     chainId: umee-1
-    url: https://rpc-umee.mzonder.com:443
+    url: https://umee-rpc.polkachu.com:443
   - chainName: gravitybridge
     chainId: gravity-bridge-3
     url: https://gravity-rpc.polkachu.com:443
   - chainName: secretnetwork
     chainId: secret-4
-    url: https://secretnetwork-rpc.highstakes.ch:443
+    url: https://rpc.mainnet.secretsaturn.net:443
   - chainName: terra2
     chainId: phoenix-1
     url: https://terra-rpc.stakely.io:443
@@ -64,22 +64,22 @@ rpc:
     url: https://rpc.comdex.one:443
   - chainName: neutron
     chainId: neutron-1
-    url: https://rpc.novel.remedy.tm.p2p.org:443
+    url: https://rpc-voidara.neutron-1.neutron.org:443
   - chainName: qwoyn
     chainId: qwoyn-1
-    url: https://qwoyn-rpc.staketab.org:443
+    url: https://rpc-qwoyn.theamsolutions.info:443
   - chainName: stargaze
     chainId: stargaze-1
-    url: https://rpc.stargaze-apis.com:443
+    url: https://rpc.stargaze.silentvalidator.com:443
   - chainName: andromeda
     chainId: andromeda-1
-    url: https://andromeda-rpc.stake-town.com:443
+    url: https://andromeda.rpc.kjnodes.com:443
   - chainName: pylons
     chainId: pylons-mainnet-1
-    url: https://pylons-rpc.noders.services:443
+    url: https://rpc.nodejumper.io:443/pylons
   - chainName: injective
     chainId: injective-1
-    url: https://rpc-injective.whispernode.com:443
+    url: https://injective-rpc.publicnode.com:443
   - chainName: dydx
     chainId: dydx-mainnet-1
     url: https://rpc-dydx.ecostake.com:443
@@ -88,22 +88,22 @@ rpc:
     url: https://rpc-akash.ecostake.com:443
   - chainName: cudos
     chainId: cudos-1
-    url: https://cudos-rpc.kleomedes.network:443
+    url: https://cudos-rpc.lavenderfive.com:443
   - chainName: celestia
     chainId: celestia
     url: https://celestia-rpc.enigma-validator.com:443
   - chainName: evmos
     chainId: evmos_9001-2
-    url: https://evmos-rpc.theamsolutions.info:443
+    url: https://rpc-evmos-01.stakeflow.io:443
   - chainName: stride
     chainId: stride-1
-    url: https://rpc-stride.whispernode.com:443
+    url: https://stride-rpc.polkachu.com:443
   - chainName: persistence
     chainId: core-1
     url: https://persistence-rpc.publicnode.com:443
   - chainName: composable
     chainId: centauri-1
-    url: https://rpc.composable.citizenweb3.com:443
+    url: https://composable-mainnet-rpc.autostake.com:443
   - chainName: empowerchain
     chainId: empowerchain-1
     url: https://rpc-empowerchain.mzonder.com:443
@@ -115,16 +115,13 @@ rpc:
     url: https://passage-rpc.staketab.org:443
   - chainName: sentinel
     chainId: sentinelhub-2
-    url: https://sentinel-rpc.publicnode.com:443
+    url: https://rpc-sentinel.chainvibes.com:443
   - chainName: dymension
     chainId: dymension_1100-1
     url: https://dymension-rpc.ibs.team:443
-  - chainName: vidulum
-    chainId: vidulum-1
-    url: https://vidulum.declab.pro:26619
   - chainName: bitsong
     chainId: bitsong-2b
-    url: https://rpc.bitsong.quokkastake.io:443
+    url: https://rpc.explorebitsong.com:443
   - chainName: coreum
     chainId: coreum-mainnet-1
     url: https://coreum-rpc.ibs.team:443
@@ -135,13 +132,16 @@ rpc:
   # testnets
   - chainName: archwaytestnet
     chainId: constantine-3
-    url: https://rpc.constantine.archway.tech:443
+    url: https://rpc.constantine.archway.io:443
   - chainName: axelartestnet
     chainId: axelar-testnet-lisbon-3
-    url: https://axelar-testnet-rpc.qubelabs.io:443
+    url: https://rpc-axelar-testnet.imperator.co:443
   - chainName: osmosistestnet
     chainId: osmo-test-5
     url: https://rpc-1.testnet.osmosis.nodes.guru:443
+  # - chainName: nobletestnet
+  #   chainId: grand-1
+  #   url: https://noble-testnet-rpc.polkachu.com:443
 accounts:
   # foundation-feegrant-astrovault
   - address: archway1gpyqzc0aerc85cpk2cm8ec6zkc95x5yqrakskv
@@ -155,3 +155,6 @@ accounts:
   - address: archway1ktka5q3cnsy3ar7qwj2huzz6qj9q4ys7h74l9y
     chainName: archway
     denom: aarch
+  - address: archway1avwvqzu9gv86g5fxx5p2xqe0w33wklt27jusdrhszwccnfnxx0rsmzz8nu
+    chainName: archway
+    denom: aarch

pkg/collector/ibc_collector.go

@@ -16,6 +16,7 @@ import (
 const (
 	clientExpiryMetricName        = "cosmos_ibc_client_expiry"
 	channelStuckPacketsMetricName = "cosmos_ibc_stuck_packets"
+	configMissingMetricName       = "cosmos_ibc_config_missing"
 )
 
 var (
@@ -48,6 +49,12 @@ var (
 		},
 		nil,
 	)
+	configMissing = prometheus.NewDesc(
+		configMissingMetricName,
+		"Returns if the rpc config is missing for a channel.",
+		[]string{"src_chain_id", "dst_chain_id", "src_chain_name", "dst_chain_name"},
+		nil,
+	)
 )
 
 type IBCCollector struct {
@@ -58,6 +65,7 @@ type IBCCollector struct {
 func (cc IBCCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- clientExpiry
 	ch <- channelStuckPackets
+	ch <- configMissing
 }
 
 func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) {

pkg/config/config.go

@@ -128,7 +128,7 @@ func (a *Account) GetBalance(ctx context.Context, rpcs *map[string]RPC) error {
 // chain_names to RPCs. It uses IBCData already extracted from
 // github IBC registry to validate config for missing RPCs and raises
 // an error if any are missing.
-func (c *Config) GetRPCsMap(ibcPaths []*IBCData) (*map[string]RPC, error) {
+func (c *Config) GetRPCsMap() (*map[string]RPC, error) {
 	rpcs := map[string]RPC{}
 
 	for _, rpc := range c.RPCs {
@@ -139,19 +139,6 @@ func (c *Config) GetRPCsMap(ibcPaths []*IBCData) (*map[string]RPC, error) {
 		rpcs[rpc.ChainName] = rpc
 	}
 
-	// Validate RPCs exist for each IBC path
-	for _, ibcPath := range ibcPaths {
-		// Check RPC for chain 1
-		if _, ok := rpcs[ibcPath.Chain1.ChainName]; !ok {
-			return &rpcs, fmt.Errorf(ErrMissingRPCConfigMsg, ibcPath.Chain1.ChainName)
-		}
-
-		// Check RPC for chain 2
-		if _, ok := rpcs[ibcPath.Chain2.ChainName]; !ok {
-			return &rpcs, fmt.Errorf(ErrMissingRPCConfigMsg, ibcPath.Chain2.ChainName)
-		}
-	}
-
 	return &rpcs, nil
 }
 
@@ -197,6 +184,7 @@ func (c *Config) getPaths(ctx context.Context, dir string, client *github.Client
 	for _, file := range ibcDir {
 		if strings.HasSuffix(*file.Path, ibcPathSuffix) {
 			log.Debug(fmt.Sprintf("Fetching IBC data for %s/%s/%s", c.GitHub.Org, c.GitHub.Repo, *file.Path))
+
 			content, _, _, err := client.Repositories.GetContents(
 				ctx,
 				c.GitHub.Org,
@@ -234,6 +222,7 @@ func (c *Config) Validate() error {
 	// https://github.com/cosmos/relayer/blob/259b1278264180a2aefc2085f1b55753849c4815/cregistry/chain_info.go#L115
 	err := validate.RegisterValidation("has_port", func(fl validator.FieldLevel) bool {
 		val := fl.Field().String()
+
 		urlParsed, err := url.Parse(val)
 		if err != nil {
 			return false
@@ -247,6 +236,7 @@ func (c *Config) Validate() error {
 		); err != nil || portNum > 65535 || portNum < 1 {
 			return false
 		}
+
 		return true
 	})
 	if err != nil {