fix(stats): seed account counter with long deadline + retry; never scan on request path (#94)

The one-time account/DID seed called CountAccounts, whose immudb Count over the 'address:' prefix is hard-capped at 30s (Accounts_helper.go) and now exceeds it on the live accounts DB -> DeadlineExceeded. Result: the counter never seeded, and getStats' fallback re-ran the same 30s Count per request and errored the whole /api/stats response.

- GetCountofRecords takes a countTimeout (<=0 => 30s default); GetMainDBCount/GetAccountsDBCount pass 30s (unchanged). New CountBuilder.GetAccountsDBCountWithTimeout + DB_OPs.CountAccountsWithTimeout expose a caller-chosen deadline.
- The one-time seed now uses a 5-min per-attempt deadline and retries with exponential backoff (30s..10m) until it succeeds or the node shuts down; it runs off the request path so a long count never delays startup.
- getStats reads ONLY the sqlite counter (O(1)); it no longer falls back to the immudb Count on the request path. Until the background seed populates the counter it reports 0 (transient) instead of erroring the endpoint.

Verified: gofmt clean; DB_OPs + explorer build; go vet ./ (package main) exit 0.

Co-authored-by: Doc <doc@local>

Author: saishibunb

DB_OPs/Accounts_helper.go

@@ -89,7 +89,11 @@ func CloseAccountsImmuClient(PooledConnection *config.PooledConnection) error {
 
 // Adapter function to get the count of records using the Native ImmuDB API
 // Get the count of accounts uisng immudb native apis
-func GetCountofRecords(PooledConnection *config.PooledConnection, ConnType int, prefix string) (int, error) {
+// GetCountofRecords counts keys with the given prefix via immudb's Count API.
+// countTimeout bounds the Count RPC; pass <= 0 for the default 30s. Large
+// prefixes (e.g. all accounts) can exceed 30s, so off-request-path callers (the
+// one-time stats seed) pass a longer budget.
+func GetCountofRecords(PooledConnection *config.PooledConnection, ConnType int, prefix string, countTimeout time.Duration) (int, error) {
 	var err error
 	var shouldReturnConnection = false
 
@@ -154,7 +158,10 @@ func GetCountofRecords(PooledConnection *config.PooledConnection, ConnType int,
 		}
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	if countTimeout <= 0 {
+		countTimeout = 30 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), countTimeout)
 	defer cancel()
 
 	// Use immudb Count API — much faster and simpler
@@ -201,9 +208,16 @@ func (cb CountBuilder) Build() (*CountBuilder, error) {
 }
 
 func (cb CountBuilder) GetMainDBCount(prefix string) (int, error) {
-	return GetCountofRecords(nil, MainImmuConn, prefix)
+	return GetCountofRecords(nil, MainImmuConn, prefix, 30*time.Second)
 }
 
 func (cb CountBuilder) GetAccountsDBCount(prefix string) (int, error) {
-	return GetCountofRecords(nil, AccountsImmuConn, prefix)
+	return GetCountofRecords(nil, AccountsImmuConn, prefix, 30*time.Second)
+}
+
+// GetAccountsDBCountWithTimeout is GetAccountsDBCount with a caller-chosen
+// deadline for the immudb Count — used by the one-time stats seed, which runs
+// off the request path and can allow minutes on a large accounts DB.
+func (cb CountBuilder) GetAccountsDBCountWithTimeout(prefix string, countTimeout time.Duration) (int, error) {
+	return GetCountofRecords(nil, AccountsImmuConn, prefix, countTimeout)
 }

DB_OPs/account_immuclient.go

@@ -1365,6 +1365,14 @@ func CountAccounts(PooledConnection *config.PooledConnection) (int, error) {
 	return count, nil
 }
 
+// CountAccountsWithTimeout is CountAccounts with a caller-chosen deadline for the
+// underlying immudb Count. The one-time explorer-stats seed uses this: it runs
+// off the request path and can allow minutes on a large accounts DB instead of
+// failing at the default 30s.
+func CountAccountsWithTimeout(countTimeout time.Duration) (int, error) {
+	return CountBuilder{}.GetAccountsDBCountWithTimeout(Prefix, countTimeout)
+}
+
 // GetTransactionsByDID retrieves all transactions associated with a given DID
 // This implementation iterates through all blocks to find matching transactions,
 // which is more efficient than fetching each transaction individually.

explorer/BlockOps.go

@@ -595,23 +595,18 @@ func (s *ImmuDBServer) getStats(c *gin.Context) {
 		return nil
 	}, local.AddToWaitGroup(GRO.ExplorerBlockOpsWaitGroup))
 
-	// Get total DIDs/accounts in a goroutine. Prefer the maintained sqlite
-	// counter (O(1)); fall back to the immudb prefix count only when the counter
-	// has not been seeded yet (first boot before the one-time seed completes).
+	// Get total DIDs/accounts from the maintained sqlite counter only (O(1)).
+	// Deliberately does NOT fall back to the immudb prefix Count on the request
+	// path: that Count is O(n) and can exceed its deadline on a large accounts DB,
+	// which would fail the whole stats response. Until the one-time background
+	// seed populates the counter, report 0 (transient) rather than erroring.
 	BlockOpsLocalGRO.Go(GRO.ExplorerBlockOpsThread, func(ctx context.Context) error {
-		if n, seeded, err := txindex.GetAccountCount(ctx); err == nil && seeded {
-			mu.Lock()
-			stats.TotalDIDs = n
-			mu.Unlock()
-			return nil
-		}
-		totalDIDs, err := DB_OPs.CountAccounts(&s.accountsdb)
+		n, _, err := txindex.GetAccountCount(ctx)
 		if err != nil {
-			handleErr(fmt.Errorf("failed to count DIDs: %w", err))
-			return fmt.Errorf("failed to count DIDs: %w", err)
+			n = 0 // counter unavailable/unseeded — never block the endpoint on a scan
 		}
 		mu.Lock()
-		stats.TotalDIDs = int64(totalDIDs)
+		stats.TotalDIDs = n
 		mu.Unlock()
 		return nil
 	}, local.AddToWaitGroup(GRO.ExplorerBlockOpsWaitGroup))

main.go

@@ -1006,28 +1006,47 @@ func main() {
 		}()
 	})
 	// One-time seed: indexing the existing DIDs/accounts is a one-shot activity.
-	// Only run the expensive immudb CountAccounts if the counter has never been
-	// seeded; once present it is maintained by the increments above. Runs in a
-	// goroutine so a first-boot seed never delays startup.
+	// Only run the expensive immudb Count if the counter has never been seeded;
+	// once present it is maintained by the increments above. Runs in a goroutine
+	// so a first-boot seed never delays startup, and retries with backoff: the
+	// immudb Count over the accounts prefix can exceed the default 30s on a large
+	// DB or under load, so the seed uses a long per-attempt deadline and keeps
+	// retrying until it succeeds (or the node shuts down).
 	go func() {
-		_, seeded, err := txindex.GetAccountCount(context.Background())
-		if err != nil {
+		if _, seeded, err := txindex.GetAccountCount(context.Background()); err != nil {
 			log.Debug().Err(err).Msg("[stats] account counter unavailable; skipping one-time seed")
 			return
-		}
-		if seeded {
+		} else if seeded {
 			return // already indexed once — keep incrementing
 		}
-		n, err := DB_OPs.CountAccounts(nil)
-		if err != nil {
-			log.Warn().Err(err).Msg("[stats] one-time account/DID count seed failed; will retry next boot")
-			return
-		}
-		if err := txindex.SetAccountCount(context.Background(), int64(n)); err != nil {
-			log.Warn().Err(err).Msg("[stats] failed to persist seeded account/DID count")
-			return
+		backoff := 30 * time.Second
+		for attempt := 1; ; attempt++ {
+			// Re-check: another path (e.g. a manual reseed) may have seeded it.
+			if _, seeded, _ := txindex.GetAccountCount(context.Background()); seeded {
+				return
+			}
+			// Long per-attempt deadline — this runs off the request path.
+			n, err := DB_OPs.CountAccountsWithTimeout(5 * time.Minute)
+			if err == nil {
+				if serr := txindex.SetAccountCount(context.Background(), int64(n)); serr != nil {
+					log.Warn().Err(serr).Msg("[stats] failed to persist seeded account/DID count")
+				} else {
+					log.Info().Int("count", n).Int("attempt", attempt).Msg("[stats] account/DID counter seeded (one-time)")
+					return
+				}
+			} else {
+				log.Warn().Err(err).Int("attempt", attempt).Dur("retry_in", backoff).
+					Msg("[stats] one-time account/DID count seed failed; retrying")
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(backoff):
+			}
+			if backoff < 10*time.Minute {
+				backoff *= 2
+			}
 		}
-		log.Info().Int("count", n).Msg("[stats] account/DID counter seeded (one-time)")
 	}()
 
 	// ── Account Sync Worker (Redis Stream) ───────────────────────────────────