Merge branch 'ratelimiting'

Author: bznein

backend/backend.go

@@ -52,7 +52,6 @@ import (
 	"github.com/BitBoxSwiss/bitbox-wallet-app/util/logging"
 	"github.com/BitBoxSwiss/bitbox-wallet-app/util/observable"
 	"github.com/BitBoxSwiss/bitbox-wallet-app/util/observable/action"
-	"github.com/BitBoxSwiss/bitbox-wallet-app/util/ratelimit"
 	"github.com/BitBoxSwiss/bitbox-wallet-app/util/socksproxy"
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/ethereum/go-ethereum/params"
@@ -290,7 +289,7 @@ func NewBackend(arguments *arguments.Arguments, environment Environment) (*Backe
 	backend.notifier = notifier
 	backend.socksProxy = backendProxy
 	backend.httpClient = hclient
-	backend.etherScanHTTPClient = ratelimit.FromTransport(hclient.Transport, etherscan.CallInterval)
+	backend.etherScanHTTPClient = hclient
 
 	ratesCache := filepath.Join(arguments.CacheDirectoryPath(), "exchangerates")
 	if err := os.MkdirAll(ratesCache, 0700); err != nil {

backend/coins/eth/etherscan/etherscan.go

@@ -35,11 +35,13 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"golang.org/x/time/rate"
 )
 
-// CallInterval is the duration between etherscan requests.
+// callsPerSec is thenumber of etherscanr equests allowed
+// per second.
 // Etherscan rate limits to one request per 0.2 seconds.
-var CallInterval = 260 * time.Millisecond
+var callsPerSec = 3.8
 
 const apiKey = "X3AFAGQT2QCAFTFPIH9VJY88H9PIQ2UWP7"
 
@@ -50,17 +52,22 @@ const ERC20GasErr = "insufficient funds for gas * price + value"
 type EtherScan struct {
 	url        string
 	httpClient *http.Client
+	limiter    *rate.Limiter
 }
 
 // NewEtherScan creates a new instance of EtherScan.
 func NewEtherScan(url string, httpClient *http.Client) *EtherScan {
 	return &EtherScan{
 		url:        url,
 		httpClient: httpClient,
+		limiter:    rate.NewLimiter(rate.Limit(callsPerSec), 1),
 	}
 }
 
-func (etherScan *EtherScan) call(params url.Values, result interface{}) error {
+func (etherScan *EtherScan) call(ctx context.Context, params url.Values, result interface{}) error {
+	if err := etherScan.limiter.Wait(ctx); err != nil {
+		return errp.WithStack(err)
+	}
 	params.Set("apikey", apiKey)
 	response, err := etherScan.httpClient.Get(etherScan.url + "?" + params.Encode())
 	if err != nil {
@@ -323,7 +330,7 @@ func (etherScan *EtherScan) Transactions(
 	result := struct {
 		Result []*Transaction
 	}{}
-	if err := etherScan.call(params, &result); err != nil {
+	if err := etherScan.call(context.TODO(), params, &result); err != nil {
 		return nil, err
 	}
 	isERC20 := erc20Token != nil
@@ -338,7 +345,7 @@ func (etherScan *EtherScan) Transactions(
 		resultInternal := struct {
 			Result []*Transaction
 		}{}
-		if err := etherScan.call(params, &resultInternal); err != nil {
+		if err := etherScan.call(context.TODO(), params, &resultInternal); err != nil {
 			return nil, err
 		}
 		var err error
@@ -353,7 +360,7 @@ func (etherScan *EtherScan) Transactions(
 
 // ----- RPC node proxy methods follow
 
-func (etherScan *EtherScan) rpcCall(params url.Values, result interface{}) error {
+func (etherScan *EtherScan) rpcCall(ctx context.Context, params url.Values, result interface{}) error {
 	params.Set("module", "proxy")
 
 	var wrapped struct {
@@ -364,7 +371,7 @@ func (etherScan *EtherScan) rpcCall(params url.Values, result interface{}) error
 		} `json:"error"`
 		Result *json.RawMessage `json:"result"`
 	}
-	if err := etherScan.call(params, &wrapped); err != nil {
+	if err := etherScan.call(ctx, params, &wrapped); err != nil {
 		return err
 	}
 	if wrapped.Error != nil {
@@ -389,7 +396,7 @@ func (etherScan *EtherScan) TransactionReceiptWithBlockNumber(
 	params.Set("action", "eth_getTransactionReceipt")
 	params.Set("txhash", hash.Hex())
 	var result *rpcclient.RPCTransactionReceipt
-	if err := etherScan.rpcCall(params, &result); err != nil {
+	if err := etherScan.rpcCall(ctx, params, &result); err != nil {
 		return nil, err
 	}
 	return result, nil
@@ -402,7 +409,7 @@ func (etherScan *EtherScan) TransactionByHash(
 	params.Set("action", "eth_getTransactionByHash")
 	params.Set("txhash", hash.Hex())
 	var result rpcclient.RPCTransaction
-	if err := etherScan.rpcCall(params, &result); err != nil {
+	if err := etherScan.rpcCall(ctx, params, &result); err != nil {
 		return nil, false, err
 	}
 	return &result.Transaction, result.BlockNumber == nil, nil
@@ -415,7 +422,7 @@ func (etherScan *EtherScan) BlockNumber(ctx context.Context) (*big.Int, error) {
 	params.Set("tag", "latest")
 	params.Set("boolean", "false")
 	var header *types.Header
-	if err := etherScan.rpcCall(params, &header); err != nil {
+	if err := etherScan.rpcCall(ctx, params, &header); err != nil {
 		return nil, err
 	}
 	return header.Number, nil
@@ -434,7 +441,7 @@ func (etherScan *EtherScan) Balance(ctx context.Context, account common.Address)
 	params.Set("action", "balance")
 	params.Set("address", account.Hex())
 	params.Set("tag", "latest")
-	if err := etherScan.call(params, &result); err != nil {
+	if err := etherScan.call(ctx, params, &result); err != nil {
 		return nil, err
 	}
 	if result.Status != "1" {
@@ -461,7 +468,7 @@ func (etherScan *EtherScan) ERC20Balance(account common.Address, erc20Token *erc
 	params.Set("address", account.Hex())
 	params.Set("contractaddress", erc20Token.ContractAddress().Hex())
 	params.Set("tag", "latest")
-	if err := etherScan.call(params, &result); err != nil {
+	if err := etherScan.call(context.TODO(), params, &result); err != nil {
 		return nil, err
 	}
 	if result.Status != "1" {
@@ -485,7 +492,7 @@ func (etherScan *EtherScan) CallContract(ctx context.Context, msg ethereum.CallM
 		panic("not implemented")
 	}
 	var result hexutil.Bytes
-	if err := etherScan.rpcCall(params, &result); err != nil {
+	if err := etherScan.rpcCall(ctx, params, &result); err != nil {
 		return nil, err
 	}
 	return result, nil
@@ -515,7 +522,7 @@ func (etherScan *EtherScan) EstimateGas(ctx context.Context, msg ethereum.CallMs
 	callMsgParams(&params, msg)
 
 	var result hexutil.Uint64
-	if err := etherScan.rpcCall(params, &result); err != nil {
+	if err := etherScan.rpcCall(ctx, params, &result); err != nil {
 		return 0, err
 	}
 	return uint64(result), nil
@@ -528,7 +535,7 @@ func (etherScan *EtherScan) PendingNonceAt(ctx context.Context, account common.A
 	params.Set("address", account.Hex())
 	params.Set("tag", "pending")
 	var result hexutil.Uint64
-	if err := etherScan.rpcCall(params, &result); err != nil {
+	if err := etherScan.rpcCall(ctx, params, &result); err != nil {
 		return 0, err
 	}
 	return uint64(result), nil
@@ -544,15 +551,15 @@ func (etherScan *EtherScan) SendTransaction(ctx context.Context, tx *types.Trans
 	params := url.Values{}
 	params.Set("action", "eth_sendRawTransaction")
 	params.Set("hex", hexutil.Encode(encodedTx))
-	return etherScan.rpcCall(params, nil)
+	return etherScan.rpcCall(ctx, params, nil)
 }
 
 // SuggestGasPrice implements rpc.Interface.
 func (etherScan *EtherScan) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
 	params := url.Values{}
 	params.Set("action", "eth_gasPrice")
 	var result hexutil.Big
-	if err := etherScan.rpcCall(params, &result); err != nil {
+	if err := etherScan.rpcCall(ctx, params, &result); err != nil {
 		return nil, err
 	}
 	return (*big.Int)(&result), nil
@@ -581,7 +588,7 @@ func (etherScan *EtherScan) FeeTargets(ctx context.Context) ([]*ethtypes.FeeTarg
 	params := url.Values{}
 	params.Set("module", "gastracker")
 	params.Set("action", "gasoracle")
-	if err := etherScan.call(params, &result); err != nil {
+	if err := etherScan.call(ctx, params, &result); err != nil {
 		return nil, err
 	}
 	// Convert string fields to int64

backend/rates/gecko.go

@@ -1,6 +1,10 @@
 package rates
 
-import "time"
+import (
+	"time"
+
+	"golang.org/x/time/rate"
+)
 
 const (
 	// See the following for docs and details: https://www.coingecko.com/en/api.
@@ -16,21 +20,19 @@ const (
 	maxGeckoRange = 364 * 24 * time.Hour
 )
 
-// apiRateLimit specifies the minimal interval between equally spaced API calls
+// apiRateLimit specifies the maximum number of API calls per second
 // to one of the supported exchange rates providers.
-func apiRateLimit(baseURL string) time.Duration {
+func apiRateLimit(baseURL string) rate.Limit {
 	switch baseURL {
 	default:
-		return time.Second // arbitrary; localhost, staging, etc.
+		return rate.Limit(1) // arbitrary; localhost, staging, etc.
 	case coingeckoAPIV3:
 		// API calls. From https://www.coingecko.com/en/api:
 		// > Generous rate limits with up to 100 requests/minute
 		// We use slightly lower value.
-		return 2 * time.Second
+		return rate.Limit(0.5)
 	case shiftGeckoMirrorAPIV3:
-		// Avoid zero to prevent unexpected panics like in time.NewTicker
-		// and leave some room to breathe.
-		return 10 * time.Millisecond
+		return rate.Limit(100)
 	}
 }
 

backend/rates/history.go

@@ -278,38 +278,38 @@ func (updater *RateUpdater) fetchGeckoMarketRange(ctx context.Context, coin, fia
 	}
 
 	// Make the call, abiding the upstream rate limits.
-	msg := fmt.Sprintf("fetch coingecko coin=%s fiat=%s start=%s", coin, fiat, timeRange.start)
 	var jsonBody struct{ Prices [][2]float64 } // [timestamp in milliseconds, value]
-	callErr := updater.geckoLimiter.Call(ctx, msg, func() error {
-		param := url.Values{
-			"from":        {strconv.FormatInt(timeRange.start.Unix(), 10)},
-			"to":          {strconv.FormatInt(timeRange.end().Unix(), 10)},
-			"vs_currency": {gfiat},
-		}
-		endpoint := fmt.Sprintf("%s/coins/%s/market_chart/range?%s", updater.coingeckoURL, gcoin, param.Encode())
-		req, err := http.NewRequest(http.MethodGet, endpoint, nil)
-		if err != nil {
-			return err
-		}
+	if err := updater.geckoLimiter.Wait(ctx); err != nil {
+		return nil, err
+	}
 
-		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-		defer cancel()
-		res, err := updater.httpClient.Do(req.WithContext(ctx))
-		if err != nil {
-			return err
-		}
-		defer res.Body.Close() //nolint:errcheck
-		if res.StatusCode != http.StatusOK {
-			return fmt.Errorf("fetchGeckoMarketRange: bad response code %d", res.StatusCode)
-		}
-		// 1Mb is more than enough for a single response, but make sure initial
-		// download with empty cache fits here. See maxGeckoRange.
-		return json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(&jsonBody)
-	})
-	if callErr != nil {
-		return nil, callErr
+	param := url.Values{
+		"from":        {strconv.FormatInt(timeRange.start.Unix(), 10)},
+		"to":          {strconv.FormatInt(timeRange.end().Unix(), 10)},
+		"vs_currency": {gfiat},
+	}
+	endpoint := fmt.Sprintf("%s/coins/%s/market_chart/range?%s", updater.coingeckoURL, gcoin, param.Encode())
+	req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	res, err := updater.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close() //nolint:errcheck
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("fetchGeckoMarketRange: bad response code %d", res.StatusCode)
 	}
 
+	// 1Mb is more than enough for a single response, but make sure initial
+	// download with empty cache fits here. See maxGeckoRange
+	if err := json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(&jsonBody); err != nil {
+		return nil, err
+	}
 	// Transform the response into a usable result.
 	rates := make([]exchangeRate, len(jsonBody.Prices))
 	for i, v := range jsonBody.Prices {