Remove base_backoff_ms

Author: garrettheel

docs/settings.rst

@@ -34,15 +34,6 @@ The number of times to retry certain failed DynamoDB API calls. The most common
 retries include ``ProvisionedThroughputExceededException`` and ``5xx`` errors.
 
 
-base_backoff_ms
----------------
-
-Default: ``25``
-
-The base number of milliseconds used for `exponential backoff and jitter
-<https://www.awsarchitectureblog.com/2015/03/backoff.html>`_ on retries.
-
-
 region
 ------
 

pynamodb/connection/base.py

@@ -236,7 +236,6 @@ def __init__(self,
                  read_timeout_seconds: Optional[float] = None,
                  connect_timeout_seconds: Optional[float] = None,
                  max_retry_attempts: Optional[int] = None,
-                 base_backoff_ms: Optional[int] = None,
                  max_pool_connections: Optional[int] = None,
                  extra_headers: Optional[Mapping[str, str]] = None):
         self._tables: Dict[str, MetaTable] = {}
@@ -264,11 +263,6 @@ def __init__(self,
         else:
             self._max_retry_attempts_exception = get_settings_value('max_retry_attempts')
 
-        if base_backoff_ms is not None:
-            self._base_backoff_ms = base_backoff_ms
-        else:
-            self._base_backoff_ms = get_settings_value('base_backoff_ms')
-
         if max_pool_connections is not None:
             self._max_pool_connections = max_pool_connections
         else:

pynamodb/connection/table.py

@@ -24,7 +24,6 @@ def __init__(
         connect_timeout_seconds: Optional[float] = None,
         read_timeout_seconds: Optional[float] = None,
         max_retry_attempts: Optional[int] = None,
-        base_backoff_ms: Optional[int] = None,
         max_pool_connections: Optional[int] = None,
         extra_headers: Optional[Mapping[str, str]] = None,
         aws_access_key_id: Optional[str] = None,
@@ -37,7 +36,6 @@ def __init__(
                                      connect_timeout_seconds=connect_timeout_seconds,
                                      read_timeout_seconds=read_timeout_seconds,
                                      max_retry_attempts=max_retry_attempts,
-                                     base_backoff_ms=base_backoff_ms,
                                      max_pool_connections=max_pool_connections,
                                      extra_headers=extra_headers)
 

pynamodb/models.py

@@ -152,8 +152,9 @@ def commit(self) -> None:
         retries = 0
         unprocessed_items = data.get(UNPROCESSED_ITEMS, {}).get(self.model.Meta.table_name)
         while unprocessed_items:
-            sleep_time = random.randint(0, self.model.Meta.base_backoff_ms * (2 ** retries)) / 1000
-            time.sleep(sleep_time)
+            # TODO: we should consider using exponential backoff here
+            # TODO: it is somewhat unintuitive that we retry unprocessed items max_retry_attempts times,
+            # since each `batch_write_item` operation is also subject to max_retry_attempts
             retries += 1
             if retries >= self.model.Meta.max_retry_attempts:
                 self.failed_operations = unprocessed_items
@@ -165,8 +166,7 @@ def commit(self) -> None:
                     put_items.append(item.get(PUT_REQUEST).get(ITEM))  # type: ignore
                 elif DELETE_REQUEST in item:
                     delete_items.append(item.get(DELETE_REQUEST).get(KEY))  # type: ignore
-            log.info("Resending %d unprocessed keys for batch operation after %d seconds sleep",
-                     len(unprocessed_items), sleep_time)
+            log.info("Resending %d unprocessed keys for batch operation (retry %d)", len(unprocessed_items), retries)
             data = self.model._get_connection().batch_write_item(
                 put_items=put_items,
                 delete_items=delete_items,
@@ -182,7 +182,6 @@ class MetaProtocol(Protocol):
     host: Optional[str]
     connect_timeout_seconds: int
     read_timeout_seconds: int
-    base_backoff_ms: int
     max_retry_attempts: int
     max_pool_connections: int
     extra_headers: Mapping[str, str]
@@ -241,8 +240,6 @@ def __init__(self, name, bases, namespace, discriminator=None) -> None:
                         setattr(attr_obj, 'connect_timeout_seconds', get_settings_value('connect_timeout_seconds'))
                     if not hasattr(attr_obj, 'read_timeout_seconds'):
                         setattr(attr_obj, 'read_timeout_seconds', get_settings_value('read_timeout_seconds'))
-                    if not hasattr(attr_obj, 'base_backoff_ms'):
-                        setattr(attr_obj, 'base_backoff_ms', get_settings_value('base_backoff_ms'))
                     if not hasattr(attr_obj, 'max_retry_attempts'):
                         setattr(attr_obj, 'max_retry_attempts', get_settings_value('max_retry_attempts'))
                     if not hasattr(attr_obj, 'max_pool_connections'):
@@ -1052,7 +1049,6 @@ def _get_connection(cls) -> TableConnection:
                                               connect_timeout_seconds=cls.Meta.connect_timeout_seconds,
                                               read_timeout_seconds=cls.Meta.read_timeout_seconds,
                                               max_retry_attempts=cls.Meta.max_retry_attempts,
-                                              base_backoff_ms=cls.Meta.base_backoff_ms,
                                               max_pool_connections=cls.Meta.max_pool_connections,
                                               extra_headers=cls.Meta.extra_headers,
                                               aws_access_key_id=cls.Meta.aws_access_key_id,

pynamodb/settings.py

@@ -12,7 +12,6 @@
     'connect_timeout_seconds': 15,
     'read_timeout_seconds': 30,
     'max_retry_attempts': 3,
-    'base_backoff_ms': 25,
     'region': None,
     'max_pool_connections': 10,
     'extra_headers': None,

tests/test_base_connection.py

@@ -1541,9 +1541,8 @@ def test_connection_make_api_call__throws_retry_disabled(send_mock):
     send_mock.side_effect = [
         botocore.exceptions.ReadTimeoutError(endpoint_url='http://lyft.com'),
     ]
-    c = Connection(read_timeout_seconds=11, base_backoff_ms=3, max_retry_attempts=0)
+    c = Connection(read_timeout_seconds=11, max_retry_attempts=0)
 
-    assert c._base_backoff_ms == 3
     with pytest.raises(botocore.exceptions.ReadTimeoutError):
         c._make_api_call('DescribeTable', {'TableName': 'MyTable'})
 

tests/test_model.py

@@ -523,14 +523,12 @@ def fake_dynamodb(*args):
         assert UserModel.Meta.connect_timeout_seconds, 15
         assert UserModel.Meta.read_timeout_seconds == 30
         assert UserModel.Meta.max_retry_attempts == 3
-        assert UserModel.Meta.base_backoff_ms == 25
         assert UserModel.Meta.max_pool_connections == 10
 
         assert UserModel._connection.connection._connect_timeout_seconds == 15
         assert UserModel._connection.connection._read_timeout_seconds == 30
         assert UserModel._connection.connection._max_retry_attempts_exception == 3
         assert UserModel._connection.connection._max_pool_connections == 10
-        assert UserModel._connection.connection._base_backoff_ms == 25
 
         with patch(PATCH_METHOD) as req:
             req.return_value = MODEL_TABLE_DATA