Introduce Dynamo model

service_capacity_modeling/capacity_planner.py

@@ -325,6 +325,92 @@ def models(self) -> Dict[str, CapacityModel]:
     def hardware_shapes(self) -> HardwareShapes:
         return self._shapes
 
+    def _plan_percentiles(
+        self,
+        model_name: str,
+        percentiles: Tuple[int, ...],
+        region: str,
+        desires: CapacityDesires,
+        lifecycles: Optional[Sequence[Lifecycle]] = None,
+        instance_families: Optional[Sequence[str]] = None,
+        drives: Optional[Sequence[str]] = None,
+        num_results: Optional[int] = None,
+        num_regions: int = 3,
+        extra_model_arguments: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[Sequence[CapacityPlan], Dict[int, Sequence[CapacityPlan]]]:
+        if model_name not in self._models:
+            raise ValueError(
+                f"model_name={model_name} does not exist. "
+                f"Try {sorted(list(self._models.keys()))}"
+            )
+
+        extra_model_arguments = extra_model_arguments or {}
+        lifecycles = lifecycles or self._default_lifecycles
+
+        model_mean_desires: Dict[str, CapacityDesires] = {}
+        model_percentile_desires: List[Dict[str, CapacityDesires]] = []
+        sorted_percentiles = sorted(percentiles)
+        for percentile in sorted_percentiles:
+            model_percentile_desires.append({})
+        for sub_model, sub_desires in self._sub_models(
+            model_name=model_name,
+            desires=desires,
+            extra_model_arguments=extra_model_arguments,
+        ):
+            percentile_inputs, mean_desires = model_desires_percentiles(
+                desires=sub_desires, percentiles=sorted_percentiles
+            )
+            model_mean_desires[sub_model] = mean_desires
+            index = 0
+            for percentile_input in percentile_inputs:
+                model_percentile_desires[index][sub_model] = percentile_input
+                index = index + 1
+
+        mean_plans = []
+        for mean_sub_model, mean_sub_desire in model_mean_desires.items():
+            mean_plans.append(
+                self._plan_certain(
+                    model_name=mean_sub_model,
+                    region=region,
+                    desires=mean_sub_desire,
+                    num_results=num_results,
+                    num_regions=num_regions,
+                    extra_model_arguments=extra_model_arguments,
+                    lifecycles=lifecycles,
+                    instance_families=instance_families,
+                    drives=drives,
+                )
+            )
+
+        mean_plan = [
+            functools.reduce(merge_plan, composed) for composed in zip(*mean_plans)
+        ]
+        percentile_plans = {}
+        for index, percentile in enumerate(sorted_percentiles):
+            percentile_plan = []
+            for percentile_sub_model, percentile_sub_desire in model_percentile_desires[
+                index
+            ].items():
+                percentile_plan.append(
+                    self._plan_certain(
+                        model_name=percentile_sub_model,
+                        region=region,
+                        desires=percentile_sub_desire,
+                        num_results=num_results,
+                        num_regions=num_regions,
+                        extra_model_arguments=extra_model_arguments,
+                        lifecycles=lifecycles,
+                        instance_families=instance_families,
+                        drives=drives,
+                    )
+                )
+            percentile_plans[percentile] = [
+                functools.reduce(merge_plan, composed)
+                for composed in zip(*percentile_plan)
+            ]
+
+        return mean_plan, percentile_plans
+
     def plan_certain(
         self,
         model_name: str,
@@ -413,28 +499,39 @@ def _plan_certain(
             allowed_drives.update(hardware.drives.keys())
 
         plans = []
-        for instance in hardware.instances.values():
-            if not _allow_instance(
-                instance, instance_families, lifecycles, allowed_platforms
-            ):
-                continue
-
-            if per_instance_mem > instance.ram_gib:
-                continue
+        if model.run_hardware_simulation():
+            for instance in hardware.instances.values():
+                if not _allow_instance(
+                    instance, instance_families, lifecycles, allowed_platforms
+                ):
+                    continue
 
-            for drive in hardware.drives.values():
-                if not _allow_drive(drive, drives, lifecycles, allowed_drives):
+                if per_instance_mem > instance.ram_gib:
                     continue
 
-                plan = model.capacity_plan(
-                    instance=instance,
-                    drive=drive,
-                    context=context,
-                    desires=desires,
-                    extra_model_arguments=extra_model_arguments,
-                )
-                if plan is not None:
-                    plans.append(plan)
+                for drive in hardware.drives.values():
+                    if not _allow_drive(drive, drives, lifecycles, allowed_drives):
+                        continue
+
+                    plan = model.capacity_plan(
+                        instance=instance,
+                        drive=drive,
+                        context=context,
+                        desires=desires,
+                        extra_model_arguments=extra_model_arguments,
+                    )
+                    if plan is not None:
+                        plans.append(plan)
+        else:
+            plan = model.capacity_plan(
+                instance=Instance.get_managed_instance(),
+                drive=Drive.get_managed_drive(),
+                context=context,
+                desires=desires,
+                extra_model_arguments=extra_model_arguments,
+            )
+            if plan is not None:
+                plans.append(plan)
 
         # lowest cost first
         plans.sort(
@@ -556,31 +653,20 @@ def plan(
 
         final_requirement = Requirements(zonal=final_zonal, regional=final_regional)
 
-        percentile_inputs, mean_desires = model_desires_percentiles(
-            desires=desires, percentiles=sorted(percentiles)
+        mean_plan, percentile_plans = self._plan_percentiles(
+            model_name=model_name,
+            percentiles=percentiles,
+            region=region,
+            desires=desires,
+            extra_model_arguments=extra_model_arguments,
+            num_regions=num_regions,
+            instance_families=instance_families,
         )
-        percentile_plans = {}
-        for index, percentile in enumerate(percentiles):
-            percentile_plans[percentile] = self.plan_certain(
-                model_name=model_name,
-                region=region,
-                desires=percentile_inputs[index],
-                extra_model_arguments=extra_model_arguments,
-                num_regions=num_regions,
-                instance_families=instance_families,
-            )
 
         result = UncertainCapacityPlan(
             requirements=final_requirement,
             least_regret=least_regret,
-            mean=self.plan_certain(
-                model_name=model_name,
-                region=region,
-                desires=mean_desires,
-                extra_model_arguments=extra_model_arguments,
-                num_regions=num_regions,
-                instance_families=instance_families,
-            ),
+            mean=mean_plan,
             percentiles=percentile_plans,
             explanation=PlanExplanation(
                 regret_params=regret_params,

service_capacity_modeling/hardware/profiles/pricing/aws/3yr-reserved.json

@@ -90,7 +90,7 @@
     },
     "services": {
       "blob.standard": {
-        "annual_cost_per_gib": "0.252",
+        "annual_cost_per_gib": 0.252,
         "annual_cost_per_write_io": "0.000005",
         "annual_cost_per_read_io": "0.0000004"
       },
@@ -102,6 +102,22 @@
       },
       "crdb_core_license": {
         "annual_cost_per_core": 0
+      },
+      "dynamo.standard": {
+        "annual_cost_per_gib": 3,
+        "annual_cost_per_read_io": 0.00013,
+        "annual_cost_per_write_io": 0.00065
+      },
+      "dynamo.standard.global": {
+        "annual_cost_per_gib": 3,
+        "annual_cost_per_read_io": 0.00013,
+        "annual_cost_per_write_io": 0.000975
+      },
+      "dynamo.backup.continuous": {
+        "annual_cost_per_gib": 2.4
+      },
+      "dynamo.transfer": {
+        "annual_cost_per_gib": [[122880, 0.09], [491520, 0.085], [1228800, 0.07], [-1, 0.05]]
       }
     },
     "zones_in_region": 3

service_capacity_modeling/hardware/profiles/shapes/aws.json

@@ -848,7 +848,19 @@
       "name": "Intra-region (within) transfer costs"
     },
     "crdb_core_license": {
-        "name": "CRDB license fee"
+      "name": "CRDB license fee"
+    },
+    "dynamo.standard": {
+      "name": "dynamo-standard"
+    },
+    "dynamo.standard.global": {
+      "name": "dynamo-standard-global"
+    },
+    "dynamo.backup.continuous": {
+      "name": "dynamo-backup-continuous"
+    },
+    "dynamo.transfer": {
+      "name": "dynamo-transfer-first120tb"
     }
   }
 }

service_capacity_modeling/interface.py

@@ -5,7 +5,7 @@
 from decimal import Decimal
 from enum import Enum
 from functools import lru_cache
-from typing import Any
+from typing import Any, Union
 from typing import cast
 from typing import Dict
 from typing import List
@@ -285,6 +285,10 @@ def annual_cost(self):
 
         return size * self.annual_cost_per_gib + r_cost + w_cost
 
+    @staticmethod
+    def get_managed_drive() -> Drive:
+        return Drive(name="managed")
+
 
 class Platform(str, Enum):
     """Represents the platform of the hardware
@@ -330,6 +334,12 @@ def family(self):
     def size(self):
         return self.name.rsplit(self.family_separator, 1)[1]
 
+    @staticmethod
+    def get_managed_instance() -> Instance:
+        return Instance(
+            name="managed.0", cpu=0, cpu_ghz=0, ram_gib=0, net_mbps=0, drive=None
+        )
+
 
 class Service(ExcludeUnsetModel):
     """Represents a cloud service, such as a blob store (S3) or
@@ -341,7 +351,7 @@ class Service(ExcludeUnsetModel):
     name: str
     size_gib: int = 0
 
-    annual_cost_per_gib: float = 0
+    annual_cost_per_gib: Union[float, List[Tuple[float, float]]] = 0
     annual_cost_per_read_io: float = 0
     annual_cost_per_write_io: float = 0
     annual_cost_per_core: float = 0
@@ -354,6 +364,26 @@ class Service(ExcludeUnsetModel):
         low=1, mid=10, high=50, confidence=0.9
     )
 
+    def annual_cost_gib(self, data_gib: float = 0):
+        if isinstance(self.annual_cost_per_gib, float):
+            return self.annual_cost_per_gib * data_gib
+        else:
+            _annual_data = data_gib
+            transfer_costs = self.annual_cost_per_gib
+            annual_cost = 0.0
+            for transfer_cost in transfer_costs:
+                if not _annual_data > 0:
+                    break
+                if transfer_cost[0] > 0:
+                    annual_cost += (
+                        min(_annual_data, transfer_cost[0]) * transfer_cost[1]
+                    )
+                    _annual_data -= transfer_cost[0]
+                else:
+                    # final remaining data transfer cost
+                    annual_cost += _annual_data * transfer_cost[1]
+        return annual_cost
+
 
 class RegionContext(ExcludeUnsetModel):
     services: Dict[str, Service] = {}
@@ -403,7 +433,7 @@ class DrivePricing(ExcludeUnsetModel):
 
 
 class ServicePricing(ExcludeUnsetModel):
-    annual_cost_per_gib: float = 0
+    annual_cost_per_gib: Union[float, List[Tuple[float, float]]] = 0
     annual_cost_per_read_io: float = 0
     annual_cost_per_write_io: float = 0
     annual_cost_per_core: float = 0

service_capacity_modeling/models/__init__.py

@@ -217,6 +217,16 @@ def extra_model_arguments_schema() -> Dict[str, Any]:
         """
         return {"type": "object"}
 
+    @staticmethod
+    def run_hardware_simulation() -> bool:
+        """Optional to skip hardware simulation
+
+        Some models, managed services, do not
+        require simulating through hardware (instances, drives).
+
+        """
+        return True
+
     @staticmethod
     def compose_with(
         user_desires: CapacityDesires,

service_capacity_modeling/models/common.py

@@ -129,24 +129,24 @@ def network_services(
     # inter region. This is the number of regions minus 1
     inter_txfer = context.services.get("net.inter.region", None)
     if inter_txfer:
-        price_per_gib = inter_txfer.annual_cost_per_gib
         result.append(
             ServiceCapacity(
                 service_type=f"{service_type}.net.inter.region",
-                annual_cost=(price_per_gib * txfer_gib * num_regions),
+                annual_cost=(inter_txfer.annual_cost_gib(txfer_gib) * num_regions),
                 service_params={"txfer_gib": txfer_gib, "num_regions": num_regions},
             )
         )
 
     # Same zone is free, but we pay for replication from our zone to others
     intra_txfer = context.services.get("net.intra.region", None)
     if intra_txfer:
-        price_per_gib = intra_txfer.annual_cost_per_gib
         result.append(
             ServiceCapacity(
                 service_type=f"{service_type}.net.intra.region",
                 annual_cost=(
-                    price_per_gib * txfer_gib * num_zones * context.num_regions
+                    intra_txfer.annual_cost_gib(txfer_gib)
+                    * num_zones
+                    * context.num_regions
                 ),
                 service_params={
                     "txfer_gib": txfer_gib,

service_capacity_modeling/models/org/netflix/__init__.py

@@ -1,6 +1,8 @@
 from .aurora import nflx_aurora_capacity_model
 from .cassandra import nflx_cassandra_capacity_model
+from .counter import nflx_counter_capacity_model
 from .crdb import nflx_cockroachdb_capacity_model
+from .ddb import nflx_ddb_capacity_model
 from .elasticsearch import nflx_elasticsearch_capacity_model
 from .elasticsearch import nflx_elasticsearch_data_capacity_model
 from .elasticsearch import nflx_elasticsearch_master_capacity_model
@@ -11,7 +13,6 @@
 from .rds import nflx_rds_capacity_model
 from .stateless_java import nflx_java_app_capacity_model
 from .time_series import nflx_time_series_capacity_model
-from .counter import nflx_counter_capacity_model
 from .zookeeper import nflx_zookeeper_capacity_model
 from .kafka import nflx_kafka_capacity_model
 
@@ -34,4 +35,5 @@ def models():
         "org.netflix.aurora": nflx_aurora_capacity_model,
         "org.netflix.postgres": nflx_postgres_capacity_model,
         "org.netflix.kafka": nflx_kafka_capacity_model,
+        "org.netflix.dynamodb": nflx_ddb_capacity_model,
     }