feat: smart staged rollouts for Android

app/components/live_release/prod_release/rollout_component.rb

@@ -34,6 +34,7 @@ def initialize(store_rollout, title: "Rollout Status", show_monitoring: true)
     :id,
     :parent_release, to: :store_rollout
   delegate :release, :platform, to: :release_platform_run
+  delegate :automatic_rollout?, to: :release_platform_run
 
   def decorated_status
     status_picker(STATUS, status)
@@ -122,7 +123,7 @@ def more_actions
           confirm: "Are you sure you want to rollout to all users?"
         },
         {
-          text: "Pause rollout",
+          text: "Pause automatic rollout",
           path: pause_store_rollout_path(id),
           scheme: :danger,
           disabled: !automatic_rollout?,

app/controllers/config/release_platforms_controller.rb

@@ -82,7 +82,8 @@ def config_params
       production_release_attributes: [
         :id,
         submissions_attributes: [
-          :id, :submission_type, :_destroy, :rollout_stages, :rollout_enabled, :finish_rollout_in_next_release
+          :id, :submission_type, :_destroy, :rollout_stages, :rollout_enabled, :finish_rollout_in_next_release,
+          :automatic_rollout
         ]
       ],
       internal_workflow_attributes: [
@@ -155,6 +156,7 @@ def update_production_release_rollout_stages(production_release_attributes)
       submission_attributes[:rollout_stages] = submission_attributes[:rollout_stages].safe_csv_parse(coerce_float: true)
     else
       submission_attributes[:finish_rollout_in_next_release] = false
+      submission_attributes[:automatic_rollout] = false
     end
   end
 

app/controllers/store_rollouts_controller.rb

@@ -46,7 +46,7 @@ def resume
   end
 
   def halt
-    if (res = Action.halt_the_store_rollout!(@store_rollout)).ok?
+    if (res = Action.halt_the_store_rollout!(@store_rollout, manually: true)).ok?
       redirect_back fallback_location: root_path, notice: t(".halt.success")
     else
       redirect_back fallback_location: root_path, flash: {error: t(".halt.failure", errors: res.error.message)}

app/jobs/halt_unhealthy_release_rollout_job.rb

@@ -0,0 +1,19 @@
+class HaltUnhealthyReleaseRolloutJob < ApplicationJob
+  queue_as :high
+
+  def perform(production_release_id, event_id)
+    production_release = ProductionRelease.find(production_release_id)
+    event = ReleaseHealthEvent.find(event_id)
+
+    if production_release.unhealthy?
+      store_rollout = production_release.store_rollout
+
+      if store_rollout.is_a?(PlayStoreRollout) && store_rollout.automatic_rollout?
+        result = Action.halt_the_store_rollout!(store_rollout)
+
+        # Mark the event as action triggered if halt was successful
+        event.update(action_triggered: true) if result.ok?
+      end
+    end
+  end
+end

app/jobs/increase_healthy_release_rollout_job.rb

@@ -0,0 +1,29 @@
+class IncreaseHealthyReleaseRolloutJob < ApplicationJob
+  queue_as :high
+
+  def perform(play_store_rollout_id)
+    rollout = PlayStoreRollout.find(play_store_rollout_id)
+    return if rollout.completed? || rollout.fully_released?
+    return unless rollout.automatic_rollout?
+
+    release = rollout.parent_release
+
+    if release.healthy?
+      if rollout.halted?
+        Action.resume_the_store_rollout!(rollout)
+      end
+
+      if rollout.started?
+        Action.increase_the_store_rollout!(rollout)
+      end
+    end
+
+    # This update is necessary so that our verify rollout job does not pick this up again
+    rollout.update!(
+      automatic_rollout_updated_at: Time.current,
+      automatic_rollout_next_update_at: Time.current + PlayStoreRollout::AUTO_ROLLOUT_RUN_INTERVAL
+    )
+
+    IncreaseHealthyReleaseRolloutJob.perform_in(PlayStoreRollout::AUTO_ROLLOUT_RUN_INTERVAL, play_store_rollout_id)
+  end
+end

app/jobs/verify_automatic_rollout_job.rb

@@ -0,0 +1,13 @@
+class VerifyAutomaticRolloutJob < ApplicationJob
+  queue_as :high
+
+  def perform
+    PlayStoreRollout.automatic_rollouts
+      .where(automatic_rollout_next_update_at: ...Time.current)
+      .where(
+        "automatic_rollout_next_update_at - automatic_rollout_updated_at >= interval '300 second'"
+      ).find_each do |rollout|
+      IncreaseHealthyReleaseRolloutJob.perform_async(rollout.id)
+    end
+  end
+end

app/libs/coordinators.rb

@@ -258,10 +258,15 @@ def self.resume_the_store_rollout!(rollout)
       Res.new { true }
     end
 
-    def self.halt_the_store_rollout!(rollout)
+    def self.halt_the_store_rollout!(rollout, manually: false)
       return Res.new { raise "release is not actionable" } unless rollout.actionable?
       return Res.new { raise "rollout is not started" } unless rollout.started?
       rollout.halt_release!
+
+      if manually && rollout.automatic_rollout?
+        rollout.disable_automatic_rollout!
+      end
+
       return Res.new { raise rollout.errors.full_messages.to_sentence } if rollout.errors?
       Res.new { true }
     end

app/models/app_store_rollout.rb

@@ -2,17 +2,20 @@
 #
 # Table name: store_rollouts
 #
-#  id                      :uuid             not null, primary key
-#  completed_at            :datetime
-#  config                  :decimal(8, 5)    default([]), not null, is an Array
-#  current_stage           :integer
-#  is_staged_rollout       :boolean          default(FALSE)
-#  status                  :string           not null
-#  type                    :string           not null
-#  created_at              :datetime         not null
-#  updated_at              :datetime         not null
-#  release_platform_run_id :uuid             not null, indexed
-#  store_submission_id     :uuid             indexed
+#  id                               :uuid             not null, primary key
+#  automatic_rollout                :boolean          default(FALSE), not null
+#  automatic_rollout_next_update_at :datetime
+#  automatic_rollout_updated_at     :datetime
+#  completed_at                     :datetime
+#  config                           :decimal(8, 5)    default([]), not null, is an Array
+#  current_stage                    :integer
+#  is_staged_rollout                :boolean          default(FALSE)
+#  status                           :string           not null
+#  type                             :string           not null
+#  created_at                       :datetime         not null
+#  updated_at                       :datetime         not null
+#  release_platform_run_id          :uuid             not null, indexed
+#  store_submission_id              :uuid             indexed
 #
 class AppStoreRollout < StoreRollout
   include Passportable
@@ -124,6 +127,10 @@ def halt_release!
     end
   end
 
+  def disable_automatic_rollout!
+    # noop for app store
+  end
+
   def release_fully!
     with_lock do
       return unless may_fully_release?

app/models/config/submission.rb

@@ -4,6 +4,7 @@
 #
 #  id                             :bigint           not null, primary key
 #  auto_promote                   :boolean          default(FALSE)
+#  automatic_rollout              :boolean          default(FALSE)
 #  finish_rollout_in_next_release :boolean          default(FALSE), not null
 #  integrable_type                :string
 #  number                         :integer          indexed, indexed => [release_step_config_id]
@@ -49,7 +50,8 @@ def as_json(options = {})
       finish_rollout_in_next_release: finish_rollout_in_next_release,
       rollout_config: {
         enabled: rollout_enabled,
-        stages: rollout_stages
+        stages: rollout_stages,
+        automatic: automatic_rollout
       }
     }
   end
@@ -78,6 +80,7 @@ def self.from_json(json, read_only: false)
     submission.submission_external = Config::SubmissionExternal.from_json(json["submission_config"])
     submission.rollout_stages = json.dig("rollout_config", "stages")
     submission.rollout_enabled = json.dig("rollout_config", "enabled")
+    submission.automatic_rollout = json.dig("rollout_config", "automatic")
     submission
   end
 

app/models/play_store_rollout.rb

@@ -2,17 +2,20 @@
 #
 # Table name: store_rollouts
 #
-#  id                      :uuid             not null, primary key
-#  completed_at            :datetime
-#  config                  :decimal(8, 5)    default([]), not null, is an Array
-#  current_stage           :integer
-#  is_staged_rollout       :boolean          default(FALSE)
-#  status                  :string           not null
-#  type                    :string           not null
-#  created_at              :datetime         not null
-#  updated_at              :datetime         not null
-#  release_platform_run_id :uuid             not null, indexed
-#  store_submission_id     :uuid             indexed
+#  id                               :uuid             not null, primary key
+#  automatic_rollout                :boolean          default(FALSE), not null
+#  automatic_rollout_next_update_at :datetime
+#  automatic_rollout_updated_at     :datetime
+#  completed_at                     :datetime
+#  config                           :decimal(8, 5)    default([]), not null, is an Array
+#  current_stage                    :integer
+#  is_staged_rollout                :boolean          default(FALSE)
+#  status                           :string           not null
+#  type                             :string           not null
+#  created_at                       :datetime         not null
+#  updated_at                       :datetime         not null
+#  release_platform_run_id          :uuid             not null, indexed
+#  store_submission_id              :uuid             indexed
 #
 class PlayStoreRollout < StoreRollout
   include Passportable
@@ -23,20 +26,32 @@ class PlayStoreRollout < StoreRollout
   STAMPABLE_REASONS = %w[
     started
     updated
+    paused
     resumed
     halted
     completed
     failed
     fully_released
   ]
 
+  AUTO_ROLLOUT_RUN_INTERVAL = 24.hours
+
   aasm safe_state_machine_params(with_lock: false) do
     state :created, initial: true
     state(*STATES.keys)
 
     event :start, after_commit: :on_start! do
       transitions from: :created, to: :started
       transitions from: :halted, to: :started
+      transitions from: :paused, to: :started
+    end
+
+    event :pause do
+      transitions from: :started, to: :paused
+    end
+
+    event :resume do
+      transitions from: :paused, to: :started
     end
 
     event :halt, after_commit: :on_halt! do
@@ -56,8 +71,6 @@ class PlayStoreRollout < StoreRollout
 
   def controllable_rollout? = true
 
-  def automatic_rollout? = false
-
   def start_release!(retry_on_review_fail: false)
     if staged_rollout?
       # return mock_start_play_store_rollout! if sandbox_mode?
@@ -128,13 +141,20 @@ def halt_release!
     end
   end
 
+  def disable_automatic_rollout!
+    update!(automatic_rollout: false)
+  end
+
   def resume_release!
     with_lock do
       return unless may_start?
 
       result = rollout(last_rollout_percentage, retry_on_review_fail: true)
       if result.ok?
         start!
+        if release_platform_run.automatic_rollout?
+          update!(automatic_rollout: true)
+        end
         event_stamp!(reason: :resumed, kind: :notice, data: stamp_data)
         notify!("Rollout was resumed", :production_rollout_resumed, notification_params)
       else
@@ -143,8 +163,19 @@ def resume_release!
     end
   end
 
-  def rollout_active?
-    provider.build_active?(submission_channel_id, build_number, raise_on_lock_error: true)
+  def pause_release!
+    with_lock do
+      return unless may_pause?
+
+      disable_automatic_rollout!
+      pause!
+      event_stamp!(reason: :paused, kind: :error, data: stamp_data)
+      notify!("Automatic rollout has been paused", :production_rollout_paused, notification_params)
+    end
+  end
+
+  def rollout_in_progress?
+    provider.build_in_progress?(submission_channel_id, build_number, raise_on_lock_error: true)
   rescue GooglePlayStoreIntegration::LockAcquisitionError => e
     errors.add(:base, e.message)
     false

app/models/play_store_submission.rb

@@ -320,7 +320,7 @@ def release_notes
   def on_prepare!
     event_stamp!(reason: :prepared, kind: :notice, data: stamp_data)
     config = conf.rollout_stages.presence || []
-    create_play_store_rollout!(release_platform_run:, config:, is_staged_rollout: staged_rollout?)
+    create_play_store_rollout!(release_platform_run:, config:, is_staged_rollout: staged_rollout?, automatic_rollout: auto_rollout?)
     play_store_rollout.start_release!(retry_on_review_fail: internal_channel?) if auto_rollout?
   end
 

app/models/release_health_event.rb

@@ -30,6 +30,7 @@ class ReleaseHealthEvent < ApplicationRecord
   delegate :notify!, to: :production_release
 
   after_create_commit :notify_health_rule_triggered
+  after_create_commit :trigger_halt_on_unhealthy
 
   private
 
@@ -59,4 +60,9 @@ def rule_triggers
   def previous_event
     production_release.release_health_events.for_rule(release_health_rule).where.not(id:).reorder("event_timestamp").last
   end
+
+  def trigger_halt_on_unhealthy
+    return unless unhealthy?
+    HaltUnhealthyReleaseRolloutJob.perform_async(production_release.id, id)
+  end
 end

app/models/release_platform_run.rb

@@ -346,6 +346,8 @@ def previously_completed_rollout_run
 
   def conf = Config::ReleasePlatform.from_json(config)
 
+  def automatic_rollout? = conf.production_release&.submissions&.first&.automatic_rollout || false
+
   private
 
   def set_config

app/models/store_rollout.rb

@@ -2,17 +2,20 @@
 #
 # Table name: store_rollouts
 #
-#  id                      :uuid             not null, primary key
-#  completed_at            :datetime
-#  config                  :decimal(8, 5)    default([]), not null, is an Array
-#  current_stage           :integer
-#  is_staged_rollout       :boolean          default(FALSE)
-#  status                  :string           not null
-#  type                    :string           not null
-#  created_at              :datetime         not null
-#  updated_at              :datetime         not null
-#  release_platform_run_id :uuid             not null, indexed
-#  store_submission_id     :uuid             indexed
+#  id                               :uuid             not null, primary key
+#  automatic_rollout                :boolean          default(FALSE), not null
+#  automatic_rollout_next_update_at :datetime
+#  automatic_rollout_updated_at     :datetime
+#  completed_at                     :datetime
+#  config                           :decimal(8, 5)    default([]), not null, is an Array
+#  current_stage                    :integer
+#  is_staged_rollout                :boolean          default(FALSE)
+#  status                           :string           not null
+#  type                             :string           not null
+#  created_at                       :datetime         not null
+#  updated_at                       :datetime         not null
+#  release_platform_run_id          :uuid             not null, indexed
+#  store_submission_id              :uuid             indexed
 #
 class StoreRollout < ApplicationRecord
   has_paper_trail
@@ -47,6 +50,7 @@ class StoreRollout < ApplicationRecord
   delegate :stale?, :actionable?, to: :parent_release
 
   scope :production, -> { joins(store_submission: :production_release) }
+  scope :automatic_rollouts, -> { where(status: [:started, :halted], is_staged_rollout: true, automatic_rollout: true) }
 
   def staged_rollout? = is_staged_rollout