feat: Add job to download captions

apps/cf/config/config.exs

@@ -14,25 +14,6 @@ config :cf,
   soft_limitations_period: 15 * 60,
   hard_limitations_period: 3 * 60 * 60
 
-# Configure scheduler
-config :cf, CF.Scheduler,
-  # Run only one instance across cluster
-  global: true,
-  debug_logging: false,
-  jobs: [
-    # credo:disable-for-lines:10
-    # Actions analysers
-    # Every minute
-    {"*/1 * * * *", {CF.Jobs.Reputation, :update, []}},
-    # Every day
-    {"@daily", {CF.Jobs.Reputation, :reset_daily_limits, []}},
-    # Every minute
-    {"*/1 * * * *", {CF.Jobs.Flags, :update, []}},
-    # Various updaters
-    # Every 5 minutes
-    {"*/5 * * * *", {CF.Jobs.Moderation, :update, []}}
-  ]
-
 # Configure mailer
 config :cf, CF.Mailer, adapter: Bamboo.MailgunAdapter
 

apps/cf/config/test.exs

@@ -14,9 +14,6 @@ config :cf,
 # Print only warnings and errors during test
 config :logger, level: :warn
 
-# Disable CRON tasks on test
-config :cf, CF.Scheduler, jobs: []
-
 # Mails
 config :cf, CF.Mailer, adapter: Bamboo.TestAdapter
 

apps/cf_jobs/config/config.exs

@@ -8,6 +8,7 @@ config :cf_jobs, CF.Jobs.Scheduler,
   jobs: [
     # Reputation
     update_reputations: [
+      # every 20 minutes
       schedule: {:extended, "*/20"},
       task: {CF.Jobs.Reputation, :update, []},
       overlap: false
@@ -19,21 +20,31 @@ config :cf_jobs, CF.Jobs.Scheduler,
     ],
     # Moderation
     update_moderation: [
+      # every 5 minutes
       schedule: "*/5 * * * *",
       task: {CF.Jobs.Moderation, :update, []},
       overlap: false
     ],
     # Flags
     update_flags: [
+      # every minute
       schedule: "*/1 * * * *",
       task: {CF.Jobs.Flags, :update, []},
       overlap: false
     ],
     # Notifications
     create_notifications: [
+      # every 5 minutes
       schedule: {:extended, "*/5"},
       task: {CF.Jobs.CreateNotifications, :update, []},
       overlap: false
+    ],
+    # Captions
+    download_captions: [
+      # every 10 minutes
+      schedule: {:extended, "*/10"},
+      task: {CF.Jobs.DownloadCaptions, :update, []},
+      overlap: false
     ]
   ]
 

apps/cf_jobs/lib/application.ex

@@ -16,7 +16,8 @@ defmodule CF.Jobs.Application do
       worker(CF.Jobs.Reputation, []),
       worker(CF.Jobs.Flags, []),
       worker(CF.Jobs.Moderation, []),
-      worker(CF.Jobs.CreateNotifications, [])
+      worker(CF.Jobs.CreateNotifications, []),
+      worker(CF.Jobs.DownloadCaptions, [])
     ]
 
     # Do not start scheduler in tests

apps/cf_jobs/lib/jobs/download_captions.ex

@@ -0,0 +1,58 @@
+defmodule CF.Jobs.DownloadCaptions do
+  @behaviour CF.Jobs.Job
+
+  require Logger
+  import Ecto.Query
+  import ScoutApm.Tracing
+
+  alias DB.Repo
+  alias DB.Schema.UserAction
+  alias DB.Schema.UsersActionsReport
+
+  alias CF.Jobs.ReportManager
+
+  @name :download_captions
+  @analyser_id UsersActionsReport.analyser_id(@name)
+
+  # --- Client API ---
+
+  def name, do: @name
+
+  def start_link() do
+    GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
+  end
+
+  def init(args) do
+    {:ok, args}
+  end
+
+  # 1 minute
+  @timeout 60_000
+  def update() do
+    GenServer.call(__MODULE__, :download_captions, @timeout)
+  end
+
+  # --- Server callbacks ---
+  @transaction_opts [type: "background", name: "download_captions"]
+  def handle_call(:download_captions, _from, _state) do
+    get_videos()
+    |> Enum.map(fn video ->
+      Logger.info("Downloading captions for video #{video.id}")
+      download_captions(video)
+    end)
+
+    {:reply, :ok, :ok}
+  end
+
+  # Get all videos that need new captions. We fetch new captions:
+  # - For any videos that doesn't have any captions yet
+  # - For videos whose captions haven't been updated in the last 30 days
+  defp get_videos() do
+    Repo.all(
+      from(v in DB.Schema.Video,
+        where: v.captions_updated_at == nil or v.captions_updated_at < DateTime.utc_now() - 30,
+        preload: [:channel]
+      )
+    )
+  end
+end

apps/db/lib/db_schema/users_actions_report.ex

@@ -34,6 +34,7 @@ defmodule DB.Schema.UsersActionsReport do
   def analyser_id(:achievements), do: 3
   def analyser_id(:votes), do: 4
   def analyser_id(:create_notifications), do: 5
+  def analyser_id(:download_captions), do: 6
 
   def status(:pending), do: 1
   def status(:running), do: 2