Start integrating playlist support (#13)

* [WIP] updated the output of VideoCollection to include playlists

* Updated source's name to collection_name; supported playlist ID/name fetching

* Hooked up collection_type to form; refactored enqueue_pending_media_downloads

* Added friendly_name to form

* Added media profile link to source view

* Updates comment

Author: kieraneglin

.iex.exs

@@ -2,6 +2,7 @@ alias Pinchflat.Repo
 
 alias Pinchflat.Tasks.Task
 alias Pinchflat.Media.MediaItem
+alias Pinchflat.Tasks.SourceTasks
 alias Pinchflat.Media.MediaMetadata
 alias Pinchflat.MediaSource.Source
 alias Pinchflat.Profiles.MediaProfile
@@ -12,3 +13,39 @@ alias Pinchflat.Profiles
 alias Pinchflat.MediaSource
 
 alias Pinchflat.MediaClient.{SourceDetails, VideoDownloader}
+
+defmodule IexHelpers do
+  def playlist_url do
+    "https://www.youtube.com/playlist?list=PLmqC3wPkeL8kSlTCcSMDD63gmSi7evcXS"
+  end
+
+  def channel_url do
+    "https://www.youtube.com/c/TheUselessTrials"
+  end
+
+  def video_url do
+    "https://www.youtube.com/watch?v=bR52O78ZIUw"
+  end
+
+  def details(type) do
+    source =
+      case type do
+        :playlist -> playlist_url()
+        :channel -> channel_url()
+      end
+
+    SourceDetails.get_source_details(source)
+  end
+
+  def ids(type) do
+    source =
+      case type do
+        :playlist -> playlist_url()
+        :channel -> channel_url()
+      end
+
+    SourceDetails.get_video_ids(source)
+  end
+end
+
+import IexHelpers

lib/pinchflat/media_client/backends/yt_dlp/video_collection.ex

@@ -4,8 +4,6 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollection do
   videos (aka: a source [ie: channels, playlists]).
   """
 
-  alias Pinchflat.MediaClient.SourceDetails
-
   @doc """
   Returns a list of strings representing the video ids in the collection.
 
@@ -28,19 +26,29 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollection do
   instead we're fetching just the first video (using playlist_end: 1)
   and parsing the source ID and name from _its_ metadata
 
-  Returns {:ok, %SourceDetails{}} | {:error, any, ...}.
+  Returns {:ok, map()} | {:error, any, ...}.
   """
   def get_source_details(source_url) do
-    opts = [:skip_download, playlist_end: 1]
+    opts = [:simulate, :skip_download, playlist_end: 1]
+    output_template = "%(.{channel,channel_id,playlist_id,playlist_title})j"
 
-    with {:ok, output} <- backend_runner().run(source_url, opts, "%(.{channel,channel_id})j"),
+    with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
          {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
-      {:ok, SourceDetails.new(parsed_json["channel_id"], parsed_json["channel"])}
+      {:ok, format_source_details(parsed_json)}
     else
       err -> err
     end
   end
 
+  defp format_source_details(response) do
+    %{
+      channel_id: response["channel_id"],
+      channel_name: response["channel"],
+      playlist_id: response["playlist_id"],
+      playlist_name: response["playlist_title"]
+    }
+  end
+
   defp backend_runner do
     Application.get_env(:pinchflat, :yt_dlp_runner)
   end

lib/pinchflat/media_client/source_details.ex

@@ -5,16 +5,9 @@ defmodule Pinchflat.MediaClient.SourceDetails do
   Technically hardcodes the yt-dlp backend for now, but should leave
   it open-ish for future expansion (just in case).
   """
-  @enforce_keys [:id, :name]
-  defstruct [:id, :name]
 
   alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection, as: YtDlpSource
 
-  @doc false
-  def new(id, name) do
-    %__MODULE__{id: id, name: name}
-  end
-
   @doc """
   Gets a source's ID and name from its URL, using the given backend.
 

lib/pinchflat/media_source.ex

@@ -100,6 +100,10 @@ defmodule Pinchflat.MediaSource do
   Note that this fetches source details as long as the `original_url` is present.
   This means that it'll go for it even if a changeset is otherwise invalid. This
   is pretty easy to change, but for MVP I'm not concerned.
+
+  IDEA: Maybe I could discern `collection_type` based on the original URL?
+  It also seems like it's a channel when the returned yt-dlp channel_id is the
+  same as the playlist_id - maybe could use that?
   """
   def change_source_from_url(%Source{} = source, attrs) do
     case change_source(source, attrs) do
@@ -115,14 +119,8 @@ defmodule Pinchflat.MediaSource do
     %Ecto.Changeset{changes: changes} = changeset
 
     case SourceDetails.get_source_details(changes.original_url) do
-      {:ok, %SourceDetails{} = source_details} ->
-        change_source(
-          source,
-          Map.merge(changes, %{
-            name: source_details.name,
-            collection_id: source_details.id
-          })
-        )
+      {:ok, source_details} ->
+        add_source_details_by_collection_type(source, changeset, source_details)
 
       {:error, runner_error, _status_code} ->
         Ecto.Changeset.add_error(
@@ -134,15 +132,35 @@ defmodule Pinchflat.MediaSource do
     end
   end
 
-  defp commit_and_start_indexing(changeset) do
-    case Repo.insert_or_update(changeset) do
-      {:ok, %Source{} = source} ->
-        maybe_run_indexing_task(changeset, source)
+  defp add_source_details_by_collection_type(source, changeset, source_details) do
+    %Ecto.Changeset{changes: changes} = changeset
+    collection_type = source.collection_type || changes[:collection_type]
+
+    collection_changes =
+      case collection_type do
+        :channel ->
+          %{
+            collection_id: source_details.channel_id,
+            collection_name: source_details.channel_name
+          }
+
+        :playlist ->
+          %{
+            collection_id: source_details.playlist_id,
+            collection_name: source_details.playlist_name
+          }
+
+        _ ->
+          %{}
+      end
 
-        {:ok, source}
+    change_source(source, Map.merge(changes, collection_changes))
+  end
 
-      err ->
-        err
+  defp commit_and_start_indexing(changeset) do
+    case Repo.insert_or_update(changeset) do
+      {:ok, %Source{} = source} -> maybe_run_indexing_task(changeset, source)
+      err -> err
     end
   end
 
@@ -159,5 +177,7 @@ defmodule Pinchflat.MediaSource do
           SourceTasks.kickoff_indexing_task(source)
         end
     end
+
+    {:ok, source}
   end
 end

lib/pinchflat/media_source/source.ex

@@ -9,14 +9,24 @@ defmodule Pinchflat.MediaSource.Source do
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Profiles.MediaProfile
 
-  @allowed_fields ~w(name collection_id collection_type index_frequency_minutes original_url media_profile_id)a
-  @required_fields @allowed_fields -- ~w(index_frequency_minutes)a
+  @allowed_fields ~w(
+    collection_name
+    collection_id
+    collection_type
+    friendly_name
+    index_frequency_minutes
+    original_url
+    media_profile_id
+  )a
+
+  @required_fields @allowed_fields -- ~w(index_frequency_minutes friendly_name)a
 
   schema "sources" do
-    field :name, :string
+    field :friendly_name, :string
+    field :collection_name, :string
     field :collection_id, :string
     field :collection_type, Ecto.Enum, values: [:channel, :playlist]
-    field :index_frequency_minutes, :integer
+    field :index_frequency_minutes, :integer, default: 60 * 24
     # This should only be used for user reference going forward
     # as the collection_id should be used for all API calls
     field :original_url, :string

lib/pinchflat/tasks/source_tasks.ex

@@ -3,9 +3,11 @@ defmodule Pinchflat.Tasks.SourceTasks do
   This module contains methods for managing tasks (workers) related to sources.
   """
 
+  alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.MediaSource.Source
   alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.Workers.VideoDownloadWorker
 
   @doc """
   Starts tasks for indexing a source's media.
@@ -30,4 +32,27 @@ defmodule Pinchflat.Tasks.SourceTasks do
       end
     end
   end
+
+  @doc """
+  Starts tasks for downloading videos for any of a sources _pending_ media items.
+
+  NOTE: this starts a download for each media item that is pending,
+  not just the ones that were indexed in this job run. This should ensure
+  that any stragglers are caught if, for some reason, they weren't enqueued
+  or somehow got de-queued.
+
+  I'm not sure of a case where this would happen, but it's cheap insurance.
+
+  Returns :ok
+  """
+  def enqueue_pending_media_downloads(%Source{} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(fn media_item ->
+      media_item
+      |> Map.take([:id])
+      |> VideoDownloadWorker.new()
+      |> Tasks.create_job_with_task(media_item)
+    end)
+  end
 end

lib/pinchflat/workers/media_indexing_worker.ex

@@ -7,10 +7,9 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
     tags: ["media_source", "media_indexing"]
 
   alias __MODULE__
-  alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.MediaSource
-  alias Pinchflat.Workers.VideoDownloadWorker
+  alias Pinchflat.Tasks.SourceTasks
 
   @impl Oban.Worker
   @doc """
@@ -49,7 +48,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
 
   defp index_media_and_reschedule(source) do
     MediaSource.index_media_items(source)
-    enqueue_video_downloads(source)
+    SourceTasks.enqueue_pending_media_downloads(source)
 
     source
     |> Map.take([:id])
@@ -60,21 +59,4 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
       {:error, :duplicate_job} -> {:ok, :job_exists}
     end
   end
-
-  # NOTE: this starts a download for each media item that is pending,
-  # not just the ones that were indexed in this job run. This should ensure
-  # that any stragglers are caught if, for some reason, they weren't enqueued
-  # or somehow got de-queued.
-  #
-  # I'm not sure of a case where this would happen, but it's cheap insurance.
-  defp enqueue_video_downloads(source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(fn media_item ->
-      media_item
-      |> Map.take([:id])
-      |> VideoDownloadWorker.new()
-      |> Tasks.create_job_with_task(media_item)
-    end)
-  end
 end

lib/pinchflat_web/controllers/media_sources/source_controller.ex

@@ -1,6 +1,7 @@
 defmodule PinchflatWeb.MediaSources.SourceController do
   use PinchflatWeb, :controller
 
+  alias Pinchflat.Repo
   alias Pinchflat.Profiles
   alias Pinchflat.MediaSource
   alias Pinchflat.MediaSource.Source
@@ -30,7 +31,10 @@ defmodule PinchflatWeb.MediaSources.SourceController do
   end
 
   def show(conn, %{"id" => id}) do
-    source = MediaSource.get_source!(id)
+    source =
+      id
+      |> MediaSource.get_source!()
+      |> Repo.preload(:media_profile)
 
     render(conn, :show, source: source)
   end

lib/pinchflat_web/controllers/media_sources/source_html.ex

@@ -24,4 +24,11 @@ defmodule PinchflatWeb.MediaSources.SourceHTML do
       {"Monthly", 30 * 24 * 60}
     ]
   end
+
+  def friendly_collection_types do
+    [
+      {"Channel", "channel"},
+      {"Playlist", "playlist"}
+    ]
+  end
 end

lib/pinchflat_web/controllers/media_sources/source_html/index.html.heex

@@ -8,8 +8,8 @@
 </.header>
 
 <.table id="sources" rows={@sources} row_click={&JS.navigate(~p"/media_sources/sources/#{&1}")}>
-  <:col :let={source} label="Name"><%= source.name %></:col>
-  <:col :let={source} label="Source"><%= source.collection_id %></:col>
+  <:col :let={source} label="Collection Name"><%= source.collection_name %></:col>
+  <:col :let={source} label="Collection ID"><%= source.collection_id %></:col>
   <:action :let={source}>
     <div class="sr-only">
       <.link navigate={~p"/media_sources/sources/#{source}"}>Show</.link>

lib/pinchflat_web/controllers/media_sources/source_html/show.html.heex

@@ -9,9 +9,18 @@
 </.header>
 
 <.list>
-  <:item title="Source Name"><%= @source.name %></:item>
-  <:item title="Source ID"><%= @source.collection_id %></:item>
-  <:item title="Original URL"><%= @source.original_url %></:item>
+  <:item title="media_profile">
+    <.link href={~p"/media_profiles/#{@source.media_profile}"}>
+      <%= @source.media_profile.name %>
+    </.link>
+  </:item>
+
+  <:item
+    :for={attr <- ~w(collection_type collection_name collection_id original_url friendly_name)a}
+    title={attr}
+  >
+    <%= Map.get(@source, attr) %>
+  </:item>
 </.list>
 
 <.back navigate={~p"/media_sources/sources"}>Back to sources</.back>

lib/pinchflat_web/controllers/media_sources/source_html/source_form.html.heex

@@ -3,14 +3,22 @@
     Oops, something went wrong! Please check the errors below.
   </.error>
 
+  <.input field={f[:friendly_name]} type="text" label="Friendly Name" />
+
   <.input
     field={f[:media_profile_id]}
     options={Enum.map(@media_profiles, &{&1.name, &1.id})}
     type="select"
     label="Media Profile"
   />
 
-  <.input field={f[:collection_type]} type="text" label="Collection Type" />
+  <.input
+    field={f[:collection_type]}
+    options={friendly_collection_types()}
+    type="select"
+    label="Collection Type"
+  />
+
   <.input field={f[:original_url]} type="text" label="Source URL" />
 
   <.input

priv/repo/migrations/20240202190351_rename_source_name_to_collection_name.exs

@@ -0,0 +1,11 @@
+defmodule Pinchflat.Repo.Migrations.RenameSourceNameToCollectionName do
+  use Ecto.Migration
+
+  def change do
+    rename table(:sources), :name, to: :collection_name
+
+    alter table(:sources) do
+      add :friendly_name, :string
+    end
+  end
+end