Media downloading indexing options (round 2) (#14)

* Adds options + option builder + metadata parsing for media thumbnails

* Added release-type options to media profile; built option parser for indexing operations

* Added new media_profile options to creation form; made show helper for rendering database items

* Added options for downloading/embedding metadata

* Adds option on sources to not download media (index only)

* reformatted docs

Author: kieraneglin

.gitignore

@@ -39,3 +39,4 @@ npm-debug.log
 /.elixir_ls
 .env
 .DS_Store
+scratchpad.md

lib/pinchflat/media/media_item.ex

@@ -10,13 +10,23 @@ defmodule Pinchflat.Media.MediaItem do
   alias Pinchflat.MediaSource.Source
   alias Pinchflat.Media.MediaMetadata
 
+  @allowed_fields ~w(
+    title
+    media_id
+    media_filepath
+    source_id
+    subtitle_filepaths
+    thumbnail_filepath
+    metadata_filepath
+  )a
   @required_fields ~w(media_id source_id)a
-  @allowed_fields ~w(title media_id media_filepath source_id subtitle_filepaths)a
 
   schema "media_items" do
     field :title, :string
     field :media_id, :string
     field :media_filepath, :string
+    field :thumbnail_filepath, :string
+    field :metadata_filepath, :string
     # This is an array of [iso-2 language, filepath] pairs. Probably could
     # be an associated record, but I don't see the benefit right now.
     # Will very likely revisit because I can't leave well-enough alone.

lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex

@@ -25,6 +25,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
     metadata_attrs
     |> Map.merge(parse_media_metadata(metadata))
     |> Map.merge(parse_subtitle_metadata(metadata))
+    |> Map.merge(parse_thumbnail_metadata(metadata))
+    |> Map.merge(parse_infojson_metadata(metadata))
   end
 
   defp parse_media_metadata(metadata) do
@@ -35,15 +37,34 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
   end
 
   defp parse_subtitle_metadata(metadata) do
-    subtitle_map = metadata["requested_subtitles"] || %{}
     # IDEA: if needed, consider filtering out subtitles that don't exist on-disk
     subtitle_filepaths =
-      subtitle_map
+      (metadata["requested_subtitles"] || %{})
       |> Enum.map(fn {lang, attrs} -> [lang, attrs["filepath"]] end)
       |> Enum.sort(fn [lang_a, _], [lang_b, _] -> lang_a < lang_b end)
 
     %{
       subtitle_filepaths: subtitle_filepaths
     }
   end
+
+  defp parse_thumbnail_metadata(metadata) do
+    thumbnail_filepath =
+      (metadata["thumbnails"] || %{})
+      # Reverse so that higher resolution thumbnails come first.
+      # This _shouldn't_ matter yet, but I'd rather default to the best
+      # in case I'm wrong.
+      |> Enum.reverse()
+      |> Enum.find_value(fn attrs -> attrs["filepath"] end)
+
+    %{
+      thumbnail_filepath: thumbnail_filepath
+    }
+  end
+
+  defp parse_infojson_metadata(metadata) do
+    %{
+      metadata_filepath: metadata["infojson_filename"]
+    }
+  end
 end

lib/pinchflat/media_client/source_details.ex

@@ -6,10 +6,14 @@ defmodule Pinchflat.MediaClient.SourceDetails do
   it open-ish for future expansion (just in case).
   """
 
+  alias Pinchflat.Repo
+  alias Pinchflat.MediaSource.Source
+
   alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection, as: YtDlpSource
+  alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder, as: YtDlpIndexOptionBuilder
 
   @doc """
-  Gets a source's ID and name from its URL, using the given backend.
+  Gets a source's ID and name from its URL using the given backend.
 
   Returns {:ok, map()} | {:error, any, ...}.
   """
@@ -18,11 +22,23 @@ defmodule Pinchflat.MediaClient.SourceDetails do
   end
 
   @doc """
-  Returns a list of video IDs for the given source URL, using the given backend.
+  Returns a list of video IDs for the given source URL OR source record using the given backend.
+
+  If passing a source record, the call to the backend may have custom options applied based on
+  the `option_builder`.
 
   Returns {:ok, list(binary())} | {:error, any, ...}.
   """
-  def get_video_ids(source_url, backend \\ :yt_dlp) do
+  def get_video_ids(sourceable, backend \\ :yt_dlp)
+
+  def get_video_ids(%Source{} = source, backend) do
+    media_profile = Repo.preload(source, :media_profile).media_profile
+    {:ok, options} = option_builder(backend).build(media_profile)
+
+    source_module(backend).get_video_ids(source.collection_id, options)
+  end
+
+  def get_video_ids(source_url, backend) when is_binary(source_url) do
     source_module(backend).get_video_ids(source_url)
   end
 
@@ -31,4 +47,10 @@ defmodule Pinchflat.MediaClient.SourceDetails do
       :yt_dlp -> YtDlpSource
     end
   end
+
+  defp option_builder(backend) do
+    case backend do
+      :yt_dlp -> YtDlpIndexOptionBuilder
+    end
+  end
 end

lib/pinchflat/media_client/video_downloader.ex

@@ -14,14 +14,18 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
   alias Pinchflat.Profiles.MediaProfile
 
   alias Pinchflat.MediaClient.Backends.YtDlp.Video, as: YtDlpVideo
-  alias Pinchflat.Profiles.Options.YtDlp.OptionBuilder, as: YtDlpOptionBuilder
+  alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
   alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: YtDlpMetadataParser
 
   @doc """
   Downloads a video for a media item, updating the media item based on the metadata
   returned by the backend. Also saves the entire metadata response to the associated
   media_metadata record.
 
+  NOTE: related methods (like the download worker) won't download if the media item's source
+  is set to not download media. However, I'm not enforcing that here since I need this for testing.
+  This may change in the future but I'm not stressed.
+
   Returns {:ok, %MediaItem{}} | {:error, any, ...any}
   """
   def download_for_media_item(%MediaItem{} = media_item, backend \\ :yt_dlp) do
@@ -52,7 +56,7 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
 
   defp option_builder(backend) do
     case backend do
-      :yt_dlp -> YtDlpOptionBuilder
+      :yt_dlp -> YtDlpDownloadOptionBuilder
     end
   end
 

lib/pinchflat/media_source/source.ex

@@ -15,6 +15,7 @@ defmodule Pinchflat.MediaSource.Source do
     collection_type
     friendly_name
     index_frequency_minutes
+    download_media
     original_url
     media_profile_id
   )a
@@ -27,6 +28,7 @@ defmodule Pinchflat.MediaSource.Source do
     field :collection_id, :string
     field :collection_type, Ecto.Enum, values: [:channel, :playlist]
     field :index_frequency_minutes, :integer, default: 60 * 24
+    field :download_media, :boolean, default: true
     # This should only be used for user reference going forward
     # as the collection_id should be used for all API calls
     field :original_url, :string

lib/pinchflat/profiles/media_profile.ex

@@ -15,18 +15,39 @@ defmodule Pinchflat.Profiles.MediaProfile do
     download_auto_subs
     embed_subs
     sub_langs
+    download_thumbnail
+    embed_thumbnail
+    download_metadata
+    embed_metadata
+    shorts_behaviour
+    livestream_behaviour
   )a
 
   @required_fields ~w(name output_path_template)a
 
   schema "media_profiles" do
     field :name, :string
     field :output_path_template, :string
+
     field :download_subs, :boolean, default: true
     field :download_auto_subs, :boolean, default: true
     field :embed_subs, :boolean, default: true
     field :sub_langs, :string, default: "en"
 
+    field :download_thumbnail, :boolean, default: true
+    field :embed_thumbnail, :boolean, default: true
+
+    field :download_metadata, :boolean, default: true
+    field :embed_metadata, :boolean, default: true
+
+    # NOTE: these do NOT speed up indexing - the indexer still has to go
+    # through the entire collection to determine if a video is a short or
+    # a livestream.
+    # NOTE: these can BOTH be set to :only which will download shorts and
+    # livestreams _only_ and ignore regular videos.
+    field :shorts_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
+    field :livestream_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
+
     has_many :sources, Source
 
     timestamps(type: :utc_datetime)

lib/pinchflat/profiles/options/yt_dlp/option_builder.ex lib/pinchflat/profiles/options/yt_dlp/download_option_builder.ex

@@ -1,6 +1,6 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
+defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder do
   @moduledoc """
-  Builds the options for yt-dlp based on the given media profile.
+  Builds the options for yt-dlp to download media based on the given media profile.
 
   IDEA: consider making this a behaviour so I can add other backends later
   """
@@ -9,7 +9,7 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
   alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
 
   @doc """
-  Builds the options for yt-dlp based on the given media profile.
+  Builds the options for yt-dlp to download media based on the given media profile.
 
   IDEA: consider adding the ability to pass in a second argument to override
         these options
@@ -24,18 +24,16 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
     built_options =
       default_options() ++
         subtitle_options(media_profile) ++
+        thumbnail_options(media_profile) ++
+        metadata_options(media_profile) ++
         output_options(media_profile)
 
     {:ok, built_options}
   end
 
   # This will be updated a lot as I add new options to profiles
   defp default_options do
-    [
-      :embed_metadata,
-      :embed_thumbnail,
-      :no_progress
-    ]
+    [:no_progress]
   end
 
   defp subtitle_options(media_profile) do
@@ -65,6 +63,30 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
     end)
   end
 
+  defp thumbnail_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_thumbnail, true} -> acc ++ [:write_thumbnail]
+        {:embed_thumbnail, true} -> acc ++ [:embed_thumbnail]
+        _ -> acc
+      end
+    end)
+  end
+
+  defp metadata_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_metadata, true} -> acc ++ [:write_info_json, :clean_info_json]
+        {:embed_metadata, true} -> acc ++ [:embed_metadata]
+        _ -> acc
+      end
+    end)
+  end
+
   defp output_options(media_profile) do
     {:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template)
 

lib/pinchflat/profiles/options/yt_dlp/index_option_builder.ex

@@ -0,0 +1,52 @@
+defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder do
+  @moduledoc """
+  Builds the options for yt-dlp to index a media source based on the given media profile.
+  """
+
+  alias Pinchflat.Profiles.MediaProfile
+
+  @doc """
+  Builds the options for yt-dlp to index a media source based on the given media profile.
+  """
+  def build(%MediaProfile{} = media_profile) do
+    built_options = release_type_options(media_profile)
+
+    {:ok, built_options}
+  end
+
+  defp release_type_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    # Appending multiple match filters treats them as an OR condition,
+    # so we have to be careful around combining `only` and `exclude` options.
+    # eg: only shorts + exclude livestreams = "any video that is a short OR is not a livestream"
+    # which will return all shorts AND normal videos.
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case {attr, media_profile} do
+        {{:shorts_behaviour, :only}, _} ->
+          acc ++ [match_filter: "original_url*=/shorts/"]
+
+        {{:livestream_behaviour, :only}, _} ->
+          acc ++ [match_filter: "was_live"]
+
+        # Since match_filter is an OR (see above), `exclude`s must be ignored entirely if the
+        # other type is set to `only`. There is also special behaviour if they're both excludes,
+        # hence why these check against `:include` alone.
+        {{:shorts_behaviour, :exclude}, %{livestream_behaviour: :include}} ->
+          acc ++ [match_filter: "original_url!*=/shorts/"]
+
+        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :include}} ->
+          acc ++ [match_filter: "!was_live"]
+
+        # Again, since it's an OR, there's a special syntax if they're both excluded
+        # to make it an AND. Note that I'm not checking for the other permutation of
+        # both excluding since this MUST get hit so adding the other version would double up.
+        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :exclude}} ->
+          acc ++ [match_filter: "!was_live & original_url!*=/shorts/"]
+
+        _ ->
+          acc
+      end
+    end)
+  end
+end

lib/pinchflat/profiles/options/yt_dlp/output_path_builder.ex

@@ -12,6 +12,11 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder do
 
   Translates liquid-style templates into yt-dlp-style templates,
   leaving yt-dlp syntax intact.
+
+  IDEA: apart from any custom options I've defined, I can support any yt-dlp
+  option by assuming `{{ identifier }}` should transform to `%(identifier)S`.
+  It's not doing anything huge, but it's nicer to type and more approachable IMO.
+  IDEA: set a default for `MediaProfile`'s `output_path_template` field
   """
   def build(template_string) do
     TemplateParser.parse(template_string, full_yt_dlp_options_map())

lib/pinchflat/tasks/source_tasks.ex

@@ -35,6 +35,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
 
   @doc """
   Starts tasks for downloading videos for any of a sources _pending_ media items.
+  Jobs are not enqueued if the source is set to not download media. This will return :ok.
 
   NOTE: this starts a download for each media item that is pending,
   not just the ones that were indexed in this job run. This should ensure
@@ -45,7 +46,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
 
   Returns :ok
   """
-  def enqueue_pending_media_downloads(%Source{} = source) do
+  def enqueue_pending_media_downloads(%Source{download_media: true} = source) do
     source
     |> Media.list_pending_media_items_for()
     |> Enum.each(fn media_item ->
@@ -55,4 +56,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
       |> Tasks.create_job_with_task(media_item)
     end)
   end
+
+  def enqueue_pending_media_downloads(%Source{download_media: false} = _source) do
+    :ok
+  end
 end