diff --git a/README.md b/README.md index 2282523..ceeeda6 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -# Membrane H264 Plugin +# Membrane H26x Plugin [![Hex.pm](https://img.shields.io/hexpm/v/membrane_h264_plugin.svg)](https://hex.pm/packages/membrane_h264_plugin) [![API Docs](https://img.shields.io/badge/api-docs-yellow.svg?style=flat)](https://hexdocs.pm/membrane_h264_plugin) [![CircleCI](https://circleci.com/gh/membraneframework/membrane_h264_plugin.svg?style=svg)](https://circleci.com/gh/membraneframework/membrane_h264_plugin) -Membrane H264 parser. -It is the Membrane element responsible for parsing the incoming H264 stream. The parsing is done as a sequence of the following steps: -* splitting the H264 stream into stream NAL units, based on the "Annex B" of the "ITU-T Rec. H.264 (01/2012)" or length prefix defined in "ISO/IEC 14496-10" +Membrane H.264 and H.265 parsers. +It is a pair of Membrane elements responsible for parsing the incoming H.264 and H.265 streams. The parsing is done as a sequence of the following steps: +* splitting the stream into stream NAL units * Parsing the NAL unit headers, so that to read the type of the NAL unit * Parsing the NAL unit body with the appropriate scheme, based on the NAL unit type read in the step before * Aggregating the NAL units into a stream of *access units* @@ -17,12 +17,12 @@ It is part of [Membrane Multimedia Framework](https://membraneframework.org). ## Installation -The package can be installed by adding `membrane_h264_plugin` to your list of dependencies in `mix.exs`: +The package can be installed by adding `membrane_h26x_plugin` to your list of dependencies in `mix.exs`: ```elixir def deps do [ - {:membrane_h264_plugin, "~> 0.9.1"} + {:membrane_h26x_plugin, "~> 0.10.0"} ] end ``` diff --git a/lib/membrane_h264_plugin/h264/decoder_configuration_record.ex b/lib/membrane_h264_plugin/h264/decoder_configuration_record.ex index ffedc46..e7c361f 100644 --- a/lib/membrane_h264_plugin/h264/decoder_configuration_record.ex +++ b/lib/membrane_h264_plugin/h264/decoder_configuration_record.ex @@ -28,7 +28,7 @@ defmodule Membrane.H264.DecoderConfigurationRecord do @doc """ Generates a DCR based on given PPSs and SPSs. """ - @spec generate([binary()], [binary()], Membrane.H26x.Parser.stream_structure()) :: + @spec generate([binary()], [binary()], Membrane.H264.Parser.stream_structure()) :: binary() | nil def generate([], _ppss, _stream_structure) do nil diff --git a/lib/membrane_h264_plugin/h264_parser.ex b/lib/membrane_h264_plugin/h264_parser.ex index 0638045..e4bcde6 100644 --- a/lib/membrane_h264_plugin/h264_parser.ex +++ b/lib/membrane_h264_plugin/h264_parser.ex @@ -112,10 +112,9 @@ defmodule Membrane.H264.Parser do output_stream_structure: [ spec: nil - | :annexb + | stream_structure() | :avc1 - | :avc3 - | {:avc1 | :avc3, nalu_length_size :: pos_integer()}, + | :avc3, default: nil, description: """ format of the outgoing H264 stream, if set to `:annexb` NALUs will be separated by @@ -151,6 +150,14 @@ defmodule Membrane.H264.Parser do """ ] + @typedoc """ + Format of the H264 stream, if set to `:annexb` NALUs will be separated by + a start code (0x(00)000001) or if set to `:avc3` or `:avc1` they will + be prefixed by their size. + """ + @type stream_structure :: + :annexb | {codec_tag :: :avc1 | :avc3, nalu_length_size :: pos_integer()} + @impl true def handle_init(ctx, opts) do output_stream_structure = diff --git a/lib/membrane_h264_plugin/h265/decoder_configuration_record.ex b/lib/membrane_h264_plugin/h265/decoder_configuration_record.ex index b83d662..d41eab5 100644 --- a/lib/membrane_h264_plugin/h265/decoder_configuration_record.ex +++ b/lib/membrane_h264_plugin/h265/decoder_configuration_record.ex @@ -48,7 +48,7 @@ defmodule Membrane.H265.DecoderConfigurationRecord do @doc """ Generates a DCR based on given PPSs, SPSs and VPSs. """ - @spec generate([NALu.t()], [NALu.t()], [NALu.t()], Membrane.H26x.Parser.stream_structure()) :: + @spec generate([NALu.t()], [NALu.t()], [NALu.t()], Membrane.H265.Parser.stream_structure()) :: binary() | nil def generate(_vpss, [], _ppss, _stream_structure) do nil diff --git a/lib/membrane_h264_plugin/h265_parser.ex b/lib/membrane_h264_plugin/h265_parser.ex index 4736db6..7850c29 100644 --- a/lib/membrane_h264_plugin/h265_parser.ex +++ b/lib/membrane_h264_plugin/h265_parser.ex @@ -110,10 +110,9 @@ defmodule Membrane.H265.Parser do output_stream_structure: [ spec: nil - | :annexb + | stream_structure() | :hvc1 - | :hev1 - | {:hvc1 | :hev1, nalu_length_size :: pos_integer()}, + | :hev1, default: nil, description: """ format of the outgoing H265 stream, if set to `:annexb` NALUs will be separated by @@ -155,6 +154,14 @@ defmodule Membrane.H265.Parser do """ ] + @typedoc """ + Format of the H265 stream, if set to `:annexb` NALUs will be separated by + a start code (0x(00)000001) or if set to `:hvc1` or `:hev1` they will be + prefixed by their size. + """ + @type stream_structure :: + :annexb | {codec_tag :: :hvc1 | :hev1, nalu_length_size :: pos_integer()} + @impl true def handle_init(ctx, opts) do output_stream_structure = diff --git a/lib/membrane_h264_plugin/h26x/nalu_parser.ex b/lib/membrane_h264_plugin/h26x/nalu_parser.ex index e852b92..874980d 100644 --- a/lib/membrane_h264_plugin/h26x/nalu_parser.ex +++ b/lib/membrane_h264_plugin/h26x/nalu_parser.ex @@ -4,7 +4,7 @@ defmodule Membrane.H26x.NALuParser do is a payload of a single NAL unit. """ - alias Membrane.H26x.{NALu, Parser} + alias Membrane.H26x.NALu alias Membrane.H26x.NALuParser.SchemeParser @annexb_prefix_code <<0, 0, 0, 1>> @@ -103,7 +103,7 @@ defmodule Membrane.H26x.NALuParser do """ @type t :: %__MODULE__{ scheme_parser_state: SchemeParser.t(), - input_stream_structure: Parser.stream_structure() + input_stream_structure: Membrane.H264.Parser.stream_structure() } @enforce_keys [:input_stream_structure] defstruct @enforce_keys ++ @@ -115,7 +115,7 @@ defmodule Membrane.H26x.NALuParser do Returns a structure holding a clear NALu parser state. `input_stream_structure` determines the prefixes of input NALU payloads. """ - @spec new(Parser.stream_structure()) :: t() + @spec new(Membrane.H264.Parser.stream_structure()) :: t() def new(input_stream_structure) do %__MODULE__{ input_stream_structure: input_stream_structure @@ -126,7 +126,8 @@ defmodule Membrane.H26x.NALuParser do Returns payload of the NALu with appropriate prefix generated based on output stream structure and prefix length. """ - @spec get_prefixed_nalu_payload(NALu.t(), Parser.stream_structure(), boolean()) :: binary() + @spec get_prefixed_nalu_payload(NALu.t(), Membrane.H264.Parser.stream_structure(), boolean()) :: + binary() def get_prefixed_nalu_payload(nalu, output_stream_structure, stable_prefixing? \\ true) do case {output_stream_structure, stable_prefixing?} do {:annexb, true} -> @@ -144,7 +145,7 @@ defmodule Membrane.H26x.NALuParser do end end - @spec unprefix_nalu_payload(binary(), Parser.stream_structure()) :: + @spec unprefix_nalu_payload(binary(), Membrane.H264.Parser.stream_structure()) :: {stripped_prefix :: binary(), payload :: binary()} def unprefix_nalu_payload(nalu_payload, :annexb) do case nalu_payload do @@ -159,7 +160,7 @@ defmodule Membrane.H26x.NALuParser do {<>, rest} end - @spec prefix_nalus_payloads([binary()], Parser.stream_structure()) :: binary() + @spec prefix_nalus_payloads([binary()], Membrane.H264.Parser.stream_structure()) :: binary() def prefix_nalus_payloads(nalus, :annexb) do Enum.join([<<>> | nalus], @annexb_prefix_code) end diff --git a/lib/membrane_h264_plugin/h26x/nalu_parser/scheme.ex b/lib/membrane_h264_plugin/h26x/nalu_parser/scheme.ex index c4e6f1e..d18df57 100644 --- a/lib/membrane_h264_plugin/h26x/nalu_parser/scheme.ex +++ b/lib/membrane_h264_plugin/h26x/nalu_parser/scheme.ex @@ -29,26 +29,16 @@ defmodule Membrane.H26x.NALuParser.Scheme do alias Membrane.H26x.NALuParser.SchemeParser @type field :: {any(), SchemeParser.field()} - @type if_clause :: {value_provider(boolean()), t()} + @type if_clause :: {SchemeParser.value_provider(boolean()), t()} @type for_loop :: - {[iterator: any(), from: value_provider(integer()), to: value_provider(integer())], t()} - @type calculate :: {any(), value_provider(any())} + {[ + iterator: any(), + from: SchemeParser.value_provider(integer()), + to: SchemeParser.value_provider(integer()) + ], t()} + @type calculate :: {any(), SchemeParser.value_provider(any())} @type execute :: (binary(), SchemeParser.t(), list(integer()) -> {binary(), SchemeParser.t()}) - @type save_as_global_state :: value_provider(any()) - - @typedoc """ - This type defines a value provider which provides values used in further - processing of a parser. - - A value provider can be either a hardcoded value, known at the compilation - time, or a tuple consisting of a lambda expression and the list of keys - mapping to some values in the parser's state. If the value provider is a tuple, - then it's first element - the lambda expression- is invoked with the arguments - being the values of the fields which are available in the parser's state under - the key names given in the parser's state, and the value used in the further - processing is the value returned by that lambda expression. - """ - @type value_provider(return_type) :: return_type | {(... -> return_type), list(any())} + @type save_as_global_state :: SchemeParser.value_provider(any()) @type directive :: {:field, field()} diff --git a/lib/membrane_h264_plugin/h26x/nalu_parser/scheme_parser.ex b/lib/membrane_h264_plugin/h26x/nalu_parser/scheme_parser.ex index 46dd78f..416d428 100644 --- a/lib/membrane_h264_plugin/h26x/nalu_parser/scheme_parser.ex +++ b/lib/membrane_h264_plugin/h26x/nalu_parser/scheme_parser.ex @@ -1,12 +1,12 @@ defmodule Membrane.H26x.NALuParser.SchemeParser do - @moduledoc false - # The module providing functions to parse the binary, - # based on the given Scheme. + @moduledoc """ + The module providing functions to parse the binary, + based on the given Scheme. + """ use Bunch.Access alias Membrane.H26x.ExpGolombConverter - alias Membrane.H26x.NALuParser.Scheme @typedoc """ A type defining the state of the scheme parser. @@ -30,6 +30,20 @@ defmodule Membrane.H26x.NALuParser.SchemeParser do @enforce_keys [:__global__, :__local__] defstruct @enforce_keys + @typedoc """ + This type defines a value provider which provides values used in further + processing of a parser. + + A value provider can be either a hardcoded value, known at the compilation + time, or a tuple consisting of a lambda expression and the list of keys + mapping to some values in the parser's state. If the value provider is a tuple, + then it's first element - the lambda expression- is invoked with the arguments + being the values of the fields which are available in the parser's state under + the key names given in the parser's state, and the value used in the further + processing is the value returned by that lambda expression. + """ + @type value_provider(return_type) :: return_type | {(... -> return_type), list(any())} + @typedoc """ A type describing the field types which can be used in NALu scheme definition. @@ -46,7 +60,7 @@ defmodule Membrane.H26x.NALuParser.SchemeParser do | :u8 | :u16 | :u16 - | {:uv, Scheme.value_provider(integer())} + | {:uv, value_provider(integer())} | :ue | :se diff --git a/lib/membrane_h264_plugin/h26x/nalu_splitter.ex b/lib/membrane_h264_plugin/h26x/nalu_splitter.ex index 8c3c1fc..2c9e4c1 100644 --- a/lib/membrane_h264_plugin/h26x/nalu_splitter.ex +++ b/lib/membrane_h264_plugin/h26x/nalu_splitter.ex @@ -22,7 +22,7 @@ defmodule Membrane.H26x.NALuSplitter do The `input_stream_structure` determines which prefix is considered as delimiting two NALUs. By default, the inner `unparsed_payload` of the state is clean, but can be set to a given binary. """ - @spec new(Parser.stream_structure(), initial_binary :: binary()) :: t() + @spec new(Membrane.H264.Parser.stream_structure(), initial_binary :: binary()) :: t() def new(input_stream_structure \\ :annexb, initial_binary \\ <<>>) do %__MODULE__{ input_stream_structure: input_stream_structure, @@ -33,9 +33,9 @@ defmodule Membrane.H26x.NALuSplitter do @doc """ Splits the binary into NALus sequence. - Takes a binary H26x stream as an input + Takes a binary H264 stream as an input and produces a list of binaries, where each binary is - a complete NALu that can be passed to the `Membrane.H26x.NALuParser.parse/4`. + a complete NALu that can be passed to the `Membrane.H264.NALuParser.parse/4`. If `assume_nalu_aligned` flag is set to `true`, input is assumed to form a complete set of NAL units and therefore all of them are returned. Otherwise, the NALu is not returned diff --git a/lib/membrane_h264_plugin/h26x_parser.ex b/lib/membrane_h264_plugin/h26x_parser.ex index 3582a3a..0515d72 100644 --- a/lib/membrane_h264_plugin/h26x_parser.ex +++ b/lib/membrane_h264_plugin/h26x_parser.ex @@ -13,7 +13,8 @@ defmodule Membrane.H26x.Parser do Stream structure of the NALUs. In case it's not `:annexb` format, it contains an information about the size of each NALU's prefix describing their length. """ - @type stream_structure :: :annexb | {codec_tag :: atom(), nalu_length_size :: pos_integer()} + @type stream_structure :: + Membrane.H264.Parser.stream_structure() | Membrane.H265.Parser.stream_structure() @type callback_context :: Membrane.Element.CallbackContext.t() @type action :: Membrane.Element.Action.t() diff --git a/mix.exs b/mix.exs index 05ead56..8e5687e 100644 --- a/mix.exs +++ b/mix.exs @@ -1,12 +1,12 @@ -defmodule Membrane.H264.Plugin.Mixfile do +defmodule Membrane.H26x.Plugin.Mixfile do use Mix.Project - @version "0.9.1" - @github_url "https://github.com/membraneframework/membrane_h264_plugin" + @version "0.10.0" + @github_url "https://github.com/membraneframework/membrane_h26x_plugin" def project do [ - app: :membrane_h264_plugin, + app: :membrane_h26x_plugin, version: @version, elixir: "~> 1.13", elixirc_paths: elixirc_paths(Mix.env()), @@ -15,11 +15,11 @@ defmodule Membrane.H264.Plugin.Mixfile do dialyzer: dialyzer(), # hex - description: "Membrane H264 parser", + description: "Membrane H.264 and H.265 parser", package: package(), # docs - name: "Membrane H264 plugin", + name: "Membrane H.264 and H.265 plugin", source_url: @github_url, homepage_url: "https://membrane.stream", docs: docs() @@ -79,8 +79,7 @@ defmodule Membrane.H264.Plugin.Mixfile do extras: ["README.md", "LICENSE"], formatters: ["html"], source_ref: "v#{@version}", - filter_modules: "Membrane\.H264\.Parser", - nest_modules_by_prefix: [Membrane.H264.Parser] + nest_modules_by_prefix: [Membrane.H264, Membrane.H265, Membrane.H26x] ] end end diff --git a/test/parser/h264/au_splitter_test.exs b/test/parser/h264/au_splitter_test.exs index 5d13ddc..6d08019 100644 --- a/test/parser/h264/au_splitter_test.exs +++ b/test/parser/h264/au_splitter_test.exs @@ -5,9 +5,7 @@ defmodule Membrane.H264.AUSplitterTest do @test_files_names ["10-720a", "10-720p"] - # These values were obtained with the use of H264.FFmpeg.Parser, available - # in the membrane_h264_ffmpeg_plugin repository. - @au_lengths_ffmpeg %{ + @au_lengths_snapshot %{ "10-720a" => [777, 146, 93, 136], "10-720p" => [25_699, 19_043, 14_379, 14_281, 14_761, 18_702, 14_735, 13_602, 12_094, 17_228] } @@ -40,7 +38,7 @@ defmodule Membrane.H264.AUSplitterTest do byte_size(payload) + byte_size(prefix) + acc end) - assert au_lengths == @au_lengths_ffmpeg[name] + assert au_lengths == @au_lengths_snapshot[name] end end diff --git a/test/parser/h265/process_all_test.exs b/test/parser/h265/process_all_test.exs index 0b9ae17..2643014 100644 --- a/test/parser/h265/process_all_test.exs +++ b/test/parser/h265/process_all_test.exs @@ -6,8 +6,8 @@ defmodule Membrane.H265.ProcessAllTest do import Membrane.ChildrenSpec import Membrane.Testing.Assertions - alias Membrane.H26x.NALuSplitter alias Membrane.H265 + alias Membrane.H26x.NALuSplitter alias Membrane.Testing.Pipeline defp make_pipeline(in_path, out_path, parameter_sets) do diff --git a/test/parser/h265/repeat_parameter_sets_test.exs b/test/parser/h265/repeat_parameter_sets_test.exs index b5ae4ba..d152d66 100644 --- a/test/parser/h265/repeat_parameter_sets_test.exs +++ b/test/parser/h265/repeat_parameter_sets_test.exs @@ -52,14 +52,14 @@ defmodule Membrane.H265.RepeatParameterSetsTest do parser_input_stream_structure \\ :annexb, parser_output_stream_structure \\ :annexb ) do - buffers = prepare_h264_buffers(data, mode, parser_input_stream_structure) + buffers = prepare_h265_buffers(data, mode, parser_input_stream_structure) assert_sink_playing(pipeline_pid, :sink) actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} Pipeline.message_child(pipeline_pid, :source, actions ++ [end_of_stream: :output]) output_buffers = - prepare_h264_buffers( + prepare_h265_buffers( File.read!(@ref_path), :au_aligned, parser_output_stream_structure @@ -112,14 +112,14 @@ defmodule Membrane.H265.RepeatParameterSetsTest do source = %H26x.Support.TestSource{mode: :bytestream} pid = make_pipeline(source) - buffers = prepare_h264_buffers(File.read!(in_path), :bytestream) + buffers = prepare_h265_buffers(File.read!(in_path), :bytestream) assert_sink_playing(pid, :sink) actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} Pipeline.message_child(pid, :source, actions ++ [end_of_stream: :output]) File.read!(ref_path) - |> prepare_h264_buffers(:au_aligned) + |> prepare_h265_buffers(:au_aligned) |> Enum.each(fn output_buffer -> assert_sink_buffer(pid, :sink, buffer) assert split_access_unit(output_buffer.payload) == split_access_unit(buffer.payload)