chore: move benchmarks into a mix task

Also, generate synthetic data using field type apparition frequency provided by Google

.formatter.exs

@@ -1,3 +1,3 @@
 [
-  inputs: ["mix.exs", "{config,lib,test,conformance,benchmarks}/**/*.{ex,exs}"]
+  inputs: ["mix.exs", "{config,lib,test,conformance,benchmark}/**/*.{ex,exs}"]
 ]

.gitignore

@@ -1,22 +1,20 @@
+*.beam
+*.benchee
+*.ez
+.DS_Store
+.lefthook-local.yml
+.tool-versions
 /_build
+/benchmark/output
+/conformance-test-runner
+/conformance_report
 /cover
 /deps
 /doc
 /docs
-erl_crash.dump
-*.ez
-*.beam
-.DS_Store
-/protobuf-*
-/conformance_report
-/protox_conformance
+/failing_tests.txt
 /priv/plts/*.plt
 /priv/plts/*.plt.hash
-/conformance-test-runner
-.tool-versions
-/benchmarks/benchmarks/
-/benchmarks/*.benchee
-/benchmarks/payloads.bin
-/benchmarks/output
-/failing_tests.txt
-.lefthook-local.yml
+/protobuf-*
+/protox_conformance
+erl_crash.dump

README.md

@@ -44,7 +44,7 @@ iex> {:ok, msg} = Msg.decode(binary)
 - [Files generation](#files-generation)
 - [Conformance](#conformance)
 - [Types mapping](#types-mapping)
-- [Benchmarks](#benchmarks)
+- [Benchmark](#benchmark)
 - [Contributing](#contributing)
 - [Credits](#credits)
 
@@ -448,13 +448,11 @@ Protobuf   | Elixir
 `enum`     | `atom() \| integer()`
 `message`  | `struct()`
 
-## Benchmarks
+## Benchmark
 
 You can launch benchmarks to see how `protox` perform:
 ```
-mix run ./benchmarks/generate_payloads.exs # first time only, generates random payloads
-mix run ./benchmarks/run.exs --lib=./benchmarks/protox.exs
-mix run ./benchmarks/load.exs
+TODO
 ```
 
 ## Contributing

benchmark/compile_benchmark_protos.ex

@@ -0,0 +1,4 @@
+defmodule Protox.CompileBenchmarkProtos do
+  @moduledoc false
+  use Protox, files: Path.wildcard("./benchmark/protos/*.proto")
+end

benchmark/mix/tasks/protox/benchmark/generate/payloads.ex

@@ -0,0 +1,76 @@
+defmodule Mix.Tasks.Protox.Benchmark.Generate.Payloads do
+  @moduledoc false
+
+  require Logger
+
+  use Mix.Task
+  use PropCheck
+
+  @nb_samples 1
+
+  @impl Mix.Task
+  @spec run(any) :: any
+  def run(_args) do
+    with {:ok, modules} <- get_benchmark_modules(),
+         {:ok, payloads} <- generate_payloads(modules),
+         {:ok, file} <- File.open("./benchmark/benchmark_payloads.bin", [:write]) do
+      IO.binwrite(file, :erlang.term_to_binary(payloads))
+      File.close(file)
+    else
+      err ->
+        IO.puts(:stderr, "Error: #{inspect(err)}")
+        exit({:shutdown, 1})
+    end
+  end
+
+  defp get_benchmark_modules() do
+    case :application.get_key(:protox, :modules) do
+      {:ok, modules} ->
+        modules =
+          Enum.filter(modules, fn mod ->
+            match?(["Protox", "Benchmark", _, "Message"], Module.split(mod))
+          end)
+
+        modules = [ProtobufTestMessages.Proto3.TestAllTypesProto3 | modules]
+
+        Logger.info("Modules: #{inspect(modules)}")
+
+        {:ok, modules}
+
+      :undefined ->
+        :error
+    end
+  end
+
+  defp generate_payloads(modules) do
+    payloads_async =
+      for module <- modules, into: %{} do
+        {module, fn -> generate_payload(module) end}
+      end
+
+    payloads =
+      payloads_async
+      |> Task.async_stream(fn {name, gen} -> {name, gen.()} end, timeout: :infinity)
+      |> Stream.map(fn {:ok, {name, payloads}} -> {name, payloads} end)
+      |> Map.new()
+
+    {:ok, payloads}
+  end
+
+  defp generate_payload(mod) do
+    Logger.info("Generating payload for #{mod}")
+
+    gen =
+      let fields <- Protox.RandomInit.generate_fields(mod) do
+        Protox.RandomInit.generate_struct(mod, fields)
+      end
+
+    Stream.repeatedly(fn -> :proper_gen.pick(gen, 5) end)
+    |> Stream.map(fn {:ok, msg} -> {msg, msg |> Protox.encode!() |> IO.iodata_to_binary()} end)
+    |> Stream.reject(fn {_msg, bytes} -> byte_size(bytes) == 0 end)
+    |> Stream.reject(fn {_msg, bytes} -> byte_size(bytes) > 16_384 * 16 end)
+    |> Stream.map(fn {msg, bytes} -> {msg, byte_size(bytes), bytes} end)
+    |> Stream.each(fn _ -> Logger.info("Payload generated for #{mod}") end)
+    |> Enum.take(@nb_samples)
+  end
+end

benchmark/mix/tasks/protox/benchmark/generate/protos.ex

@@ -0,0 +1,89 @@
+defmodule Mix.Tasks.Protox.Benchmark.Generate.Protos do
+  @moduledoc false
+
+  require Logger
+
+  use Mix.Task
+
+  # Frequencies are taken from
+  # https://github.com/protocolbuffers/protobuf/blob/336d6f04e94efebcefb5574d0c8d487bcb0d187e/benchmarks/gen_synthetic_protos.py.
+  @field_freqs [
+    {"bool", "", 8.321},
+    {"bool", "repeated", 0.033},
+    {"bytes", "", 0.809},
+    {"bytes", "repeated", 0.065},
+    {"double", "", 2.845},
+    {"double", "repeated", 0.143},
+    {"fixed32", "", 0.084},
+    {"fixed32", "repeated", 0.012},
+    {"fixed64", "", 0.204},
+    {"fixed64", "repeated", 0.027},
+    {"float", "", 2.355},
+    {"float", "repeated", 0.132},
+    {"int32", "", 6.717},
+    {"int32", "repeated", 0.366},
+    {"int64", "", 9.678},
+    {"int64", "repeated", 0.425},
+    {"sfixed32", "", 0.018},
+    {"sfixed32", "repeated", 0.005},
+    {"sfixed64", "", 0.022},
+    {"sfixed64", "repeated", 0.005},
+    {"sint32", "", 0.026},
+    {"sint32", "repeated", 0.009},
+    {"sint64", "", 0.018},
+    {"sint64", "repeated", 0.006},
+    {"string", "", 25.461},
+    {"string", "repeated", 2.606},
+    {"Enum", "", 6.16},
+    {"Enum", "repeated", 0.576},
+    {"Message", "", 22.472},
+    {"Message", "repeated", 7.766},
+    {"uint32", "", 1.289},
+    {"uint32", "repeated", 0.051},
+    {"uint64", "", 1.044},
+    {"uint64", "repeated", 0.079}
+  ]
+
+  @message_template """
+  syntax = "proto3";
+  package protox.benchmark.synthetic_<%= count %>;
+
+  enum Enum {
+    ZERO = 0;
+  }
+
+  message Message {
+  <%= for {type, label, counter} <- fields do %>
+    <%= label %> <%= type %> field_<%= counter %> = <%= counter %>;<% end %>
+  }
+  """
+
+  @impl Mix.Task
+  @spec run(any) :: any
+  def run(_args) do
+    for count <- [5, 10, 20, 50, 100] do
+      Logger.info("Generating synthetic proto with #{count} fields")
+      content = EEx.eval_string(@message_template, count: count, fields: random_choices(count))
+      File.write!("./protos/benchmark/synthetic_#{count}.proto", content)
+    end
+  end
+
+  defp random_choices(count) when count >= 0 do
+    total_weight =
+      @field_freqs
+      |> Stream.map(fn {_, _, freq} -> freq end)
+      |> Enum.sum()
+
+    cumulative_weights =
+      @field_freqs
+      |> Stream.map(fn {_, _, freq} -> freq end)
+      |> Enum.scan(&(&1 + &2))
+
+    Enum.map(1..count, fn c ->
+      rand = :rand.uniform() * total_weight
+      index = Enum.find_index(cumulative_weights, &(rand <= &1))
+      {type, label, _freq} = Enum.at(@field_freqs, index)
+      {type, label, c}
+    end)
+  end
+end

benchmark/mix/tasks/protox/benchmark/run.ex

@@ -0,0 +1,76 @@
+defmodule Mix.Tasks.Protox.Benchmark.Run do
+  @moduledoc false
+
+  use Mix.Task
+
+  @options [
+    prefix: :string
+  ]
+
+  @impl Mix.Task
+  @spec run(any) :: any
+  def run(args) do
+    with {opts, _argv, []} <- OptionParser.parse(args, strict: @options),
+         prefix <- Keyword.get(opts, :prefix, nil),
+         tag <- get_tag(prefix),
+         payloads <- get_payloads("./benchmark/benchmark_payloads.bin") do
+      IO.puts("tag=#{tag}\n")
+
+      Benchee.run(
+        %{
+          "decode" => fn input ->
+            Enum.map(input, fn {msg, _size, bytes} -> msg.__struct__.decode!(bytes) end)
+          end,
+          "encode" => fn input ->
+            Enum.map(input, fn {msg, _size, _bytes} -> msg.__struct__.encode!(msg) end)
+          end
+        },
+        inputs: payloads,
+        save: [
+          path: Path.join(["./benchmark", "#{tag}.benchee"]),
+          tag: "#{tag}"
+        ],
+        load: ["./benchmark/*.benchee"],
+        time: 10,
+        memory_time: 2,
+        formatters: [
+          {Benchee.Formatters.HTML, file: "benchmark/output/#{tag}.html"},
+          Benchee.Formatters.Console
+        ]
+      )
+    else
+      err ->
+        IO.puts(:stderr, "Error: #{inspect(err)}")
+        exit({:shutdown, 1})
+    end
+  end
+
+  defp get_tag(prefix) do
+    {hash, 0} = System.cmd("git", ["rev-parse", "--short", "HEAD"])
+    elixir_version = System.version()
+
+    erlang_version =
+      [:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]
+      |> Path.join()
+      |> File.read!()
+      |> String.trim()
+
+    tag = [elixir_version, erlang_version, hash]
+
+    tag =
+      case prefix do
+        nil -> tag
+        prefix -> [prefix | tag]
+      end
+
+    tag
+    |> Enum.map(&String.trim/1)
+    |> Enum.join("-")
+  end
+
+  def get_payloads(path) do
+    path
+    |> File.read!()
+    |> :erlang.binary_to_term()
+  end
+end

benchmark/protos/synthetic_10.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+package protox.benchmark.synthetic_10;
+
+enum Enum {
+  ZERO = 0;
+}
+
+message Message {
+  repeated Message field_1 = 1;
+  Message field_2 = 2;
+  int64 field_3 = 3;
+  Enum field_4 = 4;
+  string field_5 = 5;
+  repeated Enum field_6 = 6;
+  int64 field_7 = 7;
+  Message field_8 = 8;
+  Message field_9 = 9;
+  Message field_10 = 10;
+}