feat: download protobuf sources to compile conformance runner

ahamez · Feb 26, 2025 · b9bf8f6 · b9bf8f6
1 parent 94c7f9e
commit b9bf8f6
Show file tree

Hide file tree

Showing 10 changed files with 220 additions and 239 deletions.
diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml
@@ -1,9 +1,7 @@
 name: Elixir CI
 
 env:
-  PROTOBUF_VERSION: "21.4"
-  PROTOBUF_LIB_VERSION_MAJOR: "32"
-  PROTOBUF_LIB_VERSION_MINOR: "0.4"
+  PROTOX_PROTOBUF_VERSION: "29.2"
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   MIX_ENV: test
 
@@ -60,31 +58,12 @@ jobs:
           path: priv/plts
           key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles('**/mix.lock') }}
 
-      - name: Restore conformance-test-runner
+      - name: Restore conformance_test_runner cache
         uses: actions/cache@v4
-        id: compile-conformance-test-runner
+        id: restore_conformance_test_runner
         with:
-          path: conformance-bin
-          key: ${{ runner.os }}-protobuf-${{ env.PROTOBUF_VERSION }}
-
-      - name: Compile conformance-test-runner
-        if: steps.compile-conformance-test-runner.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p ./conformance-bin/.libs
-          wget https://github.com/protocolbuffers/protobuf/archive/v${{ env.PROTOBUF_VERSION }}.tar.gz
-          tar xf v${{ env.PROTOBUF_VERSION }}.tar.gz
-          cd protobuf-${{ env.PROTOBUF_VERSION }}
-          ./autogen.sh && ./configure --disable-maintainer-mode --disable-dependency-tracking --disable-static
-          make -C ./src protoc
-          make -C conformance
-          cp ./conformance/.libs/conformance-test-runner ../conformance-bin
-          cp ./src/.libs/libprotobuf.so.${{ env.PROTOBUF_LIB_VERSION_MAJOR }}.${{ env.PROTOBUF_LIB_VERSION_MINOR }} ../conformance-bin/.libs/libprotobuf.so.${{ env.PROTOBUF_LIB_VERSION_MAJOR }}
-
-      - name: Install protoc
-        run: |
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip
-          unzip -d protoc protoc-24.4-linux-x86_64.zip
-          echo "${PWD}/protoc/bin" >> $GITHUB_PATH
+          path: conformance_test_runner
+          key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-protobuf-${{ env.PROTOX_PROTOBUF_VERSION }}
 
       - name: Unlock all dependencies
         run: mix deps.unlock --all
@@ -96,13 +75,28 @@ jobs:
           mix local.hex --force
           mix deps.get
 
+      - name: Install protoc
+        run: |
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v${{ env.PROTOX_PROTOBUF_VERSION }}/protoc-${{ env.PROTOX_PROTOBUF_VERSION }}-linux-x86_64.zip
+          unzip -d protoc protoc-${{ env.PROTOX_PROTOBUF_VERSION }}-linux-x86_64.zip
+          echo "${PWD}/protoc/bin" >> $GITHUB_PATH
+
       - name: Compile prod with warnings as errors
         run: MIX_ENV=prod mix compile --warnings-as-errors
 
+      - name: Compile conformance-test-runner
+        if: steps.restore_conformance_test_runner.outputs.cache-hit != 'true'
+        run: |
+          mix protox.conformance --compile-only
+          cp ./deps/protobuf/conformance_test_runner ./conformance_test_runner
+
+      - name: Restore conformance-test-runner from cache
+        if: steps.restore_conformance_test_runner.outputs.cache-hit == 'true'
+        run: |
+          mkdir -p ./deps/protobuf
+          cp ./conformance_test_runner ./deps/protobuf/conformance_test_runner
+
       - name: Run tests
-        env:
-          PROTOBUF_CONFORMANCE_RUNNER: conformance-bin/conformance-test-runner
-          LD_LIBRARY_PATH: conformance-bin/.libs
         run: mix coveralls.github --include conformance
 
       - name: Check formatting

diff --git a/.github/workflows/mutation_testing.yml b/.github/workflows/mutation_testing.yml
@@ -1,6 +1,7 @@
 name: Mutation testing
 
 env:
+  PROTOX_PROTOBUF_VERSION: "29.2"
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 on:
@@ -15,8 +16,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - elixir: 1.17.3
-            otp: 27.1
+          - elixir: "1.17"
+            otp: "27"
 
     steps:
       - uses: actions/checkout@v4
@@ -37,8 +38,8 @@ jobs:
 
       - name: Install protoc
         run: |
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protoc-21.4-linux-x86_64.zip
-          unzip -d protoc protoc-21.4-linux-x86_64.zip
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v${{ env.PROTOX_PROTOBUF_VERSION }}/protoc-${{ env.PROTOX_PROTOBUF_VERSION }}-linux-x86_64.zip
+          unzip -d protoc protoc-${{ env.PROTOX_PROTOBUF_VERSION }}-linux-x86_64.zip
           echo "${PWD}/protoc/bin" >> $GITHUB_PATH
 
       - name: Run mutation testing

diff --git a/README.md b/README.md
@@ -516,16 +516,10 @@ The protox library has been thoroughly tested using the conformance checker [pro
 
 Here's how to launch the conformance tests:
 
-* Get conformance-test-runner [sources](https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.0.tar.gz).
-* Compile conformance-test-runner ([macOS and Linux only](https://github.com/protocolbuffers/protobuf/tree/master/conformance#portability)):
-    ```
-    tar xf protobuf-3.18.0.tar.gz && cd protobuf-3.18.0 && ./autogen.sh && ./configure && make -j && cd conformance && make -j
-    ```
+```
+mix protox.conformance
+```
 
-* Launch the conformance tests:
-    ```
-    mix protox.conformance --runner=/path/to/protobuf-3.18.0/conformance/conformance-test-runner
-    ```
 * A report will be generated in the directory `conformance_report` and the following text should be displayed:
 
     ```

diff --git a/conformance/failure_list.txt b/conformance/failure_list.txt
@@ -22,3 +22,67 @@ Required.Proto3.JsonInput.AnyWithValueForJsonObject.ProtobufOutput
 
 # Could not find the specs for this one
 Recommended.Proto2.JsonInput.FieldNameExtension.Validator
+
+# TMP
+Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput
+Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput
+Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInOptionalField.ProtobufOutput
+Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput
+Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInRepeatedPart.ProtobufOutput
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInOptionalField.ProtobufOutput
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedPart.ProtobufOutput
+Recommended.Proto3.ValueRejectInfNumberValue.JsonOutput
+Recommended.Proto3.ValueRejectNanNumberValue.JsonOutput
+Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput
+Required.Proto2.ProtobufInput.UnknownWireType6_Field1_Verion0
+Required.Proto2.ProtobufInput.UnknownWireType6_Field1_Verion1
+Required.Proto2.ProtobufInput.UnknownWireType6_Field1_Verion2
+Required.Proto2.ProtobufInput.UnknownWireType6_Field1_Verion3
+Required.Proto2.ProtobufInput.UnknownWireType6_Field2_Verion0
+Required.Proto2.ProtobufInput.UnknownWireType6_Field2_Verion1
+Required.Proto2.ProtobufInput.UnknownWireType6_Field2_Verion2
+Required.Proto2.ProtobufInput.UnknownWireType6_Field2_Verion3
+Required.Proto2.ProtobufInput.UnknownWireType6_Field3_Verion0
+Required.Proto2.ProtobufInput.UnknownWireType6_Field3_Verion1
+Required.Proto2.ProtobufInput.UnknownWireType6_Field3_Verion2
+Required.Proto2.ProtobufInput.UnknownWireType6_Field3_Verion3
+Required.Proto2.ProtobufInput.UnknownWireType7_Field1_Verion0
+Required.Proto2.ProtobufInput.UnknownWireType7_Field1_Verion1
+Required.Proto2.ProtobufInput.UnknownWireType7_Field1_Verion2
+Required.Proto2.ProtobufInput.UnknownWireType7_Field1_Verion3
+Required.Proto2.ProtobufInput.UnknownWireType7_Field2_Verion0
+Required.Proto2.ProtobufInput.UnknownWireType7_Field2_Verion1
+Required.Proto2.ProtobufInput.UnknownWireType7_Field2_Verion2
+Required.Proto2.ProtobufInput.UnknownWireType7_Field2_Verion3
+Required.Proto2.ProtobufInput.UnknownWireType7_Field3_Verion0
+Required.Proto2.ProtobufInput.UnknownWireType7_Field3_Verion1
+Required.Proto2.ProtobufInput.UnknownWireType7_Field3_Verion2
+Required.Proto2.ProtobufInput.UnknownWireType7_Field3_Verion3
+Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch
+Required.Proto3.ProtobufInput.UnknownWireType6_Field1_Verion0
+Required.Proto3.ProtobufInput.UnknownWireType6_Field1_Verion1
+Required.Proto3.ProtobufInput.UnknownWireType6_Field1_Verion2
+Required.Proto3.ProtobufInput.UnknownWireType6_Field1_Verion3
+Required.Proto3.ProtobufInput.UnknownWireType6_Field2_Verion0
+Required.Proto3.ProtobufInput.UnknownWireType6_Field2_Verion1
+Required.Proto3.ProtobufInput.UnknownWireType6_Field2_Verion2
+Required.Proto3.ProtobufInput.UnknownWireType6_Field2_Verion3
+Required.Proto3.ProtobufInput.UnknownWireType6_Field3_Verion0
+Required.Proto3.ProtobufInput.UnknownWireType6_Field3_Verion1
+Required.Proto3.ProtobufInput.UnknownWireType6_Field3_Verion2
+Required.Proto3.ProtobufInput.UnknownWireType6_Field3_Verion3
+Required.Proto3.ProtobufInput.UnknownWireType7_Field1_Verion0
+Required.Proto3.ProtobufInput.UnknownWireType7_Field1_Verion1
+Required.Proto3.ProtobufInput.UnknownWireType7_Field1_Verion2
+Required.Proto3.ProtobufInput.UnknownWireType7_Field1_Verion3
+Required.Proto3.ProtobufInput.UnknownWireType7_Field2_Verion0
+Required.Proto3.ProtobufInput.UnknownWireType7_Field2_Verion1
+Required.Proto3.ProtobufInput.UnknownWireType7_Field2_Verion2
+Required.Proto3.ProtobufInput.UnknownWireType7_Field2_Verion3
+Required.Proto3.ProtobufInput.UnknownWireType7_Field3_Verion0
+Required.Proto3.ProtobufInput.UnknownWireType7_Field3_Verion1
+Required.Proto3.ProtobufInput.UnknownWireType7_Field3_Verion2
+Required.Proto3.ProtobufInput.UnknownWireType7_Field3_Verion3
diff --git a/conformance/mix/tasks/protox/task.ex b/conformance/mix/tasks/protox/task.ex
@@ -6,33 +6,116 @@ defmodule Mix.Tasks.Protox.Conformance do
   @impl Mix.Task
   @spec run(any) :: any
   def run(args) do
-    with {options, _, []} <- OptionParser.parse(args, strict: [runner: :string, quiet: :boolean]),
-         {:ok, runner} <- Keyword.fetch(options, :runner),
-         quiet <- Keyword.get(options, :quiet, false),
+    with {options, _, []} <-
+           OptionParser.parse(args,
+             strict: [
+               runner: :string,
+               quiet: :boolean,
+               force_runner_build: :boolean,
+               compile_only: :boolean
+             ]
+           ),
+         {:ok, runner} <- get_runner(options),
          :ok <- Mix.Tasks.Escript.Build.run([]),
-         :ok <- launch(runner, quiet) do
+         :ok <- launch_runner(options, runner) do
       {:ok, :conformance_successful}
     else
+      # We return :ok here because it means that the conformance test was launched, not necessarily successful.
       {:error, :runner_failure} -> {:ok, :conformance_failure}
       e -> e
     end
   end
 
-  defp launch(runner, quiet) do
-    shell =
-      case quiet do
-        true -> Mix.Shell.Quiet
-        false -> Mix.Shell.IO
+  defp launch_runner(options, runner) do
+    compile_only = Keyword.get(options, :compile_only, false)
+
+    if compile_only do
+      :ok
+    else
+      shell = shell(options)
+
+      cmd =
+        shell.cmd(
+          "#{runner} --enforce_recommended --failure_list ./conformance/failure_list.txt --output_dir . ./protox_conformance"
+        )
+
+      case cmd do
+        0 -> :ok
+        1 -> {:error, :runner_failure}
+        126 -> {:error, :cannot_execute_runner}
+        127 -> {:error, :no_such_file_or_directory}
+        code -> {:error, code}
       end
+    end
+  end
+
+  defp get_runner(options) do
+    case Keyword.get(options, :runner) do
+      nil ->
+        runner_path =
+          Path.expand("#{Mix.Project.deps_paths().protobuf}/conformance_test_runner")
+
+        force_runner_build = Keyword.get(options, :force_runner_build, false)
+
+        if File.exists?(runner_path) and not force_runner_build do
+          {:ok, runner_path}
+        else
+          with :ok <- configure_runner(options),
+               :ok <- build_runner(options) do
+            {:ok, runner_path}
+          end
+        end
+
+      runner_path ->
+        {:ok, runner_path}
+    end
+  end
+
+  defp configure_runner(options) do
+    shell = shell(options)
+
+    configuration =
+      [
+        {"CMAKE_CXX_STANDARD", "14"},
+        {"protobuf_INSTALL", "OFF"},
+        {"protobuf_BUILD_TESTS", "OFF"},
+        {"protobuf_BUILD_CONFORMANCE", "ON"},
+        {"protobuf_BUILD_EXAMPLES", "OFF"},
+        {"protobuf_BUILD_PROTOBUF_BINARIES", "ON"},
+        {"protobuf_BUILD_PROTOC_BINARIES", "OFF"},
+        {"protobuf_BUILD_LIBPROTOC", "OFF"},
+        {"protobuf_BUILD_LIBUPB", "OFF"}
+      ]
+      |> Enum.map(fn {key, value} -> "-D#{key}=#{value}" end)
+      |> Enum.join(" ")
+
+    File.cd!(Mix.Project.deps_paths().protobuf, fn ->
+      cmd = shell.cmd("cmake . #{configuration}")
+
+      case cmd do
+        0 -> :ok
+        code -> {:error, code}
+      end
+    end)
+  end
+
+  defp build_runner(options) do
+    shell = shell(options)
+
+    File.cd!(Mix.Project.deps_paths().protobuf, fn ->
+      cmd = shell.cmd("cmake --build . --parallel")
+
+      case cmd do
+        0 -> :ok
+        code -> {:error, code}
+      end
+    end)
+  end
 
-    case shell.cmd(
-           "#{runner} --enforce_recommended --failure_list ./conformance/failure_list.txt ./protox_conformance"
-         ) do
-      0 -> :ok
-      1 -> {:error, :runner_failure}
-      126 -> {:error, :cannot_execute_runner}
-      127 -> {:error, :no_such_file_or_directory}
-      code -> {:error, code}
+  defp shell(options) do
+    case Keyword.get(options, :quiet, false) do
+      true -> Mix.Shell.Quiet
+      false -> Mix.Shell.IO
     end
   end
 end