Vision bench update (#1704)

* Added LMMS_eval template, exact_match_mm metric, RGB image augmentor.

* Revert "Added LMMS_eval template, exact_match_mm metric, RGB image augmentor."

This reverts commit fbf63e1.

* up to date benchmark results

* Fix ruff

Signed-off-by: elronbandel <[email protected]>

---------

Signed-off-by: elronbandel <[email protected]>
Co-authored-by: Elron Bandel <[email protected]>

Author: alfassy

examples/evaluate_llama_vision_benchmark.py

@@ -41,9 +41,9 @@
 """
 | subset   |    score | score_name      |   num_of_instances |
 |:---------|---------:|:----------------|-------------------:|
-| ALL      | 0.553122 | subsets_mean    |                120 |
-| doc_vqa  | 0.666774 | anls            |                 30 |
-| info_vqa | 0.51238  | anls            |                 30 |
+| ALL      | 0.570827 | subsets_mean    |                120 |
+| doc_vqa  | 0.704262 | anls            |                 30 |
+| info_vqa | 0.545713 | anls            |                 30 |
 | chart_qa | 0.266667 | relaxed_overall |                 30 |
 | ai2d     | 0.766667 | exact_match_mm  |                 30 |
 """

examples/evaluate_vision_default_benchmark.py

@@ -40,10 +40,10 @@
 Using LLAMA-VISION-11B
 | subset   |    score | score_name      |   num_of_instances |
 |:---------|---------:|:----------------|-------------------:|
-| ALL      | 0.384752 | subsets_mean    |                150 |
-| doc_vqa  | 0.717027 | anls            |                 30 |
-| info_vqa | 0.485069 | anls            |                 30 |
+| ALL      | 0.432395 | subsets_mean    |                150 |
+| doc_vqa  | 0.802078 | anls            |                 30 |
+| info_vqa | 0.506233 | anls            |                 30 |
 | chart_qa | 0.266667 | relaxed_overall |                 30 |
 | ai2d     | 0.1      | exact_match_mm  |                 30 |
-| websrc   | 0.355    | websrc_squad_f1 |                 30 |
+| websrc   | 0.487    | websrc_squad_f1 |                 30 |
 """

examples/evaluate_vision_full_benchmark.py

@@ -35,18 +35,17 @@
 print("Subsets scores:")
 print(results.subsets_scores.summary)
 """
-Using LLAMA-VISION-11B
 | subset                         |    score | score_name      |   num_of_instances |
 |:-------------------------------|---------:|:----------------|-------------------:|
-| ALL                            | 0.462355 | subsets_mean    |               4608 |
-| doc_vqa_default                | 0.7814   | anls            |                512 |
-| info_vqa_default               | 0.562389 | anls            |                512 |
-| chart_qa_default               | 0.197266 | relaxed_overall |                512 |
+| ALL                            | 0.482178 | subsets_mean    |               4608 |
+| doc_vqa_default                | 0.817023 | anls            |                512 |
+| info_vqa_default               | 0.530652 | anls            |                512 |
+| chart_qa_default               | 0.199219 | relaxed_overall |                512 |
 | ai2d_default                   | 0.126953 | exact_match_mm  |                512 |
-| websrc_default                 | 0.371    | websrc_squad_f1 |                512 |
-| doc_vqa_llama_vision_template  | 0.653235 | anls            |                512 |
-| info_vqa_llama_vision_template | 0.508014 | anls            |                512 |
-| chart_qa_llama_vision_template | 0.197266 | relaxed_overall |                512 |
-| ai2d_llama_vision_template     | 0.763672 | exact_match_mm  |                512 |
+| websrc_default                 | 0.465    | websrc_squad_f1 |                512 |
+| doc_vqa_llama_vision_template  | 0.715098 | anls            |                512 |
+| info_vqa_llama_vision_template | 0.514954 | anls            |                512 |
+| chart_qa_llama_vision_template | 0.201172 | relaxed_overall |                512 |
+| ai2d_llama_vision_template     | 0.769531 | exact_match_mm  |                512 |
 
 """