Skip to content

Commit b2a2074

Browse files
ljvmiranda921ljvmiranda921
committed
fixup
1 parent a8d37dc commit b2a2074

1 file changed

Lines changed: 252 additions & 0 deletions

File tree

‎community_tasks/filipino_evals.py‎

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
# ruff: noqa: F405, F403, F401
2424
from collections import OrderedDict
2525
from functools import partial
26+
from typing import Any
2627

2728
from langcodes import Language as LangCodeLanguage
2829
from langcodes import standardize_tag
@@ -515,6 +516,253 @@ def create_task(language: Language, formulation):
515516
for formulation in [MCFFormulation(), HybridFormulation()]
516517
]
517518

519+
520+
def prepare_stingray_correctness(line: dict[str, str]) -> dict[str, Any]:
521+
# lang2 is Tagalog
522+
word = line["word"]
523+
sentence = line["lang2_sentence"]
524+
question = f"Is the usage of {word} in this sentence correct? \n{sentence}"
525+
choices = ["Yes", "No"]
526+
gold_idx = choices.index(line["usage_correctness_lang2_answer"])
527+
return {"question": question, "choices": choices, "gold_idx": gold_idx}
528+
529+
530+
def prepare_stingray_semantic_appropriateness(line: dict[str, str]) -> dict[str, Any]:
531+
lang1 = line["lang1_sentence"]
532+
lang2 = line["lang2_sentence"]
533+
question = "Which sentence is more semantically appropriate?"
534+
choices = [lang1, lang2, "Both"]
535+
choice_letters = ["A", "B", "C"]
536+
gold_idx = choice_letters.index(line["semantic_appropriate_answer"])
537+
return {"question": question, "choices": choices, "gold_idx": gold_idx}
538+
539+
540+
FILIPINO_STINGRAY_CORRECTNESS_TASKS = [
541+
LightevalTaskConfig(
542+
name=f"stingraybench_correctness_tgl_{formulation.name.lower()}",
543+
prompt_function=get_mcq_prompt_function(
544+
Language.ENGLISH, # the orig instruction is in English, so we replicate it.
545+
adapter=prepare_stingray_correctness,
546+
formulation=formulation,
547+
),
548+
suite=("filbench",),
549+
hf_subset="id_tl",
550+
hf_repo="StingrayBench/StingrayBench",
551+
metric=get_metrics_for_formulation(
552+
formulation,
553+
[
554+
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
555+
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
556+
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
557+
],
558+
),
559+
hf_avail_splits=["test"],
560+
evaluation_splits=["test"],
561+
few_shots_split="test",
562+
few_shots_select="random",
563+
generation_size=-1,
564+
trust_dataset=True,
565+
version=0,
566+
)
567+
for formulation in [MCFFormulation(), HybridFormulation()]
568+
]
569+
570+
FILIPINO_STINGRAY_SEMANTIC_TAKS = [
571+
LightevalTaskConfig(
572+
name=f"stingraybench_semantic_appropriateness_tgl_{formulation.name.lower()}",
573+
prompt_function=get_mcq_prompt_function(
574+
Language.ENGLISH, # the orig instruction is in English, so we replicate it.
575+
adapter=prepare_stingray_semantic_appropriateness,
576+
formulation=formulation,
577+
),
578+
suite=("filbench",),
579+
hf_subset="id_tl",
580+
hf_repo="StingrayBench/StingrayBench",
581+
metric=get_metrics_for_formulation(
582+
formulation,
583+
[
584+
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
585+
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
586+
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
587+
],
588+
),
589+
hf_avail_splits=["test"],
590+
evaluation_splits=["test"],
591+
few_shots_split="test",
592+
few_shots_select="random",
593+
generation_size=-1,
594+
trust_dataset=True,
595+
version=0,
596+
)
597+
for formulation in [MCFFormulation(), HybridFormulation()]
598+
]
599+
600+
FILIPINO_STINGRAY_TASKS = FILIPINO_STINGRAY_SEMANTIC_TAKS + FILIPINO_STINGRAY_CORRECTNESS_TASKS
601+
602+
# Tatoeba
603+
# We follow the original translation direction from tatoeba
604+
lang_dict = {
605+
"ceb": {
606+
"subset": "ceb-eng",
607+
"source_language": Language.CEBUANO,
608+
"target_language": Language.ENGLISH,
609+
},
610+
"tgl": {
611+
"subset": "eng-tgl",
612+
"source_language": Language.ENGLISH,
613+
"target_language": Language.TAGALOG,
614+
},
615+
}
616+
617+
FILIPINO_TATOEBA_TASKS = [
618+
LightevalTaskConfig(
619+
name=f"tatoeba_{language}",
620+
prompt_function=get_translation_prompt_function(
621+
source_language=meta.get("source_language"),
622+
target_language=meta.get("target_language"),
623+
adapter=lambda line: {
624+
"source_text": line["sourceString"],
625+
"target_text": line["targetString"],
626+
},
627+
formulation=CFFormulation(),
628+
),
629+
suite=("filbench",),
630+
hf_repo="Helsinki-NLP/tatoeba_mt",
631+
hf_subset=meta.get("subset"),
632+
metric=[
633+
Metrics.rougeL,
634+
Metrics.bleu,
635+
Metrics.bleurt,
636+
Metrics.chrf,
637+
Metrics.ter,
638+
],
639+
hf_avail_splits=["test"],
640+
evaluation_splits=["test"],
641+
trust_dataset=True,
642+
generation_size=64,
643+
)
644+
for language, meta in lang_dict.items()
645+
]
646+
647+
# TICO-19
648+
FILIPINO_TICO19_TASKS = [
649+
LightevalTaskConfig(
650+
name="tico19_tgl",
651+
prompt_function=get_translation_prompt_function(
652+
source_language=Language.ENGLISH,
653+
target_language=Language.TAGALOG,
654+
adapter=lambda line: {
655+
"source_text": line["sourceString"],
656+
"target_text": line["targetString"],
657+
},
658+
formulation=CFFormulation(),
659+
),
660+
suite=("filbench",),
661+
hf_repo="gmnlp/tico19",
662+
hf_subset="en-tl",
663+
metric=[
664+
Metrics.rougeL,
665+
Metrics.bleu,
666+
Metrics.bleurt,
667+
Metrics.chrf,
668+
Metrics.ter,
669+
],
670+
hf_avail_splits=["test", "validation"],
671+
evaluation_splits=["validation"],
672+
few_shots_split=["validation"],
673+
few_shots_select="random",
674+
trust_dataset=True,
675+
generation_size=64,
676+
)
677+
]
678+
679+
# TLUnified-NER
680+
tlunified_ner_choices = ["PERSON", "ORGANIZATION", "LOCATION"]
681+
tlunified_ner_answer_idx = ["A", "B", "C"]
682+
683+
FILIPINO_TLUNIFIED_NER_TASK = [
684+
LightevalTaskConfig(
685+
name=f"tlunifiedner_tgl_{formulation.name.lower()}",
686+
hf_subset="instruction",
687+
prompt_function=get_mcq_prompt_function(
688+
Language.TAGALOG,
689+
lambda line: {
690+
"question": f"Ano ang named-entity ng salitang '{line['entity']}' sa pangungusap na ito: {line['text']}",
691+
"choices": tlunified_ner_choices,
692+
"gold_idx": tlunified_ner_answer_idx.index(line["answer"]),
693+
},
694+
formulation=formulation,
695+
),
696+
hf_repo="ljvmiranda921/tlunified-ner",
697+
hf_avail_splits=["test"],
698+
evaluation_splits=["test"],
699+
few_shots_split="test",
700+
few_shots_select="random",
701+
suite=["filbench"],
702+
generation_size=-1,
703+
trust_dataset=True,
704+
metric=get_metrics_for_formulation(
705+
formulation,
706+
[
707+
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
708+
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
709+
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
710+
],
711+
),
712+
version=0,
713+
)
714+
for formulation in [MCFFormulation(), HybridFormulation()]
715+
]
716+
717+
# Universal NER
718+
universalner_choices = ["PERSON", "ORGANIZATION", "LOCATION"]
719+
universalner_answer_idx = ["A", "B", "C"]
720+
721+
722+
def create_task(language: Language, formulation):
723+
if language == Language.CEBUANO:
724+
question = "Unsa ang ginganlan nga named-entity sa pulong '{entity}' niini nga sentence: {text}"
725+
if language == Language.TAGALOG:
726+
question = "Ano ang named-entity ng salitang '{entity}' sa pangungusap na ito: {text}"
727+
728+
return LightevalTaskConfig(
729+
name=f"universalner_{language.value}_{formulation.name.lower()}",
730+
hf_subset=language.value,
731+
prompt_function=get_mcq_prompt_function(
732+
language,
733+
lambda line: {
734+
"question": question.format(entity=line["entity"], text=line["text"]),
735+
"choices": universalner_choices,
736+
"gold_idx": universalner_answer_idx.index(line["answer"]),
737+
},
738+
formulation=formulation,
739+
),
740+
hf_repo="UD-Filipino/universalner-instruction",
741+
hf_avail_splits=["test"],
742+
evaluation_splits=["test"],
743+
few_shots_split="test",
744+
few_shots_select="random",
745+
suite=["filbench"],
746+
generation_size=-1,
747+
trust_dataset=True,
748+
metric=get_metrics_for_formulation(
749+
formulation,
750+
[
751+
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
752+
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
753+
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
754+
],
755+
),
756+
version=0,
757+
)
758+
759+
760+
FILIPINO_UNIVERSALNER_TASKS = [
761+
create_task(language, formulation)
762+
for language in [Language.CEBUANO, Language.TAGALOG]
763+
for formulation in [MCFFormulation(), HybridFormulation()]
764+
]
765+
518766
TASKS_TABLE: list[LightevalTaskConfig] = (
519767
FILIPINO_BALITA_TASKS
520768
+ FILIPINO_BELEBELE_TASKS
@@ -528,4 +776,8 @@ def create_task(language: Language, formulation):
528776
+ FILIPINO_NEWSPH_NLI_TASKS
529777
+ FILIPINO_NTREX_TASK
530778
+ FILIPINO_SIB_TASKS
779+
+ FILIPINO_STINGRAY_TASKS
780+
+ FILIPINO_TATOEBA_TASKS
781+
+ FILIPINO_TICO19_TASKS
782+
+ FILIPINO_TLUNIFIED_NER_TASK
531783
)

0 commit comments

Comments
 (0)