Skip to content

Commit 31a7156

Browse files
author
Ziyang SONG
committed
Upload EasyR1 to RetrieveR1
1 parent 77b6dfa commit 31a7156

7 files changed

Lines changed: 170 additions & 9 deletions

File tree

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
set -x
2+
export VLLM_ATTENTION_BACKEND=XFORMERS
3+
export VLLM_USE_V1=0
4+
MODEL_PATH=/home/mcb/users/zsong15/qwen_model
5+
6+
# Use single quotes to protect the entire string
7+
SYSTEM_PROMPT='You have a question that requires multi-step reasoning and information retrieval. Follow these steps - 1) FIRST, think about the reasoning process as an internal monologue. This MUST BE enclosed within <think> </think> tags. 2) THEN, identify and extract relevant information needed to answer the question. This MUST BE enclosed within <retrieval> </retrieval> tags. 3) FINALLY, provide your final answer. This MUST BE enclosed within <answer> </answer> tags.'
8+
9+
python3 -m verl.trainer.main \
10+
config=examples/grpo_example.yaml \
11+
data.train_files=hiyouga/geometry3k@train \
12+
data.val_files=hiyouga/geometry3k@test \
13+
data.system_prompt=''"$SYSTEM_PROMPT"'' \
14+
worker.actor.model.model_path=${MODEL_PATH} \
15+
worker.rollout.enable_chunked_prefill=false \
16+
trainer.experiment_name=qwen2_5_retrieval_geo \
17+
worker.reward.compute_score=retrieve \
18+
trainer.n_gpus_per_node=4

examples/run_qwen2_5_vl_7b_geo.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ set -x
33
export VLLM_ATTENTION_BACKEND=XFORMERS
44
export VLLM_USE_V1=0
55

6-
MODEL_PATH=Qwen/Qwen2.5-VL-7B-Instruct # replace it with your local file path
6+
MODEL_PATH=/home/mcb/users/zsong15/qwen_model # replace it with your local file path
77

88
SYSTEM_PROMPT="""You FIRST think about the reasoning process as an internal monologue and then provide the final answer.
99
The reasoning process MUST BE enclosed within <think> </think> tags. The final answer MUST BE put in \boxed{}."""
@@ -16,4 +16,4 @@ python3 -m verl.trainer.main \
1616
worker.actor.model.model_path=${MODEL_PATH} \
1717
worker.rollout.enable_chunked_prefill=false \
1818
trainer.experiment_name=qwen2_5_vl_7b_geo \
19-
trainer.n_gpus_per_node=8
19+
trainer.n_gpus_per_node=4

verl/trainer/config.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ class TrainerConfig:
6868
experiment_name: str = "demo"
6969
logger: Tuple[str] = ("console", "wandb")
7070
nnodes: int = 1
71-
n_gpus_per_node: int = 8
71+
n_gpus_per_node: int = 4
7272
critic_warmup: int = 0
7373
val_freq: int = -1
7474
val_before_train: bool = True

verl/utils/reward_score/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
from .math import math_compute_score
1717
from .r1v import r1v_compute_score
18+
from .retrieve import retrieve_compute_score
1819

1920

20-
__all__ = ["math_compute_score", "r1v_compute_score"]
21+
__all__ = ["math_compute_score", "r1v_compute_score", "retrieve_compute_score"]
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import re
16+
17+
18+
def retrieve_format_reward(predict_str: str) -> float:
19+
"""
20+
Check if the prediction has the required format structure:
21+
<think>...</think> followed by <retrieval>...</retrieval> and <answer>...</answer>
22+
23+
Args:
24+
predict_str: The prediction string to evaluate
25+
26+
Returns:
27+
1.0 if the format is correct, 0.0 otherwise
28+
"""
29+
# Define the pattern for the required format
30+
pattern = re.compile(r"<think>.*</think>.*<retrieval>.*</retrieval>.*<answer>.*</answer>", re.DOTALL)
31+
format_match = re.fullmatch(pattern, predict_str)
32+
return 1.0 if format_match else 0.0
33+
34+
35+
def extract_answer(predict_str: str) -> str:
36+
"""
37+
Extract the answer from the prediction string, looking for content within <answer> tags.
38+
39+
Args:
40+
predict_str: The prediction string to evaluate
41+
42+
Returns:
43+
The extracted answer or an empty string if not found
44+
"""
45+
answer_match = re.search(r"<answer>(.*?)</answer>", predict_str, re.DOTALL)
46+
if answer_match:
47+
return answer_match.group(1).strip()
48+
return ""
49+
50+
51+
def retrieve_accuracy_reward(predict_str: str, ground_truth: str) -> float:
52+
"""
53+
Check if the answer (within <answer> tags) matches the ground truth.
54+
55+
Args:
56+
predict_str: The prediction string to evaluate
57+
ground_truth: The ground truth answer
58+
59+
Returns:
60+
1.0 if the answer matches the ground truth, 0.0 otherwise
61+
"""
62+
answer = extract_answer(predict_str)
63+
64+
# Normalize both answers for comparison (lowercase, strip spaces)
65+
answer_norm = answer.lower().strip()
66+
ground_truth_norm = ground_truth.lower().strip()
67+
68+
# Check if the normalized answer matches the ground truth
69+
return 1.0 if answer_norm == ground_truth_norm else 0.0
70+
71+
72+
def retrieval_spans_in_context(predict_str: str, context: str) -> float:
73+
"""
74+
Check if all retrieval spans in the prediction are found in the context.
75+
76+
Args:
77+
predict_str: The prediction string to evaluate
78+
context: The context string to search in
79+
80+
Returns:
81+
1.0 if all retrieval spans are in the context, 0.0 otherwise
82+
"""
83+
# Extract all retrieval spans
84+
spans = re.findall(r"<retrieval>(.*?)</retrieval>", predict_str, re.DOTALL)
85+
86+
# If no retrieval spans were found, return 0.0
87+
if not spans:
88+
return 0.0
89+
90+
# Check if all spans are in the context
91+
spans_found = 0
92+
for span in spans:
93+
# Clean up the span by removing extra whitespace
94+
cleaned_span = re.sub(r'\s+', ' ', span).strip()
95+
if not cleaned_span:
96+
continue
97+
if cleaned_span in context:
98+
spans_found += 1
99+
100+
# Return a score based on the proportion of spans found
101+
if not spans:
102+
return 0.0
103+
return min(1.0, spans_found / len([s for s in spans if s.strip()]))
104+
105+
106+
def retrieve_compute_score(predict_str: str, ground_truth: str, context: str) -> float:
107+
"""
108+
Compute the combined score for retrieval-based QA evaluation.
109+
110+
Args:
111+
predict_str: The prediction string to evaluate
112+
ground_truth: The ground truth answer
113+
context: The context from which retrieval should happen
114+
115+
Returns:
116+
The combined reward score between 0.0 and 1.0
117+
"""
118+
# Calculate individual reward components
119+
format_score = retrieve_format_reward(predict_str)
120+
accuracy_score = retrieve_accuracy_reward(predict_str, ground_truth)
121+
retrieval_score = retrieval_spans_in_context(predict_str, context)
122+
123+
# Combine scores with weights (similar to math.py's weighting)
124+
return 0.7 * accuracy_score + 0.1 * format_score + 0.2 * retrieval_score

verl/workers/reward/config.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,19 @@
1515
Reward config
1616
"""
1717

18-
from dataclasses import dataclass
18+
from dataclasses import dataclass, field
19+
from typing import Any, List
1920

2021

2122
@dataclass
2223
class RewardConfig:
2324
reward_type: str = "function"
2425
compute_score: str = "math"
26+
27+
# Define valid options as a class variable
28+
valid_compute_scores: List[str] = field(default_factory=lambda: ["math", "r1v", "retrieve"], repr=False)
29+
30+
def __post_init__(self):
31+
# Validate compute_score
32+
if self.compute_score not in self.valid_compute_scores:
33+
raise ValueError(f"compute_score must be one of {self.valid_compute_scores}, got {self.compute_score}")

verl/workers/reward/custom.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
from transformers import PreTrainedTokenizer
1818

1919
from ...protocol import DataProto
20-
from ...utils.reward_score import math_compute_score, r1v_compute_score
20+
from ...utils.reward_score import math_compute_score, r1v_compute_score, retrieve_compute_score
2121

2222

2323
class CustomRewardManager:
@@ -28,6 +28,8 @@ def __init__(self, tokenizer: PreTrainedTokenizer, num_examine: int, compute_sco
2828
self.compute_score = math_compute_score
2929
elif compute_score == "r1v":
3030
self.compute_score = r1v_compute_score
31+
elif compute_score == "retrieve":
32+
self.compute_score = retrieve_compute_score
3133
else:
3234
raise NotImplementedError()
3335

@@ -53,8 +55,15 @@ def __call__(self, data: DataProto) -> torch.Tensor:
5355
response_str = self.tokenizer.decode(valid_response_ids, skip_special_tokens=True)
5456

5557
ground_truth = data_item.non_tensor_batch["ground_truth"]
56-
57-
score = self.compute_score(response_str, ground_truth)
58+
59+
# Check if we're using retrieve compute score which needs context
60+
if hasattr(self.compute_score, "__code__") and "context" in self.compute_score.__code__.co_varnames:
61+
# Get context from non_tensor_batch if available
62+
context = data_item.non_tensor_batch.get("context", "")
63+
score = self.compute_score(response_str, ground_truth, context)
64+
else:
65+
score = self.compute_score(response_str, ground_truth)
66+
5867
reward_tensor[i, valid_response_length - 1] = score
5968

6069
if already_print < self.num_examine:
@@ -64,4 +73,4 @@ def __call__(self, data: DataProto) -> torch.Tensor:
6473
print("[ground_truth]", ground_truth)
6574
print("[score]", score)
6675

67-
return reward_tensor
76+
return reward_tensor

0 commit comments

Comments
 (0)