Skip to content

Commit 6908a55

Browse files
authored
add: github CI/CD tasks (#77)
1 parent d9b563f commit 6908a55

File tree

28 files changed

+2789
-72
lines changed

28 files changed

+2789
-72
lines changed

docs/setup/github-env-setup.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ If you want to benchmark additional repositories:
7070

7171
1. Export the desired repository state:
7272
```bash
73-
python -m src.mcp_services.github.repo_exporter --repo owner/name --out ./github_state/{your_repo_name}
73+
python -m src.mcp_services.github.repo_exporter --source_repo_url owner/name --max-issues 20 --max-pulls 5
7474
```
7575
2. Open `src/mcp_services/github/state_manager.py` and add a new entry to `self.initial_state_mapping` pointing to the exported folder.
7676

src/mcp_services/filesystem/filesystem_login_helper.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ def __init__(self, state_path: Optional[Path] = None):
3333
"""
3434
super().__init__()
3535
self.state_path = (
36-
state_path or Path.home() / ".mcpbench" / "filesystem_state.json"
36+
state_path or Path.home() / ".mcpmark" / "filesystem_state.json"
3737
)
3838
logger.info("Initialized FilesystemLoginHelper (no auth required)")
3939

src/mcp_services/github/github_login_helper.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ def __init__(
3535
state_path: Path to save authentication state
3636
"""
3737
self.token = token
38-
self.state_path = state_path or Path.home() / ".mcpbench" / "github_auth.json"
38+
self.state_path = state_path or Path.home() / ".mcpmark" / "github_auth.json"
3939

4040
# Ensure state directory exists
4141
self.state_path.parent.mkdir(parents=True, exist_ok=True)

src/mcp_services/github/github_state_manager.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ def __init__(
2727
self,
2828
github_token: Union[str, List[str]],
2929
# Name of the evaluation organisation / user where temporary test repositories are created
30-
eval_org: str = "mcpleague-eval",
30+
eval_org: str = "mcpmark-eval",
3131
# Local directory that stores *exported* repository templates (produced by repo_exporter.py)
3232
templates_root: str = "./github_state",
3333
):
@@ -106,7 +106,9 @@ def __init__(
106106

107107
# Initial state mapping - categories to initial state repositories
108108
self.initial_state_mapping = {
109-
"mixeval": "JinjieNi-MixEval",
109+
"build_your_own_x": "codecrafters-io-build-your-own-x",
110+
"missing-semester": "missing-semester-missing-semester",
111+
"mcpmark-cicd": "zjwu0522-mcpmark-cicd",
110112
"harmony": "openai-harmony",
111113
"claude-code": "anthropics-claude-code",
112114
"easyr1": "hiyouga-EasyR1",
@@ -412,7 +414,10 @@ def _create_initial_state(self, task: "BaseTask") -> Optional[InitialStateInfo]:
412414
logger.info(f"Importing repository template from {template_dir} …")
413415
owner = self.eval_org if self.eval_org else self._get_authenticated_user()
414416

415-
repo_url = self._import_template_repo(template_dir, owner, True)
417+
if "mcpmark-cicd" in template_name:
418+
repo_url = self._import_template_repo(template_dir, owner, False)
419+
else:
420+
repo_url = self._import_template_repo(template_dir, owner, True)
416421

417422
# Record for cleanup later
418423
repo_name = repo_url.rstrip("/").split("/")[-1]
@@ -621,11 +626,6 @@ def get_service_config_for_agent(self) -> dict:
621626
Dictionary containing configuration needed by the agent/MCP server
622627
"""
623628
service_config = {}
624-
625-
# Rotate to next token for this agent execution
626-
if self.token_pool.pool_size > 1:
627-
self._rotate_token()
628-
logger.info(f"Rotating token for new agent execution")
629629

630630
# Add GitHub token if available
631631
if self.github_token:

src/mcp_services/github/github_task_manager.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ def _format_task_instruction(self, base_instruction: str) -> str:
9595
"""Format task instruction with GitHub-specific additions."""
9696
return (
9797
base_instruction
98-
+ "\n\nNote: Use GitHub tools to complete this task. Work systematically and verify your actions."
98+
+ "\n\nNote: Based on your understanding, solve the task all at once by yourself, don't ask for my opinions on anything."
9999
)
100100

101101
def get_task_instruction(self, task: GitHubTask) -> str:

src/mcp_services/github/repo_exporter.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -437,13 +437,13 @@ def _fetch_review_comments(number: int) -> list[dict]:
437437
parser.add_argument(
438438
"--max-issues",
439439
type=int,
440-
default=50,
440+
default=20,
441441
help="Export only the latest N issues (optional)",
442442
)
443443
parser.add_argument(
444444
"--max-pulls",
445445
type=int,
446-
default=20,
446+
default=5,
447447
help="Export only the latest N pull requests (optional)",
448448
)
449449
args = parser.parse_args()

src/mcp_services/github/repo_importer.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,7 @@ def import_repository(
494494
parser.add_argument(
495495
"--target-owner",
496496
"-o",
497-
default="mcpleague-eval",
497+
default="mcpmark-eval",
498498
help="User or organisation that will own the new repository",
499499
)
500500
args = parser.parse_args()
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Find out when the entries in the Voxel Engine section were first created by Daniel Stefanovic. After finding this information, create an ANSWER.md file in the repository with the content being the date in [YYYY]-[MM]-[DD] format (e.g., 2000-06-02).
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import sys
2+
import os
3+
import requests
4+
from typing import Dict, Optional, Tuple
5+
import base64
6+
from dotenv import load_dotenv
7+
8+
9+
def _get_github_api(
10+
endpoint: str, headers: Dict[str, str], org: str, repo: str = "build-your-own-x"
11+
) -> Tuple[bool, Optional[Dict]]:
12+
"""Make a GET request to GitHub API and return (success, response)."""
13+
url = f"https://api.github.com/repos/{org}/{repo}/{endpoint}"
14+
try:
15+
response = requests.get(url, headers=headers)
16+
if response.status_code == 200:
17+
return True, response.json()
18+
elif response.status_code == 404:
19+
return False, None
20+
else:
21+
print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
22+
return False, None
23+
except Exception as e:
24+
print(f"Exception for {endpoint}: {e}", file=sys.stderr)
25+
return False, None
26+
27+
28+
def _get_file_content(
29+
file_path: str,
30+
headers: Dict[str, str],
31+
org: str,
32+
repo: str = "build-your-own-x",
33+
ref: str = "master",
34+
) -> Optional[str]:
35+
"""Get the content of a file from the repository."""
36+
success, result = _get_github_api(
37+
f"contents/{file_path}?ref={ref}", headers, org, repo
38+
)
39+
if not success or not result:
40+
return None
41+
42+
try:
43+
content = base64.b64decode(result.get("content", "")).decode("utf-8")
44+
return content
45+
except Exception as e:
46+
print(f"Content decode error for {file_path}: {e}", file=sys.stderr)
47+
return None
48+
49+
50+
def verify_task() -> bool:
51+
"""Verify the find commit data task for Voxel Engine entries."""
52+
# Load environment variables from .mcp_env
53+
load_dotenv(".mcp_env")
54+
55+
# Get GitHub token and org
56+
github_token = os.environ.get("MCP_GITHUB_TOKEN")
57+
github_org = os.environ.get("GITHUB_EVAL_ORG")
58+
59+
if not github_token:
60+
print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
61+
return False
62+
63+
if not github_org:
64+
print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
65+
return False
66+
67+
headers = {
68+
"Authorization": f"Bearer {github_token}",
69+
"Accept": "application/vnd.github.v3+json",
70+
}
71+
72+
print("Verifying Voxel Engine commit date task...")
73+
74+
# 1. Check if ANSWER.md exists in the repository
75+
print("1. Checking if ANSWER.md exists...")
76+
content = _get_file_content("ANSWER.md", headers, github_org)
77+
if not content:
78+
print("Error: ANSWER.md not found in repository", file=sys.stderr)
79+
return False
80+
print("✓ ANSWER.md found")
81+
82+
# 2. Check the content format
83+
print("2. Checking content format...")
84+
content = content.strip()
85+
86+
# The expected date when Daniel Stefanovic added Voxel Engine entries
87+
# Based on historical records, this should be 2018-07-07
88+
expected_date = "2018-07-07"
89+
90+
# Check if the content matches the expected date format (YYYY-MM-DD)
91+
import re
92+
date_pattern = r'^\d{4}-\d{2}-\d{2}$'
93+
if not re.match(date_pattern, content):
94+
print(f"Error: Invalid date format. Expected YYYY-MM-DD, got: {content}", file=sys.stderr)
95+
return False
96+
print("✓ Date format is correct")
97+
98+
# 3. Verify the date is correct
99+
print("3. Verifying the date...")
100+
if content != expected_date:
101+
print(f"Error: Incorrect date. Expected {expected_date}, got: {content}", file=sys.stderr)
102+
return False
103+
print(f"✓ Date is correct: {content}")
104+
105+
# 4. Verify README.md contains Voxel Engine section
106+
print("4. Checking if README.md contains Voxel Engine section...")
107+
readme_content = _get_file_content("README.md", headers, github_org)
108+
if not readme_content:
109+
print("Error: README.md not found in repository", file=sys.stderr)
110+
return False
111+
112+
if "Voxel Engine" not in readme_content:
113+
print("Error: Voxel Engine section not found in README.md", file=sys.stderr)
114+
return False
115+
116+
# Check for specific Voxel Engine entries
117+
voxel_entries = [
118+
"Let's Make a Voxel Engine",
119+
"Java Voxel Engine Tutorial"
120+
]
121+
122+
for entry in voxel_entries:
123+
if entry not in readme_content:
124+
print(f"Warning: Voxel Engine entry '{entry}' not found in README.md", file=sys.stderr)
125+
126+
print("✓ Voxel Engine section found in README.md")
127+
128+
print("\n✅ All verification checks passed!")
129+
print("Task completed successfully:")
130+
print(f" - ANSWER.md created with date: {content}")
131+
print(" - Date format is correct (YYYY-MM-DD)")
132+
print(" - Date matches expected creation date for Voxel Engine entries by Daniel Stefanovic")
133+
print(" - Voxel Engine section exists in README.md")
134+
135+
return True
136+
137+
138+
if __name__ == "__main__":
139+
success = verify_task()
140+
sys.exit(0 if success else 1)
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Find out the specific commit SHA of adding an entry about "RAG for Document Search". After finding this information, create an ANSWER.md file in the repository with the content being the commit SHA (e.g., 023dfa35694db2709057488ad338afdbc89fb226).
2+
3+
Hint: It should be in an "AI model" section I think.

0 commit comments

Comments
 (0)