eval generator #2
Conversation
| info.pr_number = url.split("/pull/")[-1] | ||
|
|
||
| if info.pr_number: | ||
| if not info.pr_number.isdigit(): |
| old_dirs = [ | ||
| d for d in examples_dir.iterdir() if d.is_dir() and d.name.endswith(".old") | ||
| ] | ||
|
|
||
| for old_dir in old_dirs: | ||
| # Construct the corresponding .new directory name | ||
| base_name = old_dir.name[:-4] # remove .old | ||
| base_name = old_dir.name[:-4] |
| @@ -366,7 +362,22 @@ async def run( | |||
| log_stream, | |||
| local_worktrees, | |||
| llm_fakes, | |||
| target_dir: str = "", | |||
| target_basename: str = "", | |||
| dont_create_evals=False, | |||
There was a problem hiding this comment.
might as well type the rest too then
| @@ -378,6 +389,10 @@ async def run( | |||
| start_point = ( | |||
| await subprocess_run(["git", "rev-parse", "HEAD"], check=True) | |||
| ).strip() | |||
|
|
|||
| # Define target_dir_rel_path | |||
| target_dir_rel_path = None | |||
There was a problem hiding this comment.
Hmm, is this set somewhere to something else than None? Should it be what's in 397?
| dont_create_evals, | ||
| target_dir=target_dir, | ||
| target_dir_rel_path=target_dir_rel_path, | ||
| target_basename=target_basename, |
There was a problem hiding this comment.
These don't seem to be used as far as I can tell
| @@ -0,0 +1,301 @@ | |||
| import tempfile | |||
There was a problem hiding this comment.
can you remove the AI provided comments here that don't add value?
There was a problem hiding this comment.
Forgot to purge the test files.
| if base: | ||
| return f"// Original content of {file_path}\n// This is a placeholder for demonstration purposes" | ||
| else: | ||
| return f"// Modified content of {file_path}\n// This is a placeholder for demonstration purposes" | ||
| except Exception as e: | ||
| print(f"Approach 3 failed: {e}") | ||
|
|
||
| # If all approaches fail, return a minimal placeholder |
There was a problem hiding this comment.
Probably should have caught this before because I see it's not part of this diff, but this "Approach 3" doesn't seem to make a lot of sense -- I don't think you'd ever want placeholders for content, I think you'd just want to fail loudly. You'd also never need to put this particular code in a try/except
There was a problem hiding this comment.
yeah i'll drop this
This pull request introduces the automatic evaluation generation feature to the ai_migrate tool, enhancing its capabilities by automatically creating evaluation test cases from successful migrations.