adapt for mac

Author: zhlmmc

human_eval/evaluate_functional_correctness.py

@@ -24,5 +24,5 @@ def entry_point(
 def main():
     fire.Fire(entry_point)
 
-
-sys.exit(main())
+if __name__ == "__main__": 
+    sys.exit(main())

human_eval/evaluation.py

@@ -42,6 +42,7 @@ def evaluate_functional_correctness(
     n_workers: int = 4,
     timeout: float = 3.0,
     problem_file: str = HUMAN_EVAL,
+    ignore_incomplete: bool = True
 ):
     """
     Evaluates the functional correctness of generated samples, and writes
@@ -68,7 +69,8 @@ def evaluate_functional_correctness(
             completion_id[task_id] += 1
             n_samples += 1
 
-        assert len(completion_id) == len(problems), "Some problems are not attempted."
+        if not ignore_incomplete:
+            assert len(completion_id) == len(problems), "Some problems are not attempted."
 
         print("Running test suites...")
         for future in tqdm.tqdm(as_completed(futures), total=len(futures)):

human_eval/execution.py

@@ -4,7 +4,7 @@
 import faulthandler
 import io
 import os
-import multiprocessing
+import multiprocess
 import platform
 import signal
 import tempfile
@@ -36,11 +36,13 @@ def unsafe_execute():
 
             # Construct the check program and run it.
             check_program = (
-                problem["prompt"] + completion + "\n" +
+                #problem["prompt"] + completion + "\n" +
+                completion + "\n" +
                 problem["test"] + "\n" +
                 f"check({problem['entry_point']})"
             )
 
+            print(check_program)
             try:
                 exec_globals = {}
                 with swallow_io():
@@ -55,7 +57,7 @@ def unsafe_execute():
 # information on how OpenAI sandboxes its code, see the accompanying paper.
 # Once you have read this disclaimer and taken appropriate precautions, 
 # uncomment the following line and proceed at your own risk:
-#                         exec(check_program, exec_globals)
+                        exec(check_program, exec_globals)
                 result.append("passed")
             except TimeoutException:
                 result.append("timed out")
@@ -67,10 +69,10 @@ def unsafe_execute():
             os.rmdir = rmdir
             os.chdir = chdir
 
-    manager = multiprocessing.Manager()
+    manager = multiprocess.Manager()
     result = manager.list()
 
-    p = multiprocessing.Process(target=unsafe_execute)
+    p = multiprocess.Process(target=unsafe_execute)
     p.start()
     p.join(timeout=timeout + 1)
     if p.is_alive():