Use isolate's cg-oom-killed to detect "memory limit exceeded" conditions (#1471)

* Add separate "memory limit exceeded" message

* make test suite check for memory limit exceeded

* fix "memory limit exceeded" behavior in php

PHP has a memory_limit setting that limits the amount of memory the
interpreter even tries to allocate; this limit is 128M by default.
Exceeding this causes PHP to exit with code 255 (and print a message on
stdout).

This commit changes the limit to infinity, and lets isolate's memory
limit handle it instead. This gives the correct feedback to contestants.

Author: prandla

cms/grading/Sandbox.py

@@ -188,6 +188,7 @@ class SandboxBase(metaclass=ABCMeta):
     EXIT_SIGNAL = 'signal'
     EXIT_TIMEOUT = 'timeout'
     EXIT_TIMEOUT_WALL = 'wall timeout'
+    EXIT_MEM_LIMIT = 'memory limit exceeded'
     EXIT_NONZERO_RETURN = 'nonzero return'
 
     def __init__(
@@ -1283,7 +1284,10 @@ def get_exit_status(self) -> str:
             else:
                 return self.EXIT_TIMEOUT
         elif 'SG' in status_list:
-            return self.EXIT_SIGNAL
+            if 'cg-oom-killed' in self.log:
+                return self.EXIT_MEM_LIMIT
+            else:
+                return self.EXIT_SIGNAL
         elif 'RE' in status_list:
             return self.EXIT_NONZERO_RETURN
         # OK status is not reported in the log file, it's implicit.

cms/grading/languages/php.py

@@ -59,4 +59,4 @@ def get_evaluation_commands(
             self, executable_filename, main=None, args=None):
         """See Language.get_evaluation_commands."""
         args = args if args is not None else []
-        return [["/usr/bin/php", executable_filename] + args]
+        return [["/usr/bin/php", "-d", "memory_limit=-1", executable_filename] + args]

cms/grading/steps/evaluation.py

@@ -67,15 +67,12 @@ def N_(message: str):
                     "for example. Note that in this case the CPU time "
                     "visible in the submission details might be much smaller "
                     "than the time limit.")),
+    HumanMessage("memorylimit",
+                 N_("Memory limit exceeded"),
+                 N_("Your submission used too much memory.")),
     HumanMessage("signal",
-                 N_("Execution killed (could be triggered by violating memory "
-                    "limits)"),
-                 N_("The evaluation was killed by a signal. "
-                    "Among other things, this might be caused by exceeding "
-                    "the memory limit. Note that if this is the reason, "
-                    "the memory usage visible in the submission details is "
-                    "the usage before the allocation that caused the "
-                    "signal.")),
+                 N_("Execution killed by signal"),
+                 N_("The evaluation was killed by a signal.")),
     HumanMessage("returncode",
                  N_("Execution failed because the return code was nonzero"),
                  N_("Your submission failed because it exited with a return "
@@ -244,6 +241,7 @@ def evaluation_step_after_run(
             Sandbox.EXIT_TIMEOUT,
             Sandbox.EXIT_TIMEOUT_WALL,
             Sandbox.EXIT_NONZERO_RETURN,
+            Sandbox.EXIT_MEM_LIMIT,
             Sandbox.EXIT_SIGNAL]:
         # Evaluation succeeded, and user program was interrupted for some error
         # condition. We report the success, the task type should decide how to
@@ -288,6 +286,8 @@ def human_evaluation_message(stats: StatsDict) -> list[str]:
     elif exit_status == Sandbox.EXIT_SANDBOX_ERROR:
         # Contestants won't see this, the submission will still be evaluating.
         return []
+    elif exit_status == Sandbox.EXIT_MEM_LIMIT:
+        return [EVALUATION_MESSAGES.get("memorylimit").message]
     elif exit_status == Sandbox.EXIT_NONZERO_RETURN:
         # Don't tell which code: would be too much information!
         return [EVALUATION_MESSAGES.get("returncode").message]

cmstestsuite/Test.py

@@ -136,6 +136,13 @@ def __init__(self):
             "Execution failed because the return code was nonzero")
 
 
+class CheckMemoryLimit(CheckAbstractEvaluationFailure):
+    def __init__(self):
+        CheckAbstractEvaluationFailure.__init__(
+            self, "memory limit exceeded",
+            "Memory limit exceeded")
+
+
 class CheckUserTest(ABC):
     @abstractmethod
     def check(self, *args, **kwargs):

cmstestsuite/Tests.py

@@ -37,7 +37,7 @@
 import cmstestsuite.tasks.outputonly_comparator as outputonly_comparator
 import cmstestsuite.tasks.twosteps as twosteps
 import cmstestsuite.tasks.twosteps_comparator as twosteps_comparator
-from cmstestsuite.Test import Test, CheckOverallScore, CheckCompilationFail, \
+from cmstestsuite.Test import CheckMemoryLimit, Test, CheckOverallScore, CheckCompilationFail, \
     CheckTimeout, CheckTimeoutWall, CheckNonzeroReturn, CheckUserTestEvaluated
 
 
@@ -257,12 +257,12 @@
          task=batch_stdio, filenames=['oom-static.%l'],
          languages=(LANG_C, LANG_CPP, LANG_CPP14,
                     LANG_CPP17, LANG_CPP20, LANG_PASCAL),
-         checks=[CheckOverallScore(0, 100)]),
+         checks=[CheckOverallScore(0, 100), CheckMemoryLimit()]),
 
     Test('oom-heap',
          task=batch_stdio, filenames=['oom-heap.%l'],
          languages=ALL_LANGUAGES,
-         checks=[CheckOverallScore(0, 100)]),
+         checks=[CheckOverallScore(0, 100), CheckMemoryLimit()]),
 
     # Tasks with graders.