Add new restricted feedback mode (#1378)

Co-authored-by: Németh Zsolt <37755878+birka0@users.noreply.github.com>

Author: veluca93

cms/__init__.py

@@ -33,6 +33,7 @@
     "TOKEN_MODE_DISABLED", "TOKEN_MODE_FINITE", "TOKEN_MODE_INFINITE",
     "TOKEN_MODE_MIXED",
     "FEEDBACK_LEVEL_FULL", "FEEDBACK_LEVEL_RESTRICTED",
+    "FEEDBACK_LEVEL_OI_RESTRICTED",
     # log
     # Nothing intended for external use, no need to advertise anything.
     # conf
@@ -68,6 +69,9 @@
 # Restricted set of information (no killing signal, time or memory, testcases
 # can be omitted).
 FEEDBACK_LEVEL_RESTRICTED = "restricted"
+# Restricted set of information in accordance with the Contest Rules of
+# the International Olympiad of Informatics
+FEEDBACK_LEVEL_OI_RESTRICTED = "oi_restricted"
 
 
 from .conf import Address, ServiceCoord, ConfigError, async_config, config

cms/db/task.py

@@ -36,7 +36,7 @@
     Interval, Enum, BigInteger
 
 from cms import TOKEN_MODE_DISABLED, TOKEN_MODE_FINITE, TOKEN_MODE_INFINITE, \
-    FEEDBACK_LEVEL_FULL, FEEDBACK_LEVEL_RESTRICTED
+    FEEDBACK_LEVEL_FULL, FEEDBACK_LEVEL_RESTRICTED, FEEDBACK_LEVEL_OI_RESTRICTED
 from cmscommon.constants import \
     SCORE_MODE_MAX, SCORE_MODE_MAX_SUBTASK, SCORE_MODE_MAX_TOKENED_LAST
 from . import Codename, Filename, FilenameSchemaArray, Digest, Base, Contest
@@ -199,6 +199,7 @@ class Task(Base):
     # reverse engineer task data.
     feedback_level: str = Column(
         Enum(FEEDBACK_LEVEL_FULL, FEEDBACK_LEVEL_RESTRICTED,
+             FEEDBACK_LEVEL_OI_RESTRICTED,
              name="feedback_level"),
         nullable=False,
         default=FEEDBACK_LEVEL_RESTRICTED)

cms/grading/scoretypes/abc.py

@@ -274,9 +274,13 @@ class ScoreTypeGroup(ScoreTypeAlone):
             </thead>
             <tbody>
     {% for tc in st["testcases"] %}
-        {% if "outcome" in tc
-               and (feedback_level == FEEDBACK_LEVEL_FULL
-                    or tc["show_in_restricted_feedback"]) %}
+        {% set show_tc = "outcome" in tc
+               and ((feedback_level == FEEDBACK_LEVEL_FULL)
+               or (feedback_level == FEEDBACK_LEVEL_RESTRICTED
+               and tc["show_in_restricted_feedback"])
+               or (feedback_level == FEEDBACK_LEVEL_OI_RESTRICTED
+               and tc["show_in_oi_restricted_feedback"])) %}
+        {% if show_tc %}
             {% if tc["outcome"] == "Correct" %}
                 <tr class="correct">
             {% elif tc["outcome"] == "Not correct" %}
@@ -307,16 +311,18 @@ class ScoreTypeGroup(ScoreTypeAlone):
             {% endif %}
                 </tr>
         {% else %}
+            {% if feedback_level != FEEDBACK_LEVEL_OI_RESTRICTED %}
                 <tr class="undefined">
                     <td class="idx">{{ loop.index }}</td>
-            {% if feedback_level == FEEDBACK_LEVEL_FULL %}
+                {% if feedback_level == FEEDBACK_LEVEL_FULL %}
                     <td colspan="4">
-            {% else %}
+                {% else %}
                     <td colspan="2">
-            {% endif %}
+                {% endif %}
                         {% trans %}N/A{% endtrans %}
                     </td>
                 </tr>
+            {% endif %}
         {% endif %}
     {% endfor %}
             </tbody>
@@ -412,25 +418,33 @@ def compute_score(self, submission_result):
 
             testcases = []
             public_testcases = []
-            previous_tc_all_correct = True
+            # In "Restricted" feedback mode:
+            #   show until the first testcase with a lowest score
+            # In "OI Restricted" feedback mode:
+            #   show only the first testcase with a lowest score
+
+            tc_first_lowest_idx = None
+            tc_first_lowest_score = None
             for tc_idx in target:
+                tc_score = float(evaluations[tc_idx].outcome)
                 tc_outcome = self.get_public_outcome(
-                    float(evaluations[tc_idx].outcome), parameter)
+                    tc_score, parameter)
 
                 testcases.append({
                     "idx": tc_idx,
                     "outcome": tc_outcome,
                     "text": evaluations[tc_idx].text,
                     "time": evaluations[tc_idx].execution_time,
                     "memory": evaluations[tc_idx].execution_memory,
-                    "show_in_restricted_feedback": previous_tc_all_correct})
+                    "show_in_restricted_feedback": self.public_testcases[tc_idx],
+                    "show_in_oi_restricted_feedback": self.public_testcases[tc_idx]})
+
                 if self.public_testcases[tc_idx]:
                     public_testcases.append(testcases[-1])
-                    # Only block restricted feedback if this is the first
-                    # *public* non-correct testcase, otherwise we might be
-                    # leaking info on private testcases.
-                    if tc_outcome != "Correct":
-                        previous_tc_all_correct = False
+                    if tc_first_lowest_score is None or \
+                            tc_score < tc_first_lowest_score:
+                        tc_first_lowest_idx = tc_idx
+                        tc_first_lowest_score = tc_score
                 else:
                     public_testcases.append({"idx": tc_idx})
 
@@ -439,6 +453,15 @@ def compute_score(self, submission_result):
                 parameter)
             st_score = st_score_fraction * parameter[0]
 
+            if tc_first_lowest_idx is not None and st_score_fraction < 1.0:
+                for tc in testcases:
+                    if not self.public_testcases[tc["idx"]]:
+                        continue
+                    tc["show_in_restricted_feedback"] = (
+                        tc["idx"] <= tc_first_lowest_idx)
+                    tc["show_in_oi_restricted_feedback"] = (
+                        tc["idx"] == tc_first_lowest_idx)
+
             score += st_score
             subtasks.append({
                 "idx": st_idx + 1,

cms/server/admin/templates/task.html

@@ -137,6 +137,7 @@ <h2 id="title_task_configuration" class="toggling_on">Task configuration</h2>
         </td>
         <td>
           <select name="feedback_level">
+            <option value={{ FEEDBACK_LEVEL_OI_RESTRICTED }} {{ "selected" if task.feedback_level == FEEDBACK_LEVEL_OI_RESTRICTED else "" }}>OI Restricted</option>
             <option value={{ FEEDBACK_LEVEL_RESTRICTED }} {{ "selected" if task.feedback_level == FEEDBACK_LEVEL_RESTRICTED else "" }}>Restricted</option>
             <option value={{ FEEDBACK_LEVEL_FULL }} {{ "selected" if task.feedback_level == FEEDBACK_LEVEL_FULL else "" }}>Full</option>
           </select>

cms/server/jinja2_toolbox.py

@@ -30,7 +30,8 @@
 from jinja2.runtime import Context
 
 from cms import TOKEN_MODE_DISABLED, TOKEN_MODE_FINITE, TOKEN_MODE_INFINITE, \
-    TOKEN_MODE_MIXED, FEEDBACK_LEVEL_FULL, FEEDBACK_LEVEL_RESTRICTED
+    TOKEN_MODE_MIXED, FEEDBACK_LEVEL_FULL, FEEDBACK_LEVEL_RESTRICTED, \
+    FEEDBACK_LEVEL_OI_RESTRICTED
 from cms.db import SubmissionResult, UserTestResult
 from cms.db.task import Dataset
 from cms.grading import format_status_text
@@ -156,6 +157,7 @@ def instrument_generic_toolbox(env: Environment):
 
     env.globals["FEEDBACK_LEVEL_FULL"] = FEEDBACK_LEVEL_FULL
     env.globals["FEEDBACK_LEVEL_RESTRICTED"] = FEEDBACK_LEVEL_RESTRICTED
+    env.globals["FEEDBACK_LEVEL_OI_RESTRICTED"] = FEEDBACK_LEVEL_OI_RESTRICTED
 
     env.filters["all"] = all_
     env.filters["any"] = any_

cmscontrib/loaders/italy_yaml.py

@@ -33,7 +33,7 @@
 import yaml
 
 from cms import TOKEN_MODE_DISABLED, TOKEN_MODE_FINITE, TOKEN_MODE_INFINITE, \
-    FEEDBACK_LEVEL_FULL, FEEDBACK_LEVEL_RESTRICTED
+    FEEDBACK_LEVEL_FULL, FEEDBACK_LEVEL_RESTRICTED, FEEDBACK_LEVEL_OI_RESTRICTED
 from cms.db import Contest, User, Task, Statement, Attachment, Team, Dataset, \
     Manager, Testcase
 from cms.grading.languagemanager import LANGUAGES, HEADER_EXTS
@@ -465,6 +465,8 @@ def get_task(self, get_statement=True) -> Task | None:
             args["feedback_level"] = FEEDBACK_LEVEL_FULL
         elif conf.get("feedback_level", None) == FEEDBACK_LEVEL_RESTRICTED:
             args["feedback_level"] = FEEDBACK_LEVEL_RESTRICTED
+        elif conf.get("feedback_level", None) == FEEDBACK_LEVEL_OI_RESTRICTED:
+            args["feedback_level"] = FEEDBACK_LEVEL_OI_RESTRICTED
 
         if conf.get("score_mode", None) == SCORE_MODE_MAX:
             args["score_mode"] = SCORE_MODE_MAX

docs/Configuring a contest.rst

@@ -29,7 +29,23 @@ Each of these fields can be left unset to prevent the corresponding limitation f
 Feedback to contestants
 =======================
 
-Each testcase can be marked as public or private. After sending a submission, a contestant can always see its results on the public testcases: a brief passed / partial / not passed status for each testcase, and the partial score that is computable from the public testcases only. Note that input and output data are always hidden.
+Each testcase can be marked as public or private.
+
+On public testcases, the information that a contestant receives depends on
+the feedback mode configured for the task. More precisely:
+
+- if the feedback mode is "Full", a contestant can always see its results on
+  the public testcases: a brief passed / partial / not passed status for each
+  testcase, and the partial score that is computable from the public testcases
+  only. Note that input and output data are always hidden.
+
+- if the feedback mode is "Restricted", information is only shown about
+  testcases up to the first testcase that has the lowest score for a given
+  subtask (inclusive). 
+
+- if the feedback mode is "OI Restricted", information is only shown about
+  the first testcase that has the lowest score for a given subtask (inclusive).
+  This matches IOI contest rules.
 
 Tokens were introduced to provide contestants with limited access to the detailed results of their submissions on the private testcases as well. If a contestant uses a token on a submission, then they will be able to see its result on all testcases, and the global score.