Implement 'type' field in problem.yaml from 2023-07-draft spec (#421)

* Implement 'type' field in problem.yaml from 2023-07-draft spec

* [problem] Move determining fields interactive, multipass, and custom_output to constructor of ProblemSettings

* Rename variables `multipass` to `multi_pass`

* [skel] Read type instead of validation

* [problem] Mark TODOs for #424 (support singular output_validator/)

* [skel] Remove support for `bt new_problem --type default` in favour of `pass-fail`

* [problem] If we don't find output validator paths, abort when problem type is interactive and/or multi-pass

Author: mpsijm

bin/config.py

@@ -96,7 +96,7 @@
 grep -Ev '^(h|jobs|time|verbose)$' | sed "s/^/'/;s/$/',/" | tr '\n' ' ' | sed 's/^/ARGS_LIST: Final = [/;s/, $/]\n/'
 """
 # fmt: off
-ARGS_LIST: Final = ['1', 'add', 'all', 'answer', 'api', 'author', 'check_deterministic', 'clean', 'colors', 'contest', 'contest_id', 'contestname', 'cp', 'default_solution', 'depth', 'directory', 'error', 'force', 'force_build', 'input', 'interaction', 'interactive', 'invalid', 'kattis', 'language', 'memory', 'more', 'move_to', 'no_bar', 'no_generate', 'no_solution', 'no_solutions', 'no_testcase_sanity_checks', 'no_time_limit', 'no_validators', 'no_visualizer', 'open', 'order', 'order_from_ccs', 'overview', 'password', 'post_freeze', 'problem', 'problemname', 'remove', 'reorder', 'samples', 'sanitizer', 'skel', 'skip', 'sort', 'submissions', 'table', 'testcases', 'time_limit', 'timeout', 'token', 'tree', 'username', 'validation', 'watch', 'web', 'write']
+ARGS_LIST: Final = ['1', 'add', 'all', 'answer', 'api', 'author', 'check_deterministic', 'clean', 'colors', 'contest', 'contest_id', 'contestname', 'cp', 'default_solution', 'depth', 'directory', 'error', 'force', 'force_build', 'input', 'interaction', 'interactive', 'invalid', 'kattis', 'language', 'memory', 'more', 'move_to', 'no_bar', 'no_generate', 'no_solution', 'no_solutions', 'no_testcase_sanity_checks', 'no_time_limit', 'no_validators', 'no_visualizer', 'open', 'order', 'order_from_ccs', 'overview', 'password', 'post_freeze', 'problem', 'problemname', 'remove', 'reorder', 'samples', 'sanitizer', 'skel', 'skip', 'sort', 'submissions', 'table', 'testcases', 'time_limit', 'timeout', 'token', 'tree', 'type', 'username', 'watch', 'web', 'write']
 # fmt: on
 
 

bin/constraints.py

@@ -22,7 +22,7 @@ def check_validators(problem):
     if not in_constraints:
         warn("No constraint validation of input values found in input validators.")
     problem.validate_data(validate.Mode.ANSWER, constraints=ans_constraints)
-    if not problem.interactive and not problem.multipass and not ans_constraints:
+    if not problem.interactive and not problem.multi_pass and not ans_constraints:
         log("No constraint validation of answer values found in answer or output validators.")
     print()
 

bin/export.py

@@ -74,11 +74,11 @@ def build_samples_zip(problems, output, statement_language):
         outputdir = Path(problem.label)
 
         attachments_dir = problem.path / "attachments"
-        if (problem.interactive or problem.multipass) and not attachments_dir.is_dir():
+        if (problem.interactive or problem.multi_pass) and not attachments_dir.is_dir():
             interactive = "interactive " if problem.interactive else ""
-            multipass = "multi-pass " if problem.multipass else ""
+            multi_pass = "multi-pass " if problem.multi_pass else ""
             util.error(
-                f"{interactive}{multipass}problem {problem.name} does not have an attachments/ directory."
+                f"{interactive}{multi_pass}problem {problem.name} does not have an attachments/ directory."
             )
             continue
 
@@ -96,7 +96,7 @@ def build_samples_zip(problems, output, statement_language):
                     util.error(f"Cannot include broken file {f}.")
 
         # Add samples for non-interactive and non-multi-pass problems.
-        if not problem.interactive and not problem.multipass:
+        if not problem.interactive and not problem.multi_pass:
             samples = problem.testcases(only_samples=True)
             if samples:
                 for i in range(0, len(samples)):
@@ -128,15 +128,15 @@ def build_problem_zip(problem, output):
         ("problem_statement/*", True),
         ("submissions/accepted/**/*", True),
         ("submissions/*/**/*", False),
-        ("attachments/**/*", problem.interactive or problem.multipass),
+        ("attachments/**/*", problem.interactive or problem.multi_pass),
     ]
 
     testcases = [
         ("data/secret/**/*.in", True),
-        ("data/sample/**/*.in", not problem.interactive and not problem.multipass),
+        ("data/sample/**/*.in", not problem.interactive and not problem.multi_pass),
     ]
 
-    if problem.interactive or problem.multipass:
+    if problem.interactive or problem.multi_pass:
         # .interaction files don't need a corresponding .in
         # therefore we can handle them like all other files
         files += [("data/sample/**/*.interaction", False)]
@@ -148,7 +148,7 @@ def build_problem_zip(problem, output):
             ("data/sample/**/*.ans.statement", False),
         ]
 
-    if "custom" in problem.settings.validation:
+    if problem.custom_output:
         files.append(("output_validators/**/*", True))
 
     if config.args.kattis:

bin/fuzz.py

@@ -72,7 +72,7 @@ def _run(self, bar):
         localbar.done()
 
         # Generate .ans.
-        if not self.fuzz.problem.interactive and not self.fuzz.problem.multipass:
+        if not self.fuzz.problem.interactive and not self.fuzz.problem.multi_pass:
             if self.solution and not testcase.ans_path.is_file():
                 if testcase.ans_path.is_file():
                     testcase.ans_path.unlink()

bin/generate.py

@@ -706,11 +706,11 @@ def validate_ans(t, problem, testcase, meta_yaml, bar):
         ansfile = problem.tmpdir / "data" / t.hash / "testcase.ans"
         assert ansfile.is_file()
 
-        if problem.interactive or problem.multipass:
+        if problem.interactive or problem.multi_pass:
             if ansfile.stat().st_size != 0:
                 interactive = "interaction " if problem.interactive else ""
-                multipass = "multipass " if problem.multipass else ""
-                bar.warn(f".ans file for {interactive}{multipass}problem is expected to be empty.")
+                multi_pass = "multi-pass " if problem.multi_pass else ""
+                bar.warn(f".ans file for {interactive}{multi_pass}problem is expected to be empty.")
         else:
             size = ansfile.stat().st_size
             if (
@@ -958,8 +958,8 @@ def needed(ext):
 
             used_solution = False
             changed_ans = False
-            if problem.interactive or problem.multipass:
-                # Generate empty ans file for interactive/multipass problems
+            if problem.interactive or problem.multi_pass:
+                # Generate empty ans file for interactive/multi-pass problems
                 if ".ans" not in meta_yaml["generated_extensions"]:
                     if not ansfile.is_file() or ansfile.stat().st_size != 0:
                         ansfile.write_text("")
@@ -1829,7 +1829,7 @@ def build_program(p):
         build_programs(program.Visualizer, visualizers_used)
 
         self.problem.validators(validate.InputValidator)
-        if not self.problem.interactive and not self.problem.multipass:
+        if not self.problem.interactive and not self.problem.multi_pass:
             self.problem.validators(validate.AnswerValidator)
         self.problem.validators(validate.OutputValidator)
 

bin/interactive.py

@@ -64,7 +64,7 @@ def get_validator_command():
     validator_dir = output_validator.tmpdir
     submission_dir = run.submission.tmpdir
 
-    nextpass = run.feedbackdir / "nextpass.in" if run.problem.multipass else False
+    nextpass = run.feedbackdir / "nextpass.in" if run.problem.multi_pass else False
 
     if config.args.verbose >= 2:
         print("Validator:  ", *get_validator_command(), file=sys.stderr)
@@ -132,7 +132,7 @@ def get_validator_command():
                         _feedback(run, validator_err),
                         exec_res.err,
                         verdict,
-                        pass_id if run.problem.multipass else None,
+                        pass_id if run.problem.multi_pass else None,
                     )
                 else:
                     tle_result.timeout_expired |= max_duration >= timeout
@@ -170,7 +170,7 @@ def get_validator_command():
                 _feedback(run, validator_err),
                 exec_res.err,
                 verdict,
-                pass_id if run.problem.multipass else None,
+                pass_id if run.problem.multi_pass else None,
             )
         else:
             tle_result.duration = max_duration
@@ -401,7 +401,7 @@ def kill_handler_function():
                     val_err,
                     team_err,
                     verdict,
-                    pass_id if run.problem.multipass else None,
+                    pass_id if run.problem.multi_pass else None,
                 )
             else:
                 tle_result.timeout_expired |= aborted
@@ -433,7 +433,7 @@ def kill_handler_function():
             val_err,
             team_err,
             verdict,
-            pass_id if run.problem.multipass else None,
+            pass_id if run.problem.multi_pass else None,
         )
     else:
         tle_result.duration = max_duration

bin/latex.py

@@ -104,10 +104,10 @@ def flush():
                         last = line[0]
                 flush()
             else:
-                assert problem.multipass
+                assert problem.multi_pass
 
-                multipass_dir = builddir / "multipass"
-                multipass_dir.mkdir(exist_ok=True)
+                multi_pass_dir = builddir / "multi_pass"
+                multi_pass_dir.mkdir(exist_ok=True)
 
                 lines = sample.read_text()
                 last = "<"
@@ -121,8 +121,8 @@ def flush():
                 def flush():
                     nonlocal current_sample
 
-                    in_path = multipass_dir / f"{sample_name}-{pass_id:02}.in"
-                    out_path = multipass_dir / f"{sample_name}-{pass_id:02}.out"
+                    in_path = multi_pass_dir / f"{sample_name}-{pass_id:02}.in"
+                    out_path = multi_pass_dir / f"{sample_name}-{pass_id:02}.out"
                     in_path.write_text(cur_in)
                     out_path.write_text(cur_out)
 

bin/problem.py

@@ -22,6 +22,23 @@
 from colorama import Fore, Style
 
 
+# Parse validation mode (only for legacy problem format version)
+def parse_legacy_validation(mode: str) -> set[str]:
+    if mode == "default":
+        return {mode}
+    else:
+        ok = True
+        parsed = set()
+        for part in mode.split():
+            if part in ["custom", "interactive", "multi-pass"] and part not in parsed:
+                parsed.add(part)
+            else:
+                ok = False
+        if "custom" not in parsed or not ok:
+            fatal(f"Unrecognised validation mode {mode}.")
+        return parsed
+
+
 class ProblemLimits:
     def __init__(
         self,
@@ -104,7 +121,12 @@ def __init__(
 
 
 class ProblemSettings:
-    def __init__(self, yamldata: dict[str, Any], legacy_time_limit: Optional[float] = None):
+    def __init__(
+        self,
+        yamldata: dict[str, Any],
+        problem: "Problem",
+        legacy_time_limit: Optional[float] = None,
+    ):
         assert isinstance(yamldata, dict)
 
         if "name" in yamldata and isinstance(yamldata["name"], str):
@@ -120,7 +142,28 @@ def __init__(self, yamldata: dict[str, Any], legacy_time_limit: Optional[float]
         )
         if not self.is_legacy() and self.problem_format_version != "2023-07-draft":
             fatal(f"problem_format_version {self.problem_format_version} not supported")
-        # TODO: also support 'type'. For now, we only support legacy 'validation'.
+
+        if self.is_legacy():
+            mode = parse_legacy_validation(parse_setting(yamldata, "validation", "default"))
+        else:
+            if "validation" in yamldata:
+                warn(
+                    "problem.yaml: 'validation' is removed in 2023-07-draft, please use 'type' instead"
+                )
+            mode = set(parse_setting(yamldata, "type", "pass-fail").split(" "))
+        self.interactive: bool = "interactive" in mode
+        self.multi_pass: bool = "multi-pass" in mode
+        self.custom_output: bool = (
+            self.interactive
+            or self.multi_pass
+            or (
+                "custom" in mode
+                if self.is_legacy()
+                # TODO #424: output_validator should be singular, but DOMjudge does not support this yet, so this should be fixed during export.
+                else (problem.path / "output_validators").exists()
+            )
+        )
+
         self.name: dict[str, str] = parse_setting(yamldata, "name", {"en": ""})
         self.uuid: str = parse_setting(yamldata, "uuid", "")
         self.author: str = parse_setting(yamldata, "author", "")
@@ -129,7 +172,6 @@ def __init__(self, yamldata: dict[str, Any], legacy_time_limit: Optional[float]
         self.license: str = parse_setting(yamldata, "license", "unknown")
         self.rights_owner: str = parse_setting(yamldata, "rights_owner", "")
         self.limits = ProblemLimits(parse_setting(yamldata, "limits", {}), self, legacy_time_limit)
-        self.validation: str = parse_setting(yamldata, "validation", "default")
         self.validator_flags: list[str] = parse_setting(yamldata, "validator_flags", [])
         self.keywords: str = parse_setting(yamldata, "keywords", "")
 
@@ -279,19 +321,20 @@ def _read_settings(self):
             yaml_path.write_text(raw)
             log("Added new UUID to problem.yaml")
 
-        self.settings = ProblemSettings(yaml_data, self._get_legacy_time_limit(yaml_data))
-        self.limits = self.settings.limits
+        self.settings = ProblemSettings(yaml_data, self, self._get_legacy_time_limit(yaml_data))
 
-        mode = parse_validation(self.settings.validation)
-        self.interactive = "interactive" in mode
-        self.multipass = "multi-pass" in mode
+        # Aliasing fields makes life easier for us 😛
+        self.limits: ProblemLimits = self.settings.limits
+        self.interactive: bool = self.settings.interactive
+        self.multi_pass: bool = self.settings.multi_pass
+        self.custom_output: bool = self.settings.custom_output
 
         # Handle dependencies...
         has_validation_passes = self.limits.validation_passes is not None
-        if self.multipass and not has_validation_passes:
+        if self.multi_pass and not has_validation_passes:
             self.limits.validation_passes = 2
-        if not self.multipass and has_validation_passes:
-            warn("limit: validation_passes is only used for multipass problems. SKIPPED.")
+        if not self.multi_pass and has_validation_passes:
+            warn("limit: validation_passes is only used for multi_pass problems. SKIPPED.")
 
     def _parse_testdata_yaml(p, path, bar):
         assert path.is_relative_to(p.path / "data")
@@ -444,14 +487,14 @@ def testcases(
         for f in in_paths:
             t = testcase.Testcase(p, f, print_warn=True)
             if (
-                (p.interactive or p.multipass)
+                (p.interactive or p.multi_pass)
                 and mode == validate.Mode.INVALID
                 and t.root in ["invalid_answers", "invalid_outputs"]
             ):
                 msg = ""
                 if p.interactive:
                     msg += " interactive"
-                if p.multipass:
+                if p.multi_pass:
                     msg += " multi-pass"
                 warn(f"Found file {f} for {mode} validation in{msg} problem. Skipping.")
                 continue
@@ -504,7 +547,7 @@ def statement_samples(p) -> list[Path | tuple[Path, Path]]:
         # Non-interactive and Non-multi-pass problems should not have .interaction files.
         # On the other hand, interactive problems are allowed to have .{in,ans}.statement files,
         # so that they can emulate a non-interactive problem with on-the-fly generated input.
-        if not p.interactive and not p.multipass:
+        if not p.interactive and not p.multi_pass:
             if len(interaction_paths) != 0:
                 warn(
                     f"Non-interactive/Non-multi-pass problem {p.name} should not have data/sample/*.interaction files."
@@ -672,13 +715,18 @@ def _validators(
             return problem._validators_cache[key]
 
         assert hasattr(cls, "source_dirs")
+        # TODO #424: We should not support multiple output validators inside output_validator/.
         paths = [p for source_dir in cls.source_dirs for p in glob(problem.path / source_dir, "*")]
 
         # Handle default output validation
-        if cls == validate.OutputValidator and problem.settings.validation == "default":
-            if paths:
+        if cls == validate.OutputValidator:
+            if problem.settings.is_legacy() and not problem.custom_output and paths:
                 error("Validation is default but custom output validator exists (ignoring it)")
-            paths = [config.TOOLS_ROOT / "support" / "default_output_validator.cpp"]
+                paths = []
+            if not paths:
+                if problem.custom_output:
+                    fatal("Problem validation type requires output_validators/")
+                paths = [config.TOOLS_ROOT / "support" / "default_output_validator.cpp"]
 
         # TODO: Instead of checking file contents, maybe specify this in generators.yaml?
         def has_constraints_checking(f):
@@ -906,12 +954,12 @@ def validate_data(problem, mode: validate.Mode, constraints: dict | bool | None
             constraints = {}
         assert constraints is None or isinstance(constraints, dict)
 
-        if (problem.interactive or problem.multipass) and mode == validate.Mode.ANSWER:
+        if (problem.interactive or problem.multi_pass) and mode == validate.Mode.ANSWER:
             if (problem.path / "answer_validators").exists():
                 msg = ""
                 if problem.interactive:
                     msg += " interactive"
-                if problem.multipass:
+                if problem.multi_pass:
                     msg += " multi-pass"
                 log(f"Not running answer_validators for{msg} problems.")
             return True
@@ -925,12 +973,12 @@ def validate_data(problem, mode: validate.Mode, constraints: dict | bool | None
                 testcases = problem.testcases(mode=mode)
             case validate.Mode.ANSWER:
                 assert not problem.interactive
-                assert not problem.multipass
+                assert not problem.multi_pass
                 problem.validators(validate.AnswerValidator, check_constraints=check_constraints)
                 testcases = problem.testcases(mode=mode)
             case validate.Mode.INVALID:
                 problem.validators(validate.InputValidator)
-                if not problem.interactive and not problem.multipass:
+                if not problem.interactive and not problem.multi_pass:
                     problem.validators(validate.AnswerValidator)
                 testcases = problem.testcases(mode=mode)
             case _: