Merge pull request #431 from RagnarGrootKoerkamp/constants-final

Implement constants

Author: mzuenni

bin/config.py

@@ -34,9 +34,20 @@
 # When --table is set, this threshold determines the number of identical profiles needed to get flagged.
 TABLE_THRESHOLD: Final[int] = 4
 
-FILE_NAME_REGEX: Final[str] = "[a-zA-Z0-9][a-zA-Z0-9_.-]*[a-zA-Z0-9]"
+FILE_NAME_REGEX: Final[str] = "[a-zA-Z0-9][a-zA-Z0-9_.-]{0,253}[a-zA-Z0-9]"
 COMPILED_FILE_NAME_REGEX: Final[re.Pattern[str]] = re.compile(FILE_NAME_REGEX)
 
+CONSTANT_NAME_REGEX = "[a-zA-Z_][a-zA-Z0-9_]*"
+COMPILED_CONSTANT_NAME_REGEX: Final[re.Pattern[str]] = re.compile(CONSTANT_NAME_REGEX)
+CONSTANT_SUBSTITUTE_REGEX: Final[re.Pattern[str]] = re.compile(
+    f"\\{{\\{{({CONSTANT_NAME_REGEX})\\}}\\}}"
+)
+
+BAPCTOOLS_SUBSTITUTE_REGEX: Final[re.Pattern[str]] = re.compile(
+    f"\\{{%({CONSTANT_NAME_REGEX})%\\}}"
+)
+
+
 KNOWN_TESTCASE_EXTENSIONS: Final[Sequence[str]] = [
     ".in",
     ".ans",

bin/export.py

@@ -12,31 +12,6 @@
 from problem import Problem
 
 
-# Replace \problemname{...} by the value of `name:` in problems.yaml in all .tex files.
-# This is needed because Kattis is currently still running the legacy version of the problem spec,
-# rather than 2023-07-draft.
-def fix_problem_name_cmd(problem):
-    reverts = []
-    for f in (problem.path / "problem_statement").iterdir():
-        if f.is_file() and f.suffix == ".tex" and len(f.suffixes) >= 2:
-            lang = f.suffixes[-2][1:]
-            t = f.read_text()
-            match = re.search(r"\\problemname\{\s*(\\problemyamlname)?\s*\}", t)
-            if match:
-                if lang in problem.settings.name:
-                    reverts.append((f, t))
-                    t = t.replace(match[0], r"\problemname{" + problem.settings.name[lang] + "}")
-                    f.write_text(t)
-                else:
-                    util.error(f"{f}: no name set for language {lang}.")
-
-    def revert():
-        for f, t in reverts:
-            f.write_text(t)
-
-    return revert
-
-
 def force_single_language(problems):
     if config.args.languages and len(config.args.languages) == 1:
         statement_language = config.args.languages[0]
@@ -115,18 +90,13 @@ def build_samples_zip(problems, output, statement_language):
 
 
 def build_problem_zip(problem: Problem, output: Path):
-    """Make DOMjudge ZIP file for specified problem."""
+    """Make DOMjudge/Kattis ZIP file for specified problem."""
 
     # Add problem PDF for only one language to the zip file (note that Kattis export does not include PDF)
     statement_language = None if config.args.kattis else force_single_language([problem])
 
-    deprecated = [  # may be removed at some point.
-        "domjudge-problem.ini",
-    ]
-
-    write_file_strs: list[tuple[str, str]] = []
-
     files = [
+        ("problem.yaml", True),
         ("problem_statement/*", True),
         ("submissions/accepted/**/*", True),
         ("submissions/*/**/*", False),
@@ -156,56 +126,21 @@ def build_problem_zip(problem: Problem, output: Path):
     if config.args.kattis:
         files.append(("input_validators/**/*", True))
 
-    print("Preparing to make ZIP file for problem dir %s" % problem.path, file=sys.stderr)
-
-    # DOMjudge does not support 'type' in problem.yaml nor 'output_validator_args' in testdata.yaml yet.
-    # TODO: Remove this once it does.
-    problem_yaml_str = (problem.path / "problem.yaml").read_text()
-    if not config.args.kattis:
-        validator_flags = " ".join(
-            problem.get_testdata_yaml(
-                problem.path / "data",
-                "output_validator_args",
-                PrintBar("Getting validator_flags for legacy DOMjudge export"),
-            )
-        )
-        if validator_flags:
-            validator_flags = "validator_flags: " + validator_flags + "\n"
-        write_file_strs.append(
-            (
-                "problem.yaml",
-                f"""{problem_yaml_str}\nvalidation: {
-                    "custom interactive"
-                    if problem.interactive
-                    else "custom multi-pass"
-                    if problem.multi_pass
-                    else "custom"
-                    if problem.custom_output
-                    else "default"
-                }\n{validator_flags}""",
-            )
-        )
-    else:
-        write_file_strs.append(("problem.yaml", problem_yaml_str))
-
-    # DOMjudge does not support 'limits.time_limit' in problem.yaml yet.
-    # TODO: Remove this once it does.
-    if not config.args.kattis:
-        write_file_strs.append((".timelimit", str(problem.limits.time_limit)))
+    message("preparing zip file content", "Zip", problem.path, color_type=MessageType.LOG)
 
-    # Warn for all deprecated files but still add them to the files list
-    for pattern in deprecated:
-        files.append((pattern, False))
-        # Only include hidden files if the pattern starts with a '.'.
-        paths = list(util.glob(problem.path, pattern, include_hidden=pattern[0] == "."))
-        if len(paths) > 0:
-            addition = ""
-            if len(paths) > 1:
-                addition = f" and {len(paths) - 1} more"
-            util.warn(f'Found deprecated file "{paths[0]}"{addition}.')
+    # prepare files inside dir
+    export_dir = problem.tmpdir / "export"
+    if export_dir.exists():
+        shutil.rmtree(export_dir)
+    # For Kattis, prepend the problem shortname to all files.
+    if config.args.kattis:
+        export_dir /= problem.name
+    export_dir.mkdir(parents=True, exist_ok=True)
 
-    # Build list of files to store in ZIP file.
-    copyfiles = set()
+    def add_file(path, source):
+        path = export_dir / path
+        path.parent.mkdir(parents=True, exist_ok=True)
+        ensure_symlink(path, source)
 
     # Include all files beside testcases
     for pattern, required in files:
@@ -214,22 +149,17 @@ def build_problem_zip(problem: Problem, output: Path):
         if required and len(paths) == 0:
             util.error(f"No matches for required path {pattern}.")
         for f in paths:
-            # NOTE: Directories are skipped because ZIP only supports files.
             if f.is_file():
                 out = f.relative_to(problem.path)
                 out = remove_language_suffix(out, statement_language)
-                # For Kattis, prepend the problem shortname to all files.
-                if config.args.kattis:
-                    out = problem.name / out
-                copyfiles.add((f, out))
+                add_file(out, f)
 
     # Include all testcases (specified by a .in file) and copy all related files
     for pattern, required in testcases:
         paths = list(util.glob(problem.path, pattern))
         if required and len(paths) == 0:
             util.error(f"No matches for required path {pattern}.")
         for f in paths:
-            # NOTE: Directories are skipped because ZIP only supports files.
             if f.is_file():
                 if not f.with_suffix(".ans").is_file():
                     util.warn(f"No answer file found for {f}, skipping.")
@@ -238,31 +168,98 @@ def build_problem_zip(problem: Problem, output: Path):
                         f2 = f.with_suffix(ext)
                         if f2.is_file():
                             out = f2.relative_to(problem.path)
-                            # For Kattis, prepend the problem shortname to all files.
-                            if config.args.kattis:
-                                out = problem.name / out
-                            copyfiles.add((f2, out))
+                            add_file(out, f2)
 
-    # Build .ZIP file.
-    print("writing ZIP file:", output, file=sys.stderr)
+    # DOMjudge does not support 'type' in problem.yaml nor 'output_validator_args' in testdata.yaml yet.
+    # TODO: Remove this once it does.
+    if not config.args.kattis:
+        yaml_path = export_dir / "problem.yaml"
+        yaml_data = [yaml_path.read_text(), "\nvalidation:"]
+        if problem.custom_output:
+            yaml_data.append(" custom")
+            if problem.interactive:
+                yaml_data.append(" interactive")
+            if problem.multi_pass:
+                yaml_data.append(" multi-pass")
+        else:
+            yaml_data.append(" default")
+        yaml_data.append("\n")
+
+        validator_flags = " ".join(
+            problem.get_testdata_yaml(
+                problem.path / "data",
+                "output_validator_args",
+                PrintBar("Getting validator_flags for legacy DOMjudge export"),
+            )
+        )
+        if validator_flags:
+            yaml_data.append(f"validator_flags: {validator_flags}\n")
+
+        yaml_path.unlink()
+        yaml_path.write_text("".join(yaml_data))
+
+    # DOMjudge does not support 'limits.time_limit' in problem.yaml yet.
+    # TODO: Remove this once it does.
+    if not config.args.kattis:
+        (export_dir / ".timelimit").write_text(str(problem.limits.time_limit))
 
-    revert_problem_name_cmd = fix_problem_name_cmd(problem)
+    # Replace \problemname{...} by the value of `name:` in problems.yaml in all .tex files.
+    # This is needed because Kattis is currently still running the legacy version of the problem spec,
+    # rather than 2023-07-draft.
+    for f in (export_dir / "problem_statement").iterdir():
+        if f.is_file() and f.suffix == ".tex" and len(f.suffixes) >= 2:
+            lang = f.suffixes[-2][1:]
+            t = f.read_text()
+            match = re.search(r"\\problemname\{\s*(\\problemyamlname)?\s*\}", t)
+            if match:
+                if lang in problem.settings.name:
+                    t = t.replace(match[0], r"\problemname{" + problem.settings.name[lang] + "}")
+                    f.unlink()
+                    f.write_text(t)
+                else:
+                    util.error(f"{f}: no name set for language {lang}.")
+
+    # DOMjudge does not support constants.
+    # TODO: Remove this if it ever does.
+    if problem.settings.constants:
+        constants_supported = [
+            "data/**/testdata.yaml",
+            "output_validators/**/*",
+            "input_validators/**/*",
+            # "problem_statement/*", uses \constants
+            # "submissions/*/**/*", removed support?
+        ]
+        for pattern in constants_supported:
+            for f in export_dir.glob(pattern):
+                if f.is_file() and util.has_substitute(f, config.CONSTANT_SUBSTITUTE_REGEX):
+                    text = f.read_text()
+                    text = util.substitute(
+                        text,
+                        problem.settings.constants,
+                        pattern=config.CONSTANT_SUBSTITUTE_REGEX,
+                        bar=util.PrintBar("Zip"),
+                    )
+                    f.unlink()
+                    f.write_text(text)
 
+    # Build .ZIP file.
+    message("writing zip file", "Zip", output, color_type=MessageType.LOG)
     try:
         zf = zipfile.ZipFile(output, mode="w", compression=zipfile.ZIP_DEFLATED, allowZip64=False)
 
-        for source, target in sorted(copyfiles):
-            zf.write(source, target, compress_type=zipfile.ZIP_DEFLATED)
-        for target_file, content in sorted(write_file_strs):
-            zf.writestr(target_file, content, compress_type=zipfile.ZIP_DEFLATED)
+        export_dir = problem.tmpdir / "export"
+        for f in sorted(export_dir.rglob("*")):
+            # NOTE: Directories are skipped because ZIP only supports files.
+            if f.is_file():
+                name = f.relative_to(export_dir)
+                zf.write(f, name, compress_type=zipfile.ZIP_DEFLATED)
 
         # Done.
         zf.close()
-        print("done", file=sys.stderr)
+        message("done", "Zip", color_type=MessageType.LOG)
         print(file=sys.stderr)
-
-    finally:
-        revert_problem_name_cmd()
+    except Exception:
+        return False
 
     return True
 

bin/generate.py

@@ -268,7 +268,7 @@ def default_solution_path(generator_config):
     if config.args.default_solution:
         if generator_config.has_yaml:
             message(
-                f"""--default-solution Ignored. Set the default solution in the generator.yaml!
+                f"""--default-solution Ignored. Set the default solution in the generators.yaml!
 solution: /{config.args.default_solution}""",
                 "generators.yaml",
                 color_type=MessageType.WARN,
@@ -297,7 +297,7 @@ def default_solution_path(generator_config):
             raw = f"solution: /{solution.relative_to(problem.path)}\n" + raw
             yaml_path.write_text(raw)
             message(
-                f"No solution specified. {solution_short_path} added as default solution in the generator.yaml",
+                f"No solution specified. {solution_short_path} added as default solution in the generators.yaml",
                 "generators.yaml",
                 color_type=MessageType.LOG,
             )
@@ -522,8 +522,15 @@ def __init__(self, problem, generator_config, key, name: str, yaml, parent, coun
                     if len(yaml["generate"]) == 0:
                         raise ParseException("`generate` must not be empty.")
 
-                    # replace count
+                    # first replace {{constants}}
                     command_string = yaml["generate"]
+                    command_string = substitute(
+                        command_string,
+                        problem.settings.constants,
+                        pattern=config.CONSTANT_SUBSTITUTE_REGEX,
+                    )
+
+                    # then replace {count} and {seed}
                     if "{count}" in command_string:
                         if "count" in yaml:
                             command_string = command_string.replace(
@@ -907,6 +914,7 @@ def generate_from_rule():
                         bar.error(f"Hardcoded {ext} data must not be empty!")
                         return False
                     else:
+                        # substitute in contents? -> No!
                         infile.with_suffix(ext).write_text(contents)
 
                 # Step 4: Error if infile was not generated.

bin/latex.py

@@ -164,9 +164,20 @@ def flush():
     samples_file_path.write_text("".join(samples_data))
 
 
+def create_constants_file(problem: "problem.Problem", language: str) -> None:
+    constant_data: list[str] = []
+    for key, item in problem.settings.constants.items():
+        constant_data.append(f"\\expandafter\\def\\csname constants_{key}\\endcsname{{{item}}}\n")
+
+    builddir = latex_builddir(problem, language)
+    constants_file_path = builddir / "constants.tex"
+    constants_file_path.write_text("".join(constant_data))
+
+
 # Steps needed for both problem and contest compilation.
 def prepare_problem(problem: "problem.Problem", language: str):
     create_samples_file(problem, language)
+    create_constants_file(problem, language)
 
 
 def get_tl(problem: "problem.Problem"):
@@ -350,7 +361,7 @@ def run_latexmk(stdout, stderr):
 
 # 1. Copy the latex/problem.tex file to tmpdir/<problem>/latex/<language>/problem.tex,
 #    substituting variables.
-# 2. Create tmpdir/<problem>/latex/<language>/samples.tex.
+# 2. Create tmpdir/<problem>/latex/<language>/{samples,constants}.tex.
 # 3. Run latexmk and link the resulting <build_type>.<language>.pdf into the problem directory.
 def build_problem_pdf(
     problem: "problem.Problem", language: str, build_type=PdfType.PROBLEM, web=False
@@ -374,6 +385,7 @@ def build_problem_pdf(
         local_data if local_data.is_file() else config.TOOLS_ROOT / "latex" / main_file,
         builddir / main_file,
         problem_data(problem, language),
+        bar=bar,
     )
 
     return build_latex_pdf(builddir, builddir / main_file, language, bar, problem.path)
@@ -465,6 +477,7 @@ def build_contest_pdf(
         ),
         builddir / "contest_data.tex",
         config_data,
+        bar=bar,
     )
 
     problems_data = ""
@@ -489,6 +502,7 @@ def build_contest_pdf(
         if build_type == PdfType.PROBLEM:
             prepare_problem(prob, language)
         else:  # i.e. for SOLUTION and PROBLEM_SLIDE
+            create_constants_file(prob, language)
             tex_no_lang = prob.path / "problem_statement" / f"{build_type.value}.tex"
             tex_with_lang = prob.path / "problem_statement" / f"{build_type.value}.{language}.tex"
             if tex_with_lang.is_file():
@@ -507,6 +521,7 @@ def build_contest_pdf(
         problems_data += substitute(
             per_problem_data_tex,
             problem_data(prob, language),
+            bar=bar,
         )
 
     if solutions: