Legacy export (#441)

* make legacy export an explicit command

* [export] Add all the directories!

* [export] Legacy: remove solution/ and problem_slide/ from export dir

* [export] Make answer_validators/ not required

Apparently, `bt validate` also doesn't require them?

* prepend problem name

* fix test

* handle languages

* use ngerman

* add german to wsl

* keep languages in sync

* keep languages in sync

* [export] Add comment explaining why name in problems.yaml can also be str

* [doc] Improve singular/plural in explanation of `--languages`

* [export][latex] Rename --languages flag to --lang

* [test][export] Add assertions for which PDFs should be in the ZIPs

* [test] TestContest.test_zip: also remove constituent zip files after test completes

---------

Co-authored-by: Maarten Sijm <[email protected]>

Author: mzuenni

.github/workflows/ci.yaml

@@ -50,5 +50,6 @@ jobs:
                 lmodern
                 texlive-science
                 latexmk
+                texlive-lang-german
       - shell: wsl-bash {0}
         run: pytest

bin/config.py

@@ -109,7 +109,6 @@
     "jobs": (os.cpu_count() or 1) // 2,
     "time": 600,  # Used for `bt fuzz`
     "verbose": 0,
-    "languages": None,
 }
 
 
@@ -120,7 +119,7 @@
 grep -Ev '^(h|jobs|time|verbose)$' | sed "s/^/'/;s/$/',/" | tr '\n' ' ' | sed 's/^/ARGS_LIST: Final[Sequence[str]] = [/;s/, $/]\n/'
 """
 # fmt: off
-ARGS_LIST: Final[Sequence[str]] = ['1', 'add', 'all', 'answer', 'api', 'author', 'check_deterministic', 'clean', 'colors', 'contest', 'contest_id', 'contestname', 'cp', 'default_solution', 'depth', 'directory', 'error', 'force', 'force_build', 'generic', 'input', 'interaction', 'interactive', 'invalid', 'kattis', 'language', 'memory', 'more', 'move_to', 'no_bar', 'no_generate', 'no_solution', 'no_solutions', 'no_testcase_sanity_checks', 'no_time_limit', 'no_validators', 'no_visualizer', 'open', 'order', 'order_from_ccs', 'overview', 'password', 'post_freeze', 'problem', 'problemname', 'remove', 'reorder', 'samples', 'sanitizer', 'skel', 'skip', 'sort', 'submissions', 'table', 'testcases', 'time_limit', 'timeout', 'token', 'tree', 'type', 'username', 'valid_output', 'watch', 'web', 'write']
+ARGS_LIST: Final[Sequence[str]] = ['1', 'add', 'all', 'answer', 'api', 'author', 'check_deterministic', 'clean', 'colors', 'contest', 'contest_id', 'contestname', 'cp', 'default_solution', 'depth', 'directory', 'error', 'force', 'force_build', 'generic', 'input', 'interaction', 'interactive', 'invalid', 'kattis', 'lang', 'legacy', 'memory', 'more', 'move_to', 'no_bar', 'no_generate', 'no_solution', 'no_solutions', 'no_testcase_sanity_checks', 'no_time_limit', 'no_validators', 'no_visualizer', 'open', 'order', 'order_from_ccs', 'overview', 'password', 'post_freeze', 'problem', 'problemname', 'remove', 'reorder', 'samples', 'sanitizer', 'skel', 'skip', 'sort', 'submissions', 'table', 'testcases', 'time_limit', 'timeout', 'token', 'tree', 'type', 'username', 'valid_output', 'watch', 'web', 'write']
 # fmt: on
 
 

bin/export.py

@@ -396,21 +396,21 @@ def build_problem_pdf(problem: "Problem", language: str, build_type=PdfType.PROB
 
 def build_problem_pdfs(problem: "Problem", build_type=PdfType.PROBLEM, web=False):
     """Build PDFs for various languages. If list of languages is specified,
-    (either via config files or --language arguments), build those. Otherwise
+    (either via config files or --lang arguments), build those. Otherwise
     build all languages for which there is a statement latex source.
     """
-    if config.args.languages is not None:
-        for lang in config.args.languages:
+    if config.args.lang is not None:
+        for lang in config.args.lang:
             if lang not in problem.statement_languages:
                 message(
                     f"No statement source for language {lang}",
                     problem.name,
                     color_type=MessageType.FATAL,
                 )
-        languages = config.args.languages
+        languages = config.args.lang
     else:
         languages = problem.statement_languages
-        # For solutions or problem slides, filter for `<build_type>.<language>.tex` files that exist.
+        # For solutions or problem slides, filter for `<build_type>.<lang>.tex` files that exist.
         if build_type != PdfType.PROBLEM:
             filtered_languages = []
             for lang in languages:
@@ -424,7 +424,7 @@ def build_problem_pdfs(problem: "Problem", build_type=PdfType.PROBLEM, web=False
                     )
             languages = filtered_languages
     if config.args.watch and len(languages) > 1:
-        fatal("--watch does not work with multiple languages. Please use --language")
+        fatal("--watch does not work with multiple languages. Please use --lang")
     return all([build_problem_pdf(problem, lang, build_type, web) for lang in languages])
 
 
@@ -551,8 +551,8 @@ def build_contest_pdfs(contest, problems, tmpdir, lang=None, build_type=PdfType.
         message(
             "No statement language present in every problem.", contest, color_type=MessageType.FATAL
         )
-    if config.args.languages is not None:
-        languages = config.args.languages
+    if config.args.lang is not None:
+        languages = config.args.lang
         for lang in set(languages) - statement_languages:
             message(
                 f"Unable to build all statements for language {lang}",
@@ -563,7 +563,7 @@ def build_contest_pdfs(contest, problems, tmpdir, lang=None, build_type=PdfType.
         languages = statement_languages
     if config.args.watch and len(languages) > 1:
         message(
-            "--watch does not work with multiple languages. Please use --language",
+            "--watch does not work with multiple languages. Please use --lang",
             contest,
             color_type=MessageType.FATAL,
         )

bin/latex.py

@@ -5,7 +5,6 @@
 # Local imports
 import config
 import latex
-from export import force_single_language
 from problem import Problem
 from util import *
 import contest
@@ -140,7 +139,7 @@ def new_problem():
     if config.args.problem:
         fatal("--problem does not work for new_problem.")
 
-    statement_languages = config.args.languages if config.args.languages else ["en"]
+    statement_languages = config.args.lang if config.args.lang else ["en"]
     main_language = "en" if "en" in statement_languages else statement_languages[0]
 
     problemname = {
@@ -288,11 +287,6 @@ def rename_problem(problem):
     data["name"] = newname
     write_yaml(data, problem_yaml)
 
-    # DOMjudge does not yet support multilingual problems.yaml files.
-    statement_language = force_single_language([problem])
-    if isinstance(newname, dict):
-        newname = newname[statement_language]
-
     problems_yaml = Path("problems.yaml")
     if problems_yaml.is_file():
         data = read_yaml(problems_yaml) or []

bin/skel.py

@@ -343,9 +343,7 @@ def build_parser():
         action="store_true",
         help="Copy the output pdf instead of symlinking it.",
     )
-    global_parser.add_argument(
-        "--language", dest="languages", action="append", help="Set language."
-    )
+    global_parser.add_argument("--lang", nargs="+", help="Languages to include.")
 
     subparsers = parser.add_subparsers(
         title="actions", dest="action", parser_class=SuppressingParser
@@ -808,12 +806,22 @@ def build_parser():
         action="store_true",
         help="Make a zip more following the kattis problemarchive.com format.",
     )
+    zipparser.add_argument(
+        "--legacy",
+        action="store_true",
+        help="Make a zip more following the legacy format.",
+    )
     zipparser.add_argument("--no-solutions", action="store_true", help="Do not compile solutions")
 
     # Build a zip with all samples.
-    subparsers.add_parser(
+    samplezipparser = subparsers.add_parser(
         "samplezip", parents=[global_parser], help="Create zip file of all samples."
     )
+    samplezipparser.add_argument(
+        "--legacy",
+        action="store_true",
+        help="Make a zip more following the legacy format.",
+    )
 
     subparsers.add_parser(
         "gitlabci",
@@ -831,6 +839,11 @@ def build_parser():
         action="store",
         help="Contest ID to use when writing to the API. Defaults to value of contest_id in contest.yaml.",
     )
+    exportparser.add_argument(
+        "--legacy",
+        action="store_true",
+        help="Make export more following the legacy format.",
+    )
 
     updateproblemsyamlparser = subparsers.add_parser(
         "update_problems_yaml",
@@ -846,6 +859,11 @@ def build_parser():
         action="store_true",
         help="Sort the problems by id.",
     )
+    updateproblemsyamlparser.add_argument(
+        "--legacy",
+        action="store_true",
+        help="Make problems.yaml more following the legacy format.",
+    )
 
     # Print the corresponding temporary directory.
     tmpparser = subparsers.add_parser(
@@ -1009,8 +1027,8 @@ def run_parsed_arguments(args):
         sampleout = Path("samples.zip")
         if level == "problem":
             sampleout = problems[0].path / sampleout
-        statement_language = export.force_single_language(problems)
-        export.build_samples_zip(problems, sampleout, statement_language)
+        languages = export.select_languages(problems)
+        export.build_samples_zip(problems, sampleout, languages)
         return
 
     if action == "rename_problem":
@@ -1142,10 +1160,16 @@ def run_parsed_arguments(args):
                 config.args = old_args
 
                 if not config.args.kattis:
-                    # Make sure that all problems use the same language for the PDFs
-                    export.force_single_language(problems)
-
                     success &= latex.build_problem_pdfs(problem)
+                    if not config.args.no_solutions:
+                        success &= latex.build_problem_pdfs(
+                            problem, build_type=latex.PdfType.SOLUTION
+                        )
+
+                    if problem.path.glob(str(latex.PdfType.PROBLEM_SLIDE.path("*"))):
+                        success &= latex.build_problem_pdfs(
+                            problem, build_type=latex.PdfType.PROBLEM_SLIDE
+                        )
 
                 if not config.args.force:
                     success &= problem.validate_data(validate.Mode.INPUT, constraints={})
@@ -1159,10 +1183,8 @@ def run_parsed_arguments(args):
             print(file=sys.stderr)
 
     if action in ["export"]:
-        # Add contest PDF for only one language to DOMjudge
-        statement_language = export.force_single_language(problems)
-
-        export.export_contest_and_problems(problems, statement_language)
+        languages = export.select_languages(problems)
+        export.export_contest_and_problems(problems, languages)
 
     if level == "problemset":
         print(f"{Style.BRIGHT}CONTEST {contest}{Style.RESET_ALL}", file=sys.stderr)
@@ -1190,48 +1212,53 @@ def run_parsed_arguments(args):
             )
 
         if action in ["zip"]:
-            statement_language = None
+            languages = []
             if not config.args.kattis:
-                # Add contest/solutions PDF for only one language to the zip file
-                statement_language = export.force_single_language(problems)
+                languages = export.select_languages(problems)
 
-                success &= latex.build_contest_pdfs(contest, problems, tmpdir, statement_language)
-                success &= latex.build_contest_pdfs(
-                    contest, problems, tmpdir, statement_language, web=True
-                )
-                if not config.args.no_solutions:
-                    success &= latex.build_contest_pdf(
-                        contest,
-                        problems,
-                        tmpdir,
-                        statement_language,
-                        build_type=latex.PdfType.SOLUTION,
-                    )
-                    success &= latex.build_contest_pdf(
-                        contest,
-                        problems,
-                        tmpdir,
-                        statement_language,
-                        build_type=latex.PdfType.SOLUTION,
-                        web=True,
-                    )
                 # Only build the problem slides if at least one problem has the TeX for it
                 slideglob = latex.PdfType.PROBLEM_SLIDE.path("*")
-                if any(problem.path.glob(str(slideglob)) for problem in problems):
-                    success &= latex.build_contest_pdf(
-                        contest,
-                        problems,
-                        tmpdir,
-                        statement_language,
-                        build_type=latex.PdfType.PROBLEM_SLIDE,
+                build_problem_slides = any(
+                    problem.path.glob(str(slideglob)) for problem in problems
+                )
+
+                for language in languages:
+                    success &= latex.build_contest_pdfs(contest, problems, tmpdir, language)
+                    success &= latex.build_contest_pdfs(
+                        contest, problems, tmpdir, language, web=True
                     )
-                else:
+                    if not config.args.no_solutions:
+                        success &= latex.build_contest_pdf(
+                            contest,
+                            problems,
+                            tmpdir,
+                            language,
+                            build_type=latex.PdfType.SOLUTION,
+                        )
+                        success &= latex.build_contest_pdf(
+                            contest,
+                            problems,
+                            tmpdir,
+                            language,
+                            build_type=latex.PdfType.SOLUTION,
+                            web=True,
+                        )
+                    if build_problem_slides:
+                        success &= latex.build_contest_pdf(
+                            contest,
+                            problems,
+                            tmpdir,
+                            language,
+                            build_type=latex.PdfType.PROBLEM_SLIDE,
+                        )
+
+                if not build_problem_slides:
                     log(f"No problem has {slideglob.name}, skipping problem slides")
 
             outfile = contest + ".zip"
             if config.args.kattis:
                 outfile = contest + "-kattis.zip"
-            export.build_contest_zip(problems, problem_zips, outfile, statement_language)
+            export.build_contest_zip(problems, problem_zips, outfile, languages)
 
         if action in ["update_problems_yaml"]:
             export.update_problems_yaml(

bin/tools.py

@@ -388,7 +388,6 @@ def _upgrade(problem_path: Path, bar: ProgressBar) -> None:
     upgrade_statement(problem_path, bar)
     upgrade_format_validators(problem_path, bar)
     upgrade_output_validators(problem_path, bar)
-    # update .in.statement?
     upgrade_problem_yaml(problem_path, bar)
 
     bar.done()

bin/upgrade.py

@@ -55,7 +55,7 @@ The flags below work for any subcommand:
 - `--no-bar`: Disable showing progress bars. This is useful when running in non-interactive contexts (such as CI jobs) or on platforms/terminals that don't handle the progress bars well.
 - `--error`/`-e`: show full output of failing commands using `--error`. The default is to show a short snippet only.
 - `--force-build`: Force rebuilding binaries instead of reusing cached version.
-- `--language <LANG>`: select a single language to use. `<LANG>` should be a language code like `en` or `nl`.
+- `--lang`: select languages to use for LaTeX commands. The languages should be specified by language codes like `en` or `nl`.
 
 # Problem development
 

doc/commands.md

@@ -15,18 +15,18 @@ Here, `LANG` is a two-letter language code, see
 
 It is expected that the languages keys in the metadata and statement files agree.
 
-The default language for BAPCtools is English, but multiple languages can be specified at various points of the tool, typically using the `--language` flag or configuration files.
+The default language for BAPCtools is English, but multiple languages can be specified at various points of the tool, typically using the `--lang` flag or configuration files.
 
 ## Creating a contest
 
 In short,
 
-1. configure `languages` in `.bapctools.yaml`.
+1. configure `lang` in `.bapctools.yaml`.
 2. add a skeleton for `problem.LANG.tex` in `skel/problem/statement`.
 
-### Configure `language`
+### Configure `lang`
 
-To create a contest supporting French, Dutch, and Luxembourgish, set the configurartion key `languages` to the list `['nl', 'fr', 'lt']`.
+To create a contest supporting French, Dutch, and Luxembourgish, set the configurartion key `lang` to the list `['nl', 'fr', 'lt']`.
 Configuration keys can be set in many ways, see **Personal configuration file** in the BAPCtools documentation, but an easy way is to create a new contest:
 
 ```sh
@@ -36,7 +36,7 @@ bt new_contest
 and then create or extend the file `<contestdirectory>/.bapctools.yaml` with
 
 ```yaml
-languages:
+lang:
   - nl
   - fr
   - lt
@@ -82,13 +82,13 @@ To create a problem,
 bt new_problem
 ```
 
-will look for the `languages` configuration (for instance, at contest level) and use that by default.
+will look for the `lang` configuration (for instance, at contest level) and use that by default.
 Thus, if the contest is set up as above, you need to do nothing extra.
 
 With arguments, or outside of a contest directory,
 
 ```sh
-bt new_problem --language en --language fr
+bt new_problem --lang en fr
 ```
 
 creates a problem with two languages, English and French.
@@ -108,7 +108,7 @@ creates PDFs for every problem language statement `problem.xy.tex`.
 With arguments,
 
 ```sh
-bt pdf --language en --language fr
+bt pdf --lang en fr
 ```
 
 produces PDFs for English and French.
@@ -117,7 +117,7 @@ The resulting PDFs are named `<problemdirectory>/problem.xy.pdf`.
 
 ## Solution PDF
 
-Similarly, `bt solutions [--language en --language fr]` creates
+Similarly, `bt solutions [--lang en fr]` creates
 `<problemdirectory>/solution.xy.pdf` for the given languages, defaulting to
 all available `solution.xy.tex` files.
 

doc/multiple_languages.md

@@ -1,4 +1,4 @@
-\newcommand{\langbabel}{german}
+\newcommand{\langbabel}{ngerman}
 
 % bapc.cls
 \newcommand{\langblank}{Diese Seite wurde absichtlich leer gelassen.}

latex/lang/de.tex

@@ -1,6 +1,8 @@
 problem_format_version: 2023-07-draft
 type: pass-fail
-name: Identity
+name:
+  en: Identity
+  de: Identität
 credits:
   authors: Ragnar Groot Koerkamp
 uuid: a7d29d67-9b0b-4fd4-ae56-ab2cad5919ab