Use toml for configuration

Author: prandla

Dockerfile

@@ -49,11 +49,11 @@ RUN sudo pip3 install --break-system-packages .
 
 RUN sudo python3 prerequisites.py --yes --cmsuser=cmsuser install
 
-RUN sed 's|/cmsuser:your_password_here@localhost:5432/cmsdb"|/postgres@testdb:5432/cmsdbfortesting"|' ./config/cms.conf.sample \
-    | sudo tee /usr/local/etc/cms-testdb.conf
-RUN sed -e 's|/cmsuser:your_password_here@localhost:5432/cmsdb"|/postgres@devdb:5432/cmsdb"|' -e 's/127.0.0.1/0.0.0.0/' ./config/cms.conf.sample \
-    | sudo tee /usr/local/etc/cms-devdb.conf
-RUN sed 's/127.0.0.1/0.0.0.0/' ./config/cms.ranking.conf.sample | sudo tee /usr/local/etc/cms.ranking.conf
+RUN sed 's|/cmsuser:your_password_here@localhost:5432/cmsdb"|/postgres@testdb:5432/cmsdbfortesting"|' ./config/cms.sample.toml \
+    | sudo tee /usr/local/etc/cms-testdb.toml
+RUN sed -e 's|/cmsuser:your_password_here@localhost:5432/cmsdb"|/postgres@devdb:5432/cmsdb"|' -e 's/127.0.0.1/0.0.0.0/' ./config/cms.sample.toml \
+    | sudo tee /usr/local/etc/cms-devdb.toml
+RUN sed 's/127.0.0.1/0.0.0.0/' ./config/cms_ranking.sample.toml | sudo tee /usr/local/etc/cms_ranking.toml
 
 ENV LANG C.UTF-8
 

cms/conf.py

@@ -22,7 +22,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import errno
-import json
+import tomllib
 import logging
 import os
 import sys
@@ -81,7 +81,7 @@ class AsyncConfig:
 class Config:
     """This class will contain the configuration for CMS. This needs
     to be populated at the initilization stage. This is loaded by
-    default with some sane data. See cms.conf.sample in the config
+    default with some sane data. See cms.sample.toml in the config
     directory for information on the meaning of the fields.
 
     """
@@ -166,13 +166,21 @@ def __init__(self):
         self.max_jobs_per_user = 10
         self.pdf_printing_allowed = False
 
+        # PrometheusExporter
+        self.prometheus_listen_address = "127.0.0.1"
+        self.prometheus_listen_port = 8811
+
+        # TelegramBot
+        self.telegram_bot_token = None
+        self.telegram_bot_chat_id = None
+
         self.log_dir = os.path.join("/", "var", "local", "log", "cms")
         self.cache_dir = os.path.join("/", "var", "local", "cache", "cms")
         self.data_dir = os.path.join("/", "var", "local", "lib", "cms")
         self.run_dir = os.path.join("/", "var", "local", "run", "cms")
         paths = [
-            os.path.join("/", "usr", "local", "etc", "cms.conf"),
-            os.path.join("/", "etc", "cms.conf"),
+            os.path.join("/", "usr", "local", "etc", "cms.toml"),
+            os.path.join("/", "etc", "cms.toml"),
         ]
 
         # Allow user to override config file path using environment
@@ -202,23 +210,21 @@ def _load(self, paths: list[str]):
 
     def _load_unique(self, path: str):
         """Populate the Config class with everything that sits inside
-        the JSON file path (usually something like /etc/cms.conf). The
+        the TOML file path (usually something like /etc/cms.toml). The
         only pieces of data treated differently are the elements of
         core_services and other_services that are sent to async
         config.
 
-        Services whose name begins with an underscore are ignored, so
-        they can be commented out in the configuration file.
-
-        path: the path of the JSON config file.
+        path: the path of the TOML config file.
 
         """
         # Load config file.
         try:
-            with open(path, 'rt', encoding='utf-8') as f:
-                data = json.load(f)
+            with open(path, 'rb') as f:
+                data = tomllib.load(f)
         except FileNotFoundError:
-            logger.debug("Couldn't find config file %s.", path)
+            logger.debug("Couldn't find config file %s (maybe you need to "
+                         "convert it from cms.conf to cms.toml?).", path)
             return False
         except OSError as error:
             logger.warning("I/O error while opening file %s: [%s] %s",
@@ -257,6 +263,8 @@ def _load_unique(self, path: str):
 
         # Put everything else in self.
         for key, value in data.items():
+            if not hasattr(self, key):
+                logger.warning("Unrecognized key %s in config!", key)
             setattr(self, key, value)
 
         return True

cms/db/util.py

@@ -74,7 +74,7 @@ def test_db_connection():
     except OperationalError as e:
         logger.error(e)
         raise ConfigError("Operational error while talking to the DB. "
-                          "Is the connection string in cms.conf correct?")
+                          "Is the connection string in cms.toml correct?")
 
 
 def get_contest_list(session: "Session | None" = None) -> list[Contest]:

cms/io/service.py

@@ -97,7 +97,7 @@ def __init__(self, shard: int = 0):
             address = get_service_address(self._my_coord)
         except KeyError:
             raise ConfigError("Unable to find address for service %r. "
-                              "Is it specified in core_services in cms.conf?" %
+                              "Is it specified in core_services in cms.toml?" %
                               (self._my_coord,))
 
         self.rpc_server = StreamServer(address, self._connection_handler)
@@ -197,7 +197,7 @@ def connect_to(
                 # the service was optional.
                 if must_be_present:
                     raise ConfigError("Missing address and port for %s "
-                                      "in cms.conf." % (coord, ))
+                                      "in cms.toml." % (coord, ))
                 else:
                     service = FakeRemoteServiceClient(coord, None)
             service.connect()

cms/server/admin/templates/base.html

@@ -85,7 +85,7 @@ <h1 class="child">{{ contest.name }}</h1>
         </div>
         {% if config.secret_key == config.secret_key_default %}
         <div class="secret_notice">
-          Change secret_key in cms.conf!<br/>
+          Change secret_key in cms.toml!<br/>
           For example,<br/>
           {{ get_hex_random_key() }}
         </div>

cms/server/contest/server.py

@@ -80,7 +80,7 @@ def __init__(self, shard: int, contest_id: int | None = None):
             raise ConfigError("Wrong shard number for %s, or missing "
                               "address/port configuration. Please check "
                               "contest_listen_address and contest_listen_port "
-                              "in cms.conf." % __name__)
+                              "in cms.toml." % __name__)
 
         self.contest_id = contest_id
 

cmsranking/Config.py

@@ -17,10 +17,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import errno
-import json
 import logging
 import os
 import sys
+import tomllib
 import typing
 
 import pkg_resources
@@ -63,8 +63,8 @@ def __init__(self):
         self.log_dir = os.path.join("/", "var", "local", "log", "cms", "ranking")
         self.lib_dir = os.path.join("/", "var", "local", "lib", "cms", "ranking")
         self.conf_paths = [
-            os.path.join("/", "usr", "local", "etc", "cms.ranking.conf"),
-            os.path.join("/", "etc", "cms.ranking.conf"),
+            os.path.join("/", "usr", "local", "etc", "cms_ranking.toml"),
+            os.path.join("/", "etc", "cms_ranking.toml"),
         ]
 
         # Allow users to override config file path using environment
@@ -121,7 +121,7 @@ def _load_many(self, conf_paths: list[str]) -> bool:
         """
         for conf_path in conf_paths:
             try:
-                with open(conf_path, "rt", encoding="utf-8") as conf_fobj:
+                with open(conf_path, "rb") as conf_fobj:
                     logger.info("Using config file %s.", conf_path)
                     return self._load_one(conf_fobj)
             except FileNotFoundError:
@@ -135,10 +135,10 @@ def _load_many(self, conf_paths: list[str]) -> bool:
         logger.warning("No config file found, using hardcoded defaults.")
         return True
 
-    def _load_one(self, conf_fobj: typing.TextIO) -> bool:
+    def _load_one(self, conf_fobj: typing.BinaryIO) -> bool:
         """Populate config parameters from the given file.
 
-        Parse it as JSON and store in self all configuration properties
+        Parse it as TOML and store in self all configuration properties
         it defines. Log critical message and return False if anything
         goes wrong or seems odd.
 
@@ -148,19 +148,16 @@ def _load_one(self, conf_fobj: typing.TextIO) -> bool:
         """
         # Parse config file.
         try:
-            data = json.load(conf_fobj)
+            data = tomllib.load(conf_fobj)
         except ValueError:
-            logger.critical("Config file is invalid JSON.")
+            logger.critical("Config file is invalid TOML.")
             return False
 
         # Store every config property.
         for key, value in data.items():
-            if key.startswith("_"):
-                continue
             if not hasattr(self, key):
                 logger.critical("Invalid field %s in config file, maybe a "
-                                "typo? (use leading underscore to ignore).",
-                                key)
+                                "typo?", key)
                 return False
             setattr(self, key, value)
         return True

cmsranking/RankingWebServer.py

@@ -526,7 +526,7 @@ def main() -> int:
     """
     parser = argparse.ArgumentParser(
         description="Ranking for CMS.")
-    parser.add_argument("--config", type=argparse.FileType("rt"),
+    parser.add_argument("--config", type=argparse.FileType("rb"),
                         help="override config file")
     parser.add_argument("-d", "--drop", action="store_true",
                         help="drop the data already stored")

cmstestsuite/functionaltestframework.py

@@ -27,6 +27,7 @@
 import re
 import sys
 import time
+import tomllib
 
 from cmstestsuite import CONFIG, TestException, sh
 from cmstestsuite.web import Browser
@@ -73,7 +74,7 @@ def __new__(cls):
         return FunctionalTestFramework.__instance
 
     def __init__(self):
-        # This holds the decoded-JSON of the cms.conf configuration file.
+        # This holds the decoded-TOML of the cms.toml configuration file.
         # Lazily loaded, to be accessed through the getter method.
         self._cms_config = None
 
@@ -141,8 +142,8 @@ def initialize_aws(self) -> str:
 
     def get_cms_config(self):
         if self._cms_config is None:
-            with open("%(CONFIG_PATH)s" % CONFIG, "rt", encoding="utf-8") as f:
-                self._cms_config = json.load(f)
+            with open("%(CONFIG_PATH)s" % CONFIG, "rb") as f:
+                self._cms_config = tomllib.load(f)
         return self._cms_config
 
     def admin_req(self, path, args=None, files=None):

cmstestsuite/testrunner.py

@@ -94,9 +94,10 @@ def load_cms_conf(self):
         except subprocess.CalledProcessError:
             git_root = None
         CONFIG["TEST_DIR"] = git_root
-        CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"]
+        # TODO: this probably doesn't work: config/cms.toml doesn't even exist
+        CONFIG["CONFIG_PATH"] = "%s/config/cms.toml" % CONFIG["TEST_DIR"]
         if CONFIG["TEST_DIR"] is None:
-            CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf"
+            CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.toml"
 
         # Override CMS config path when environment variable is present
         CMS_CONFIG_ENV_VAR = "CMS_CONFIG"