Introduce structured YAML settings for Recursor.

Mostly written in Rust, using CXX and Serde

Code generation is used to generate both the old style config tables as
the new Rust based code. The code generation also produces the code
to covert old styel to new style and documentation.

Th main entry point for code generationo is settings/generate.py,
using the table table.py

Existing configs continue to work as before.

.github/actions/spell-check/expect.txt

@@ -451,6 +451,7 @@ fulltoc
 fullycapable
 Furnell
 Fusl
+fwzones
 FYhvws
 FZq
 gaba
@@ -1121,7 +1122,9 @@ rdata
 rdqueries
 rdynamic
 reconnections
+recordcache
 recursor
+recursord
 recursordist
 Recursordoc
 Recuweb
@@ -1512,6 +1515,7 @@ webbased
 webdocs
 webhandler
 webpassword
+webservice
 Webspider
 Wegener
 Weimer
@@ -1559,6 +1563,8 @@ xpf
 XRecord
 XXXXXX
 yahttp
+yamlconversion
+yamlsettings
 Yehuda
 yeswehack
 Yiu

.github/workflows/build-and-test-all.yml

@@ -120,6 +120,7 @@ jobs:
           path: ~/.ccache
           key: recursor-${{ matrix.sanitizers }}-ccache-${{ steps.get-stamp.outputs.stamp }}
           restore-keys: recursor-${{ matrix.sanitizers }}-ccache-
+      - run: inv ci-install-rust ${{ env.REPO_HOME }}
       - run: inv ci-autoconf
       - run: inv ci-rec-configure
       - run: inv ci-rec-make-bear

.github/workflows/codeql-analysis.yml

@@ -104,6 +104,7 @@ jobs:
                 libwslay-dev \
                 libyaml-cpp-dev \
                 ragel \
+                rustc \
                 unixodbc-dev
 
     - name: Build auth
@@ -133,6 +134,8 @@ jobs:
         autoreconf -vfi
         ./configure --enable-unit-tests --enable-nod --enable-dnstap CFLAGS='-O0' CXXFLAGS='-O0'
         make -j8 -C ext
+        make -j8 -C settings
+        make -j8 -C settings/rust
         make htmlfiles.h
         make -j4 pdns_recursor rec_control
 

builder-support/debian/recursor/debian-buster/rules

@@ -40,6 +40,7 @@ override_dh_auto_install:
 	install -d debian/pdns-recursor/usr/share/pdns-recursor/snmp
 	install -m 644 -t debian/pdns-recursor/usr/share/pdns-recursor/snmp RECURSOR-MIB.txt
 	rm -f debian/pdns-recursor/etc/powerdns/recursor.conf-dist
+	rm -f debian/pdns-recursor/etc/powerdns/recursor.yml-dist
 	./pdns_recursor --no-config --config=default | sed \
 		-e 's!^# config-dir=.*!config-dir=/etc/powerdns!' \
 		-e 's!^# hint-file=.*!&\nhint-file=/usr/share/dns/root.hints!' \

builder-support/dockerfiles/Dockerfile.debbuild-prepare

@@ -1,11 +1,16 @@
 FROM dist-base as package-builder
 ARG APT_URL
-RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends devscripts dpkg-dev build-essential python3-venv equivs
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends devscripts dpkg-dev build-essential python3-venv equivs curl
 
 RUN mkdir /dist /pdns
 WORKDIR /pdns
 
 ADD builder/helpers/ /pdns/builder/helpers/
+ADD builder-support/helpers/ /pdns/builder-support/helpers/
+
+@IF [ -n "$M_recursor$M_all" ]
+RUN /pdns/builder-support/helpers/install_rust.sh
+@ENDIF
 
 # Used for -p option to only build specific packages
 ARG BUILDER_PACKAGE_MATCH

builder-support/dockerfiles/Dockerfile.rpmbuild

@@ -1,10 +1,10 @@
 FROM dist-base as package-builder
 RUN touch /var/lib/rpm/* && if $(grep -q 'release 7' /etc/redhat-release); then \
       yum upgrade -y && \
-      yum install -y rpm-build rpmdevtools python2 python3 "@Development Tools"; \
+      yum install -y rpm-build rpmdevtools python2 python3 curl "@Development Tools"; \
     else \
       yum upgrade -y && \
-      yum install -y rpm-build rpmdevtools python3 "@Development Tools"; \
+      yum install -y rpm-build rpmdevtools python3 curl "@Development Tools"; \
     fi
 
 RUN mkdir /dist /pdns
@@ -13,6 +13,11 @@ RUN rpmdev-setuptree
 
 # Only ADD/COPY the files you really need for efficient docker caching.
 ADD builder/helpers/ /pdns/builder/helpers/
+ADD builder-support/helpers/ /pdns/builder-support/helpers/
+
+@IF [ -n "$M_recursor$M_all" ]
+RUN /pdns/builder-support/helpers/install_rust.sh
+@ENDIF
 
 # Used for -p option to only build specific spec files
 ARG BUILDER_PACKAGE_MATCH

builder-support/helpers/install_rust.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+set -e
+
+ARCH=$(arch)
+
+# Default version
+RUST_VERSION=rust-1.72.0-$ARCH-unknown-linux-gnu
+
+if [ $# -ge 1 ]; then
+    RUST_VERSION=$1
+    shift
+fi
+
+SITE=https://downloads.powerdns.com/rust
+RUST_TARBALL=$RUST_VERSION.tar.gz
+
+SHA256SUM_x86_64=f2bbe23e685852104fd48d0e34ac981b0917e76c62cfcd6d0ac5283e4710c7b9
+
+NAME=SHA256SUM_$ARCH
+eval VALUE=\$$NAME
+if [ -z "$VALUE" ]; then
+    echo "$0: No SHA256 defined for $ARCH" > /dev/stderr
+    exit 1
+fi
+
+# Procedure to update the Rust tarball:
+# 1. Download tarball and signature (.asc) file from
+#    https://forge.rust-lang.org/infra/other-installation-methods.html "Standalone installers" section
+# 2. Import Rust signing key into your gpg if not already done so
+# 3. Run gpg --verify $RUST_TARBALL.asc and make sure it is OK
+# 4. Run sha256sum $RUST_TARBALL and set SHA256SUM above, don't forget to update RUST_VERSION as well
+# 5. Make $RUST_TARBALL available from https://downloads.powerdns.com/rust
+#
+cd /tmp
+echo $0: Downloading $RUST_TARBALL
+
+curl -o $RUST_TARBALL $SITE/$RUST_TARBALL
+# Line below should echo two spaces between digest and name
+echo $VALUE"  "$RUST_TARBALL | sha256sum -c -
+tar -zxf $RUST_TARBALL
+cd $RUST_VERSION
+./install.sh --prefix=/usr
+cd ..
+rm -rf $RUST_TARBALL $RUST_VERSION

builder-support/specs/pdns-recursor.spec

@@ -125,4 +125,5 @@ systemctl daemon-reload ||:
 %dir %{_sysconfdir}/%{name}
 %dir %{_sysconfdir}/%{name}/recursor.d
 %config(noreplace) %{_sysconfdir}/%{name}/recursor.conf
+%config %{_sysconfdir}/%{name}/recursor.yml-dist
 %doc README

pdns/arguments.cc

@@ -81,11 +81,6 @@ vector<string> ArgvMap::list()
   return ret;
 }
 
-string ArgvMap::getHelp(const string& item)
-{
-  return helpmap[item];
-}
-
 string& ArgvMap::set(const string& var, const string& help)
 {
   helpmap[var] = help;
@@ -366,7 +361,8 @@ static const map<string, string> deprecateList = {
   {"xpf-rr-code", "Proxy Protocol"},
 };
 
-void ArgvMap::warnIfDeprecated(const string& var)
+// NOLINTNEXTLINE(readability-convert-member-functions-to-static): accesses d_log (compiled out in auth, hence clang-tidy message)
+void ArgvMap::warnIfDeprecated(const string& var) const
 {
   const auto msg = deprecateList.find(var);
   if (msg != deprecateList.end()) {
@@ -375,6 +371,12 @@ void ArgvMap::warnIfDeprecated(const string& var)
   }
 }
 
+string ArgvMap::isDeprecated(const string& var)
+{
+  const auto msg = deprecateList.find(var);
+  return msg != deprecateList.end() ? msg->second : "";
+}
+
 void ArgvMap::parseOne(const string& arg, const string& parseOnly, bool lax)
 {
   string var;
@@ -475,7 +477,7 @@ void ArgvMap::preParse(int& argc, char** argv, const string& arg)
   }
 }
 
-bool ArgvMap::parseFile(const char* fname, const string& arg, bool lax)
+bool ArgvMap::parseFile(const string& fname, const string& arg, bool lax)
 {
   string line;
   string pline;
@@ -523,19 +525,19 @@ bool ArgvMap::parseFile(const char* fname, const string& arg, bool lax)
   return true;
 }
 
-bool ArgvMap::preParseFile(const char* fname, const string& arg, const string& theDefault)
+bool ArgvMap::preParseFile(const string& fname, const string& arg, const string& theDefault)
 {
   d_params[arg] = theDefault;
 
   return parseFile(fname, arg, false);
 }
 
-bool ArgvMap::file(const char* fname, bool lax)
+bool ArgvMap::file(const string& fname, bool lax)
 {
   return file(fname, lax, false);
 }
 
-bool ArgvMap::file(const char* fname, bool lax, bool included)
+bool ArgvMap::file(const string& fname, bool lax, bool included)
 {
   if (!parmIsset("include-dir")) { // inject include-dir
     set("include-dir", "Directory to include configuration files from");
@@ -550,7 +552,7 @@ bool ArgvMap::file(const char* fname, bool lax, bool included)
   // handle include here (avoid re-include)
   if (!included && !d_params["include-dir"].empty()) {
     std::vector<std::string> extraConfigs;
-    gatherIncludes(extraConfigs);
+    gatherIncludes(d_params["include-dir"], ".conf", extraConfigs);
     for (const std::string& filename : extraConfigs) {
       if (!file(filename.c_str(), lax, true)) {
         SLOG(g_log << Logger::Error << filename << " could not be parsed" << std::endl,
@@ -563,30 +565,31 @@ bool ArgvMap::file(const char* fname, bool lax, bool included)
   return true;
 }
 
-void ArgvMap::gatherIncludes(std::vector<std::string>& extraConfigs)
+// NOLINTNEXTLINE(readability-convert-member-functions-to-static): accesses d_log (compiled out in auth, hence clang-tidy message)
+void ArgvMap::gatherIncludes(const std::string& directory, const std::string& suffix, std::vector<std::string>& extraConfigs)
 {
-  extraConfigs.clear();
-  if (d_params["include-dir"].empty()) {
+  if (directory.empty()) {
     return; // nothing to do
   }
 
-  DIR* dir = nullptr;
-  if ((dir = opendir(d_params["include-dir"].c_str())) == nullptr) {
+  auto dir = std::unique_ptr<DIR, decltype(&closedir)>(opendir(directory.c_str()), closedir);
+  if (dir == nullptr) {
     int err = errno;
-    string msg = d_params["include-dir"] + " is not accessible: " + stringerror(err);
+    string msg = directory + " is not accessible: " + stringerror(err);
     SLOG(g_log << Logger::Error << msg << std::endl,
-         d_log->error(Logr::Error, err, "Directory is not accessible", "name", Logging::Loggable(d_params["include-dir"])));
+         d_log->error(Logr::Error, err, "Directory is not accessible", "name", Logging::Loggable(directory)));
     throw ArgException(msg);
   }
 
+  std::vector<std::string> vec;
   struct dirent* ent = nullptr;
-  while ((ent = readdir(dir)) != nullptr) { // NOLINT(concurrency-mt-unsafe): see Linux man page
+  while ((ent = readdir(dir.get())) != nullptr) { // NOLINT(concurrency-mt-unsafe): see Linux man page
     if (ent->d_name[0] == '.') {
       continue; // skip any dots
     }
-    if (boost::ends_with(ent->d_name, ".conf")) {
+    if (boost::ends_with(ent->d_name, suffix)) {
       // build name
-      string name = d_params["include-dir"] + "/" + ent->d_name; // NOLINT: Posix API
+      string name = directory + "/" + ent->d_name; // NOLINT: Posix API
       // ensure it's readable file
       struct stat statInfo
       {
@@ -595,12 +598,11 @@ void ArgvMap::gatherIncludes(std::vector<std::string>& extraConfigs)
         string msg = name + " is not a regular file";
         SLOG(g_log << Logger::Error << msg << std::endl,
              d_log->info(Logr::Error, "Unable to open non-regular file", "name", Logging::Loggable(name)));
-        closedir(dir);
         throw ArgException(msg);
       }
-      extraConfigs.push_back(name);
+      vec.emplace_back(name);
     }
   }
-  std::sort(extraConfigs.begin(), extraConfigs.end(), CIStringComparePOSIX());
-  closedir(dir);
+  std::sort(vec.begin(), vec.end(), CIStringComparePOSIX());
+  extraConfigs.insert(extraConfigs.end(), vec.begin(), vec.end());
 }

pdns/arguments.hh

@@ -81,15 +81,15 @@ public:
     parse(argc, argv, true);
   }
   void preParse(int& argc, char** argv, const string& arg); //!< use this to preparse a single var
-  bool preParseFile(const char* fname, const string& arg, const string& theDefault = ""); //!< use this to preparse a single var in configuration
+  bool preParseFile(const string& fname, const string& arg, const string& theDefault = ""); //!< use this to preparse a single var in configuration
 
-  bool file(const char* fname, bool lax = false); //!< Parses a file with parameters
-  bool file(const char* fname, bool lax, bool included);
-  bool laxFile(const char* fname)
+  bool file(const string& fname, bool lax = false); //!< Parses a file with parameters
+  bool file(const string& fname, bool lax, bool included);
+  bool laxFile(const string& fname)
   {
     return file(fname, true);
   }
-  bool parseFile(const char* fname, const string& arg, bool lax); //<! parse one line
+  bool parseFile(const string& fname, const string& arg, bool lax); //<! parse one line
   bool parmIsset(const string& var); //!< Checks if a parameter is set to *a* value
   bool mustDo(const string& var); //!< if a switch is given, if we must do something (--help)
   int asNum(const string& arg, int def = 0); //!< return a variable value as a number or the default if the variable is empty
@@ -109,23 +109,32 @@ public:
   void setDefaults();
 
   vector<string> list();
-  string getHelp(const string& item);
-
+  [[nodiscard]] string getHelp(const string& item) const
+  {
+    auto iter = helpmap.find(item);
+    return iter == helpmap.end() ? "" : iter->second;
+  }
+  [[nodiscard]] string getDefault(const string& item) const
+  {
+    auto iter = defaultmap.find(item);
+    return iter == defaultmap.end() ? "" : iter->second;
+  }
   using param_t = map<string, string>; //!< use this if you need to know the content of the map
 
   param_t::const_iterator begin(); //!< iterator semantics
   param_t::const_iterator end(); //!< iterator semantics
   const string& operator[](const string&); //!< iterator semantics
   const vector<string>& getCommands();
-  void gatherIncludes(std::vector<std::string>& extraConfigs);
+  void gatherIncludes(const std::string& dir, const std::string& suffix, std::vector<std::string>& extraConfigs);
+  void warnIfDeprecated(const string& var) const;
+  [[nodiscard]] static string isDeprecated(const string& var);
 #ifdef RECURSOR
   void setSLog(Logr::log_t log)
   {
     d_log = log;
   }
 #endif
 private:
-  void warnIfDeprecated(const string& var);
   void parseOne(const string& arg, const string& parseOnly = "", bool lax = false);
   static string formatOne(bool running, bool full, const string& var, const string& help, const string& theDefault, const string& current);
   map<string, string> d_params;

pdns/recursordist/.gitignore

@@ -34,6 +34,7 @@
 /rec_control
 /pdns-recursor-*
 /recursor.conf-dist
+/recursor.yml-dist
 /ext/Makefile
 /ext/Makefile.in
 /dnsmessage.pb.cc