WIP: implement a secret vars store in nixpkgs

nixos/modules/module-list.nix

@@ -1709,6 +1709,8 @@
   ./system/boot/timesyncd.nix
   ./system/boot/tmp.nix
   ./system/boot/uvesafb.nix
+  ./system/vars/options.nix
+  ./system/vars/on-machine-backend.nix
   ./system/etc/etc-activation.nix
   ./tasks/auto-upgrade.nix
   ./tasks/bcache.nix

nixos/modules/services/networking/syncthing.nix

@@ -621,6 +621,21 @@ in {
 
   config = mkIf cfg.enable {
 
+    vars.generators.syncthing = {
+      files."cert.pem" = {};
+      files."key.pem" = {};
+      files."syncthing.pub".secret = false;
+      runtimeInputs = [
+        pkgs.coreutils
+        pkgs.gnugrep
+        pkgs.syncthing
+      ];
+      script = ''
+        syncthing generate --config "$out"
+        < "$out"/config.xml grep -oP '(?<=<device id=")[^"]+' | uniq > "$out"/syncthing.pub
+      '';
+    };
+
     networking.firewall = mkIf cfg.openDefaultPorts {
       allowedTCPPorts = [ 22000 ];
       allowedUDPPorts = [ 21027 22000 ];

nixos/modules/system/vars/on-machine-backend.nix

@@ -0,0 +1,140 @@
+# we use this vars backend as an example backend.
+# this generates a script which creates the values at the expected path.
+# this script has to be run manually (I guess after updating the system) to generate the required vars.
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.vars.settings.on-machine;
+  sortedGenerators =
+    (lib.toposort (a: b: builtins.elem a.name b.dependencies) (lib.attrValues config.vars.generators))
+    .result;
+
+  promptCmd = {
+    hidden = "read -sr prompt_value";
+    line = "read -r prompt_value";
+    multiline = ''
+      echo 'press control-d to finish'
+      prompt_value=$(cat)
+    '';
+  };
+  generate-vars = pkgs.writeShellApplication {
+    name = "generate-vars";
+    text = ''
+      set -efuo pipefail
+
+      PATH=${lib.makeBinPath [ pkgs.coreutils ]}
+
+      # make the output directory overridable
+      OUT_DIR=''${OUT_DIR:-${cfg.fileLocation}}
+
+      # check if all files are present or all files are missing
+      # if not, they are in an incosistent state and we bail out
+      ${lib.concatMapStringsSep "\n" (gen: ''
+        all_files_missing=true
+        all_files_present=true
+        ${lib.concatMapStringsSep "\n" (file: ''
+          if test -e ${file.path} ; then
+            all_files_missing=false
+          else
+            all_files_present=false
+          fi
+        '') (lib.attrValues gen.files)}
+
+        if [ $all_files_missing = false ] && [ $all_files_present = false ] ; then
+          echo "Inconsistent state for generator ${gen.name}"
+          exit 1
+        fi
+        if [ $all_files_present = true ] ; then
+          echo "All secrets for ${gen.name} are present"
+        elif [ $all_files_missing = true ] ; then
+
+          # prompts
+          prompts=$(mktemp -d)
+          trap 'rm -rf $prompts' EXIT
+          export prompts
+          mkdir -p "$prompts"
+          ${lib.concatMapStringsSep "\n" (prompt: ''
+            echo ${lib.escapeShellArg prompt.description}
+            ${promptCmd.${prompt.type}}
+            echo -n "$prompt_value" > "$prompts"/${prompt.name}
+          '') (lib.attrValues gen.prompts)}
+          echo "Generating vars for ${gen.name}"
+
+          # dependencies
+          in=$(mktemp -d)
+          trap 'rm -rf $in' EXIT
+          export in
+          mkdir -p "$in"
+          ${lib.concatMapStringsSep "\n" (input: ''
+            mkdir -p "$in"/${input}
+            ${lib.concatMapStringsSep "\n" (file: ''
+              cp "$OUT_DIR"/${
+                if file.secret then "secret" else "public"
+              }/${input}/${file.name} "$in"/${input}/${file.name}
+            '') (lib.attrValues config.vars.generators.${input}.files)}
+          '') gen.dependencies}
+
+          # outputs
+          out=$(mktemp -d)
+          trap 'rm -rf $out' EXIT
+          export out
+          mkdir -p "$out"
+
+          (
+            # prepare PATH
+            unset PATH
+            ${lib.optionalString (gen.runtimeInputs != [ ]) ''
+              PATH=${lib.makeBinPath gen.runtimeInputs}
+              export PATH
+            ''}
+
+            # actually run the generator
+            ${gen.script}
+          )
+
+          # check if all files got generated
+          ${lib.concatMapStringsSep "\n" (file: ''
+            if ! test -e "$out"/${file.name} ; then
+              echo 'generator ${gen.name} failed to generate ${file.name}'
+              exit 1
+            fi
+          '') (lib.attrValues gen.files)}
+
+          # move the files to the correct location
+          ${lib.concatMapStringsSep "\n" (file: ''
+            OUT_FILE="$OUT_DIR"/${if file.secret then "secret" else "public"}/${file.generator}/${file.name}
+            mkdir -p "$(dirname "$OUT_FILE")"
+            mv "$out"/'${file.name}' "$OUT_FILE"
+          '') (lib.attrValues gen.files)}
+          rm -rf "$out"
+        fi
+      '') sortedGenerators}
+    '';
+  };
+in
+{
+  options.vars.settings.on-machine = {
+    enable = lib.mkEnableOption "Enable on-machine vars backend";
+    fileLocation = lib.mkOption {
+      type = lib.types.str;
+      default = "/etc/vars";
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    vars.settings.fileModule = file: {
+      path =
+        if file.config.secret then
+          "${cfg.fileLocation}/secret/${file.config.generator}/${file.config.name}"
+        else
+          "${cfg.fileLocation}/public/${file.config.generator}/${file.config.name}";
+    };
+    environment.systemPackages = [
+      generate-vars
+    ];
+    system.build.generate-vars = generate-vars;
+  };
+}

nixos/modules/system/vars/options.nix

@@ -0,0 +1,184 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+{
+  options.vars = {
+    settings = {
+      fileModule = lib.mkOption {
+        type = lib.types.deferredModule;
+        internal = true;
+        description = ''
+          A module to be imported in every vars.files.<name> submodule.
+          Used by backends to define the `path` attribute.
+
+          Takes the file as an arument and returns maybe an attrset with should at least contain the `path` attribute.
+          Can be used to set other file attributes as well, like `value`.
+        '';
+        default = { };
+      };
+    };
+    generators = lib.mkOption {
+      description = ''
+        A set of generators that can be used to generate files.
+        Generators are scripts that produce files based on the values of other generators and user input.
+        Each generator is expected to produce a set of files under a directory.
+      '';
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (generator: {
+          options = {
+            name = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                The name of the generator.
+                This name will be used to refer to the generator in other generators.
+              '';
+              readOnly = true;
+              default = generator.config._module.args.name;
+              defaultText = "Name of the generator";
+            };
+
+            dependencies = lib.mkOption {
+              description = ''
+                A list of other generators that this generator depends on.
+                The output values of these generators will be available to the generator script as files.
+                For example, the file 'file1' of a dependency named 'dep1' will be available via $in/dep1/file1.
+              '';
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+            };
+            files = lib.mkOption {
+              description = ''
+                A set of files to generate.
+                The generator 'script' is expected to produce exactly these files under $out.
+              '';
+              defaultText = "attrs of files";
+              type = lib.types.attrsOf (
+                lib.types.submodule (file: {
+                  imports = [
+                    config.vars.settings.fileModule
+                  ];
+                  options = {
+                    name = lib.mkOption {
+                      type = lib.types.str;
+                      description = ''
+                        name of the public fact
+                      '';
+                      readOnly = true;
+                      default = file.config._module.args.name;
+                      defaultText = "Name of the file";
+                    };
+                    generator = lib.mkOption {
+                      description = ''
+                        The generator that produces the file.
+                        This is the name of another generator.
+                      '';
+                      type = lib.types.str;
+                      readOnly = true;
+                      internal = true;
+                      default = generator.config.name;
+                      defaultText = "Name of the generator";
+                    };
+                    deploy = lib.mkOption {
+                      description = ''
+                        Whether the file should be deployed to the target machine.
+
+                        Enable this if the generated file is only used as an input to other generators.
+                      '';
+                      type = lib.types.bool;
+                      default = true;
+                    };
+                    secret = lib.mkOption {
+                      description = ''
+                        Whether the file should be treated as a secret.
+                      '';
+                      type = lib.types.bool;
+                      default = true;
+                    };
+                    path = lib.mkOption {
+                      description = ''
+                        The path to the file containing the content of the generated value.
+                        This will be set automatically
+                      '';
+                      type = lib.types.str;
+                    };
+                  };
+                })
+              );
+            };
+            prompts = lib.mkOption {
+              description = ''
+                A set of prompts to ask the user for values.
+                Prompts are available to the generator script as files.
+                For example, a prompt named 'prompt1' will be available via $prompts/prompt1
+              '';
+              default = { };
+              type = lib.types.attrsOf (
+                lib.types.submodule (prompt: {
+                  options = {
+                    name = lib.mkOption {
+                      description = ''
+                        The name of the prompt.
+                        This name will be used to refer to the prompt in the generator script.
+                      '';
+                      type = lib.types.str;
+                      default = prompt.config._module.args.name;
+                      defaultText = "Name of the prompt";
+                    };
+                    description = lib.mkOption {
+                      description = ''
+                        The description of the prompted value
+                      '';
+                      type = lib.types.str;
+                      example = "SSH private key";
+                      default = prompt.config._module.args.name;
+                      defaultText = "Name of the prompt";
+                    };
+                    type = lib.mkOption {
+                      description = ''
+                        The input type of the prompt.
+                        The following types are available:
+                          - hidden: A hidden text (e.g. password)
+                          - line: A single line of text
+                          - multiline: A multiline text
+                      '';
+                      type = lib.types.enum [
+                        "hidden"
+                        "line"
+                        "multiline"
+                      ];
+                      default = "line";
+                    };
+                  };
+                })
+              );
+            };
+            runtimeInputs = lib.mkOption {
+              description = ''
+                A list of packages that the generator script requires.
+                These packages will be available in the PATH when the script is run.
+              '';
+              type = lib.types.listOf lib.types.package;
+              default = [ pkgs.coreutils ];
+            };
+            script = lib.mkOption {
+              description = ''
+                The script to run to generate the files.
+                The script will be run with the following environment variables:
+                  - $in: The directory containing the output values of all declared dependencies
+                  - $out: The output directory to put the generated files
+                  - $prompts: The directory containing the prompted values as files
+                The script should produce the files specified in the 'files' attribute under $out.
+              '';
+              type = lib.types.either lib.types.str lib.types.path;
+              default = "";
+            };
+          };
+        })
+      );
+    };
+  };
+}

nixos/tests/all-tests.nix

@@ -1132,6 +1132,7 @@ in {
   user-home-mode = handleTest ./user-home-mode.nix {};
   ustreamer = handleTest ./ustreamer.nix {};
   uwsgi = handleTest ./uwsgi.nix {};
+  vars = handleTest ./vars.nix {};
   v2ray = handleTest ./v2ray.nix {};
   varnish60 = handleTest ./varnish.nix { package = pkgs.varnish60; };
   varnish75 = handleTest ./varnish.nix { package = pkgs.varnish75; };