New custom-persist feature

Makefile

@@ -154,6 +154,8 @@ install-systemd: install-init
 	install -m 0644 vm-systemd/xendriverdomain.service $(DESTDIR)/etc/systemd/system/
 	install -m 0644 vm-systemd/80-qubes-vif.link $(DESTDIR)$(SYSLIBDIR)/systemd/network/
 	install -m 0644 vm-systemd/30_resolved-no-mdns-or-llmnr.conf $(DESTDIR)$(SYSLIBDIR)/systemd/resolved.conf.d/
+	install -m 0644 vm-systemd/home.mount $(DESTDIR)$(SYSLIBDIR)/systemd/system/
+	install -m 0644 vm-systemd/usr-local.mount $(DESTDIR)$(SYSLIBDIR)/systemd/system/
 
 .PHONY: install-sysvinit
 install-sysvinit: install-init

debian/qubes-core-agent.install

@@ -86,6 +86,7 @@ lib/systemd/system/org.cups.cupsd.path.d/30_qubes.conf
 lib/systemd/system/org.cups.cupsd.service.d/30_qubes.conf
 lib/systemd/system/org.cups.cupsd.socket.d/30_qubes.conf
 lib/systemd/system/dev-xvdc1-swap.service
+lib/systemd/system/qubes-bind-dirs.service
 lib/systemd/system/qubes-early-vm-config.service
 lib/systemd/system/qubes-misc-post.service
 lib/systemd/system/qubes-mount-dirs.service
@@ -107,6 +108,8 @@ lib/systemd/system/sysinit.target.d/30_qubes.conf
 lib/systemd/system/systemd-timesyncd.service.d/30_qubes.conf
 lib/systemd/system/systemd-logind.service.d/30_qubes.conf
 lib/systemd/resolved.conf.d/30_resolved-no-mdns-or-llmnr.conf
+lib/systemd/system/home.mount
+lib/systemd/system/usr-local.mount
 usr/lib/sysctl.d/20-qubes-core.conf
 usr/lib/systemd/user/tracker-extract-3.service.d/30_qubes.conf
 usr/lib/systemd/user/tracker-miner-fs-3.service.d/30_qubes.conf

filesystem/fstab

@@ -6,8 +6,6 @@
 
 /dev/mapper/dmroot   /              ext4    defaults,discard,noatime    1 1
 /dev/xvdb            /rw            auto    noauto,defaults,discard,nosuid,nodev    1 2
-/rw/home             /home          none    noauto,bind,defaults,nosuid,nodev        0 0
-/rw/usrlocal         /usr/local     none    noauto,bind,defaults        0 0
 /dev/xvdc1           swap           swap    defaults                    0 0
 tmpfs                /dev/shm       tmpfs   defaults,size=1G            0 0
 devpts               /dev/pts       devpts  gid=5,mode=620              0 0

init/functions

@@ -23,7 +23,9 @@ qsvc() {
 }
 
 under_systemd() {
-    pidof systemd >/dev/null 2>&1
+    local init_command_name
+    read -r init_command_name < /proc/1/comm
+    [ "$init_command_name" = "systemd" ]
 }
 
 systemd_version_changed() {
@@ -57,6 +59,10 @@ is_appvm() {
     [ "$(qubes_vm_type)" = "AppVM" ]
 }
 
+is_custom_persist_enabled() {
+    [ -f "/var/run/qubes-service/custom-persist" ]
+}
+
 is_proxyvm() {
     [ "$(qubes_vm_type)" = "ProxyVM" ]
 }
@@ -246,3 +252,15 @@ initialize_home() {
         for waitpid in $waitpids ; do wait "$waitpid" ; done ; waitpids=
     done
 }
+
+disable_persistent_home() {
+    echo "Disabling persistent /home"
+    mkdir /run/systemd/system/home.mount.d
+    echo -e '[Mount]\nWhat=/home' > /run/systemd/system/home.mount.d/30_qubes.conf
+}
+
+disable_persistent_usrlocal() {
+    echo "Disabling persistent /usr/local"
+    mkdir /run/systemd/system/usr-local.mount.d
+    echo -e '[Mount]\nWhat=/usr/local' > /run/systemd/system/usr-local.mount.d/30_qubes.conf
+}

qubes-rpc/prepare-suspend

@@ -16,7 +16,7 @@ MODULES_BLACKLIST=""
 if [ -r /etc/qubes-suspend-module-blacklist ]; then
     MODULES_BLACKLIST="$MODULES_BLACKLIST $(grep -v '^#' /etc/qubes-suspend-module-blacklist)"
 fi
-if [ -r /rw/config/suspend-module-blacklist ]; then
+if [ -r /rw/config/suspend-module-blacklist ] && ! is_custom_persist_enabled; then
     MODULES_BLACKLIST="$MODULES_BLACKLIST $(grep -v '^#' /rw/config/suspend-module-blacklist)"
 fi
 

qubesagent/firewall.py

@@ -86,10 +86,15 @@ def get_connected_ips(self, family):
             return []
         return ips.decode().split()
 
+    def is_custom_persist_enabled(self) -> bool:
+        """Check if the feature custom-persist is enabled on the current VM"""
+        return os.path.isfile('/var/run/qubes-service/custom-persist')
+
     def run_firewall_dir(self):
         """Run scripts dir contents, before user script"""
-        script_dir_paths = ['/etc/qubes/qubes-firewall.d',
-                            '/rw/config/qubes-firewall.d']
+        script_dir_paths = ['/etc/qubes/qubes-firewall.d']
+        if not self.is_custom_persist_enabled():
+            script_dir_paths.append('/rw/config/qubes-firewall.d')
         for script_dir_path in script_dir_paths:
             if not os.path.isdir(script_dir_path):
                 continue
@@ -328,7 +333,8 @@ def main(self):
         self.terminate_requested = False
         self.init()
         self.run_firewall_dir()
-        self.run_user_script()
+        if not self.is_custom_persist_enabled():
+            self.run_user_script()
         self.sd_notify('READY=1')
         self.qdb.watch('/qubes-firewall/')
         self.qdb.watch('/connected-ips')

rpm_spec/core-agent.spec.in

@@ -1260,6 +1260,8 @@ The Qubes core startup configuration for SystemD init.
 %defattr(-,root,root,-)
 /etc/systemd/system/xendriverdomain.service
 %_unitdir/dev-xvdc1-swap.service
+%_unitdir/home.mount
+%_unitdir/qubes-bind-dirs.service
 %_unitdir/qubes-misc-post.service
 %_unitdir/qubes-mount-dirs.service
 %_unitdir/qubes-rootfs-resize.service
@@ -1271,6 +1273,7 @@ The Qubes core startup configuration for SystemD init.
 %_unitdir/qubes-sync-time.timer
 %_unitdir/[email protected]
 %_unitdir/qubes-updates-proxy-forwarder.socket
+%_unitdir/usr-local.mount
 %{_unitdir}-preset/%qubes_preset_file
 %_modulesloaddir/qubes-core.conf
 %dir %_unitdir/boot.automount.d

vm-systemd/75-qubes-vm.preset

@@ -96,6 +96,7 @@ enable qubes-network.service
 enable qubes-network-uplink.service
 enable qubes-qrexec-agent.service
 enable qubes-mount-dirs.service
+enable qubes-bind-dirs.service
 enable qubes-rootfs-resize.service
 enable qubes-firewall.service
 enable qubes-meminfo-writer.service

vm-systemd/NetworkManager.service.d/30_qubes.conf

@@ -1,7 +1,7 @@
 [Unit]
 ConditionPathExists=/var/run/qubes-service/network-manager
 # For /rw
-After=qubes-mount-dirs.service
+After=qubes-bind-dirs.service
 # For /var/run/qubes-service
 After=qubes-sysinit.service
 # For configuration of qubes-provided interfaces

vm-systemd/bind-dirs.sh

@@ -29,6 +29,8 @@ shopt -s nullglob dotglob
 # shellcheck source=init/functions
 source /usr/lib/qubes/init/functions
 
+readonly DEFAULT_RW_BIND_DIR="/rw/bind-dirs"
+
 prerequisite() {
    if is_fully_persistent ; then
       echo "No TemplateBasedVM/DisposableVM detected. Exiting."
@@ -37,7 +39,7 @@ prerequisite() {
 }
 
 init() {
-   [ -n "$rw_dest_dir" ] || rw_dest_dir="/rw/bind-dirs"
+   [ -n "$rw_dest_dir" ] || rw_dest_dir="$DEFAULT_RW_BIND_DIR"
    [ -n "$symlink_level_max" ] || symlink_level_max="10"
    mkdir --parents "$rw_dest_dir"
 }
@@ -49,6 +51,21 @@ legacy() {
    true
 }
 
+rw_from_ro() {
+  ro="$1"
+  # special cases for files/dirs in /home or /usr/local
+  if [[ "$ro" =~ ^/home/ ]]; then
+    # use /rw/home for /home/... binds
+    rw="/rw${ro}"
+  elif [[ "$ro" =~ ^/usr/local/ ]]; then
+    # use /rw/usrlocal for /usr/local/... binds
+    rw="/rw/usrlocal/$(echo "$ro" | cut -d/ -f4-)"
+  else
+    [ -z "$rw_dest_dir" ] && rw="${DEFAULT_RW_BIND_DIR}${ro}" || rw="${rw_dest_dir}${ro}"
+  fi
+  echo "$rw"
+}
+
 bind_dirs() {
    ## legend
    ## fso: file system object
@@ -77,7 +94,7 @@ bind_dirs() {
       done
 
       true "fso_ro: $fso_ro"
-      fso_rw="${rw_dest_dir}${fso_ro}"
+      fso_rw="$(rw_from_ro "$fso_ro")"
 
       # Make sure fso_ro is not mounted.
       umount "$fso_ro" 2> /dev/null || true
@@ -91,13 +108,19 @@ bind_dirs() {
          if [ ! -e "$fso_ro" ]; then
             ## Create empty file or directory if path exists in /rw to allow to bind mount none existing files/dirs.
             test -d "$fso_rw" && mkdir --parents "$fso_ro"
-            test -f "$fso_rw" && touch "$fso_ro"
+            if [ -f "$fso_rw" ]; then
+              parent_directory="$(dirname "$fso_ro")"
+              test -d "$parent_directory" || mkdir --parents "$parent_directory"
+              touch "$fso_ro"
+            fi
          fi
       else
          if [ -d "$fso_ro" ] || [ -f "$fso_ro" ]; then
             ## Initially copy over data directories to /rw if rw directory does not exist.
-            echo "Initializing $rw_dest_dir with files from $fso_ro" >&2
-            cp --archive --recursive --parents "$fso_ro" "$rw_dest_dir"
+            echo "Initializing $fso_rw with files from $fso_ro" >&2
+            parent_directory="$(dirname "$fso_rw")"
+            test -d "$parent_directory" || mkdir --parents "$parent_directory"
+            cp --archive --recursive "$fso_ro" "$fso_rw"
          else
             echo "$fso_ro is neither a directory nor a file and the path does not exist below /rw, skipping."
             continue
@@ -118,7 +141,12 @@ main() {
 }
 
 binds=()
-for source_folder in /usr/lib/qubes-bind-dirs.d /etc/qubes-bind-dirs.d /rw/config/qubes-bind-dirs.d ; do
+sources=( "/usr/lib/qubes-bind-dirs.d" "/etc/qubes-bind-dirs.d" )
+if [ ! -f "/var/run/qubes-service/custom-persist" ]; then
+    sources+=( "/rw/config/qubes-bind-dirs.d" )
+fi
+
+for source_folder in "${sources[@]}"; do
    true "source_folder: $source_folder"
    if [ ! -d "$source_folder" ]; then
       continue
@@ -130,6 +158,52 @@ for source_folder in /usr/lib/qubes-bind-dirs.d /etc/qubes-bind-dirs.d /rw/confi
    done
 done
 
+# read binds in QubesDB if custom-persist feature is enabled
+if is_custom_persist_enabled; then
+  while read -r qubes_persist_entry; do
+    [[ "$qubes_persist_entry" =~ =\ (.*)$ ]] || continue
+    target="${BASH_REMATCH[1]}"
+
+    # if the first char is not a slash, options should be extracted from
+    # the value
+    if [[ "$target" != /* ]]; then
+      resource_type="$(echo "$target" | cut -d':' -f1)"
+      owner="$(echo "$target" | cut -d':' -f2)"
+      group="$(echo "$target" | cut -d':' -f3)"
+      mode="$(echo "$target" | cut -d':' -f4)"
+      path="$(echo "$target" | cut -d':' -f5-)"
+
+      if [ -z "$path" ] || [[ "$path" != /* ]]; then
+        echo "Skipping invalid custom-persist value '${target}'" >&2
+        continue
+      fi
+
+      rw_path="$(rw_from_ro "${path}")"
+      # create resource if it does not exist
+      if ! [ -e "${path}" ] && ! [ -e "$rw_path" ]; then
+        if [ "$resource_type" = "file" ]; then
+          # for files, we need to create parent directories
+          parent_directory="$(dirname "$rw_path")"
+          echo "custom-persist: pre-creating file ${rw_path} with rights ${owner}:${group} ${mode}"
+          [ -d "$parent_directory" ] || mkdir -p "${parent_directory}"
+          touch "${rw_path}"
+        elif [ "$resource_type" = "dir" ]; then
+          echo "custom-persist: pre-creating directory ${rw_path} with rights ${owner}:${group} ${mode}"
+          mkdir -p "${rw_path}"
+        else
+          echo "Invalid entry ${target}, skipping"
+          continue
+        fi
+        chown "$owner":"$group" "${rw_path}"
+        chmod "$mode" "${rw_path}"
+      fi
+      target="$path"
+    fi
+    [[ "$target" =~ ^(\/home|\/usr\/local)$ ]] && continue
+    binds+=( "$target" )
+  done <<< "$(qubesdb-multiread /persist/)"
+fi
+
 main "$@"
 
 true "OK: END."

vm-systemd/home.mount

@@ -0,0 +1,5 @@
+[Mount]
+What=/rw/home
+Where=/home
+Type=none
+Options=noauto,bind,defaults,nosuid,nodev

vm-systemd/misc-post.sh

@@ -11,9 +11,11 @@ if [ -n "$(ls -A /usr/local/lib 2>/dev/null)" ] || \
     ldconfig
 fi
 
-for rc in /rw/config/rc.local.d/*.rc /rw/config/rc.local; do
-    [ -f "${rc}" ] || continue
-    [ -x "${rc}" ] || continue
-    "${rc}"
-done
+if ! is_custom_persist_enabled; then
+  for rc in /rw/config/rc.local.d/*.rc /rw/config/rc.local; do
+      [ -f "${rc}" ] || continue
+      [ -x "${rc}" ] || continue
+      "${rc}"
+  done
+fi
 unset rc

vm-systemd/mount-dirs.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Source Qubes library.
 # shellcheck source=init/functions
@@ -10,9 +10,31 @@ set -e
 if [ -e /dev/xvdb ] ; then mount /rw ; fi
 /usr/lib/qubes/init/setup-rw.sh
 
-initialize_home "/rw/home" ifneeded
-echo "Mounting /rw/home onto /home" >&2
-mount /home
-echo "Mounting /rw/usrlocal onto /usr/local" >&2
-mount /usr/local
-/usr/lib/qubes/init/bind-dirs.sh
+if is_custom_persist_enabled; then
+  mount_home=false
+  mount_usr_local=false
+
+  while read -r qubes_persist_entry; do
+    [[ "$qubes_persist_entry" =~ \=\ /home$ ]] && mount_home=true
+    [[ "$qubes_persist_entry" =~ \=\ /usr/local$ ]] && mount_usr_local=true
+  done <<< "$(qubesdb-multiread /persist/)"
+else
+  mount_home=true
+  mount_usr_local=true
+fi
+
+if $mount_home; then
+  initialize_home "/rw/home" ifneeded
+  under_systemd || mount -o noauto,bind,defaults,nosuid,nodev /rw/home /home
+else
+  under_systemd && disable_persistent_home
+  initialize_home "/home" unconditionally
+fi
+
+if $mount_usr_local; then
+  under_systemd || mount -o noauto,bind,defaults /rw/usrlocal /usr/local
+else
+  under_systemd && disable_persistent_usrlocal
+fi
+
+under_systemd && systemctl daemon-reload || exit 0

vm-systemd/qubes-bind-dirs.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Mount configured bind files and directories
+After=qubes-mount-dirs.service local-fs.target rw.mount home.mount usr-local.mount
+Before=qubes-gui-agent.service
+DefaultDependencies=no
+Requires=qubes-mount-dirs.service home.mount usr-local.mount
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/lib/qubes/init/bind-dirs.sh
+
+[Install]
+WantedBy=multi-user.target

vm-systemd/qubes-misc-post.service

@@ -1,6 +1,6 @@
 [Unit]
 Description=Qubes misc post-boot actions
-After=network-pre.target qubes-mount-dirs.service qubes-network.service qubes-firewall.service
+After=network-pre.target qubes-bind-dirs.service qubes-network.service qubes-firewall.service
 
 [Service]
 Type=oneshot

vm-systemd/qubes-mount-dirs.service

@@ -6,7 +6,7 @@ Description=Initialize and mount /rw and /home
 After=qubes-sysinit.service dev-xvdb.device
 [email protected]
 DefaultDependencies=no
-Before=local-fs.target rw.mount home.mount qubes-gui-agent.service
+Before=local-fs.target rw.mount usr-local.mount home.mount qubes-gui-agent.service
 
 [Service]
 Type=oneshot

vm-systemd/usr-local.mount

@@ -0,0 +1,5 @@
+[Mount]
+What=/rw/usrlocal
+Where=/usr/local
+Type=none
+Options=noauto,bind,defaults