updater: dom0 non-iteractive updater

dom0-updates/qubes-dom0-update

@@ -28,6 +28,7 @@ if [ "$1" = "--help" ]; then
     echo "    --force-xen-upgrade  force major Xen upgrade even if some qubes are running"
     echo "    --console    does nothing; ignored for backward compatibility"
     echo "    --show-output  does nothing; ignored for backward compatibility"
+    echo "    --quiet      do not print anything to stdout"
     echo "    --preserve-terminal  does nothing; ignored for backward compatibility"
     echo "    --skip-boot-check  does not check if /boot & /boot/efi should be mounted"
     echo "    --switch-audio-server-to=(pulseaudio|pipewire) switch audio daemon to pipewire or pulseaudio"
@@ -46,6 +47,7 @@ YUM_OPTS=()
 UPDATEVM_OPTS=()
 QVMTEMPLATE_OPTS=()
 GUI=
+PROGRESS_REPORTING=
 CHECK_ONLY=
 CLEAN=
 TEMPLATE=
@@ -79,6 +81,12 @@ while [ $# -gt 0 ]; do
         --show-output)
             # ignore
             ;;
+        --quiet)
+            exec > /dev/null
+            ;;
+        --just-print-progress)
+            PROGRESS_REPORTING=1
+            ;;
         --check-only)
             CHECK_ONLY=1
             UPDATEVM_OPTS+=( "$1" )
@@ -310,29 +318,102 @@ qvm-run --nogui -q -- "$UPDATEVM" "rm -rf -- '$dom0_updates_dir/etc' '$dom0_upda
    exit "$status"
 }
 
-CMD="/usr/lib/qubes/qubes-download-dom0-updates.sh --doit --nogui"
+QVMRUN_OPTS=(--quiet --filter-escape-chars --nogui --pass-io)
 
-# We avoid using bash’s own facilities for this, as they produce $'\n'-style
-# strings in certain cases.  These are not portable, whereas the string produced
-# by the following is.
-for i in "${UPDATEVM_OPTS[@]}"; do CMD+=" '${i//\'/\'\\\'\'}'"; done
+progress_agent_version="4.3"
 
-QVMRUN_OPTS=(--quiet --filter-escape-chars --nogui --pass-io)
-if [[ -t 1 ]] && [[ -t 2 ]]; then
-    # Use ‘script’ to emulate a TTY, so that we get status bars and other
-    # progress output.  Since stdout and stderr are both terminals, qvm-run
-    # will automatically sanitize them, but we explicitly tell it to anyway
-    # as a precaution.
-    #
-    # We MUST NOT use ‘exec script’ here.  That causes ‘script’ to
-    # inherit the child processes of the shell.  ‘script’ mishandles
-    # this and enters an infinite loop.
-    CMD="script --quiet --return --command '${CMD//\'/\'\\\'\'}' /dev/null"
+get_base_vm() {
+    # Resolve base VM (TemplateVM or StandaloneVM)
+    local vm="$1"
+    while true; do
+        # If it's a TemplateVM or StandaloneVM, stop
+        if qvm-check --template "$vm" &>/dev/null || qvm-check --standalone "$vm" &>/dev/null; then
+            echo "$vm"
+            return
+        fi
+        # Try to get its template
+        vm=$(qvm-prefs "$vm" template 2>/dev/null) || return 1
+    done
+}
+version_check() {
+    # Compare version numbers (e.g., 4.2 < 4.3)
+    awk 'BEGIN {exit !(ARGV[1] < ARGV[2])}' "$1" "$2"
+}
+base_vm=$(get_base_vm $UPDATEVM)
+OLD_VERSION=0
+if [ -n "$base_vm" ]; then
+    agent_version=$(qvm-features "$base_vm" qubes-agent-version 2>/dev/null)
+
+    if [ -n "$agent_version" ] && version_check "$agent_version" "$progress_agent_version"; then
+        OLD_VERSION=1
+    fi
 fi
 
-qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null
+
+if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "0" ]; then
+    CMD="/usr/lib/qubes/qubes-download-dom0-updates-init.sh"
+    qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD"
+
+    update_agent_log="/var/log/qubes/qubes-update"
+    qvm-run --nogui -q -u root -- "$UPDATEVM" "user=\$(qubesdb-read /default-user) && chown -R -- \"\$user:qubes\" '$update_agent_log'" || exit 1
+
+    # "--no-cleanup" is needed since fakeroot cannot remove entrypoint
+    qubes-vm-update --force-update --targets "$UPDATEVM" --signal-no-updates --just-print-progress --display-name dom0 --download-only --no-cleanup --show-output --log=DEBUG
+    RETCODE=$?
+    if [ "$RETCODE" -eq 100 ]; then
+        echo "$(hostname):out: Nothing to do."
+        echo "$(hostname) done no_updates" >&2
+        exit 100
+    fi
+    if [ "$RETCODE" -ne 0 ]; then
+        echo "$(hostname) done error" >&2
+        exit $RETCODE
+    fi
+
+    # qubes-vm-update leaves the downloaded packages with root ownership
+    qvm-run --nogui -q -u root -- "$UPDATEVM" "user=\$(qubesdb-read /default-user) && chown -R -- \"\$user:qubes\" '$dom0_updates_dir'" || exit 1
+
+    CMD="/usr/lib/qubes/qubes-download-dom0-updates-finish.sh"
+    qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD"
+else
+    if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "1" ]; then
+        echo "$(hostname):out: Progress reporting requires updateVM based on a template with Qubes 4.3 packages."
+    fi
+    CMD="/usr/lib/qubes/qubes-download-dom0-updates.sh --doit --nogui"
+
+    # We avoid using bash’s own facilities for this, as they produce $'\n'-style
+    # strings in certain cases.  These are not portable, whereas the string produced
+    # by the following is.
+    for i in "${UPDATEVM_OPTS[@]}"; do CMD+=" '${i//\'/\'\\\'\'}'"; done
+
+    if [[ -t 1 ]] && [[ -t 2 ]]; then
+        # Use ‘script’ to emulate a TTY, so that we get status bars and other
+        # progress output.  Since stdout and stderr are both terminals, qvm-run
+        # will automatically sanitize them, but we explicitly tell it to anyway
+        # as a precaution.
+        #
+        # We MUST NOT use ‘exec script’ here.  That causes ‘script’ to
+        # inherit the child processes of the shell.  ‘script’ mishandles
+        # this and enters an infinite loop.
+        CMD="script --quiet --return --command '${CMD//\'/\'\\\'\'}' /dev/null"
+    fi
+    if [ "$PROGRESS_REPORTING" == "1" ]; then
+        qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null | sed "s/``^/$(hostname):out: /"
+        # "consume" the last empty line
+        echo ""
+    else
+        qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null
+    fi
+fi
 
 RETCODE=$?
+if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "1" ]; then
+    echo "$(hostname) updating 50.0" >&2
+    if [ "$RETCODE" -ne 0 ]; then
+        echo "$(hostname) done error" >&2
+        exit $RETCODE
+    fi
+fi
 if [[ "$REMOTE_ONLY" = '1' ]] || [ "$RETCODE" -ne 0 ]; then
     if [ "$CHECK_ONLY" = '1' ]; then
         if [ "$RETCODE" -eq 100 ]; then
@@ -401,6 +482,9 @@ elif [ -f /var/lib/qubes/updates/repodata/repomd.xml ]; then
         # refresh packagekit metadata, GUI utilities use it
         pkcon refresh force
         $guiapp
+    elif [ "$PROGRESS_REPORTING" == 1 ]; then
+        # report progress to the user
+        qubes-vm-update --no-refresh --targets dom0 --force-update --log=DEBUG --just-print-progress --show-output
     else
         dnf check-update ||
         if [ $? -eq 100 ]; then # Run dnf with options

vmupdate/agent/entrypoint.py

@@ -8,6 +8,7 @@
 from source.utils import get_os_data
 from source.log_congfig import init_logs
 from source.common.exit_codes import EXIT
+from source.common.package_manager import AgentType
 
 
 def main(args=None):
@@ -21,18 +22,31 @@ def main(args=None):
     os_data = get_os_data()
 
     log.debug("Selecting package manager.")
+    agent_type = AgentType.VM
+    if os_data["id"] == "qubes":
+        agent_type = AgentType.DOM0
+    if args.download_only:
+        agent_type = AgentType.UPDATE_VM
     pkg_mng = get_package_manager(
-        os_data, log, log_handler, log_level, args.no_progress)
+        os_data, log, log_handler, log_level, agent_type, args.no_progress)
 
     log.debug("Running upgrades.")
     return_code = pkg_mng.upgrade(refresh=not args.no_refresh,
                                   hard_fail=not args.force_upgrade,
                                   remove_obsolete=not args.leave_obsolete,
-                                  print_streams=args.show_output
+                                  print_streams=args.show_output,
                                   )
 
-    log.debug("Notify dom0 about upgrades.")
-    os.system("/usr/lib/qubes/upgrades-status-notify")
+    if not pkg_mng.PROGRESS_REPORTING and not args.no_progress:
+        # even if progress reporting is unavailable we want info that update finished
+        if agent_type is AgentType.UPDATE_VM:
+            print(f"{55:.2f}", flush=True, file=sys.stderr)
+        else:
+            print(f"{100:.2f}", flush=True, file=sys.stderr)
+
+    if agent_type is AgentType.VM:
+        log.debug("Notify dom0 about upgrades.")
+        os.system("/usr/lib/qubes/upgrades-status-notify")
 
     if not args.no_cleanup:
         return_code = max(pkg_mng.clean(), return_code)
@@ -49,7 +63,7 @@ def parse_args(args):
     return args
 
 
-def get_package_manager(os_data, log, log_handler, log_level, no_progress):
+def get_package_manager(os_data, log, log_handler, log_level, agent_type, no_progress):
     """
     Returns instance of `PackageManager`.
 
@@ -59,57 +73,113 @@ def get_package_manager(os_data, log, log_handler, log_level, no_progress):
     requirements = {}
     # plugins MUST be applied before import anything from package managers.
     # in case of apt configuration is loaded on `import apt`.
-    for plugin in plugins.entrypoints:
-        plugin(os_data, log, requirements=requirements)
-
-    if os_data["os_family"] == "Debian":
-        try:
-            from source.apt.apt_api import APT as PackageManager
-        except ImportError:
-            log.warning("Failed to load apt with progress bar. Using apt cli.")
-            # no progress reporting
-            no_progress = True
-            print(f"Progress reporting not supported.", flush=True)
-
-        if no_progress:
-            from source.apt.apt_cli import APTCLI as PackageManager
-    elif os_data["os_family"] == "RedHat":
-        try:
-            version = int(os_data["release"].split(".")[0])
-        except ValueError:
-            version = 99  # fedora changed its version
-
-        loaded = False
-        if version >= 41:
-            try:
-                from source.dnf.dnf5_api import DNF as PackageManager
-                loaded = True
-            except ImportError:
-                log.warning("Failed to load dnf5.")
-
-        if not loaded:
-            try:
-                from source.dnf.dnf_api import DNF as PackageManager
-                loaded = True
-            except ImportError:
-                log.warning(
-                    "Failed to load dnf with progress bar. Using dnf cli.")
-                print(f"Progress reporting not supported.", flush=True)
-
-        if no_progress or not loaded:
-            from source.dnf.dnf_cli import DNFCLI as PackageManager
+    if agent_type is not AgentType.UPDATE_VM:
+        for plugin in plugins.entrypoints:
+            plugin(os_data, log, requirements=requirements)
+
+    if os_data["os_family"] == "RedHat" or agent_type is AgentType.UPDATE_VM:
+        PackageManager = import_rhel_package_manager(os_data, log, no_progress)
+    elif os_data["os_family"] == "Debian":
+        PackageManager = import_debian_package_manager(log, no_progress)
     elif os_data["os_family"] == "ArchLinux":
         from source.pacman.pacman_cli import PACMANCLI as PackageManager
         print(f"Progress reporting not supported.", flush=True)
+    elif os_data["os_family"] == "Qubes":
+        PackageManager = import_dom0_package_manager(os_data, log, no_progress)
     else:
         raise NotImplementedError(
             "Only Debian, RedHat and ArchLinux based OS is supported.")
 
-    pkg_mng = PackageManager(log_handler, log_level)
+    pkg_mng = PackageManager(log_handler, log_level, agent_type)
     pkg_mng.requirements = requirements
     return pkg_mng
 
 
+def import_rhel_package_manager(os_data, log, no_progress):
+    """
+    Import dnf package manager.
+    """
+    dnf5_fedora_version = 41
+    if os_data["os_family"] == "RedHat":
+        try:
+            version = int(os_data["release"].split(".")[0])
+        except ValueError:
+            version = 99  # fedora changed its version
+    else:
+        version = dnf5_fedora_version  # try to use whatever is available, starting from dnf5
+
+    loaded = False
+    if version >= dnf5_fedora_version:
+        try:
+            from source.dnf.dnf5_api import DNF5 as PackageManager
+            loaded = True
+        except ImportError:
+            log.warning("Failed to load dnf5.")
+
+    if not loaded:
+        try:
+            from source.dnf.dnf_api import DNF as PackageManager
+            loaded = True
+            log.debug("Using dnf python API for progress reporting.")
+        except ImportError:
+            print(f"Progress reporting not supported.", flush=True)
+
+    if no_progress or not loaded:
+        log.warning(
+            "Failed to load dnf with progress bar. Using dnf cli.")
+        from source.dnf.dnf_cli import DNFCLI as PackageManager
+
+    return PackageManager
+
+
+def import_debian_package_manager(log, no_progress):
+    """
+    Import apt package manager.
+    """
+    loaded = False
+    try:
+        from source.apt.apt_api import APT as PackageManager
+        loaded = True
+    except ImportError:
+        log.warning("Failed to load apt with progress bar. Using apt cli.")
+        print(f"Progress reporting not supported.", flush=True)
+
+    if no_progress or not loaded:
+        from source.apt.apt_cli import APTCLI as PackageManager
+
+    return PackageManager
+
+
+def import_dom0_package_manager(os_data, log, no_progress):
+    """
+    Import dnf package manager for dom0.
+    """
+    major, minor = os_data["release"].split(".")
+    major, minor = int(major), int(minor)
+    loaded = False
+    if major >= 5 or (major == 4 and minor >= 3):
+        try:
+            from source.dnf.dnf5_api import DNF5 as PackageManager
+            loaded = True
+        except ImportError:
+            log.warning("Failed to load dnf5.")
+
+    if not loaded:
+        try:
+            from source.dnf.dnf_api import DNF as PackageManager
+            loaded = True
+            log.debug("Using dnf python API for progress reporting.")
+        except ImportError:
+            print(f"Progress reporting not supported.", flush=True)
+
+    if no_progress or not loaded:
+        log.warning(
+            "Failed to load dnf with progress bar. Using dnf cli.")
+        from source.dnf.dnf_cli import DNFCLI as PackageManager
+
+    return PackageManager
+
+
 if __name__ == '__main__':
     try:
         sys.exit(main())

vmupdate/agent/source/apt/apt_api.py

@@ -26,6 +26,7 @@
 import apt.progress.base
 import apt_pkg
 
+from source.common.package_manager import AgentType
 from source.common.process_result import ProcessResult
 from source.common.exit_codes import EXIT
 from source.common.progress_reporter import ProgressReporter, Progress
@@ -34,8 +35,10 @@
 
 
 class APT(APTCLI):
-    def __init__(self, log_handler, log_level):
-        super().__init__(log_handler, log_level)
+    PROGRESS_REPORTING = True
+
+    def __init__(self, log_handler, log_level, agent_type: AgentType):
+        super().__init__(log_handler, log_level, agent_type)
         self.apt_cache = apt.Cache()
         update = FetchProgress(
             weight=4, log=self.log, refresh=True)  # 4% of total time