GRE-226: activate and roll back qualified releases

crates/obscam/tests/deployment.rs

@@ -69,12 +69,16 @@ fn live_resource_controller_preflight_fails_closed() {
         .find("verify-memory-controller")
         .expect("live memory-controller preflight");
     let first_install = live_install
-        .find("install_owned_file")
-        .expect("first owned-file mutation");
+        .find("stage_release")
+        .expect("first release mutation");
     assert!(preflight < first_install);
-    assert!(installer.contains(
-        "839e6b2077c5687c12eaa9b8f0fd5dbb0ad05fd92691d600c1ba865f69150c7f pre-memory-controller"
-    ));
+    assert!(preflight < live_install.find("recover_interrupted_activation").unwrap());
+    assert!(
+        preflight
+            < live_install
+                .find("resume_or_preflight_installation")
+                .unwrap()
+    );
 }
 
 #[test]
@@ -228,7 +232,7 @@ fn installed_command_covers_all_required_read_only_diagnostics() {
     }
     assert!(diagnostics.contains("--lines=100"));
     assert!(!diagnostics.contains("wg1"));
-    assert!(guide.contains("/usr/local/libexec/obscam/obscam-diagnostics"));
+    assert!(guide.contains("/usr/local/bin/obscam-diagnostics"));
 }
 
 #[test]
@@ -302,7 +306,7 @@ fn mediamtx_startup_rejects_binary_or_configuration_drift() {
     assert_eq!(
         directive_values(&unit, "ExecStartPre"),
         [
-            "/usr/local/libexec/obscam/verify-mediamtx /usr/local/libexec/obscam/mediamtx /etc/obscam/mediamtx.yml /usr/local/share/obscam/mediamtx-linux-arm64.sha256 /usr/local/share/obscam/mediamtx-config.sha256"
+            "/opt/obscam/active/bin/verify-mediamtx /opt/obscam/active/bin/mediamtx /opt/obscam/active/config/mediamtx.yml /opt/obscam/active/config/mediamtx-linux-arm64.sha256 /opt/obscam/active/config/mediamtx-config.sha256"
         ]
     );
     assert!(verifier.contains("expected_version=v1.19.3"));
@@ -334,7 +338,7 @@ fn media_namespace_contains_only_a_private_point_to_point_link() {
     assert!(!setup.contains("ip route show"));
     assert_eq!(
         directive_values(&unit, "ExecStart"),
-        ["/usr/local/libexec/obscam/setup-media-network start"]
+        ["/opt/obscam/active/bin/setup-media-network start"]
     );
     assert_eq!(
         directive_values(&unit, "CapabilityBoundingSet"),
@@ -350,6 +354,75 @@ fn media_namespace_contains_only_a_private_point_to_point_link() {
     assert!(!unit.contains("ProtectHome="));
 }
 
+#[test]
+fn service_runtime_and_host_configuration_follow_one_active_release() {
+    let obscam = repository_file("deploy/systemd/obscam.service");
+    let mediamtx = repository_file("deploy/systemd/obscam-mediamtx.service");
+    let network = repository_file("deploy/systemd/obscam-media-network.service");
+    let installer = repository_file("deploy/install-appliance");
+    let verifier = repository_file("deploy/verify-release");
+
+    assert_eq!(
+        directive_values(&obscam, "ExecStart"),
+        ["/opt/obscam/active/bin/obscam"]
+    );
+    assert_eq!(
+        directive_values(&obscam, "EnvironmentFile"),
+        ["/etc/obscam/host.env"]
+    );
+    assert_eq!(
+        directive_values(&obscam, "Environment"),
+        ["LD_LIBRARY_PATH=/opt/obscam/active/lib", "RUST_LOG=info"]
+    );
+    for unit in [&mediamtx, &network] {
+        assert!(unit.contains("/opt/obscam/active/"));
+        assert!(!unit.contains("/usr/local/libexec/obscam/"));
+    }
+    for member in [
+        "browser/index.html",
+        "browser/app.js",
+        "browser/styles.css",
+        "lib/libASICamera2.so.1.41",
+        "systemd/obscam.service",
+    ] {
+        assert!(verifier.contains(member));
+    }
+    assert!(installer.contains("mv -Tf \"$temp\" \"$directory/$name\""));
+    assert!(installer.contains("activation-pending"));
+    assert!(installer.contains("prior_previous"));
+    assert!(installer.contains("stop_candidate_without_prior_release"));
+    assert!(installer.contains("systemctl stop obscam.target"));
+    assert!(installer.contains("systemctl is-active --quiet obscam.target"));
+}
+
+#[test]
+fn post_activation_gate_is_bounded_and_exercises_public_runtime_contracts() {
+    let checks = repository_file("deploy/check-release");
+
+    for evidence in [
+        "systemctl is-active",
+        "/api/v1/health",
+        "/assets/app.js",
+        "/assets/styles.css",
+        "169.254.218.2:9998/metrics",
+        "name=\"obscam\"",
+        "systemctl restart obscam-mediamtx.service",
+        "systemctl restart obscam.service",
+        "MainPID",
+        "/proc/$obscam_pid/exe",
+        "/proc/$obscam_pid/maps",
+        "OBSCAM_BIND_ADDRESS",
+        "health_origin",
+    ] {
+        assert!(
+            checks.contains(evidence),
+            "missing post-activation evidence: {evidence}"
+        );
+    }
+    assert!(checks.contains("check_timeout_seconds=30"));
+    assert!(checks.contains("--max-time 5"));
+}
+
 #[test]
 fn media_forwarding_is_limited_to_approved_addresses_and_ports() {
     let rules = repository_file("deploy/obscam-media.nft");

deploy/check-release

@@ -0,0 +1,135 @@
+#!/bin/sh
+set -eu
+
+root=/
+if [ "${1:-}" = --root ]; then
+    [ "$#" -eq 2 ] || exit 2
+    root=$2
+elif [ "$#" -ne 0 ]; then
+    exit 2
+fi
+root=${root%/}
+[ -n "$root" ] || root=/
+
+target() {
+    if [ "$root" = / ]; then printf '%s\n' "$1"; else printf '%s%s\n' "$root" "$1"; fi
+}
+
+active=$(target /opt/obscam/active)
+[ -L "$active" ] || {
+    echo "check-release: active release link is missing" >&2
+    exit 1
+}
+release_link=$(readlink "$active")
+case "$release_link" in releases/*) ;; *) exit 1 ;; esac
+release=$(target "/opt/obscam/$release_link")
+"$release/bin/verify-release" "$release"
+"$release/bin/validate-host-config" "$(target /etc/obscam/host.env)"
+
+# A staged-root check proves the complete immutable and host-configuration
+# contract. Live service, HTTP, media, and recovery checks run only on the host.
+[ "$root" != / ] && exit 0
+
+check_timeout_seconds=30
+bind_address=$(sed -n 's/^OBSCAM_BIND_ADDRESS=//p' "$(target /etc/obscam/host.env)")
+case "$bind_address" in
+    \[*\]:*)
+        health_host=${bind_address#\[}
+        health_host=${health_host%%\]*}
+        health_port=${bind_address##*\]:}
+        if [ "$health_host" = :: ]; then health_host=::1; fi
+        health_origin="http://[$health_host]:$health_port"
+        ;;
+    *)
+        health_host=${bind_address%:*}
+        health_port=${bind_address##*:}
+        if [ "$health_host" = 0.0.0.0 ]; then health_host=127.0.0.1; fi
+        health_origin="http://$health_host:$health_port"
+        ;;
+esac
+
+services_ready() {
+    systemctl is-active --quiet obscam.target obscam.service obscam-mediamtx.service obscam-media-network.service
+}
+
+health_ready() {
+    health_json=$(curl --noproxy '*' --fail --silent --show-error --max-time 5 "$health_origin/api/v1/health") || return 1
+    for health_component in capture encoder relay; do
+        printf '%s\n' "$health_json" | grep -Eq "\"$health_component\":\\{[^}]*\"state\":\"ready\"" || return 1
+    done
+}
+
+media_ready() {
+    curl --fail --silent --show-error --max-time 5 http://169.254.218.2:9998/metrics |
+        grep -Eq '^paths\{.*name="obscam".*\}[[:space:]]+1$'
+}
+
+wait_for() {
+    wait_description=$1
+    wait_check=$2
+    wait_deadline=$(($(date +%s) + check_timeout_seconds))
+    until "$wait_check"; do
+        [ "$(date +%s)" -lt "$wait_deadline" ] || {
+            echo "check-release: timed out waiting for $wait_description" >&2
+            return 1
+        }
+        sleep 1
+    done
+}
+
+wait_for "service readiness" services_ready
+wait_for "local health" health_ready
+wait_for "media publication" media_ready
+
+check_tmp=$(mktemp -d)
+trap 'rm -rf "$check_tmp"' EXIT HUP INT TERM
+asset_matches() {
+    asset_url=$1
+    asset_file=$2
+    curl --noproxy '*' --fail --silent --show-error --max-time 5 "$asset_url" >"$check_tmp/asset"
+    cmp -s "$check_tmp/asset" "$release/browser/$asset_file"
+}
+asset_matches "$health_origin/" index.html
+asset_matches "$health_origin/assets/app.js" app.js
+asset_matches "$health_origin/assets/styles.css" styles.css
+
+main_pid() {
+    systemctl show "$1" --property=MainPID --value
+}
+
+obscam_pid=$(main_pid obscam.service)
+mediamtx_pid=$(main_pid obscam-mediamtx.service)
+case "$obscam_pid" in ''|*[!0-9]*) exit 1 ;; esac
+case "$mediamtx_pid" in ''|*[!0-9]*) exit 1 ;; esac
+[ "$obscam_pid" -gt 0 ] && [ "$mediamtx_pid" -gt 0 ]
+[ "$(readlink -f "/proc/$obscam_pid/exe")" = "$(readlink -f "$release/bin/obscam")" ] || {
+    echo "check-release: running ObsCam binary is outside the active release" >&2
+    exit 1
+}
+[ "$(readlink -f "/proc/$mediamtx_pid/exe")" = "$(readlink -f "$release/bin/mediamtx")" ] || {
+    echo "check-release: running MediaMTX binary is outside the active release" >&2
+    exit 1
+}
+grep -F "$(readlink -f "$release/lib/libASICamera2.so.1.41")" "/proc/$obscam_pid/maps" >/dev/null || {
+    echo "check-release: running ObsCam process does not map the qualified ZWO SDK" >&2
+    exit 1
+}
+
+systemctl restart obscam-mediamtx.service
+wait_for "relay recovery" media_ready
+[ "$(main_pid obscam.service)" = "$obscam_pid" ] || {
+    echo "check-release: relay recovery restarted ObsCam" >&2
+    exit 1
+}
+restarted_mediamtx_pid=$(main_pid obscam-mediamtx.service)
+[ "$restarted_mediamtx_pid" -gt 0 ] && [ "$restarted_mediamtx_pid" != "$mediamtx_pid" ]
+
+systemctl restart obscam.service
+wait_for "ObsCam recovery" health_ready
+wait_for "publication after ObsCam recovery" media_ready
+[ "$(main_pid obscam-mediamtx.service)" = "$restarted_mediamtx_pid" ] || {
+    echo "check-release: ObsCam recovery restarted MediaMTX" >&2
+    exit 1
+}
+restarted_obscam_pid=$(main_pid obscam.service)
+[ "$restarted_obscam_pid" -gt 0 ] && [ "$restarted_obscam_pid" != "$obscam_pid" ]

deploy/host.env.example

@@ -0,0 +1,7 @@
+OBSCAM_BIND_ADDRESS=0.0.0.0:8080
+OBSCAM_WHEP_PORT=8889
+OBSCAM_WHEP_PATH=/obscam/whep
+OBSCAM_CAMERA_SOURCE=production
+OBSCAM_DEFAULT_EXPOSURE_MS=500
+OBSCAM_DEFAULT_GAIN=100
+OBSCAM_DEFAULT_TREATMENT=monochrome