clean

Author: adamshiervani

failsafe.go

@@ -0,0 +1,106 @@
+package kvm
+
+import (
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/jetkvm/kvm/internal/supervisor"
+)
+
+const (
+	failsafeDefaultLastCrashPath = "/userdata/jetkvm/crashdump/last-crash.log"
+	failsafeFile                 = "/userdata/jetkvm/.enablefailsafe"
+	failsafeLastCrashEnv         = "JETKVM_LAST_ERROR_PATH"
+	failsafeEnv                  = "JETKVM_FORCE_FAILSAFE"
+)
+
+var (
+	failsafeOnce       sync.Once
+	failsafeCrashLog   = ""
+	failsafeModeActive = false
+	failsafeModeReason = ""
+)
+
+type FailsafeModeNotification struct {
+	Active bool   `json:"active"`
+	Reason string `json:"reason"`
+}
+
+// this function has side effects and can be only executed once
+func checkFailsafeReason() {
+	failsafeOnce.Do(func() {
+		// check if the failsafe environment variable is set
+		if os.Getenv(failsafeEnv) == "1" {
+			failsafeModeActive = true
+			failsafeModeReason = "failsafe_env_set"
+			return
+		}
+
+		// check if the failsafe file exists
+		if _, err := os.Stat(failsafeFile); err == nil {
+			failsafeModeActive = true
+			failsafeModeReason = "failsafe_file_exists"
+			_ = os.Remove(failsafeFile)
+			return
+		}
+
+		// get the last crash log path from the environment variable
+		lastCrashPath := os.Getenv(failsafeLastCrashEnv)
+		if lastCrashPath == "" {
+			lastCrashPath = failsafeDefaultLastCrashPath
+		}
+
+		// check if the last crash log file exists
+		l := failsafeLogger.With().Str("path", lastCrashPath).Logger()
+		fi, err := os.Lstat(lastCrashPath)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				l.Warn().Err(err).Msg("failed to stat last crash log")
+			}
+			return
+		}
+
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			l.Warn().Msg("last crash log is not a symlink, ignoring")
+			return
+		}
+
+		// open the last crash log file and find if it contains the string "panic"
+		content, err := os.ReadFile(lastCrashPath)
+		if err != nil {
+			l.Warn().Err(err).Msg("failed to read last crash log")
+			return
+		}
+
+		// unlink the last crash log file
+		failsafeCrashLog = string(content)
+		_ = os.Remove(lastCrashPath)
+
+		// TODO: read the goroutine stack trace and check which goroutine is panicking
+		failsafeModeActive = true
+		if strings.Contains(failsafeCrashLog, supervisor.FailsafeReasonVideoMaxRestartAttemptsReached) {
+			failsafeModeReason = "video"
+			return
+		}
+		if strings.Contains(failsafeCrashLog, "runtime.cgocall") {
+			failsafeModeReason = "video"
+			return
+		} else {
+			failsafeModeReason = "unknown"
+		}
+	})
+}
+
+func notifyFailsafeMode(session *Session) {
+	if !failsafeModeActive || session == nil {
+		return
+	}
+
+	jsonRpcLogger.Info().Str("reason", failsafeModeReason).Msg("sending failsafe mode notification")
+
+	writeJSONRPCEvent("failsafeMode", FailsafeModeNotification{
+		Active: true,
+		Reason: failsafeModeReason,
+	}, session)
+}

log.go

@@ -11,6 +11,7 @@ func ErrorfL(l *zerolog.Logger, format string, err error, args ...any) error {
 
 var (
 	logger          = logging.GetSubsystemLogger("jetkvm")
+	failsafeLogger  = logging.GetSubsystemLogger("failsafe")
 	networkLogger   = logging.GetSubsystemLogger("network")
 	cloudLogger     = logging.GetSubsystemLogger("cloud")
 	websocketLogger = logging.GetSubsystemLogger("websocket")

main.go

@@ -30,7 +30,7 @@ func Main() {
 
 	logger.Log().Msg("JetKVM Starting Up")
 
-	initFailsafe()
+	checkFailsafeReason()
 	if failsafeModeActive {
 		procPrefix = "jetkvm: [app+failsafe]"
 		logger.Warn().Str("reason", failsafeModeReason).Msg("failsafe mode activated")