Add stack trace information around several recover() calls (#54881)

* Add stack trace information around several recover() calls

* downgrade panic logs to warnings

Author: Tener

lib/srv/db/mysql/proxy.go

@@ -23,6 +23,7 @@ import (
 	"crypto/tls"
 	"log/slog"
 	"net"
+	"runtime/debug"
 	"time"
 
 	"github.com/go-mysql-org/go-mysql/mysql"
@@ -81,7 +82,7 @@ func (p *Proxy) HandleConnection(ctx context.Context, clientConn net.Conn) (err
 	// has a chance to close the connection from its side.
 	defer func() {
 		if r := recover(); r != nil {
-			p.Log.WarnContext(ctx, "Recovered in MySQL proxy while handling connectionv.", "from", clientConn.RemoteAddr(), "to", r)
+			p.Log.WarnContext(ctx, "Recovered in MySQL proxy while handling connectionv.", "from", clientConn.RemoteAddr(), "problem", r, "stack", debug.Stack())
 			err = trace.BadParameter("failed to handle MySQL client connection")
 		}
 		if err != nil {

lib/srv/db/server.go

@@ -24,6 +24,7 @@ import (
 	"errors"
 	"log/slog"
 	"net"
+	"runtime/debug"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -1189,7 +1190,7 @@ func (s *Server) handleConnection(ctx context.Context, clientConn net.Conn) erro
 
 	defer func() {
 		if r := recover(); r != nil {
-			s.log.WarnContext(ctx, "Recovered while handling DB connection.", "from", clientConn.RemoteAddr(), "to", r)
+			s.log.WarnContext(ctx, "Recovered while handling DB connection.", "from", clientConn.RemoteAddr(), "problem", r, "stack", debug.Stack())
 			err = trace.BadParameter("failed to handle client connection")
 		}
 		if err != nil {

lib/srv/db/sqlserver/engine.go

@@ -23,6 +23,7 @@ import (
 	"context"
 	"io"
 	"net"
+	"runtime/debug"
 
 	"github.com/gravitational/trace"
 
@@ -132,7 +133,7 @@ func (e *Engine) HandleConnection(ctx context.Context, sessionCtx *common.Sessio
 func (e *Engine) receiveFromClient(clientConn, serverConn io.ReadWriteCloser, clientErrCh chan<- error, sessionCtx *common.Session) {
 	defer func() {
 		if r := recover(); r != nil {
-			e.Log.ErrorContext(e.Context, "Recovered while handling DB connection", "recover", r)
+			e.Log.WarnContext(e.Context, "Recovered while handling DB connection", "problem", r, "stack", debug.Stack())
 			err := trace.BadParameter("failed to handle client connection")
 			e.SendError(err)
 		}
@@ -167,7 +168,7 @@ func (e *Engine) receiveFromClient(clientConn, serverConn io.ReadWriteCloser, cl
 			sqlPacket, err := e.toSQLPacket(initialPacketHeader, p, &chunkData)
 			switch {
 			case err != nil:
-				e.Log.ErrorContext(e.Context, "Failed to parse SQLServer packet.", "error", err)
+				e.Log.WarnContext(e.Context, "Failed to parse SQLServer packet.", "error", err)
 				e.emitMalformedPacket(e.Context, sessionCtx, p)
 			default:
 				e.auditPacket(e.Context, sessionCtx, sqlPacket)

lib/srv/db/sqlserver/protocol/packet.go

@@ -22,6 +22,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"io"
+	"runtime/debug"
 
 	"github.com/gravitational/trace"
 )
@@ -139,7 +140,7 @@ func NewBasicPacket(header PacketHeader, data []byte) (*BasicPacket, error) {
 func ToSQLPacket(p *BasicPacket) (out Packet, err error) {
 	defer func() {
 		if r := recover(); r != nil {
-			err = trace.BadParameter("failed to convert packet to SQL packet: %v", r)
+			err = trace.BadParameter("failed to convert packet to SQL packet: %v @ %v", r, debug.Stack())
 		}
 	}()