fix(control-plane): implement graceful shutdown on SIGTERM/SIGINT (#715)

* fix(control-plane): implement graceful shutdown (#427)

Replace the select{} no-op with real signal handling and HTTP server
drain so SIGTERM during rolling deploys no longer kills in-flight
requests.

Changes:
- Install signal.NotifyContext for SIGINT/SIGTERM in runServer
- Replace gin Router.Run() with net/http.Server for Shutdown() support
- Call server.Stop() on signal: drains HTTP connections, stops background
  goroutines (presence manager, health monitor, cleanup, OTel, etc.)
- Add configurable shutdown_timeout (default 30s) via YAML and
  AGENTFIELD_SHUTDOWN_TIMEOUT env var
- Remove literal '// TODO: Implement graceful shutdown' comments
- Add nil guard for healthMonitor.Stop() to prevent panic on empty server

Closes #427

* test: add coverage for graceful shutdown paths

- Test AGENTFIELD_SHUTDOWN_TIMEOUT env override and defaults
- Test Stop() on empty server (nil safety)
- Test HTTP server graceful shutdown with active listener
- Test HTTP server shutdown timeout + force close
- Test defaultWaitForShutdown unblocks on SIGINT (Linux/macOS only)

* fix(control-plane): tighten graceful shutdown lifecycle

---------

Co-authored-by: santoshkumarradha <instrument.santosh@gmail.com>

Author: 7vignesh

control-plane/cmd/agentfield-server/main.go

@@ -1,14 +1,17 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"os"
 	"os/exec"
+	"os/signal"
 	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/Agent-Field/agentfield/control-plane/internal/cli"
@@ -36,7 +39,7 @@ var (
 	buildUIFunc               = buildUI
 	openBrowserFunc           = openBrowser
 	sleepFunc                 = time.Sleep
-	waitForShutdownFunc       = func() { select {} }
+	waitForShutdownFunc       = defaultWaitForShutdown
 	commandRunner             = defaultCommandRunner
 	browserLauncher           = defaultBrowserLauncher
 	startAgentFieldServerFunc = defaultStartAgentFieldServer
@@ -236,10 +239,17 @@ func runServer(cmd *cobra.Command, args []string) {
 	}
 
 	fmt.Printf("AgentField server running. Press Ctrl+C to exit.\n")
-	// Keep main goroutine alive
+
+	// Wait for shutdown signal
 	waitForShutdownFunc()
 
-	// TODO: Implement graceful shutdown
+	// Graceful shutdown
+	fmt.Println("\nShutdown signal received, draining connections...")
+	if err := agentfieldServer.Stop(); err != nil {
+		log.Printf("Error during shutdown: %v", err)
+		os.Exit(1)
+	}
+	fmt.Println("Server stopped gracefully.")
 }
 
 // loadConfig loads configuration from file and environment variables.
@@ -498,6 +508,13 @@ func defaultStartAgentFieldServer(s *server.AgentFieldServer) error {
 	return s.Start()
 }
 
+// defaultWaitForShutdown blocks until SIGINT or SIGTERM is received.
+func defaultWaitForShutdown() {
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+	<-ctx.Done()
+}
+
 func openBrowser(url string) {
 	var err error
 	switch runtime.GOOS {

control-plane/cmd/agentfield-server/main_test.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
@@ -387,3 +388,33 @@ func TestOpenBrowserUsesLauncher(t *testing.T) {
 		t.Fatal("expected browserLauncher to be invoked")
 	}
 }
+
+func TestDefaultWaitForShutdown(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("sending SIGINT to self is not supported on Windows")
+	}
+
+	// defaultWaitForShutdown should unblock when SIGINT is sent to the process
+	done := make(chan struct{})
+	go func() {
+		defaultWaitForShutdown()
+		close(done)
+	}()
+
+	// Send SIGINT to self
+	time.Sleep(50 * time.Millisecond)
+	p, err := os.FindProcess(os.Getpid())
+	if err != nil {
+		t.Fatalf("failed to find self process: %v", err)
+	}
+	if err := p.Signal(syscall.SIGINT); err != nil {
+		t.Fatalf("failed to send SIGINT: %v", err)
+	}
+
+	select {
+	case <-done:
+		// success
+	case <-time.After(3 * time.Second):
+		t.Fatal("defaultWaitForShutdown did not unblock after SIGINT")
+	}
+}

control-plane/config/agentfield.yaml

@@ -69,6 +69,7 @@ type UIConfig struct {
 // AgentFieldConfig holds the core AgentField server configuration.
 type AgentFieldConfig struct {
 	Port             int                    `yaml:"port"`
+	ShutdownTimeout  time.Duration          `yaml:"shutdown_timeout" mapstructure:"shutdown_timeout"`
 	ARD              ARDConfig              `yaml:"ard" mapstructure:"ard"`
 	Registration     RegistrationConfig     `yaml:"registration" mapstructure:"registration"`
 	NodeHealth       NodeHealthConfig       `yaml:"node_health" mapstructure:"node_health"`
@@ -485,6 +486,9 @@ func ApplyDefaults(cfg *Config) {
 	if cfg.Telemetry.Timeout <= 0 {
 		cfg.Telemetry.Timeout = 800 * time.Millisecond
 	}
+	if cfg.AgentField.ShutdownTimeout <= 0 {
+		cfg.AgentField.ShutdownTimeout = 30 * time.Second
+	}
 	if cfg.Logging.Level == "" {
 		cfg.Logging.Level = "info"
 	}
@@ -573,6 +577,13 @@ func ApplyEnvOverrides(cfg *Config) {
 		}
 	}
 
+	// Shutdown timeout override
+	if val := os.Getenv("AGENTFIELD_SHUTDOWN_TIMEOUT"); val != "" {
+		if d, err := time.ParseDuration(val); err == nil {
+			cfg.AgentField.ShutdownTimeout = d
+		}
+	}
+
 	// Node health monitoring overrides
 	if val := os.Getenv("AGENTFIELD_HEALTH_CHECK_INTERVAL"); val != "" {
 		if d, err := time.ParseDuration(val); err == nil {

control-plane/internal/config/config.go

@@ -567,3 +567,39 @@ func TestLoggingEnvOverrides(t *testing.T) {
 		}
 	})
 }
+
+func TestShutdownTimeoutDefaults(t *testing.T) {
+	t.Parallel()
+
+	cfg := Config{}
+	ApplyDefaults(&cfg)
+
+	if cfg.AgentField.ShutdownTimeout != 30*time.Second {
+		t.Fatalf("expected default shutdown timeout 30s, got %v", cfg.AgentField.ShutdownTimeout)
+	}
+}
+
+func TestShutdownTimeoutEnvOverride(t *testing.T) {
+	os.Setenv("AGENTFIELD_SHUTDOWN_TIMEOUT", "45s")
+	defer os.Unsetenv("AGENTFIELD_SHUTDOWN_TIMEOUT")
+
+	cfg := Config{}
+	ApplyEnvOverrides(&cfg)
+
+	if cfg.AgentField.ShutdownTimeout != 45*time.Second {
+		t.Fatalf("expected shutdown timeout 45s, got %v", cfg.AgentField.ShutdownTimeout)
+	}
+}
+
+func TestShutdownTimeoutEnvOverrideIgnoresInvalidValue(t *testing.T) {
+	os.Setenv("AGENTFIELD_SHUTDOWN_TIMEOUT", "not-a-duration")
+	defer os.Unsetenv("AGENTFIELD_SHUTDOWN_TIMEOUT")
+
+	cfg := Config{}
+	cfg.AgentField.ShutdownTimeout = 15 * time.Second
+	ApplyEnvOverrides(&cfg)
+
+	if cfg.AgentField.ShutdownTimeout != 15*time.Second {
+		t.Fatalf("expected invalid shutdown timeout env value to be ignored, got %v", cfg.AgentField.ShutdownTimeout)
+	}
+}

control-plane/internal/config/config_additional_test.go

@@ -0,0 +1,90 @@
+package server
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/Agent-Field/agentfield/control-plane/internal/config"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStopGracefulShutdownOnEmptyServer(t *testing.T) {
+	// Stop() should not panic on a zero-value server (all fields nil)
+	s := &AgentFieldServer{}
+	err := s.Stop()
+	require.NoError(t, err)
+}
+
+func TestStopGracefulShutdownWithHTTPServer(t *testing.T) {
+	// Create a minimal server with an httpServer that's already listening
+	cfg := &config.Config{}
+	cfg.AgentField.ShutdownTimeout = 2 * time.Second
+
+	srv := &AgentFieldServer{
+		config: cfg,
+		httpServer: &http.Server{
+			Addr: ":0", // random port
+		},
+	}
+
+	// Start listening in background
+	ln, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", ":0")
+	if err != nil {
+		t.Fatalf("failed to listen: %v", err)
+	}
+	srv.httpServer.Addr = ln.Addr().String()
+
+	go func() {
+		_ = srv.httpServer.Serve(ln)
+	}()
+
+	// Give server a moment to start
+	time.Sleep(50 * time.Millisecond)
+
+	// Stop should shut down gracefully
+	err = srv.Stop()
+	require.NoError(t, err)
+}
+
+func TestStopHTTPServerShutdownTimeout(t *testing.T) {
+	// Test that a very short timeout causes force close
+	cfg := &config.Config{}
+	cfg.AgentField.ShutdownTimeout = 1 * time.Nanosecond // impossibly short
+
+	srv := &AgentFieldServer{
+		config: cfg,
+		httpServer: &http.Server{
+			Addr: ":0",
+		},
+	}
+
+	// Start listening with a handler that holds connections open
+	ln, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", ":0")
+	if err != nil {
+		t.Fatalf("failed to listen: %v", err)
+	}
+	srv.httpServer.Addr = ln.Addr().String()
+	srv.httpServer.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(5 * time.Second) // simulate long-running request
+		w.WriteHeader(200)
+	})
+
+	go func() {
+		_ = srv.httpServer.Serve(ln)
+	}()
+	time.Sleep(50 * time.Millisecond)
+
+	// Make a request that will be in-flight during shutdown
+	go func() {
+		client := &http.Client{Timeout: 10 * time.Second}
+		_, _ = client.Get("http://" + ln.Addr().String() + "/")
+	}()
+	time.Sleep(20 * time.Millisecond)
+
+	// Stop with impossibly short timeout — should return error
+	err = srv.Stop()
+	require.Error(t, err)
+}

control-plane/internal/server/graceful_shutdown_test.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -96,6 +97,10 @@ type AgentFieldServer struct {
 	kb         *knowledgebase.KB
 	// Native scope-aware RAG knowledge store (embed-on-write/search).
 	knowledgeService *knowledge.Service
+	// HTTP server for graceful shutdown support
+	httpServerMu sync.RWMutex
+	httpServer   *http.Server
+	stopping     bool
 }
 
 // NewAgentFieldServer creates a new instance of the AgentFieldServer.
@@ -649,9 +654,59 @@ func (s *AgentFieldServer) Start() error {
 		return fmt.Errorf("failed to start admin gRPC server: %w", err)
 	}
 
-	// TODO: Implement WebSocket, gRPC
-	// Start HTTP server
-	return s.Router.Run(":" + strconv.Itoa(s.config.AgentField.Port))
+	// Start HTTP server (using net/http.Server for graceful shutdown support)
+	addr := ":" + strconv.Itoa(s.config.AgentField.Port)
+	httpServer := &http.Server{
+		Addr:    addr,
+		Handler: s.Router,
+	}
+	if !s.setHTTPServer(httpServer) {
+		return nil
+	}
+	if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		return fmt.Errorf("failed to start HTTP server on %s: %w", addr, err)
+	}
+	return nil
+}
+
+func (s *AgentFieldServer) setHTTPServer(httpServer *http.Server) bool {
+	s.httpServerMu.Lock()
+	defer s.httpServerMu.Unlock()
+	if s.stopping {
+		return false
+	}
+	s.httpServer = httpServer
+	return true
+}
+
+func (s *AgentFieldServer) getHTTPServer() *http.Server {
+	s.httpServerMu.RLock()
+	defer s.httpServerMu.RUnlock()
+	return s.httpServer
+}
+
+func (s *AgentFieldServer) shutdownHTTPServer() error {
+	httpServer := s.getHTTPServer()
+	if httpServer == nil {
+		return nil
+	}
+
+	var shutdownTimeout time.Duration
+	if s.config != nil {
+		shutdownTimeout = s.config.AgentField.ShutdownTimeout
+	}
+	if shutdownTimeout <= 0 {
+		shutdownTimeout = 30 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+	defer cancel()
+	if err := httpServer.Shutdown(ctx); err != nil {
+		logger.Logger.Error().Err(err).Msg("HTTP server shutdown timed out, forcing close")
+		_ = httpServer.Close()
+		return err
+	}
+	logger.Logger.Info().Msg("HTTP server shut down gracefully")
+	return nil
 }
 
 func (s *AgentFieldServer) startAdminGRPCServer() error {
@@ -714,6 +769,12 @@ func (s *AgentFieldServer) ListReasoners(ctx context.Context, _ *adminpb.ListRea
 
 // Stop gracefully shuts down the AgentFieldServer.
 func (s *AgentFieldServer) Stop() error {
+	s.httpServerMu.Lock()
+	s.stopping = true
+	s.httpServerMu.Unlock()
+
+	httpShutdownErr := s.shutdownHTTPServer()
+
 	if s.adminGRPCServer != nil {
 		s.adminGRPCServer.GracefulStop()
 	}
@@ -731,7 +792,9 @@ func (s *AgentFieldServer) Stop() error {
 	}
 
 	// Stop health monitor service
-	s.healthMonitor.Stop()
+	if s.healthMonitor != nil {
+		s.healthMonitor.Stop()
+	}
 
 	// Stop execution cleanup service
 	if s.cleanupService != nil {
@@ -784,8 +847,8 @@ func (s *AgentFieldServer) Stop() error {
 		}
 	}
 
-	// TODO: Implement graceful shutdown for HTTP, WebSocket, gRPC
-	return nil
+	// TODO: Implement graceful shutdown for WebSocket
+	return httpShutdownErr
 }
 
 // setupRoutes composes the full HTTP surface by delegating to focused