fix(orch): raise kernel I/O timeout above backend fetch timeout to prevent EIO

packages/orchestrator/pkg/sandbox/nbd/path_direct.go

@@ -24,7 +24,15 @@ import (
 )
 
 const (
-	connectTimeout = 30 * time.Second
+	// ioTimeout is the per-request timeout for the kernel NBD driver.
+	// Must be greater than the backend fetch timeout (60s in streaming_chunk.go)
+	// so the dispatch handler has time to respond before the kernel declares
+	// the connection dead and returns EIO to the guest.
+	ioTimeout = 90 * time.Second
+
+	// deadconnTimeout is how long the kernel waits after an I/O timeout
+	// before declaring the NBD connection dead.
+	deadconnTimeout = 30 * time.Second
 
 	// disconnectTimeout should not be necessary if the disconnect is reliable
 	disconnectTimeout = 30 * time.Second
@@ -37,9 +45,11 @@ type DirectPathMount struct {
 	devicePool   *DevicePool
 	featureFlags *featureflags.Client
 
-	Backend     block.Device
-	deviceIndex uint32
-	blockSize   uint64
+	Backend         block.Device
+	deviceIndex     uint32
+	blockSize       uint64
+	ioTimeout       time.Duration
+	deadconnTimeout time.Duration
 
 	dispatchers []*Dispatch
 	socksClient []*os.File
@@ -48,16 +58,37 @@ type DirectPathMount struct {
 	handlersWg sync.WaitGroup
 }
 
-func NewDirectPathMount(b block.Device, devicePool *DevicePool, featureFlags *featureflags.Client) *DirectPathMount {
-	return &DirectPathMount{
-		Backend:      b,
-		blockSize:    4096,
-		devicePool:   devicePool,
-		featureFlags: featureFlags,
-		socksClient:  make([]*os.File, 0),
-		socksServer:  make([]io.Closer, 0),
-		deviceIndex:  math.MaxUint32,
+// MountOption configures a DirectPathMount.
+type MountOption func(*DirectPathMount)
+
+// WithIOTimeout overrides the kernel NBD I/O timeout (default 90s).
+func WithIOTimeout(d time.Duration) MountOption {
+	return func(m *DirectPathMount) { m.ioTimeout = d }
+}
+
+// WithDeadconnTimeout overrides the kernel NBD dead-connection timeout (default 30s).
+func WithDeadconnTimeout(d time.Duration) MountOption {
+	return func(m *DirectPathMount) { m.deadconnTimeout = d }
+}
+
+func NewDirectPathMount(b block.Device, devicePool *DevicePool, featureFlags *featureflags.Client, opts ...MountOption) *DirectPathMount {
+	m := &DirectPathMount{
+		Backend:         b,
+		blockSize:       4096,
+		devicePool:      devicePool,
+		featureFlags:    featureFlags,
+		socksClient:     make([]*os.File, 0),
+		socksServer:     make([]io.Closer, 0),
+		deviceIndex:     math.MaxUint32,
+		ioTimeout:       ioTimeout,
+		deadconnTimeout: deadconnTimeout,
+	}
+
+	for _, o := range opts {
+		o(m)
 	}
+
+	return m
 }
 
 func (d *DirectPathMount) Open(ctx context.Context) (retDeviceIndex uint32, err error) {
@@ -141,8 +172,8 @@ func (d *DirectPathMount) Open(ctx context.Context) (retDeviceIndex uint32, err
 
 		var opts []nbdnl.ConnectOption
 		opts = append(opts, nbdnl.WithBlockSize(d.blockSize))
-		opts = append(opts, nbdnl.WithTimeout(connectTimeout))
-		opts = append(opts, nbdnl.WithDeadconnTimeout(connectTimeout))
+		opts = append(opts, nbdnl.WithTimeout(d.ioTimeout))
+		opts = append(opts, nbdnl.WithDeadconnTimeout(d.deadconnTimeout))
 
 		serverFlags := nbdnl.FlagHasFlags | nbdnl.FlagCanMulticonn
 

packages/orchestrator/pkg/sandbox/nbd/path_direct_slow_test.go

@@ -0,0 +1,167 @@
+package nbd_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block"
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/nbd"
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/nbd/testutils"
+	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
+	"github.com/e2b-dev/infra/packages/shared/pkg/storage/header"
+)
+
+// SlowDevice wraps a ReadonlyDevice and adds a configurable delay to every
+// ReadAt call. Used to simulate slow GCS/NFS backends in tests.
+type SlowDevice struct {
+	inner     block.ReadonlyDevice
+	readDelay time.Duration
+}
+
+var _ block.ReadonlyDevice = (*SlowDevice)(nil)
+
+func NewSlowDevice(inner block.ReadonlyDevice, readDelay time.Duration) *SlowDevice {
+	return &SlowDevice{inner: inner, readDelay: readDelay}
+}
+
+func (s *SlowDevice) ReadAt(ctx context.Context, p []byte, off int64) (int, error) {
+	select {
+	case <-time.After(s.readDelay):
+	case <-ctx.Done():
+		return 0, ctx.Err()
+	}
+
+	return s.inner.ReadAt(ctx, p, off)
+}
+
+func (s *SlowDevice) Size(ctx context.Context) (int64, error) {
+	return s.inner.Size(ctx)
+}
+
+func (s *SlowDevice) BlockSize() int64 {
+	return s.inner.BlockSize()
+}
+
+func (s *SlowDevice) Slice(ctx context.Context, off, length int64) ([]byte, error) {
+	return s.inner.Slice(ctx, off, length)
+}
+
+func (s *SlowDevice) Header() *header.Header {
+	return s.inner.Header()
+}
+
+func (s *SlowDevice) Close() error {
+	return s.inner.Close()
+}
+
+// TestSlowBackend_ShortTimeout reproduces the EIO bug: when the
+// backend read takes longer than the kernel NBD I/O timeout, the kernel
+// declares the connection dead and all I/O returns EIO.
+func TestSlowBackend_ShortTimeout(t *testing.T) {
+	t.Parallel()
+
+	if os.Geteuid() != 0 {
+		t.Skip("the nbd requires root privileges to run")
+	}
+
+	featureFlags, err := featureflags.NewClient()
+	require.NoError(t, err)
+
+	const (
+		size      = int64(10 * 1024 * 1024)
+		blockSize = header.RootfsBlockSize
+	)
+
+	emptyDevice, err := testutils.NewZeroDevice(size, blockSize)
+	require.NoError(t, err)
+
+	// Backend delays every read by 8 seconds — longer than the kernel timeout below.
+	slowDevice := NewSlowDevice(emptyDevice, 8*time.Second)
+
+	cowCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("test-slow-timeout-%s", uuid.New().String()))
+	t.Cleanup(func() { os.RemoveAll(cowCachePath) })
+
+	cache, err := block.NewCache(size, blockSize, cowCachePath, false)
+	require.NoError(t, err)
+
+	overlay := block.NewOverlay(slowDevice, cache)
+	t.Cleanup(func() { overlay.Close() })
+
+	// Kernel I/O timeout of 5s + deadconn 5s = 10s total.
+	// The 8s backend delay exceeds the 5s I/O timeout, so the kernel
+	// will declare the connection dead and return EIO.
+	devicePath, cleanup, err := testutils.GetNBDDevice(
+		context.Background(), overlay, featureFlags,
+		nbd.WithIOTimeout(5*time.Second),
+		nbd.WithDeadconnTimeout(5*time.Second),
+	)
+	t.Cleanup(func() { cleanup.Run(t.Context(), 30*time.Second) })
+	require.NoError(t, err)
+
+	deviceFile, err := os.OpenFile(devicePath, os.O_RDONLY, 0)
+	require.NoError(t, err)
+	t.Cleanup(func() { deviceFile.Close() })
+
+	buf := make([]byte, 4096)
+	_, err = deviceFile.ReadAt(buf, 0)
+	require.Error(t, err, "expected EIO from kernel timeout, but read succeeded")
+	t.Logf("got expected error: %v", err)
+}
+
+// TestSlowBackend_SufficientTimeout proves the fix: with a kernel timeout
+// longer than the backend delay, reads succeed.
+func TestSlowBackend_SufficientTimeout(t *testing.T) {
+	t.Parallel()
+
+	if os.Geteuid() != 0 {
+		t.Skip("the nbd requires root privileges to run")
+	}
+
+	featureFlags, err := featureflags.NewClient()
+	require.NoError(t, err)
+
+	const (
+		size      = int64(10 * 1024 * 1024)
+		blockSize = header.RootfsBlockSize
+	)
+
+	emptyDevice, err := testutils.NewZeroDevice(size, blockSize)
+	require.NoError(t, err)
+
+	// Backend delays every read by 3 seconds.
+	slowDevice := NewSlowDevice(emptyDevice, 3*time.Second)
+
+	cowCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("test-slow-ok-%s", uuid.New().String()))
+	t.Cleanup(func() { os.RemoveAll(cowCachePath) })
+
+	cache, err := block.NewCache(size, blockSize, cowCachePath, false)
+	require.NoError(t, err)
+
+	overlay := block.NewOverlay(slowDevice, cache)
+	t.Cleanup(func() { overlay.Close() })
+
+	// Kernel I/O timeout of 30s — well above the 3s backend delay.
+	devicePath, cleanup, err := testutils.GetNBDDevice(
+		context.Background(), overlay, featureFlags,
+		nbd.WithIOTimeout(30*time.Second),
+		nbd.WithDeadconnTimeout(30*time.Second),
+	)
+	t.Cleanup(func() { cleanup.Run(t.Context(), 30*time.Second) })
+	require.NoError(t, err)
+
+	deviceFile, err := os.OpenFile(devicePath, os.O_RDONLY, 0)
+	require.NoError(t, err)
+	t.Cleanup(func() { deviceFile.Close() })
+
+	buf := make([]byte, 4096)
+	n, err := deviceFile.ReadAt(buf, 0)
+	require.NoError(t, err, "read should succeed when timeout > backend delay")
+	require.Equal(t, 4096, n)
+}

packages/orchestrator/pkg/sandbox/nbd/testutils/nbd_device.go

@@ -10,7 +10,7 @@ import (
 	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
 )
 
-func GetNBDDevice(ctx context.Context, backend block.Device, featureFlags *featureflags.Client) (nbd.DevicePath, *Cleaner, error) {
+func GetNBDDevice(ctx context.Context, backend block.Device, featureFlags *featureflags.Client, mountOpts ...nbd.MountOption) (nbd.DevicePath, *Cleaner, error) {
 	var cleaner Cleaner
 
 	devicePool, err := nbd.NewDevicePool(64)
@@ -44,7 +44,7 @@ func GetNBDDevice(ctx context.Context, backend block.Device, featureFlags *featu
 		close(poolClosed)
 	}()
 
-	mnt := nbd.NewDirectPathMount(backend, devicePool, featureFlags)
+	mnt := nbd.NewDirectPathMount(backend, devicePool, featureFlags, mountOpts...)
 
 	mntIndex, err := mnt.Open(ctx)
 	if err != nil {