update

Author: satvu

src/WebJobs.Script.WebHost/WebScriptHostExceptionHandler.cs

@@ -54,7 +54,7 @@ public async Task OnTimeoutExceptionAsync(ExceptionDispatchInfo exceptionInfo, T
             {
                 _logger.LogWarning($"A function timeout has occurred. Restarting worker process executing invocationId '{timeoutException.InstanceId}'.", exceptionInfo.SourceException);
                 // If invocation id is not found in any of the workers => worker is already disposed. No action needed.
-                await functionInvocationDispatcher.RestartWorkerWithInvocationIdAsync(timeoutException.InstanceId.ToString());
+                await functionInvocationDispatcher.RestartWorkerWithInvocationIdAsync(timeoutException.InstanceId.ToString(), timeoutException);
                 _logger.LogWarning("Restart of language worker process(es) completed.", exceptionInfo.SourceException);
             }
             else

src/WebJobs.Script/Workers/Http/HttpFunctionInvocationDispatcher.cs

@@ -206,7 +206,7 @@ public Task ShutdownAsync()
             return Task.CompletedTask;
         }
 
-        public Task<bool> RestartWorkerWithInvocationIdAsync(string invocationId)
+        public Task<bool> RestartWorkerWithInvocationIdAsync(string invocationId, Exception exception = default)
         {
             // Since there's only one channel for httpworker
             DisposeAndRestartWorkerChannel(_httpWorkerChannel.Id);

src/WebJobs.Script/Workers/IFunctionInvocationDispatcher.cs

@@ -23,7 +23,7 @@ public interface IFunctionInvocationDispatcher : IDisposable
 
         Task ShutdownAsync();
 
-        Task<bool> RestartWorkerWithInvocationIdAsync(string invocationId);
+        Task<bool> RestartWorkerWithInvocationIdAsync(string invocationId, Exception exception = default);
 
         Task StartWorkerChannel();
 

src/WebJobs.Script/Workers/Rpc/FunctionRegistration/RpcFunctionInvocationDispatcher.cs

@@ -677,7 +677,7 @@ public void Dispose()
             Dispose(true);
         }
 
-        public async Task<bool> RestartWorkerWithInvocationIdAsync(string invocationId)
+        public async Task<bool> RestartWorkerWithInvocationIdAsync(string invocationId, Exception e = default)
         {
             // Dispose and restart errored channel with the particular invocation id
             var channels = await GetInitializedWorkerChannelsAsync();
@@ -686,7 +686,7 @@ public async Task<bool> RestartWorkerWithInvocationIdAsync(string invocationId)
                 if (channel.IsExecutingInvocation(invocationId))
                 {
                     _logger.LogDebug($"Restarting channel with workerId: '{channel.Id}' that is executing invocation: '{invocationId}' and timed out.");
-                    await DisposeAndRestartWorkerChannel(_workerRuntime, channel.Id, new TimeoutException($"Executing invocation `{invocationId}` timed out"));
+                    await DisposeAndRestartWorkerChannel(_workerRuntime, channel.Id, e);
                     return true;
                 }
             }

test/WebJobs.Script.Tests/Workers/Rpc/RpcFunctionInvocationDispatcherTests.cs

@@ -9,6 +9,7 @@
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Azure.WebJobs.Host.Executors.Internal;
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Description;
@@ -559,25 +560,21 @@ public async Task FunctionDispatcher_RestartOfTimedOutChannels_WebHostFailsCurre
 
             SetUpMocksForTimeoutTests(mockWebHostChannelManager, mockJobHostChannelManager, mockChannel, invocationId, true);
 
-            // Setup for initialization of a new channel
             var workerConfig = new RpcWorkerConfig
             {
                 Description = new RpcWorkerDescription { Language = "test" },
                 CountOptions = new WorkerProcessCountOptions { ProcessCount = 1 }
             };
 
-            // Create the dispatcher with our mocks
             var dispatcher = GetTestFunctionDispatcher(mockwebHostLanguageWorkerChannelManager: mockWebHostChannelManager, mockJobHostLanguageWorkerChannelManager: mockJobHostChannelManager);
 
-            // Act
-            var result = await dispatcher.RestartWorkerWithInvocationIdAsync(invocationId);
+            var result = await dispatcher.RestartWorkerWithInvocationIdAsync(invocationId, new FunctionTimeoutException());
 
-            // Assert
             Assert.True(result);
             mockWebHostChannelManager.Verify(m => m.ShutdownChannelIfExistsAsync(
                 It.IsAny<string>(),
                 It.Is<string>(id => id == "testChannelId"),
-                It.Is<Exception>(ex => ex is TimeoutException && ex.Message == $"Executing invocation `{invocationId}` timed out")),
+                It.Is<Exception>(ex => ex is FunctionTimeoutException)),
                 Times.Once);
         }
 
@@ -591,24 +588,20 @@ public async Task FunctionDispatcher_RestartOfTimedOutChannels_JobHostFailsCurre
 
             SetUpMocksForTimeoutTests(mockWebHostChannelManager, mockJobHostChannelManager, mockChannel, invocationId, false);
 
-            // Setup for initialization of a new channel
             var workerConfig = new RpcWorkerConfig
             {
                 Description = new RpcWorkerDescription { Language = "test" },
                 CountOptions = new WorkerProcessCountOptions { ProcessCount = 1 }
             };
 
-            // Create the dispatcher with our mocks
             var dispatcher = GetTestFunctionDispatcher(mockwebHostLanguageWorkerChannelManager: mockWebHostChannelManager, mockJobHostLanguageWorkerChannelManager: mockJobHostChannelManager);
 
-            // Act
-            var result = await dispatcher.RestartWorkerWithInvocationIdAsync(invocationId);
+            var result = await dispatcher.RestartWorkerWithInvocationIdAsync(invocationId, new FunctionTimeoutException());
 
-            // Assert
             Assert.True(result);
             mockJobHostChannelManager.Verify(m => m.ShutdownChannelIfExistsAsync(
                 It.Is<string>(id => id == "testChannelId"),
-                It.Is<Exception>(ex => ex is TimeoutException && ex.Message == $"Executing invocation `{invocationId}` timed out")),
+                It.Is<Exception>(ex => ex is FunctionTimeoutException)),
                 Times.Once);
         }
 
@@ -956,13 +949,13 @@ private void SetUpMocksForTimeoutTests(Mock<IWebHostRpcWorkerChannelManager> moc
             mockWebHostChannelManager.Setup(m => m.ShutdownChannelIfExistsAsync(
                 It.IsAny<string>(),
                 It.Is<string>(id => id == "testChannelId"),
-                It.Is<Exception>(ex => ex is TimeoutException && ex.Message == $"Executing invocation `{invocationId}` timed out")))
+                It.Is<Exception>(ex => ex is FunctionTimeoutException)))
                 .ReturnsAsync(webHostShutdownSucceeds);
 
             // Set up JobHost ShutdownChannelIfExistsAsync to be called with the right exception type
             mockJobHostChannelManager.Setup(m => m.ShutdownChannelIfExistsAsync(
                 It.Is<string>(id => id == "testChannelId"),
-                It.Is<Exception>(ex => ex is TimeoutException && ex.Message == $"Executing invocation `{invocationId}` timed out")))
+                It.Is<Exception>(ex => ex is FunctionTimeoutException)))
                 .ReturnsAsync(!webHostShutdownSucceeds);
         }
     }