Merge pull request #782 from Project-MONAI/AC-2127

neildsouth · web-flow · commit ad938c746438 · 2023-05-05T11:55:02.000+01:00
changing from workflowInstanceId to workflowId
diff --git a/src/WorkflowManager/Contracts/Migrations/M001_ExecutionStats_addVersion.cs b/src/WorkflowManager/Contracts/Migrations/M001_ExecutionStats_addVersion.cs
@@ -14,16 +14,15 @@
  * limitations under the License.
  */
 
-using System;
 using Monai.Deploy.WorkflowManager.Contracts.Models;
 using Mongo.Migration.Migrations.Document;
 using MongoDB.Bson;
 
 namespace Monai.Deploy.WorkflowManager.Contracts.Migrations
 {
-    public class M001_TaskExecutionStats_addVersion : DocumentMigration<ExecutionStats>
+    public class M001_ExecutionStats_addVersion : DocumentMigration<ExecutionStats>
     {
-        public M001_TaskExecutionStats_addVersion() : base("1.0.0") { }
+        public M001_ExecutionStats_addVersion() : base("1.0.0") { }
 
         public override void Up(BsonDocument document)
         {
@@ -35,8 +34,8 @@ public override void Down(BsonDocument document)
             {
                 document.Remove("Version");
             }
-            catch (Exception)
-            {
+            catch
+            {  // can ignore we dont want failures stopping startup !
             }
         }
     }
diff --git a/src/WorkflowManager/Contracts/Migrations/M001_Payload_addVerion.cs b/src/WorkflowManager/Contracts/Migrations/M001_Payload_addVerion.cs
@@ -13,7 +13,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-using System;
 using Monai.Deploy.WorkflowManager.Contracts.Models;
 using Mongo.Migration.Migrations.Document;
 using MongoDB.Bson;
@@ -34,8 +33,8 @@ public override void Down(BsonDocument document)
             {
                 document.Remove("Version");
             }
-            catch (Exception)
-            {
+            catch
+            {  // can ignore we dont want failures stopping startup !
             }
         }
     }
diff --git a/src/WorkflowManager/Contracts/Migrations/M001_WorkflowInstance_addVerion.cs b/src/WorkflowManager/Contracts/Migrations/M001_WorkflowInstance_addVerion.cs
@@ -13,7 +13,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-using System;
 using Monai.Deploy.WorkflowManager.Contracts.Models;
 using Mongo.Migration.Migrations.Document;
 using MongoDB.Bson;
@@ -34,8 +33,8 @@ public override void Down(BsonDocument document)
             {
                 document.Remove("Version");
             }
-            catch (Exception)
-            {
+            catch
+            {  // can ignore we dont want failures stopping startup !
             }
         }
     }
diff --git a/src/WorkflowManager/Contracts/Migrations/M001_WorkflowRevision_addVerion.cs b/src/WorkflowManager/Contracts/Migrations/M001_WorkflowRevision_addVerion.cs
@@ -13,7 +13,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-using System;
 using Monai.Deploy.WorkflowManager.Contracts.Models;
 using Mongo.Migration.Migrations.Document;
 using MongoDB.Bson;
@@ -34,8 +33,8 @@ public override void Down(BsonDocument document)
             {
                 document.Remove("Version");
             }
-            catch (Exception)
-            {
+            catch
+            {  // can ignore we dont want failures stopping startup !
             }
         }
     }
diff --git a/src/WorkflowManager/Contracts/Migrations/M002_ExecutionStats_addWorkflowId.cs b/src/WorkflowManager/Contracts/Migrations/M002_ExecutionStats_addWorkflowId.cs
@@ -0,0 +1,42 @@
+﻿/*
+ * Copyright 2022 MONAI Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Monai.Deploy.WorkflowManager.Contracts.Models;
+using Mongo.Migration.Migrations.Document;
+using MongoDB.Bson;
+
+namespace Monai.Deploy.WorkflowManager.Contracts.Migrations
+{
+    public class M002_ExecutionStats_addWorkflowId : DocumentMigration<ExecutionStats>
+    {
+        public M002_ExecutionStats_addWorkflowId() : base("1.0.1") { }
+
+        public override void Up(BsonDocument document)
+        {
+            // empty, but this will make all objects re-saved with a workflowId
+        }
+        public override void Down(BsonDocument document)
+        {
+            try
+            {
+                document.Remove("WorkflowId");
+            }
+            catch
+            {  // can ignore we dont want failures stopping startup !
+            }
+        }
+    }
+}
diff --git a/src/WorkflowManager/Contracts/Migrations/M002_Payload_addPayloadDeleted.cs b/src/WorkflowManager/Contracts/Migrations/M002_Payload_addPayloadDeleted.cs
@@ -13,7 +13,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-using System;
 using Monai.Deploy.WorkflowManager.Contracts.Models;
 using Mongo.Migration.Migrations.Document;
 using MongoDB.Bson;
@@ -35,8 +34,8 @@ public override void Down(BsonDocument document)
             {
                 document.Remove("PayloadDeleted");
             }
-            catch (Exception)
-            {
+            catch
+            {  // can ignore we dont want failures stopping startup !
             }
         }
     }
diff --git a/src/WorkflowManager/Contracts/Models/ExecutionStats.cs b/src/WorkflowManager/Contracts/Models/ExecutionStats.cs
@@ -26,7 +26,7 @@
 
 namespace Monai.Deploy.WorkflowManager.Contracts.Models
 {
-    [CollectionLocation("ExecutionStats"), RuntimeVersion("1.0.0")]
+    [CollectionLocation("ExecutionStats"), RuntimeVersion("1.0.1")]
     public class ExecutionStats : IDocument
     {
         /// <summary>
@@ -40,7 +40,7 @@ public class ExecutionStats : IDocument
         /// Gets or sets Db version.
         /// </summary>
         [JsonConverter(typeof(DocumentVersionConvert)), BsonSerializer(typeof(DocumentVersionConverBson))]
-        public DocumentVersion Version { get; set; } = new DocumentVersion(1, 0, 0);
+        public DocumentVersion Version { get; set; } = new DocumentVersion(1, 0, 1);
 
         /// <summary>
         /// the correlationId of the event
@@ -49,6 +49,13 @@ public class ExecutionStats : IDocument
         [Required]
         public string CorrelationId { get; set; } = "";
 
+        /// <summary>
+        /// The id of the workflow
+        /// </summary>
+        [JsonProperty(PropertyName = "workflow_id")]
+        [Required]
+        public string WorkflowId { get; set; } = "";
+
         /// <summary>
         /// the workflow Instance that triggered the event
         /// </summary>
@@ -117,7 +124,7 @@ public ExecutionStats()
 
         }
 
-        public ExecutionStats(TaskExecution execution, string correlationId)
+        public ExecutionStats(TaskExecution execution, string workflowId, string correlationId)
         {
             Guard.Against.Null(execution, "dispatchInfo");
             CorrelationId = correlationId;
@@ -126,26 +133,29 @@ public ExecutionStats(TaskExecution execution, string correlationId)
             TaskId = execution.TaskId;
             StartedUTC = execution.TaskStartTime.ToUniversalTime();
             Status = execution.Status.ToString();
+            WorkflowId = workflowId;
         }
 
-        public ExecutionStats(TaskUpdateEvent taskUpdateEvent)
+        public ExecutionStats(TaskUpdateEvent taskUpdateEvent, string workflowId)
         {
             Guard.Against.Null(taskUpdateEvent, "taskUpdateEvent");
             CorrelationId = taskUpdateEvent.CorrelationId;
             WorkflowInstanceId = taskUpdateEvent.WorkflowInstanceId;
             ExecutionId = taskUpdateEvent.ExecutionId;
             TaskId = taskUpdateEvent.TaskId;
             Status = taskUpdateEvent.Status.ToString();
+            WorkflowId = workflowId;
         }
 
-        public ExecutionStats(TaskCancellationEvent taskCanceledEvent, string correlationId)
+        public ExecutionStats(TaskCancellationEvent taskCanceledEvent, string workflowId, string correlationId)
         {
             Guard.Against.Null(taskCanceledEvent, "taskCanceledEvent");
             CorrelationId = correlationId;
             WorkflowInstanceId = taskCanceledEvent.WorkflowInstanceId;
             ExecutionId = taskCanceledEvent.ExecutionId;
             TaskId = taskCanceledEvent.TaskId;
             Status = TaskExecutionStatus.Failed.ToString();
+            WorkflowId = workflowId;
         }
     }
 }
diff --git a/src/WorkflowManager/Database/Interfaces/ITaskExecutionStatsRepository.cs b/src/WorkflowManager/Database/Interfaces/ITaskExecutionStatsRepository.cs
@@ -29,37 +29,37 @@ public interface ITaskExecutionStatsRepository
         /// </summary>
         /// <param name="taskDispatchEvent">A TaskDispatchEvent to create.</param>
         /// <returns></returns>
-        Task CreateAsync(TaskExecution TaskExecutionInfo, string correlationId);
+        Task CreateAsync(TaskExecution TaskExecutionInfo, string workflowId, string correlationId);
 
         /// <summary>
         /// Updates status of a task dispatch event in the database.
         /// </summary>
         /// <param name="taskDispatchEvent">A TaskDispatchEvent to update.</param>
         /// <returns></returns>
-        Task UpdateExecutionStatsAsync(TaskExecution taskUpdateEvent, TaskExecutionStatus? status = null);
+        Task UpdateExecutionStatsAsync(TaskExecution taskUpdateEvent, string workflowId, TaskExecutionStatus? status = null);
 
         /// <summary>
         /// Updates status of a task now its been canceled.
         /// </summary>
         /// <param name="TaskCanceledException">A TaskCanceledException to update.</param>
         /// <returns></returns
-        Task UpdateExecutionStatsAsync(TaskCancellationEvent taskCanceledEvent, string correlationId);
+        Task UpdateExecutionStatsAsync(TaskCancellationEvent taskCanceledEvent, string workflowId, string correlationId);
 
         /// <summary>
         /// Returns paged entries between the two given dates.
         /// </summary>
         /// <param name="startTime">start of the range.</param>
         /// <param name="endTime">end of the range.</param>
         /// <returns>a collections of stats</returns>
-        Task<IEnumerable<ExecutionStats>> GetStatsAsync(DateTime startTime, DateTime endTime, int PageSize = 10, int PageNumber = 1, string workflowInstanceId = "", string taskId = "");
+        Task<IEnumerable<ExecutionStats>> GetStatsAsync(DateTime startTime, DateTime endTime, int PageSize = 10, int PageNumber = 1, string workflowId = "", string taskId = "");
 
         /// <summary>
         /// Return the total number of stats between the dates
         /// </summary>
         /// <param name="startTime">start of the range.</param>
         /// <param name="endTime">end of the range.</param>
         /// <returns>The count of all records in range</returns>
-        //Task<long> GetStatsCountAsync(DateTime startTime, DateTime endTime, string workflowInstanceId = "", string taskId = "");
+        //Task<long> GetStatsCountAsync(DateTime startTime, DateTime endTime, string workflowId = "", string taskId = "");
 
         /// <summary>
         /// Return the count of the entries with this status, or all if no status given
@@ -68,23 +68,23 @@ public interface ITaskExecutionStatsRepository
         /// <param name="endTime">end of the range.</param>
         /// <param name="status">the status to get count of, or string.empty</param>
         /// <returns>The count of all records in range</returns>
-        Task<long> GetStatsStatusCountAsync(DateTime start, DateTime endTime, string status = "", string workflowInstanceId = "", string taskId = "");
+        Task<long> GetStatsStatusCountAsync(DateTime start, DateTime endTime, string status = "", string workflowId = "", string taskId = "");
 
         /// <summary>
         /// Returns all stats in Failed or PartialFail status.
         /// </summary>
         /// <param name="startTime">start of the range.</param>
         /// <param name="endTime">end of the range.</param>
         /// <returns>All stats that failed or partially failed</returns>
-        Task<long> GetStatsStatusFailedCountAsync(DateTime startTime, DateTime endTime, string workflowInstanceId = "", string taskId = "");
+        Task<long> GetStatsStatusFailedCountAsync(DateTime startTime, DateTime endTime, string workflowId = "", string taskId = "");
 
         /// <summary>
         /// Calculates the average exection time for the given range
         /// </summary>
         /// <param name="startTime">start of the range.</param>
         /// <param name="endTime">end of the range.</param>
         /// <returns>the average exection times in the time range</returns>
-        Task<(double avgTotalExecution, double avgArgoExecution)> GetAverageStats(DateTime startTime, DateTime endTime, string workflowInstanceId = "", string taskId = "");
+        Task<(double avgTotalExecution, double avgArgoExecution)> GetAverageStats(DateTime startTime, DateTime endTime, string workflowId = "", string taskId = "");
 
     }
 }
diff --git a/src/WorkflowManager/Database/Repositories/TaskExecutionStatsRepository.cs b/src/WorkflowManager/Database/Repositories/TaskExecutionStatsRepository.cs
diff --git a/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs
diff --git a/tests/UnitTests/Database.Tests/TaskExecutionStatsRepositoryTests.cs b/tests/UnitTests/Database.Tests/TaskExecutionStatsRepositoryTests.cs

Original file line number	Diff line number	Diff line change
`@@ -14,16 +14,15 @@`
`14`	`14`	`* limitations under the License.`
`15`	`15`	`*/`
`16`	`16`
`17`		`-using System;`
`18`	`17`	`using Monai.Deploy.WorkflowManager.Contracts.Models;`
`19`	`18`	`using Mongo.Migration.Migrations.Document;`
`20`	`19`	`using MongoDB.Bson;`
`21`	`20`
`22`	`21`	`namespace Monai.Deploy.WorkflowManager.Contracts.Migrations`
`23`	`22`	`{`
`24`		`- public class M001_TaskExecutionStats_addVersion : DocumentMigration<ExecutionStats>`
	`23`	`+ public class M001_ExecutionStats_addVersion : DocumentMigration<ExecutionStats>`
`25`	`24`	`{`
`26`		`- public M001_TaskExecutionStats_addVersion() : base("1.0.0") { }`
	`25`	`+ public M001_ExecutionStats_addVersion() : base("1.0.0") { }`
`27`	`26`
`28`	`27`	`public override void Up(BsonDocument document)`
`29`	`28`	`{`
`@@ -35,8 +34,8 @@ public override void Down(BsonDocument document)`
`35`	`34`	`{`
`36`	`35`	`document.Remove("Version");`
`37`	`36`	`}`
`38`		`- catch (Exception)`
`39`		`- {`
	`37`	`+ catch`
	`38`	`+ { // can ignore we dont want failures stopping startup !`
`40`	`39`	`}`
`41`	`40`	`}`
`42`	`41`	`}`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,6 @@`
`13`	`13`	`// See the License for the specific language governing permissions and`
`14`	`14`	`// limitations under the License.`
`15`	`15`
`16`		`-using System;`
`17`	`16`	`using Monai.Deploy.WorkflowManager.Contracts.Models;`
`18`	`17`	`using Mongo.Migration.Migrations.Document;`
`19`	`18`	`using MongoDB.Bson;`
`@@ -34,8 +33,8 @@ public override void Down(BsonDocument document)`
`34`	`33`	`{`
`35`	`34`	`document.Remove("Version");`
`36`	`35`	`}`
`37`		`- catch (Exception)`
`38`		`- {`
	`36`	`+ catch`
	`37`	`+ { // can ignore we dont want failures stopping startup !`
`39`	`38`	`}`
`40`	`39`	`}`
`41`	`40`	`}`