Skip to content

Commit 862ba21

Browse files
authored
Merge pull request #980 from Project-MONAI/nds-stats
adding new dailyStats endpont
2 parents c44cded + 1e4ae01 commit 862ba21

File tree

7 files changed

+251
-56
lines changed

7 files changed

+251
-56
lines changed
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright 2023 MONAI Consortium
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
namespace Monai.Deploy.WorkflowManager.Common.Contracts.Models
18+
{
19+
public enum ApplicationReviewStatus
20+
{
21+
Approved,
22+
Rejected,
23+
Cancelled,
24+
AwaitingReview
25+
}
26+
}

src/WorkflowManager/Contracts/Models/ExecutionStatDTO.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,5 +36,4 @@ public ExecutionStatDTO(ExecutionStats stats)
3636
public double ExecutionDurationSeconds { get; set; }
3737
public string Status { get; set; } = "Created";
3838
}
39-
4039
}
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright 2023 MONAI Consortium
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
using System;
18+
using Newtonsoft.Json;
19+
20+
namespace Monai.Deploy.WorkflowManager.Common.Contracts.Models
21+
{
22+
public class ExecutionStatDayOverview
23+
{
24+
[JsonProperty("date")]
25+
public DateOnly Date { get; set; }
26+
[JsonProperty("total_executions")]
27+
public int TotalExecutions { get; set; }
28+
[JsonProperty("total_failures")]
29+
public int TotalFailures { get; set; }
30+
[JsonProperty("total_approvals")]
31+
public int TotalApprovals { get; set; }
32+
[JsonProperty("total_rejections")]
33+
public int TotalRejections { get; set; }
34+
[JsonProperty("total_cancelled")]
35+
public int TotalCancelled { get; set; }
36+
[JsonProperty("total_awaiting_review")]
37+
public int TotalAwaitingReview { get; set; }
38+
}
39+
}

src/WorkflowManager/Database/Interfaces/ITaskExecutionStatsRepository.cs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,15 @@ public interface ITaskExecutionStatsRepository
5252
/// <returns></returns>
5353
Task UpdateExecutionStatsAsync(TaskCancellationEvent taskCanceledEvent, string workflowId, string correlationId);
5454

55+
/// <summary>
56+
/// Returns all entries between the two given dates
57+
/// </summary>
58+
/// <param name="startTime">start of the range.</param>
59+
/// <param name="endTime">end of the range.</param>
60+
/// <param name="workflowId">optional workflow id.</param>
61+
/// <param name="taskId">optional task id.</param>
62+
/// <returns>a collections of stats</returns>
63+
Task<IEnumerable<ExecutionStats>> GetAllStatsAsync(DateTime startTime, DateTime endTime, string workflowId = "", string taskId = "");
5564
/// <summary>
5665
/// Returns paged entries between the two given dates
5766
/// </summary>
@@ -62,7 +71,7 @@ public interface ITaskExecutionStatsRepository
6271
/// <param name="workflowId">optional workflow id.</param>
6372
/// <param name="taskId">optional task id.</param>
6473
/// <returns>a collections of stats</returns>
65-
Task<IEnumerable<ExecutionStats>> GetStatsAsync(DateTime startTime, DateTime endTime, int pageSize = 10, int pageNumber = 1, string workflowId = "", string taskId = "");
74+
Task<IEnumerable<ExecutionStats>> GetStatsAsync(DateTime startTime, DateTime endTime, int? pageSize = 10, int? pageNumber = 1, string workflowId = "", string taskId = "");
6675

6776
/// <summary>
6877
/// Return the count of the entries with this status, or all if no status given.

src/WorkflowManager/Database/Repositories/TaskExecutionStatsRepository.cs

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
using System.Linq;
2020
using System.Linq.Expressions;
2121
using System.Threading.Tasks;
22-
using Ardalis.GuardClauses;
2322
using Microsoft.Extensions.Logging;
2423
using Microsoft.Extensions.Options;
2524
using Monai.Deploy.Messaging.Events;
@@ -40,11 +39,7 @@ public TaskExecutionStatsRepository(
4039
IOptions<ExecutionStatsDatabaseSettings> databaseSettings,
4140
ILogger<TaskExecutionStatsRepository> logger)
4241
{
43-
if (client == null)
44-
{
45-
throw new ArgumentNullException(nameof(client));
46-
}
47-
42+
_ = client ?? throw new ArgumentNullException(nameof(client));
4843
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
4944
var mongoDatabase = client.GetDatabase(databaseSettings.Value.DatabaseName, null);
5045
_taskExecutionStatsCollection = mongoDatabase.GetCollection<ExecutionStats>("ExecutionStats", null);
@@ -149,17 +144,24 @@ await _taskExecutionStatsCollection.UpdateOneAsync(o =>
149144
}
150145
}
151146

152-
public async Task<IEnumerable<ExecutionStats>> GetStatsAsync(DateTime startTime, DateTime endTime, int pageSize = 10, int pageNumber = 1, string workflowId = "", string taskId = "")
147+
public async Task<IEnumerable<ExecutionStats>> GetAllStatsAsync(DateTime startTime, DateTime endTime, string workflowId = "", string taskId = "")
148+
{
149+
return await GetStatsAsync(startTime, endTime, null, null, workflowId, taskId);
150+
}
151+
152+
public async Task<IEnumerable<ExecutionStats>> GetStatsAsync(DateTime startTime, DateTime endTime, int? pageSize = 10, int? pageNumber = 1, string workflowId = "", string taskId = "")
153153
{
154154
CreateFilter(startTime, endTime, workflowId, taskId, out var builder, out var filter);
155155

156156
filter &= builder.Where(GetExecutedTasksFilter());
157157

158-
var result = await _taskExecutionStatsCollection.Find(filter)
159-
.Limit(pageSize)
160-
.Skip((pageNumber - 1) * pageSize)
161-
.ToListAsync();
162-
return result;
158+
var result = _taskExecutionStatsCollection.Find(filter);
159+
if (pageSize is not null)
160+
{
161+
result = result.Limit(pageSize).Skip((pageNumber - 1) * pageSize);
162+
}
163+
164+
return await result.ToListAsync();
163165
}
164166

165167
private static ExecutionStats ExposeExecutionStats(ExecutionStats taskExecutionStats, TaskExecution taskUpdateEvent)

src/WorkflowManager/WorkflowManager/Controllers/TaskStatsController.cs

Lines changed: 101 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,59 @@ public async Task<IActionResult> GetOverviewAsync([FromQuery] DateTime startTime
113113
}
114114
}
115115

116+
/// <summary>
117+
/// Get execution daily stats for a given time period.
118+
/// </summary>
119+
/// <param name="filter">TimeFiler defining start and end times, plus paging options.</param>
120+
/// <param name="workflowId">WorkflowId if you want stats just for a given workflow. (both workflowId and TaskId must be given, if you give one).</param>
121+
/// <returns>a paged obect with all the stat details.</returns>
122+
[ProducesResponseType(typeof(StatsPagedResponse<List<ExecutionStatDTO>>), StatusCodes.Status200OK)]
123+
[ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status500InternalServerError)]
124+
[HttpGet("dailystats")]
125+
public async Task<IActionResult> GetDailyStatsAsync([FromQuery] TimeFilter filter, string workflowId = "")
126+
{
127+
SetUpFilter(filter, out var route, out var pageSize, out var validFilter);
128+
129+
try
130+
{
131+
var allStats = await _repository.GetAllStatsAsync(filter.StartTime, filter.EndTime, workflowId, string.Empty);
132+
var statsDto = allStats
133+
.OrderBy(a => a.StartedUTC)
134+
.GroupBy(s => s.StartedUTC.Date)
135+
.Select(g => new ExecutionStatDayOverview
136+
{
137+
Date = DateOnly.FromDateTime(g.Key.Date),
138+
TotalExecutions = g.Count(),
139+
TotalFailures = g.Count(i => string.Compare(i.Status, "Failed", true) == 0),
140+
TotalApprovals = g.Count(i => string.Compare(i.Status, ApplicationReviewStatus.Approved.ToString(), true) == 0),
141+
TotalRejections = g.Count(i => string.Compare(i.Status, ApplicationReviewStatus.Rejected.ToString(), true) == 0),
142+
TotalCancelled = g.Count(i => string.Compare(i.Status, ApplicationReviewStatus.Cancelled.ToString(), true) == 0),
143+
TotalAwaitingReview = g.Count(i => string.Compare(i.Status, ApplicationReviewStatus.AwaitingReview.ToString(), true) == 0),
144+
});
145+
146+
var pagedStats = statsDto.Skip((filter.PageNumber - 1) * pageSize).Take(pageSize);
147+
148+
var res = CreateStatsPagedResponse(pagedStats, validFilter, statsDto.Count(), _uriService, route);
149+
var (avgTotalExecution, avgArgoExecution) = await _repository.GetAverageStats(filter.StartTime, filter.EndTime, workflowId, string.Empty);
150+
151+
res.PeriodStart = filter.StartTime;
152+
res.PeriodEnd = filter.EndTime;
153+
res.TotalExecutions = allStats.Count();
154+
res.TotalSucceeded = statsDto.Sum(s => s.TotalApprovals);
155+
res.TotalFailures = statsDto.Sum(s => s.TotalFailures);
156+
res.TotalInprogress = statsDto.Sum(s => s.TotalAwaitingReview);
157+
res.AverageTotalExecutionSeconds = Math.Round(avgTotalExecution, 2);
158+
res.AverageArgoExecutionSeconds = Math.Round(avgArgoExecution, 2);
159+
160+
return Ok(res);
161+
}
162+
catch (Exception e)
163+
{
164+
_logger.GetStatsAsyncError(e);
165+
return Problem($"Unexpected error occurred: {e.Message}", $"tasks/stats", InternalServerError);
166+
}
167+
}
168+
116169
/// <summary>
117170
/// Get execution stats for a given time period.
118171
/// </summary>
@@ -133,63 +186,71 @@ public async Task<IActionResult> GetStatsAsync([FromQuery] TimeFilter filter, st
133186
return Problem("Failed to validate ids, not a valid guid", "tasks/stats/", BadRequest);
134187
}
135188

136-
if (filter.EndTime == default)
137-
{
138-
filter.EndTime = DateTime.Now;
139-
}
189+
SetUpFilter(filter, out var route, out var pageSize, out var validFilter);
140190

141-
if (filter.StartTime == default)
191+
try
142192
{
143-
filter.StartTime = new DateTime(2023, 1, 1);
144-
}
145-
146-
var route = Request?.Path.Value ?? string.Empty;
147-
var pageSize = filter.PageSize ?? Options.Value.EndpointSettings?.DefaultPageSize ?? 10;
148-
var max = Options.Value.EndpointSettings?.MaxPageSize ?? 20;
149-
var validFilter = new PaginationFilter(filter.PageNumber, pageSize, max);
193+
var allStats = await _repository.GetStatsAsync(filter.StartTime, filter.EndTime, pageSize, filter.PageNumber, workflowId, taskId);
194+
var statsDto = allStats
195+
.OrderBy(a => a.StartedUTC)
196+
.Select(s => new ExecutionStatDTO(s));
150197

151-
try
198+
var res = await GatherPagedStats(filter, workflowId, taskId, route, validFilter, statsDto);
199+
return Ok(res);
200+
}
201+
catch (Exception e)
152202
{
153-
workflowId ??= string.Empty;
154-
taskId ??= string.Empty;
155-
var allStats = _repository.GetStatsAsync(filter.StartTime, filter.EndTime, pageSize, filter.PageNumber, workflowId, taskId);
203+
_logger.GetStatsAsyncError(e);
204+
return Problem($"Unexpected error occurred: {e.Message}", $"tasks/stats", InternalServerError);
205+
}
206+
}
156207

157-
var successes = _repository.GetStatsStatusSucceededCountAsync(filter.StartTime, filter.EndTime, workflowId, taskId);
208+
private async Task<StatsPagedResponse<IEnumerable<T>>> GatherPagedStats<T>(TimeFilter filter, string workflowId, string taskId, string route, PaginationFilter validFilter, IEnumerable<T> statsDto)
209+
{
210+
workflowId ??= string.Empty;
211+
taskId ??= string.Empty;
158212

159-
var fails = _repository.GetStatsStatusFailedCountAsync(filter.StartTime, filter.EndTime, workflowId, taskId);
213+
var successes = _repository.GetStatsStatusSucceededCountAsync(filter.StartTime, filter.EndTime, workflowId, taskId);
160214

161-
var rangeCount = _repository.GetStatsTotalCompleteExecutionsCountAsync(filter.StartTime, filter.EndTime, workflowId, taskId);
215+
var fails = _repository.GetStatsStatusFailedCountAsync(filter.StartTime, filter.EndTime, workflowId, taskId);
162216

163-
var stats = _repository.GetAverageStats(filter.StartTime, filter.EndTime, workflowId, taskId);
217+
var rangeCount = _repository.GetStatsTotalCompleteExecutionsCountAsync(filter.StartTime, filter.EndTime, workflowId, taskId);
164218

165-
var running = _repository.GetStatsStatusCountAsync(filter.StartTime, filter.EndTime, TaskExecutionStatus.Accepted.ToString(), workflowId, taskId);
219+
var stats = _repository.GetAverageStats(filter.StartTime, filter.EndTime, workflowId, taskId);
166220

167-
await Task.WhenAll(allStats, fails, rangeCount, stats, running);
221+
var running = _repository.GetStatsStatusCountAsync(filter.StartTime, filter.EndTime, TaskExecutionStatus.Accepted.ToString(), workflowId, taskId);
168222

169-
ExecutionStatDTO[] statsDto;
223+
await Task.WhenAll(fails, rangeCount, stats, running);
170224

171-
statsDto = allStats.Result
172-
.OrderBy(a => a.StartedUTC)
173-
.Select(s => new ExecutionStatDTO(s))
174-
.ToArray();
225+
var res = CreateStatsPagedResponse(statsDto, validFilter, rangeCount.Result, _uriService, route);
175226

176-
var res = CreateStatsPagedResponse(statsDto, validFilter, rangeCount.Result, _uriService, route);
227+
res.PeriodStart = filter.StartTime;
228+
res.PeriodEnd = filter.EndTime;
229+
res.TotalExecutions = rangeCount.Result;
230+
res.TotalSucceeded = successes.Result;
231+
res.TotalFailures = fails.Result;
232+
res.TotalInprogress = running.Result;
233+
res.AverageTotalExecutionSeconds = Math.Round(stats.Result.avgTotalExecution, 2);
234+
res.AverageArgoExecutionSeconds = Math.Round(stats.Result.avgArgoExecution, 2);
235+
return res;
236+
}
177237

178-
res.PeriodStart = filter.StartTime;
179-
res.PeriodEnd = filter.EndTime;
180-
res.TotalExecutions = rangeCount.Result;
181-
res.TotalSucceeded = successes.Result;
182-
res.TotalFailures = fails.Result;
183-
res.TotalInprogress = running.Result;
184-
res.AverageTotalExecutionSeconds = Math.Round(stats.Result.avgTotalExecution, 2);
185-
res.AverageArgoExecutionSeconds = Math.Round(stats.Result.avgArgoExecution, 2);
186-
return Ok(res);
238+
private void SetUpFilter(TimeFilter filter, out string route, out int pageSize, out PaginationFilter validFilter)
239+
{
240+
if (filter.EndTime == default)
241+
{
242+
filter.EndTime = DateTime.Now;
187243
}
188-
catch (Exception e)
244+
245+
if (filter.StartTime == default)
189246
{
190-
_logger.GetStatsAsyncError(e);
191-
return Problem($"Unexpected error occurred: {e.Message}", $"tasks/stats", InternalServerError);
247+
filter.StartTime = new DateTime(2023, 1, 1);
192248
}
249+
250+
route = Request?.Path.Value ?? string.Empty;
251+
pageSize = filter.PageSize ?? Options.Value.EndpointSettings?.DefaultPageSize ?? 10;
252+
var max = Options.Value.EndpointSettings?.MaxPageSize ?? 20;
253+
validFilter = new PaginationFilter(filter.PageNumber, pageSize, max);
193254
}
194255
}
195256
}

0 commit comments

Comments
 (0)