Merge branch 'develop' into gitversion

Author: Dusty-Meg

.github/workflows/build.yml

@@ -124,7 +124,7 @@ jobs:
 
       - name: Anchore Container Scan
         id: anchore-scan
-        uses: anchore/[email protected].1
+        uses: anchore/[email protected].2
         continue-on-error: true
         if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.head_ref, 'release/') }}
         with:

.github/workflows/nightly.yml

@@ -78,7 +78,7 @@ jobs:
 
       - name: Anchore Container Scan
         id: anchore-scan
-        uses: anchore/[email protected].1
+        uses: anchore/[email protected].2
         with:
           image: ${{ fromJSON(steps.meta.outputs.json).tags[0] }}
           fail-build: true

.github/workflows/security.yml

@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
 
       - name: TruffleHog OSS
-        uses: trufflesecurity/trufflehog@main
+        uses: trufflesecurity/trufflehog@v3.23.0
         with:
           path: ./
           base: ${{ github.event.repository.default_branch }}

.github/workflows/test.yml

@@ -251,7 +251,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-      run: dotnet sonarscanner begin /k:"Project-MONAI_monai-deploy-workflow-manager" /o:"project-monai" /d:sonar.login="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.opencover.reportsPaths="../**/coverage.opencover.xml" /d:sonar.coverage.exclusions="src/WorkflowManager/Database/Repositories/**/*"
+      run: dotnet sonarscanner begin /k:"Project-MONAI_monai-deploy-workflow-manager" /o:"project-monai" /d:sonar.login="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.opencover.reportsPaths="../**/coverage.opencover.xml" /d:sonar.coverage.exclusions="src/WorkflowManager/Database/Repositories/**/*,src/TaskManager/Database/TaskDispatchEventRepository.cs"
       working-directory: ./src
 
     - name: Restore Solution

docs/changelog.md

@@ -16,3 +16,7 @@
 
 
 # Changelog
+
+Renamed the (Generated) Argo client to ArgoGeneratedClient, added a new ArgoClient using just the methods used by this codebase.
+
+Enhanced the ArgoClient -> Argo_Get_WorkflowLogsAsync method to decode the json better and make the logs extracted from Argo more readable.

guidelines/mwm-input.md

@@ -66,7 +66,7 @@ The message is a JSON document with the following fields:
 
 ###### Dicom Metadata File
 To improve processing time, the MWM expects the DICOM metadata tags to be available in a structured JSON object.
-This object is used for [metadata evaluation](mwm-workflow-spec.md#dicom-tags).
+This object is used for [metadata evaluation](/docs/setup/mwm-workflow-spec.md#dicom-tags).
 
 The format is as follows:
 

guidelines/mwm-sadd.md

@@ -44,7 +44,7 @@ The scope of this document is limited to the design of the MONAI Workflow Manage
 ## Reference Documents
 
 - [MONAI Workflow Manager Requirements](mwm-srs.md)
-- [Workflow Specification Language](mwm-workflow-spec.md)
+- [Workflow Specification Language](/docs/setup/mwm-workflow-spec.md)
 - [Input Payload Specification](mwm-input.md)
 - [Storage Specification](mwm-storage.md)
 
@@ -112,7 +112,7 @@ The Workflows API allows clients to register Clinical Workflows. It will validat
 ##### POST /workflows
 This endpoint saves the a workflow.
 ###### Body
-A workflow definition as per the [workflow definition spec](mwm-workflow-spec.md).
+A workflow definition as per the [workflow definition spec](/docs/setup/mwm-workflow-spec.md).
 ![register](static/mwm-workflows-register.png)
 
 #### Responses
@@ -124,7 +124,7 @@ A workflow definition as per the [workflow definition spec](mwm-workflow-spec.md
 ##### PUT /workflows/WORKFLOW_ID
 Updates a workflow with the workflow specification sent in the body. This will create a new version of the workflow.
 ###### Body
-A workflow definition as per the [workflow definition spec](mwm-workflow-spec.md).
+A workflow definition as per the [workflow definition spec](/docs/setup/mwm-workflow-spec.md).
 May include a partial schema - the included attributes will be updated in the Workflow.
 
 ![update](static/mwm-workflows-update.png)
@@ -160,7 +160,7 @@ May include a partial schema - the included attributes will be updated in the Wo
 
 - `200`
   - Description: The workflow definition.
-  - Content: The [workflow spec](mwm-workflow-spec.md)
+  - Content: The [workflow spec](/docs/setup/mwm-workflow-spec.md)
 
 ---
 #### Tasks API
@@ -193,7 +193,7 @@ For more details & the event schema see [input](mwm-input.md#notification-messag
 ![payloadlistener](static/mwm-payload-listener.png)
 
 #### Workflow Executer
-The workflow executer is responsible for running a workflow - scheduling tasks tasks, passing metadata from one task to the other, and evaluating [Evaluators](mwm-workflow-spec.md#evaluators). Finally, it will hand over responsibility to the Export Notification service when the workflow is complete.
+The workflow executer is responsible for running a workflow - scheduling tasks tasks, passing metadata from one task to the other, and evaluating [Evaluators](/docs/setup/mwm-workflow-spec.md#evaluators). Finally, it will hand over responsibility to the Export Notification service when the workflow is complete.
 
 #### Task Executer
 The task executer is responsible for running tasks. It is invoked by the Event Broker Adapter whenever there is a new event in the task dispatch or task callback queues.
@@ -222,12 +222,12 @@ Responsibilities of plugins:
 * Adding task output files to the output directory
 
 #### Export Service
-The export service is used when a Task has an [output destination](mwm-workflow-spec.md#destinations) external to the system (ie not another task). The export services publishes an output event to a Pub/Sub service. Once published, this export operation is considered complete.
+The export service is used when a Task has an [output destination](/docs/setup/mwm-workflow-spec.md#destinations) external to the system (ie not another task). The export services publishes an output event to a Pub/Sub service. Once published, this export operation is considered complete.
 
 #### Data Retention Service
 
 ##### Retention Policies
-As part of the [Workflow definition](mwm-workflow-spec.md), _retention policies_ can be applied to data generated by workflows and tasks. The Data Retention service is responsible for enforcing those policies.
+As part of the [Workflow definition](/docs/setup/mwm-workflow-spec.md), _retention policies_ can be applied to data generated by workflows and tasks. The Data Retention service is responsible for enforcing those policies.
 
 By default, data is retained for `15` days after the workflow execution has completed.
 

guidelines/mwm-storage.md

@@ -36,7 +36,7 @@ The MWM will be responsible for managing access to objects. It will have root le
 The informatics gateway will have write-only access to a specific storage bucket. This will be used to save the original input, and a reference will then be sent to the MWM.
 
 #### Workflow Manager Task Plugins
-Task plugins, as executable code within the MWM process, will theoretically have access to the entire object store. In practice, when executed they will receive references to specific [artifacts](mwm-workflow-spec.md#artifacts).
+Task plugins, as executable code within the MWM process, will theoretically have access to the entire object store. In practice, when executed they will receive references to specific [artifacts](/docs/setup/mwm-workflow-spec.md#artifacts).
 
 It is the plugins' responsibility to ensure Applications have access to the artifacts they need. How they do so depends on the plugin's architecture - they could send files as payloads in an HTTP call, or they could generate temporary access tokens and send these along with the object reference to app servers.
 

src/TaskManager/API/ITaskDispatchEventService.cs

@@ -39,5 +39,13 @@ public interface ITaskDispatchEventService
         /// </summary>
         /// <param name="taskExecutionId">Task execution ID associated with the event</param>
         Task<TaskDispatchEventInfo?> GetByTaskExecutionIdAsync(string taskExecutionId);
+
+        /// <summary>
+        /// Updates task plugin arguments of a task dispatch event in the database.
+        /// </summary>
+        /// <param name="taskDispatchEvent">A TaskDispatchEvent to update.</param>
+        /// <param name="pluginArgs">The plugin arguments to update.</param>
+        /// <returns>Returns the updated TaskDispatchEventInfo.</returns>
+        Task<TaskDispatchEventInfo> UpdateTaskPluginArgsAsync(TaskDispatchEventInfo taskDispatchEvent, Dictionary<string, string> pluginArgs);
     }
 }

src/TaskManager/Database/ITaskDispatchEventRepository.cs

@@ -46,5 +46,13 @@ public interface ITaskDispatchEventRepository
         /// <param name="taskExecutionId">Task execution ID associated with the event</param>
         /// <returns></returns>
         Task<bool> RemoveAsync(string taskExecutionId);
+
+        /// <summary>
+        /// Updates the plugin args of a task dispatch event in the database.
+        /// </summary>
+        /// <param name="taskDispatchEventInfo">A TaskDispatchEvent to update.</param>
+        /// <param name="pluginArgs">The plugin arguments to update.</param>
+        /// <returns>The updated TaskDispatchEventInfo.</returns>
+        Task<TaskDispatchEventInfo> UpdateTaskPluginArgsAsync(TaskDispatchEventInfo taskDispatchEventInfo, Dictionary<string, string> pluginArgs);
     }
 }

src/TaskManager/Database/TaskDispatchEventRepository.cs

@@ -109,5 +109,22 @@ await _taskDispatchEventCollection.DeleteOneAsync(
                 return false;
             }
         }
+
+        public async Task<TaskDispatchEventInfo> UpdateTaskPluginArgsAsync(TaskDispatchEventInfo taskDispatchEventInfo, Dictionary<string, string> pluginArgs)
+        {
+            Guard.Against.Null(taskDispatchEventInfo);
+            Guard.Against.Null(pluginArgs);
+
+            try
+            {
+                await _taskDispatchEventCollection.FindOneAndUpdateAsync(i => i.Id == taskDispatchEventInfo.Id, Builders<TaskDispatchEventInfo>.Update.Set(p => p.Event.TaskPluginArguments, pluginArgs)).ConfigureAwait(false);
+                return await GetByTaskExecutionIdAsync(taskDispatchEventInfo.Event.ExecutionId).ConfigureAwait(false);
+            }
+            catch (Exception e)
+            {
+                _logger.DatabaseException(nameof(UpdateTaskPluginArgsAsync), e);
+                return default;
+            }
+        }
     }
 }