diff --git a/docs/data/classes/reference.tools.tfsworkitemtypevalidatortool.yaml b/docs/data/classes/reference.tools.tfsworkitemtypevalidatortool.yaml index 46cd1ed83..cdc8d1072 100644 --- a/docs/data/classes/reference.tools.tfsworkitemtypevalidatortool.yaml +++ b/docs/data/classes/reference.tools.tfsworkitemtypevalidatortool.yaml @@ -53,19 +53,10 @@ configurationSamples: { "$type": "TfsWorkItemTypeValidatorToolOptions", "Enabled": true, - "IncludeWorkItemtypes": [], - "ExcludeWorkItemtypes": [], + "IncludeWorkItemTypes": [], + "ExcludeWorkItemTypes": [], "ExcludeDefaultWorkItemTypes": true, - "SourceFieldMappings": { - "User Story": { - "Microsoft.VSTS.Common.Prirucka": "Custom.Prirucka" - } - }, - "FixedTargetFields": { - "User Story": [ - "Custom.Prirucka" - ] - } + "ExcludeSourceFields": {} } sampleFor: MigrationTools.Tools.TfsWorkItemTypeValidatorToolOptions description: >- @@ -83,48 +74,36 @@ options: - parameterName: ExcludeDefaultWorkItemTypes type: Boolean description: >- - If `true`, some work item types will be automatically added to `ExcludeWorkItemtypes` list. + If `true`, some work item types will be automatically added to `ExcludeWorkItemTypes` list. Work item types excluded by default are: Code Review Request, Code Review Response, Feedback Request, Feedback Response, Shared Parameter, Shared Steps. defaultValue: missing XML code comments isRequired: false dotNetType: System.Boolean, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e -- parameterName: ExcludeWorkItemtypes - type: List - description: List of work item types which will be excluded from validation. - defaultValue: missing XML code comments - isRequired: false - dotNetType: System.Collections.Generic.List`1[[System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e]], System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e -- parameterName: FixedTargetFields +- parameterName: ExcludeSourceFields type: Dictionary description: >- - List of target fields that are considered as `fixed`. - A field marked as fixed will not stop the migration if differences are found. - Instead of a warning, only an informational message will be logged. + List of fields in source work itemt types, that are excluded from validation. + Fields excluded from validation are still validated and all found issues are logged. + But the result of the validation is 'valid' and the issues are logged as information instead of warning. - Use this list when you already know about the differences and have resolved them, - for example by using `FieldMappingTool`. - - The key is the target work item type name. - You can also use `*` to define fixed fields that apply to all work item types. + The key is the source work item type name. + You can also use `*` to exclude fields from all source work item types. defaultValue: null isRequired: false dotNetType: System.Collections.Generic.Dictionary`2[[System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[System.Collections.Generic.List`1[[System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e]], System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e]], System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e -- parameterName: IncludeWorkItemtypes +- parameterName: ExcludeWorkItemTypes type: List - description: List of work item types which will be validated. If this list is empty, all work item types will be validated. - defaultValue: null + description: List of work item types which will be excluded from validation. + defaultValue: missing XML code comments isRequired: false dotNetType: System.Collections.Generic.List`1[[System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e]], System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e -- parameterName: SourceFieldMappings - type: Dictionary - description: >- - Field reference name mappings. Key is work item type name, value is dictionary of mapping source filed name to - target field name. Target field name can be empty string to indicate that this field will not be validated in target. - As work item type name, you can use `*` to define mappings which will be applied to all work item types. +- parameterName: IncludeWorkItemTypes + type: List + description: List of work item types which will be validated. If this list is empty, all work item types will be validated. defaultValue: null isRequired: false - dotNetType: System.Collections.Generic.Dictionary`2[[System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[System.Collections.Generic.Dictionary`2[[System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e]], System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e]], System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e + dotNetType: System.Collections.Generic.List`1[[System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e]], System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e status: missing XML code comments processingTarget: missing XML code comments classFile: src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorTool.cs diff --git a/docs/static/schema/configuration.schema.json b/docs/static/schema/configuration.schema.json index e10717524..3d4d08795 100644 --- a/docs/static/schema/configuration.schema.json +++ b/docs/static/schema/configuration.schema.json @@ -2182,22 +2182,6 @@ } } }, - "TfsChangeSetMappingTool": { - "title": "TfsChangeSetMappingTool", - "description": "missing XML code comments", - "type": "object", - "properties": { - "ChangeSetMappingFile": { - "description": "missing XML code comments", - "type": "string" - }, - "Enabled": { - "description": "If set to `true` then the tool will run. Set to `false` and the processor will not run.", - "type": "boolean", - "default": "true" - } - } - }, "TfsEmbededImagesTool": { "title": "TfsEmbededImagesTool", "description": "missing XML code comments", @@ -2234,6 +2218,22 @@ } } }, + "TfsChangeSetMappingTool": { + "title": "TfsChangeSetMappingTool", + "description": "missing XML code comments", + "type": "object", + "properties": { + "Enabled": { + "description": "If set to `true` then the tool will run. Set to `false` and the processor will not run.", + "type": "boolean", + "default": "true" + }, + "ChangeSetMappingFile": { + "description": "missing XML code comments", + "type": "string" + } + } + }, "TfsNodeStructureTool": { "title": "TfsNodeStructureTool", "description": "Tool for creating missing area and iteration path nodes in the target project during migration. Configurable through TfsNodeStructureToolOptions to specify which node types to create.", @@ -2469,20 +2469,11 @@ "default": "true" }, "ExcludeDefaultWorkItemTypes": { - "description": "If `true`, some work item types will be automatically added to `ExcludeWorkItemtypes` list.\r\n Work item types excluded by default are: Code Review Request, Code Review Response, Feedback Request,\r\n Feedback Response, Shared Parameter, Shared Steps.", + "description": "If `true`, some work item types will be automatically added to `ExcludeWorkItemTypes` list.\r\n Work item types excluded by default are: Code Review Request, Code Review Response, Feedback Request,\r\n Feedback Response, Shared Parameter, Shared Steps.", "type": "boolean" }, - "ExcludeWorkItemtypes": { - "description": "List of work item types which will be excluded from validation.", - "type": "array", - "prefixItems": [ - { - "type": "string" - } - ] - }, - "FixedTargetFields": { - "description": "List of target fields that are considered as `fixed`.\r\n A field marked as fixed will not stop the migration if differences are found.\r\n Instead of a warning, only an informational message will be logged.\r\n\n Use this list when you already know about the differences and have resolved them,\r\n for example by using `FieldMappingTool`.\r\n\n The key is the target work item type name.\r\n You can also use `*` to define fixed fields that apply to all work item types.", + "ExcludeSourceFields": { + "description": "List of fields in source work itemt types, that are excluded from validation.\r\n Fields excluded from validation are still validated and all found issues are logged.\r\n But the result of the validation is 'valid' and the issues are logged as information instead of warning.\r\n\n The key is the source work item type name.\r\n You can also use `*` to exclude fields from all source work item types.", "type": "object", "default": "null", "additionalProperties": { @@ -2494,26 +2485,24 @@ ] } }, - "IncludeWorkItemtypes": { - "description": "List of work item types which will be validated. If this list is empty, all work item types will be validated.", + "ExcludeWorkItemTypes": { + "description": "List of work item types which will be excluded from validation.", "type": "array", - "default": "null", "prefixItems": [ { "type": "string" } ] }, - "SourceFieldMappings": { - "description": "Field reference name mappings. Key is work item type name, value is dictionary of mapping source filed name to\r\n target field name. Target field name can be empty string to indicate that this field will not be validated in target.\r\n As work item type name, you can use `*` to define mappings which will be applied to all work item types.", - "type": "object", + "IncludeWorkItemTypes": { + "description": "List of work item types which will be validated. If this list is empty, all work item types will be validated.", + "type": "array", "default": "null", - "additionalProperties": { - "type": "object", - "additionalProperties": { + "prefixItems": [ + { "type": "string" } - } + ] } } }, diff --git a/docs/static/schema/schema.tools.tfsworkitemtypevalidatortool.json b/docs/static/schema/schema.tools.tfsworkitemtypevalidatortool.json index 20265a697..a54f9059a 100644 --- a/docs/static/schema/schema.tools.tfsworkitemtypevalidatortool.json +++ b/docs/static/schema/schema.tools.tfsworkitemtypevalidatortool.json @@ -11,27 +11,22 @@ "default": "true" }, "ExcludeDefaultWorkItemTypes": { - "description": "If `true`, some work item types will be automatically added to `ExcludeWorkItemtypes` list.\r\n Work item types excluded by default are: Code Review Request, Code Review Response, Feedback Request,\r\n Feedback Response, Shared Parameter, Shared Steps.", + "description": "If `true`, some work item types will be automatically added to `ExcludeWorkItemTypes` list.\r\n Work item types excluded by default are: Code Review Request, Code Review Response, Feedback Request,\r\n Feedback Response, Shared Parameter, Shared Steps.", "type": "boolean" }, - "ExcludeWorkItemtypes": { - "description": "List of work item types which will be excluded from validation.", - "type": "array" - }, - "FixedTargetFields": { - "description": "List of target fields that are considered as `fixed`.\r\n A field marked as fixed will not stop the migration if differences are found.\r\n Instead of a warning, only an informational message will be logged.\r\n\n Use this list when you already know about the differences and have resolved them,\r\n for example by using `FieldMappingTool`.\r\n\n The key is the target work item type name.\r\n You can also use `*` to define fixed fields that apply to all work item types.", + "ExcludeSourceFields": { + "description": "List of fields in source work itemt types, that are excluded from validation.\r\n Fields excluded from validation are still validated and all found issues are logged.\r\n But the result of the validation is 'valid' and the issues are logged as information instead of warning.\r\n\n The key is the source work item type name.\r\n You can also use `*` to exclude fields from all source work item types.", "type": "object", "default": "null" }, - "IncludeWorkItemtypes": { + "ExcludeWorkItemTypes": { + "description": "List of work item types which will be excluded from validation.", + "type": "array" + }, + "IncludeWorkItemTypes": { "description": "List of work item types which will be validated. If this list is empty, all work item types will be validated.", "type": "array", "default": "null" - }, - "SourceFieldMappings": { - "description": "Field reference name mappings. Key is work item type name, value is dictionary of mapping source filed name to\r\n target field name. Target field name can be empty string to indicate that this field will not be validated in target.\r\n As work item type name, you can use `*` to define mappings which will be applied to all work item types.", - "type": "object", - "default": "null" } } } \ No newline at end of file diff --git a/src/MigrationTools.Clients.TfsObjectModel.Tests/Tools/FieldReferenceNameMappingToolOptionsTests.cs b/src/MigrationTools.Clients.TfsObjectModel.Tests/Tools/FieldReferenceNameMappingToolOptionsTests.cs deleted file mode 100644 index 9eff910c3..000000000 --- a/src/MigrationTools.Clients.TfsObjectModel.Tests/Tools/FieldReferenceNameMappingToolOptionsTests.cs +++ /dev/null @@ -1,112 +0,0 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using MigrationTools.Tools; - -namespace MigrationTools.Tests.Tools -{ - [TestClass] - public class FieldReferenceNameMappingToolOptionsTests - { - [TestMethod] - public void FieldMappingsMustNotBeNullWhenNormalized() - { - TfsWorkItemTypeValidatorToolOptions options = new() { Enabled = true }; - options.SourceFieldMappings = null; - options.Normalize(); - - Assert.IsNotNull(options.SourceFieldMappings); - } - - [TestMethod] - public void ShouldLookupValueCaseInsensitively() - { - TfsWorkItemTypeValidatorToolOptions options = new() - { - Enabled = true, - SourceFieldMappings = new() - { - ["wit"] = new() - { - ["source"] = "target" - } - } - }; - options.Normalize(); - - Assert.AreEqual("target", options.GetTargetFieldName("wit", "source", out bool _)); - Assert.AreEqual("target", options.GetTargetFieldName("WIT", "source", out bool _)); - Assert.AreEqual("target", options.GetTargetFieldName("wit", "SOURCE", out bool _)); - Assert.AreEqual("target", options.GetTargetFieldName("WIT", "SOURCE", out bool _)); - } - - [TestMethod] - public void ShouldReturnSourceValueIfNotMapped() - { - TfsWorkItemTypeValidatorToolOptions options = new() - { - Enabled = true, - SourceFieldMappings = new() - { - ["wit"] = new() - { - ["source"] = "target" - } - } - }; - options.Normalize(); - - Assert.AreEqual("not-mapped-source", options.GetTargetFieldName("wit", "not-mapped-source", out bool _)); - Assert.AreEqual("source", options.GetTargetFieldName("not-mapped-wit", "source", out bool _)); - } - - [TestMethod] - public void ShouldReturnEmptyStringIfMappedToEmptyString() - { - TfsWorkItemTypeValidatorToolOptions options = new() - { - Enabled = true, - SourceFieldMappings = new() - { - ["wit"] = new() - { - ["source"] = "target", - ["source-null"] = "" - } - } - }; - - options.Normalize(); - - Assert.AreEqual("target", options.GetTargetFieldName("wit", "source", out bool _)); - Assert.IsEmpty(options.GetTargetFieldName("wit", "source-null", out bool _)); - } - - [TestMethod] - public void ShouldHandleAllWorkItemTypes() - { - TfsWorkItemTypeValidatorToolOptions options = new() - { - Enabled = true, - SourceFieldMappings = new() - { - [TfsWorkItemTypeValidatorToolOptions.AllWorkItemTypes] = new() - { - ["source-all"] = "target-all" - }, - ["wit"] = new() - { - ["source"] = "target" - }, - ["wit2"] = new() - { - ["source-all"] = "target-wit2" - } - } - }; - options.Normalize(); - - Assert.AreEqual("target-wit2", options.GetTargetFieldName("wit2", "source-all", out bool _)); - Assert.AreEqual("target-all", options.GetTargetFieldName("wit", "source-all", out bool _)); - Assert.AreEqual("target-all", options.GetTargetFieldName("not-mapped-wit", "source-all", out bool _)); - } - } -} diff --git a/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs b/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs index e3b39707c..3355c9ed9 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs @@ -259,11 +259,13 @@ private void ValidateWorkItemTypes() .ToProject() .WorkItemTypes .Cast() + .OrderBy(wit => wit.Name) .ToList(); var targetWits = Target.WorkItems.Project .ToProject() .WorkItemTypes .Cast() + .OrderBy(wit => wit.Name) .ToList(); // Reflected work item ID field is mandatory for migration, so it is validated even if the validator tool is disabled. diff --git a/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemTypeValidatorProcessor.cs b/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemTypeValidatorProcessor.cs index 6d44d49bb..8d80761af 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemTypeValidatorProcessor.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemTypeValidatorProcessor.cs @@ -41,11 +41,13 @@ protected override void InternalExecute() .ToProject() .WorkItemTypes .Cast() + .OrderBy(wit => wit.Name) .ToList(); List targetWits = Target.WorkItems.Project .ToProject() .WorkItemTypes .Cast() + .OrderBy(wit => wit.Name) .ToList(); bool containsReflectedWorkItemId = CommonTools.WorkItemTypeValidatorTool .ValidateReflectedWorkItemIdField(sourceWits, targetWits, Target.Options.ReflectedWorkItemIdField); diff --git a/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMappingToolExtentions.cs b/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMappingToolExtentions.cs new file mode 100644 index 000000000..53d1df7e7 --- /dev/null +++ b/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMappingToolExtentions.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MigrationTools.FieldMaps.AzureDevops.ObjectModel; +using MigrationTools.Tools.Infrastructure; +using MigrationTools.Tools.Interfaces; + +namespace MigrationTools.Tools +{ + internal static class FieldMappingToolExtentions + { + /// + /// Returns all defined field maps of type for work item type . + /// If no field map is defined for given work item type, empty collection is returned. + /// + /// Type of field maps to look for. + /// Field mapping tool. + /// Work item type name. + public static IEnumerable GetFieldMaps( + this IFieldMappingTool fieldMappingTool, + string witName) + where TFieldMap : IFieldMap + { + List allMaps = fieldMappingTool.GetFieldMappings(witName); + return allMaps + .Where(fm => fm is TFieldMap) + .Cast(); + } + + /// + /// Return defined field maps of type for work item type , + /// which are defined for source field . If + /// is set, only field maps with given mode are returned. + /// + /// Field mapping tool. + /// Work item type name. + /// Source field reference name. + /// Field map mode. + public static IEnumerable GetFieldToFieldMaps( + this IFieldMappingTool fieldMappingTool, + string witName, + string sourceFieldReferenceName, + FieldMapMode? mapMode) + { + IEnumerable allMaps = fieldMappingTool.GetFieldMaps(witName) + .Where(fvm => sourceFieldReferenceName.Equals(fvm.Config.sourceField, StringComparison.OrdinalIgnoreCase)); + + return mapMode.HasValue + ? allMaps.Where(fvm => fvm.Config.fieldMapMode == mapMode.Value) + : allMaps; + } + + /// + /// Returns all all defined field maps in for source field . + /// + /// Field mapping tool. + /// Work item type name. + /// Source field reference name. + public static IEnumerable<(string sourceFieldName, string targetFieldName)> GetFieldToFieldMultiMaps( + this IFieldMappingTool fieldMappingTool, + string witName, + string sourceFieldReferenceName) + { + List<(string, string)> fieldMaps = []; + + foreach (FieldToFieldMultiMap multiMap in fieldMappingTool.GetFieldMaps(witName)) + { + // Iterating through dictionary to be able to ignore casing of field names. + foreach (KeyValuePair mapping in multiMap.Config.SourceToTargetMappings) + { + if (sourceFieldReferenceName.Equals(mapping.Key, StringComparison.OrdinalIgnoreCase)) + { + fieldMaps.Add((mapping.Key, mapping.Value)); + break; + } + } + } + + return fieldMaps; + } + + /// + /// Returns defined field maps of type for work item type , + /// which are defined between fields and . + /// + /// Field mapping tool. + /// Work item type name. + /// Source field reference name. + /// Target field reference name. + public static IEnumerable GetFieldValueMaps( + this IFieldMappingTool fieldMappingTool, + string witName, + string sourceFieldReferenceName, + string targetFieldReferenceName) + => fieldMappingTool.GetFieldMaps(witName) + .Where(fvm => sourceFieldReferenceName.Equals(fvm.Config.sourceField, StringComparison.OrdinalIgnoreCase) + && targetFieldReferenceName.Equals(fvm.Config.targetField, StringComparison.OrdinalIgnoreCase)); + + /// + /// Returns all defined value mappings for work item type which are defined between + /// fields and . + /// + /// Field mapping tool. + /// Work item type name. + /// Source field reference name. + /// Target field reference name. + /// Dictionary with mappings source field values to target field values. + /// Thrown when there are defined more than one target values for + /// specific source value. + public static Dictionary GetFieldValueMappings( + this IFieldMappingTool fieldMappingTool, + string witName, + string sourceFieldReferenceName, + string targetFieldReferenceName) + { + Dictionary result = new(StringComparer.OrdinalIgnoreCase); + + IEnumerable fieldValueMaps = fieldMappingTool + .GetFieldValueMaps(witName, sourceFieldReferenceName, targetFieldReferenceName); + foreach (FieldValueMap fieldValueMap in fieldValueMaps) + { + foreach (KeyValuePair map in fieldValueMap.Config.valueMapping) + { + string sourceValue = map.Key; + string targetValue = map.Value; + if (result.TryGetValue(sourceValue, out string existingTargetValue) + && !existingTargetValue.Equals(targetValue, StringComparison.OrdinalIgnoreCase)) + { + string msg = $"Conflict in field value mapping for field '{sourceFieldReferenceName}'" + + $" to field '{targetFieldReferenceName}' in work item type '{witName}':" + + $" Value '{sourceValue}' maps to both '{existingTargetValue}' and '{targetValue}'."; + throw new InvalidOperationException(msg); + } + result[sourceValue] = targetValue; + } + } + return result; + } + } +} diff --git a/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldToFieldMap.cs b/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldToFieldMap.cs index a9b17aaa1..bbaf17410 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldToFieldMap.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldToFieldMap.cs @@ -1,13 +1,8 @@ using System; -using System.Windows.Forms; using Microsoft.Extensions.Logging; -using Microsoft.IdentityModel.Tokens; using Microsoft.TeamFoundation.WorkItemTracking.Client; -using MigrationTools.DataContracts.Pipelines; using MigrationTools.Tools; using MigrationTools.Tools.Infrastructure; -using static System.Windows.Forms.VisualStyles.VisualStyleElement.TextBox; -using static Microsoft.TeamFoundation.Client.CommandLine.Options; namespace MigrationTools.FieldMaps.AzureDevops.ObjectModel; @@ -19,7 +14,7 @@ namespace MigrationTools.FieldMaps.AzureDevops.ObjectModel; public class FieldToFieldMap : FieldMapBase { public override string MappingDisplayName => $"{Config.sourceField} {Config.targetField}"; - private FieldToFieldMapOptions Config => (FieldToFieldMapOptions)_Config; + internal FieldToFieldMapOptions Config => (FieldToFieldMapOptions)_Config; /// /// Initializes a new instance of the FieldToFieldMap class. @@ -87,13 +82,13 @@ private bool IsValid(WorkItem source, WorkItem target) case FieldMapMode.SourceToTarget: if (!source.Fields.Contains(Config.sourceField)) { - Log.LogWarning("FieldToFieldMap: [VALIDATION FAILED] Source field '{SourceField}' does not exist on source WorkItem {SourceId}. Please verify the field name is correct and exists in the source work item type. Available fields can be checked in Azure DevOps work item customization.", + Log.LogWarning("FieldToFieldMap: [VALIDATION FAILED] Source field '{SourceField}' does not exist on source WorkItem {SourceId}. Please verify the field name is correct and exists in the source work item type. Available fields can be checked in Azure DevOps work item customization.", Config.sourceField, source.Id); valid = false; } if (!target.Fields.Contains(Config.targetField)) { - Log.LogWarning("FieldToFieldMap: [VALIDATION FAILED] Target field '{TargetField}' does not exist on target WorkItem {TargetId}. Please verify the field name is correct and exists in the target work item type. You may need to add this field to the target work item type or update your field mapping configuration.", + Log.LogWarning("FieldToFieldMap: [VALIDATION FAILED] Target field '{TargetField}' does not exist on target WorkItem {TargetId}. Please verify the field name is correct and exists in the target work item type. You may need to add this field to the target work item type or update your field mapping configuration.", Config.targetField, target.Id); valid = false; } @@ -101,13 +96,13 @@ private bool IsValid(WorkItem source, WorkItem target) case FieldMapMode.TargetToTarget: if (!target.Fields.Contains(Config.sourceField)) { - Log.LogWarning("FieldToFieldMap: [VALIDATION FAILED] Source field '{SourceField}' does not exist on target WorkItem {TargetId}. In TargetToTarget mode, both source and target fields must exist on the target work item. Please verify the source field name is correct.", + Log.LogWarning("FieldToFieldMap: [VALIDATION FAILED] Source field '{SourceField}' does not exist on target WorkItem {TargetId}. In TargetToTarget mode, both source and target fields must exist on the target work item. Please verify the source field name is correct.", Config.sourceField, target.Id); valid = false; } if (!target.Fields.Contains(Config.targetField)) { - Log.LogWarning("FieldToFieldMap: [VALIDATION FAILED] Target field '{TargetField}' does not exist on target WorkItem {TargetId}. In TargetToTarget mode, both source and target fields must exist on the target work item. Please verify the target field name is correct.", + Log.LogWarning("FieldToFieldMap: [VALIDATION FAILED] Target field '{TargetField}' does not exist on target WorkItem {TargetId}. In TargetToTarget mode, both source and target fields must exist on the target work item. Please verify the target field name is correct.", Config.targetField, target.Id); valid = false; } diff --git a/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldValueMap.cs b/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldValueMap.cs index 93be345cf..3471d652a 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldValueMap.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldValueMap.cs @@ -1,8 +1,6 @@ using System; using Microsoft.Extensions.Logging; using Microsoft.TeamFoundation.WorkItemTracking.Client; -using MigrationTools.Tools.Infrastructure; -using MigrationTools._EngineV1.DataContracts; using MigrationTools.DataContracts; using MigrationTools.Tools; @@ -22,7 +20,7 @@ public FieldValueMap(ILogger logger, ITelemetryLogger telemetryLo { } - private FieldValueMapOptions Config { get { return (FieldValueMapOptions)_Config; } } + internal FieldValueMapOptions Config { get { return (FieldValueMapOptions)_Config; } } public override string MappingDisplayName => $"{Config.sourceField} {Config.targetField}"; @@ -64,4 +62,4 @@ internal override void InternalExecute(WorkItemData source, WorkItemData target) } } } -} \ No newline at end of file +} diff --git a/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldtoFieldMultiMap.cs b/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldtoFieldMultiMap.cs index 68b37b378..205aeeb66 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldtoFieldMultiMap.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Tools/FieldMappingTool/FieldMaps/FieldtoFieldMultiMap.cs @@ -1,7 +1,6 @@ using System.Collections.Generic; using Microsoft.Extensions.Logging; using Microsoft.TeamFoundation.WorkItemTracking.Client; -using MigrationTools._EngineV1.Configuration; using MigrationTools.Tools; using MigrationTools.Tools.Infrastructure; @@ -14,7 +13,7 @@ public FieldToFieldMultiMap(ILogger logger, ITelemetryLogg } public override string MappingDisplayName => string.Empty; - private FieldToFieldMultiMapOptions Config { get { return (FieldToFieldMultiMapOptions)_Config; } } + internal FieldToFieldMultiMapOptions Config { get { return (FieldToFieldMultiMapOptions)_Config; } } public override void Configure(IFieldMapOptions config) { @@ -57,4 +56,4 @@ private void mapFields(Dictionary fieldMap, WorkItem source, Wor } } } -} \ No newline at end of file +} diff --git a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorTool.cs b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorTool.cs index 2558fc60d..566239347 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorTool.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorTool.cs @@ -16,16 +16,61 @@ namespace MigrationTools.Tools /// public class TfsWorkItemTypeValidatorTool : Tool { + private class WitMapping + { + public WitMapping( + WorkItemType sourceWit, + string expectedTargetWitName, + WorkItemType? targetWit, + bool isMapped) + { + SourceWit = sourceWit; + ExpectedTargetWitName = expectedTargetWitName; + TargetWit = targetWit; + IsMapped = isMapped; + } + + public WorkItemType SourceWit { get; } + public string ExpectedTargetWitName { get; } + public WorkItemType? TargetWit { get; } + public bool HasTargetWit => TargetWit is not null; + public bool IsMapped { get; } + } + + private class FieldMapping + { + public FieldMapping( + FieldDefinition sourceField, + string expectedTargetFieldName, + FieldDefinition? targetField, + bool isMapped) + { + SourceField = sourceField; + ExpectedTargetFieldName = expectedTargetFieldName; + TargetField = targetField; + IsMapped = isMapped; + } + + public FieldDefinition SourceField { get; } + public string ExpectedTargetFieldName { get; } + public FieldDefinition? TargetField { get; } + public bool HasTargetField => TargetField is not null; + public bool IsMapped { get; } + } + private readonly IWorkItemTypeMappingTool _witMappingTool; + private readonly CommonTools _commonTools; public TfsWorkItemTypeValidatorTool( IOptions options, IWorkItemTypeMappingTool witMappingTool, + CommonTools commonTools, IServiceProvider services, ILogger logger, ITelemetryLogger telemetry) : base(options, services, logger, telemetry) { + _commonTools = commonTools ?? throw new ArgumentNullException(nameof(commonTools)); _witMappingTool = witMappingTool ?? throw new ArgumentNullException(nameof(witMappingTool)); } @@ -37,17 +82,27 @@ public bool ValidateReflectedWorkItemIdField( Log.LogInformation("Validating presence of reflected work item ID field '{reflectedWorkItemIdField}'" + " in target work item types.", reflectedWorkItemIdField); bool isValid = true; - List wits = GetTargetWitsToValidate(sourceWits, targetWits); - foreach (WorkItemType targetWit in wits) + List witPairs = GetWitsToValidate(sourceWits, targetWits); + foreach (WitMapping witPair in witPairs) { + if (!witPair.HasTargetWit) + { + continue; + } + WorkItemType targetWit = witPair.TargetWit; + if (witPair.IsMapped) + { + Log.LogInformation("Work item type '{sourceWit}' is mapped to '{targetWit}'.", + witPair.SourceWit.Name, witPair.ExpectedTargetWitName); + } if (targetWit.FieldDefinitions.Contains(reflectedWorkItemIdField)) { - Log.LogDebug(" '{targetWit}' contains reflected work item ID field '{fieldName}'.", + Log.LogDebug("'{targetWit}' contains reflected work item ID field '{fieldName}'.", targetWit.Name, reflectedWorkItemIdField); } else { - Log.LogError(" '{targetWit}' does not contain reflected work item ID field '{fieldName}'.", + Log.LogError("'{targetWit}' does not contain reflected work item ID field '{fieldName}'.", targetWit.Name, reflectedWorkItemIdField); isValid = false; } @@ -56,25 +111,29 @@ public bool ValidateReflectedWorkItemIdField( return isValid; } - private List GetTargetWitsToValidate(List sourceWits, List targetWits) + // Returns list of target work item types with respect to work item type mapping. + private List GetWitsToValidate(List sourceWits, List targetWits) { - List targetWitsToValidate = []; + List witMappings = []; foreach (WorkItemType sourceWit in sourceWits) { - string sourceWitName = sourceWit.Name; - if (!ShouldValidateWorkItemType(sourceWitName)) - { - continue; - } - string targetWitName = GetTargetWorkItemType(sourceWitName); - WorkItemType targetWit = targetWits - .FirstOrDefault(wit => wit.Name.Equals(targetWitName, StringComparison.OrdinalIgnoreCase)); - if (targetWit is not null) + if (ShouldValidateWorkItemType(sourceWit.Name)) { - targetWitsToValidate.Add(targetWit); + bool isMapped = false; + if (_witMappingTool.Mappings.TryGetValue(sourceWit.Name, out string targetWitName)) + { + isMapped = true; + } + else + { + targetWitName = sourceWit.Name; + } + WorkItemType? targetWit = targetWits + .FirstOrDefault(wit => wit.Name.Equals(targetWitName, StringComparison.OrdinalIgnoreCase)); + witMappings.Add(new WitMapping(sourceWit, targetWitName, targetWit, isMapped)); } } - return targetWitsToValidate; + return witMappings; } public bool ValidateWorkItemTypes(List sourceWits, List targetWits) @@ -83,20 +142,19 @@ public bool ValidateWorkItemTypes(List sourceWits, List targetWitNames = targetWits.Select(wit => wit.Name).ToList(); bool isValid = true; - foreach (WorkItemType sourceWit in sourceWits) + List witPairs = GetWitsToValidate(sourceWits, targetWits); + foreach (WitMapping witPair in witPairs) { - string sourceWitName = sourceWit.Name; - if (!ShouldValidateWorkItemType(sourceWitName)) + string sourceWitName = witPair.SourceWit.Name; + Log.LogInformation("Start validation of work item type '{sourceWit}'.", sourceWitName); + if (witPair.IsMapped) { - continue; + Log.LogInformation(" Work item type '{sourceWit}' is mapped to '{targetWit}'.", + sourceWitName, witPair.ExpectedTargetWitName); } - Log.LogInformation("Validating work item type '{sourceWit}'", sourceWitName); - string targetWitName = GetTargetWorkItemType(sourceWitName); - WorkItemType targetWit = targetWits - .FirstOrDefault(wit => wit.Name.Equals(targetWitName, StringComparison.OrdinalIgnoreCase)); - if (targetWit is null) + if (!witPair.HasTargetWit) { - Log.LogWarning("Work item type '{targetWit}' does not exist in target system.", targetWitName); + Log.LogWarning("Work item type '{targetWit}' does not exist in target system.", witPair.ExpectedTargetWitName); if (TryFindSimilarWorkItemType(sourceWitName, targetWitNames, out string suggestedName)) { Log.LogInformation(" Suggested mapping: '{0}' – '{1}'", sourceWitName, suggestedName); @@ -105,11 +163,12 @@ public bool ValidateWorkItemTypes(List sourceWits, List f.ReferenceName, f => f, StringComparer.OrdinalIgnoreCase); foreach (FieldDefinition sourceField in sourceFields) { - string sourceFieldName = sourceField.ReferenceName; - string targetFieldName = GetTargetFieldName(targetWit.Name, sourceFieldName); - if (string.IsNullOrEmpty(targetFieldName)) + List fieldsToValidate = GetTargetFieldsToValidate(targetWit.Name, sourceField, targetFields); + foreach (FieldMapping fieldPair in fieldsToValidate) { - continue; - } - - if (targetFields.ContainsKey(targetFieldName)) - { - if (sourceField.IsIdentity) + Log.LogInformation(" Validating source field '{sourceFieldReferenceName}' ({sourceFieldName}).", + sourceField.ReferenceName, sourceField.Name); + bool fieldResult = true; + bool fieldIsExcluded = Options.IsSourceFieldExcluded(sourceWit.Name, sourceField.ReferenceName); + LogLevel logLevel = fieldIsExcluded ? LogLevel.Information : LogLevel.Warning; + if (fieldPair.IsMapped) + { + Log.LogInformation(" Source field '{sourceFieldName}' is mapped to '{targetFieldName}'.", + fieldPair.SourceField.ReferenceName, fieldPair.ExpectedTargetFieldName); + } + if (fieldPair.HasTargetField) { - const string message = " Source field '{sourceFieldName}' is identity field." - + " Validation is not performed on identity fields, because they usually differ in allowed values."; - Log.LogDebug(message, sourceFieldName); + if (sourceField.IsIdentity) + { + const string message = " Source field '{sourceFieldName}' is identity field." + + " Validation is not performed on identity fields, because they usually differ in allowed values."; + Log.LogDebug(message, fieldPair.SourceField.ReferenceName); + } + else + { + fieldResult = ValidateField(sourceField, fieldPair.TargetField, targetWit.Name, logLevel); + } } else { - result &= ValidateField(sourceField, targetFields[targetFieldName], targetWit.Name); + LogMissingField(targetWit, sourceField, fieldPair.ExpectedTargetFieldName, logLevel); + fieldResult = false; + } + + if (!fieldResult) + { + if (fieldIsExcluded) + { + Log.LogInformation(" Field '{sourceFieldName}' is excluded from validation, so it is considered valid.", + fieldPair.SourceField.ReferenceName); + } + else + { + result = false; + } } - } - else - { - Log.LogWarning(" Missing field '{targetFieldName}' in '{targetWit}'.", targetFieldName, targetWit.Name); - Log.LogInformation(" Source field reference name: {sourceFieldReferenceName}", sourceFieldName); - Log.LogInformation(" Source field name: {sourceFieldName}", sourceField.Name); - Log.LogInformation(" Field type: {fieldType}", sourceField.FieldType); - (string valueType, List allowedValues) = GetAllowedValues(sourceField); - Log.LogInformation(" Allowed values: {allowedValues}", string.Join(", ", allowedValues.Select(v => $"'{v}'"))); - Log.LogInformation(" Allowed values type: {allowedValuesType}", valueType); - result = false; } } if (result) { - Log.LogInformation(" All fields are either present or mapped."); + Log.LogInformation(" All fields are either present or mapped for '{targetWit}'.", targetWit.Name); } return result; } - private bool ValidateField(FieldDefinition sourceField, FieldDefinition targetField, string targetWitName) + private List GetTargetFieldsToValidate( + string targetWitName, + FieldDefinition sourceField, + Dictionary targetFields) { - // If target field is in 'FixedTargetFields' list, it means, that user resolved this filed somehow. - // For example by value mapping. So any discrepancies found will be logged just as information. - // Otherwise, discrepancies are logged as warning. - LogLevel logLevel = Options.IsFieldFixed(targetWitName, targetField.ReferenceName) - ? LogLevel.Information - : LogLevel.Warning; - bool isValid = ValidateFieldType(sourceField, targetField, logLevel); - isValid &= ValidateFieldAllowedValues(sourceField, targetField, logLevel); - if (isValid) + List result = []; + + foreach (FieldToFieldMap fieldToFieldMap in _commonTools.FieldMappingTool + .GetFieldToFieldMaps(targetWitName, sourceField.ReferenceName, FieldMapMode.SourceToTarget)) + { + targetFields.TryGetValue(fieldToFieldMap.Config.targetField, out FieldDefinition targetField); + result.Add(new FieldMapping(sourceField, fieldToFieldMap.Config.targetField, targetField, true)); + } + + foreach ((string _, string targetFieldName) in _commonTools.FieldMappingTool + .GetFieldToFieldMultiMaps(targetWitName, sourceField.ReferenceName)) { - Log.LogDebug(" Target field '{targetFieldName}' exists in '{targetWit}' and is valid.", - targetField.ReferenceName, targetWitName); + targetFields.TryGetValue(targetFieldName, out FieldDefinition targetField); + result.Add(new FieldMapping(sourceField, targetFieldName, targetField, true)); } - else if (logLevel == LogLevel.Information) + + if (result.Count == 0) + { + // If no field mapping is configured for this source field, just use the same field in the target. + targetFields.TryGetValue(sourceField.ReferenceName, out FieldDefinition targetField); + result.Add(new FieldMapping(sourceField, sourceField.ReferenceName, targetField, false)); + } + + return result; + } + + private void LogMissingField( + WorkItemType targetWit, + FieldDefinition sourceField, + string targetFieldName, + LogLevel logLevel) + { + Log.Log(logLevel, " Missing field '{targetFieldName}' in '{targetWit}'.", targetFieldName, targetWit.Name); + Log.LogInformation(" Source field reference name: {sourceFieldReferenceName}", sourceField.ReferenceName); + Log.LogInformation(" Source field name: {sourceFieldName}", sourceField.Name); + Log.LogInformation(" Field type: {fieldType}", sourceField.FieldType); + (string valueType, List allowedValues) = GetAllowedValues(sourceField); + LogAllowedValues(" Allowed values: {allowedValues}", allowedValues); + Log.LogInformation(" Allowed values type: {allowedValuesType}", valueType); + } + + private bool ValidateField( + FieldDefinition sourceField, + FieldDefinition targetField, + string targetWitName, + LogLevel logLevel) + { + bool isValid = ValidateFieldType(sourceField, targetField, logLevel); + isValid &= ValidateFieldAllowedValues(targetWitName, sourceField, targetField, logLevel); + if (isValid) { - Log.LogInformation(" Target field '{targetFieldName}' in '{targetWit}' is considered valid," - + $" because it is listed in '{nameof(Options.FixedTargetFields)}'.", - targetField.ReferenceName, targetWitName, sourceField.ReferenceName); + Log.LogDebug(" Target field '{targetFieldReferenceName}' ({targetFieldName}) exists in '{targetWit}' and is valid.", + targetField.ReferenceName, targetField.Name, targetWitName); } - return (logLevel == LogLevel.Information) || isValid; + return isValid; } private bool ValidateFieldType(FieldDefinition sourceField, FieldDefinition targetField, LogLevel logLevel) @@ -192,7 +303,7 @@ private bool ValidateFieldType(FieldDefinition sourceField, FieldDefinition targ if (sourceField.FieldType != targetField.FieldType) { Log.Log(logLevel, - " Source field '{sourceField}' and target field '{targetField}' have different types:" + " Source field '{sourceField}' and target field '{targetField}' have different types:" + " source = '{sourceFieldType}', target = '{targetFieldType}'.", sourceField.ReferenceName, targetField.ReferenceName, sourceField.FieldType, targetField.FieldType); return false; @@ -200,7 +311,11 @@ private bool ValidateFieldType(FieldDefinition sourceField, FieldDefinition targ return true; } - private bool ValidateFieldAllowedValues(FieldDefinition sourceField, FieldDefinition targetField, LogLevel logLevel) + private bool ValidateFieldAllowedValues( + string targetWitName, + FieldDefinition sourceField, + FieldDefinition targetField, + LogLevel logLevel) { bool isValid = true; (string sourceValueType, List sourceAllowedValues) = GetAllowedValues(sourceField); @@ -209,27 +324,72 @@ private bool ValidateFieldAllowedValues(FieldDefinition sourceField, FieldDefini { isValid = false; Log.Log(logLevel, - " Source field '{sourceField}' and target field '{targetField}' have different allowed values types:" + " Source field '{sourceField}' and target field '{targetField}' have different allowed values types:" + " source = '{sourceFieldAllowedValueType}', target = '{targetFieldAllowedValueType}'.", sourceField.ReferenceName, targetField.ReferenceName, sourceValueType, targetValueType); } - if (!DoesTargetContainsAllSourceValues(sourceAllowedValues, targetAllowedValues)) + if (!ValidateAllowedValues(targetWitName, sourceField.ReferenceName, sourceAllowedValues, + targetField.ReferenceName, targetAllowedValues, out List missingValues, logLevel)) { isValid = false; Log.Log(logLevel, - " Source field '{sourceField}' and target field '{targetField}' have different allowed values.", + " Source field '{sourceField}' and target field '{targetField}' have different allowed values.", sourceField.ReferenceName, targetField.ReferenceName); - Log.LogInformation(" Source allowed values: {sourceAllowedValues}", - string.Join(", ", sourceAllowedValues.Select(val => $"'{val}'"))); - Log.LogInformation(" Target allowed values: {targetAllowedValues}", - string.Join(", ", targetAllowedValues.Select(val => $"'{val}'"))); + LogAllowedValues(" Source allowed values: {sourceAllowedValues}", sourceAllowedValues); + LogAllowedValues(" Target allowed values: {targetAllowedValues}", targetAllowedValues); + LogAllowedValues(" Missing values in target are: {missingValues}", missingValues); } return isValid; + } - private bool DoesTargetContainsAllSourceValues(List sourceAllowedValues, List targetAllowedValues) => - sourceAllowedValues.Except(targetAllowedValues, StringComparer.OrdinalIgnoreCase).Count() == 0; + private void LogAllowedValues(string message, List values) + => Log.LogInformation(message, string.Join(", ", values.Select(value => $"'{value}'"))); + + private bool ValidateAllowedValues( + string targetWitName, + string sourceFieldReferenceName, + List sourceAllowedValues, + string targetFieldReferenceName, + List targetAllowedValues, + out List missingValues, + LogLevel logLevel) + { + missingValues = sourceAllowedValues + .Except(targetAllowedValues, StringComparer.OrdinalIgnoreCase) + .ToList(); + if (missingValues.Count > 0) + { + Log.LogDebug(" Allowed values in target field do not match allowed values in source. Checking field value maps."); + LogAllowedValues(" Missing values are: {missingValues}", missingValues); + List mappedValues = []; + Dictionary valueMaps = _commonTools.FieldMappingTool + .GetFieldValueMappings(targetWitName, sourceFieldReferenceName, targetFieldReferenceName); + foreach (string missingValue in missingValues) + { + if (valueMaps.TryGetValue(missingValue, out string mappedValue)) + { + if (targetAllowedValues.Contains(mappedValue, StringComparer.OrdinalIgnoreCase)) + { + mappedValues.Add(missingValue); + Log.LogDebug(" Value '{missingValue}' is mapped to '{mappedValue}', which exists in target.", + missingValue, mappedValue); + } + else + { + Log.Log(logLevel, " Value '{missingValue}' is mapped to '{mappedValue}', which does not exists in target." + + $" This is probably invalid '{nameof(FieldValueMap)}' configuration.", + missingValue, mappedValue); + } + } + } + missingValues = missingValues + .Except(mappedValues, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + return missingValues.Count == 0; + } private (string valueType, List allowedValues) GetAllowedValues(FieldDefinition field) @@ -240,21 +400,22 @@ private bool DoesTargetContainsAllSourceValues(List sourceAllowedValues, { allowedValues.Add(field.AllowedValues[i]); } + allowedValues.Sort(StringComparer.OrdinalIgnoreCase); return (valueType, allowedValues); } private bool ShouldValidateWorkItemType(string workItemTypeName) { - if ((Options.IncludeWorkItemtypes.Count > 0) - && !Options.IncludeWorkItemtypes.Contains(workItemTypeName, StringComparer.OrdinalIgnoreCase)) + if ((Options.IncludeWorkItemTypes.Count > 0) + && !Options.IncludeWorkItemTypes.Contains(workItemTypeName, StringComparer.OrdinalIgnoreCase)) { Log.LogInformation( "Skipping validation of work item type '{sourceWit}' because it is not included in validation list.", workItemTypeName); return false; } - else if ((Options.ExcludeWorkItemtypes.Count > 0) - && Options.ExcludeWorkItemtypes.Contains(workItemTypeName, StringComparer.OrdinalIgnoreCase)) + else if ((Options.ExcludeWorkItemTypes.Count > 0) + && Options.ExcludeWorkItemTypes.Contains(workItemTypeName, StringComparer.OrdinalIgnoreCase)) { Log.LogInformation( "Skipping validation of work item type '{sourceWit}' because it is excluded from validation list.", @@ -264,30 +425,6 @@ private bool ShouldValidateWorkItemType(string workItemTypeName) return true; } - private string GetTargetWorkItemType(string sourceWit) - { - string targetWit = sourceWit; - if (_witMappingTool.Mappings.ContainsKey(sourceWit)) - { - targetWit = _witMappingTool.Mappings[sourceWit]; - Log.LogInformation(" This work item type is mapped to '{targetWit}' in target.", targetWit); - } - return targetWit; - } - - private string GetTargetFieldName(string targetWitName, string sourceFieldName) - { - string targetFieldName = Options.GetTargetFieldName(targetWitName, sourceFieldName, out bool isMapped); - if (isMapped) - { - string message = string.IsNullOrEmpty(targetFieldName) - ? " Source field '{sourceFieldName}' is mapped as empty string, so it is not validated in target." - : " Source field '{sourceFieldName}' is mapped as '{targetFieldName}' in target."; - Log.LogInformation(message, sourceFieldName, targetFieldName); - } - return targetFieldName; - } - private void LogWorkItemTypes(ICollection sourceWits, ICollection targetWits) { Log.LogInformation( @@ -321,16 +458,35 @@ private void LogValidationResult(bool isValid) return; } - const string message1 = "Some work item types or their fields are not present in the target system (see previous logs)." - + " Either add these fields into target work items, or map source fields to other target fields" - + $" in options ({nameof(TfsWorkItemTypeValidatorToolOptions.SourceFieldMappings)})."; - Log.LogError(message1); - const string message2 = "If you have some field mappings defined for validation, do not forget also to configure" - + $" proper field mapping in {nameof(FieldMappingTool)} so data will preserved during migration."; - Log.LogInformation(message2); - const string message3 = "If you have different allowed values in some field, either update target field to match" - + $" allowed values from source, or configure {nameof(FieldValueMap)} in {nameof(FieldMappingTool)}."; - Log.LogInformation(message3); + Log.LogError("Some work item types or their fields are not valid in the target system (see previous logs)."); + + Log.LogInformation("If the work item type does not exist in target system, you can:"); + Log.LogInformation(" - Create it there."); + Log.LogInformation(" - Configure mapping to another work item type which exists in target using" + + $" '{nameof(WorkItemTypeMappingTool)}' configuration."); + Log.LogInformation(" - Exclude it from validation. To configure which work item types are validated, use either" + + $" '{nameof(TfsWorkItemTypeValidatorToolOptions.IncludeWorkItemTypes)}' or" + + $" '{nameof(TfsWorkItemTypeValidatorToolOptions.ExcludeWorkItemTypes)}'" + + $" of '{nameof(TfsWorkItemTypeValidatorTool)}' configuration (but not both at the same time)."); + + Log.LogInformation("If field is missing in target, you can:"); + Log.LogInformation(" - Add missing field to the target work item type."); + Log.LogInformation($" - Configure field mapping using '{nameof(FieldToFieldMultiMap)}'" + + $" in '{nameof(FieldMappingTool)}' configuration." + + " This is simpler method for source to target field mapping, which allows to map multiple fields at once."); + Log.LogInformation($" - Configure field mapping using '{nameof(FieldToFieldMap)}'" + + $" in '{nameof(FieldMappingTool)}' configuration." + + $" Mapping mode must be of type '{nameof(FieldMapMode.SourceToTarget)}'."); + + Log.LogInformation("If allowed values of the source and target fields do not match, you can:"); + Log.LogInformation(" - Add missing allowed values to the target field."); + Log.LogInformation($" - Configure value mapping. Add field maps of type '{nameof(FieldValueMap)}'" + + $" to '{nameof(FieldMappingTool)}' configuration."); + + Log.LogInformation("To exclude field from validation, just configure it in" + + $" '{nameof(TfsWorkItemTypeValidatorToolOptions.ExcludeSourceFields)}'" + + $" of '{nameof(TfsWorkItemTypeValidatorTool)}' configuration." + + " If field is excluded from validation, all the issues are still logged, just the result of validation is 'valid'."); } private static bool TryFindSimilarWorkItemType( diff --git a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorToolOptions.cs b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorToolOptions.cs index d5028d8ec..c167267ef 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorToolOptions.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorToolOptions.cs @@ -14,7 +14,7 @@ public class TfsWorkItemTypeValidatorToolOptions : ToolOptions private static readonly StringComparer _normalizedComparer = StringComparer.OrdinalIgnoreCase; private bool _isNormalized = false; - private static string[] _defaultExcludedWorkItemTypes = [ + private static readonly string[] _defaultExcludedWorkItemTypes = [ "Code Review Request", "Code Review Response", "Feedback Request", @@ -27,45 +27,33 @@ public class TfsWorkItemTypeValidatorToolOptions : ToolOptions /// List of work item types which will be validated. If this list is empty, all work item types will be validated. /// /// null - public List IncludeWorkItemtypes { get; set; } = []; + public List IncludeWorkItemTypes { get; set; } = []; /// /// List of work item types which will be excluded from validation. /// - public List ExcludeWorkItemtypes { get; set; } = []; + public List ExcludeWorkItemTypes { get; set; } = []; /// - /// If , some work item types will be automatically added to list. + /// If , some work item types will be automatically added to list. /// Work item types excluded by default are: Code Review Request, Code Review Response, Feedback Request, /// Feedback Response, Shared Parameter, Shared Steps. /// public bool ExcludeDefaultWorkItemTypes { get; set; } = true; - /// - /// Field reference name mappings. Key is work item type name, value is dictionary of mapping source filed name to - /// target field name. Target field name can be empty string to indicate that this field will not be validated in target. - /// As work item type name, you can use * to define mappings which will be applied to all work item types. - /// - /// null - public Dictionary> SourceFieldMappings { get; set; } = []; - /// /// - /// List of target fields that are considered as fixed. - /// A field marked as fixed will not stop the migration if differences are found. - /// Instead of a warning, only an informational message will be logged. + /// List of fields in source work itemt types, that are excluded from validation. + /// Fields excluded from validation are still validated and all found issues are logged. + /// But the result of the validation is 'valid' and the issues are logged as information instead of warning. /// /// - /// Use this list when you already know about the differences and have resolved them, - /// for example by using . - /// - /// - /// The key is the target work item type name. - /// You can also use * to define fixed fields that apply to all work item types. + /// The key is the source work item type name. + /// You can also use * to exclude fields from all source work item types. /// /// /// null - public Dictionary> FixedTargetFields { get; set; } = []; + public Dictionary> ExcludeSourceFields { get; set; } = []; /// /// Normalizes properties, that all of them are set (not ) and all dictionaries uses @@ -78,110 +66,49 @@ public void Normalize() return; } - Dictionary> oldMappings = SourceFieldMappings; - Dictionary> newMappings = new(_normalizedComparer); - if (oldMappings is not null) - { - foreach (KeyValuePair> mapping in oldMappings) - { - Dictionary normalizedValues = new(_normalizedComparer); - foreach (KeyValuePair fieldMapping in mapping.Value) - { - normalizedValues[fieldMapping.Key.Trim()] = fieldMapping.Value.Trim(); - } - newMappings[mapping.Key.Trim()] = normalizedValues; - } - } - - Dictionary> oldFixedFields = FixedTargetFields; - Dictionary> newFixedFields = new(_normalizedComparer); - if (oldFixedFields is not null) + Dictionary> oldExcludedFields = ExcludeSourceFields; + Dictionary> newExcludedFields = new(_normalizedComparer); + if (oldExcludedFields is not null) { - foreach (KeyValuePair> mapping in oldFixedFields) + foreach (KeyValuePair> mapping in oldExcludedFields) { - newFixedFields[mapping.Key.Trim()] = mapping.Value; + newExcludedFields[mapping.Key.Trim()] = mapping.Value; } } - IncludeWorkItemtypes ??= []; - ExcludeWorkItemtypes ??= []; + IncludeWorkItemTypes ??= []; + ExcludeWorkItemTypes ??= []; if (ExcludeDefaultWorkItemTypes) { foreach (string defaultExcludedWit in _defaultExcludedWorkItemTypes) { - if (!ExcludeWorkItemtypes.Contains(defaultExcludedWit, _normalizedComparer)) + if (!ExcludeWorkItemTypes.Contains(defaultExcludedWit, _normalizedComparer)) { - ExcludeWorkItemtypes.Add(defaultExcludedWit); + ExcludeWorkItemTypes.Add(defaultExcludedWit); } } } - FixedTargetFields = newFixedFields; - SourceFieldMappings = newMappings; + ExcludeSourceFields = newExcludedFields; _isNormalized = true; } /// - /// Returns true, if field from work item type + /// Returns true, if field from work item type /// is in list of fixed target fields. Handles also fields defined for all work item types (*). /// /// Work item type name. - /// Target field reference name. - public bool IsFieldFixed(string workItemType, string targetFieldName) + /// Target field reference name. + public bool IsSourceFieldExcluded(string workItemType, string fieldReferenceName) { - if (FixedTargetFields.TryGetValue(workItemType, out List fixedFields)) + if (ExcludeSourceFields.TryGetValue(workItemType, out List excludedFields)) { - return fixedFields.Contains(targetFieldName, _normalizedComparer); + return excludedFields.Contains(fieldReferenceName, _normalizedComparer); } - if (FixedTargetFields.TryGetValue(AllWorkItemTypes, out fixedFields)) + if (ExcludeSourceFields.TryGetValue(AllWorkItemTypes, out excludedFields)) { - return fixedFields.Contains(targetFieldName, _normalizedComparer); - } - return false; - } - - /// - /// Search for mapped target field name for given . If there is no mapping for source - /// field name, its value is returned as target field name. - /// Handles also mappings defined for all work item types (*). - /// - /// Work item type name. - /// Source field reference name. - /// Flag if returned value was mapped, or just returned the original. - /// - /// Returns: - /// - /// Target field name if it is foung in mappings. This can be empty string, which means that the source - /// field is not mapped. - /// Source filed name if there is no mapping defined for this field. - /// - /// - public string GetTargetFieldName(string workItemType, string sourceFieldName, out bool isMapped) - { - if (TryGetTargetFieldName(workItemType, sourceFieldName, out string targetFieldName)) - { - isMapped = true; - return targetFieldName; - } - if (TryGetTargetFieldName(AllWorkItemTypes, sourceFieldName, out targetFieldName)) - { - isMapped = true; - return targetFieldName; - } - isMapped = false; - return sourceFieldName; - } - - private bool TryGetTargetFieldName(string workItemType, string sourceFieldName, out string targetFieldName) - { - if (SourceFieldMappings.TryGetValue(workItemType, out Dictionary mappings)) - { - if (mappings.TryGetValue(sourceFieldName, out targetFieldName)) - { - return true; - } + return excludedFields.Contains(fieldReferenceName, _normalizedComparer); } - targetFieldName = string.Empty; return false; } } diff --git a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorToolOptionsValidator.cs b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorToolOptionsValidator.cs index 425b3832e..d772e0358 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorToolOptionsValidator.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsWorkItemTypeValidatorToolOptionsValidator.cs @@ -6,14 +6,14 @@ public class TfsWorkItemTypeValidatorToolOptionsValidator : IValidateOptions 0) && (excludedCount > 0)) { - const string msg = $"'{nameof(options.IncludeWorkItemtypes)}' and '{nameof(options.ExcludeWorkItemtypes)}'" + const string msg = $"'{nameof(options.IncludeWorkItemTypes)}' and '{nameof(options.ExcludeWorkItemTypes)}'" + $" cannot be set both at the same time." - + $" If '{nameof(options.IncludeWorkItemtypes)}' list is not empty," + + $" If '{nameof(options.IncludeWorkItemTypes)}' list is not empty," + $" '{nameof(options.ExcludeDefaultWorkItemTypes)}' must be set to 'false'."; return ValidateOptionsResult.Fail(msg); } diff --git a/src/MigrationTools.Shadows/Services/TelemetryLoggerFake.cs b/src/MigrationTools.Shadows/Services/TelemetryLoggerFake.cs index 925444917..71dc0ccb0 100644 --- a/src/MigrationTools.Shadows/Services/TelemetryLoggerFake.cs +++ b/src/MigrationTools.Shadows/Services/TelemetryLoggerFake.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace MigrationTools.Services.Shadows +namespace MigrationTools.Services.Shadows { public class TelemetryLoggerFake : ITelemetryLogger { @@ -10,12 +6,10 @@ public class TelemetryLoggerFake : ITelemetryLogger public void TrackException(Exception ex, IDictionary properties = null) { - throw new NotImplementedException(); } public void TrackException(Exception ex, IEnumerable> properties = null) { - } } } diff --git a/src/MigrationTools.Shadows/Tools/MockFieldMappingTool.cs b/src/MigrationTools.Shadows/Tools/MockFieldMappingTool.cs index 03a2ee962..1268e62fc 100644 --- a/src/MigrationTools.Shadows/Tools/MockFieldMappingTool.cs +++ b/src/MigrationTools.Shadows/Tools/MockFieldMappingTool.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MigrationTools.DataContracts; +using MigrationTools.DataContracts; using MigrationTools.Tools.Infrastructure; using MigrationTools.Tools.Interfaces; @@ -18,6 +13,8 @@ public void AddFieldMap(string workItemTypeName, IFieldMap fieldToTagFieldMap) throw new NotImplementedException(); } + public List GetFieldMappings(string witName) => []; + public void ApplyFieldMappings(WorkItemData source, WorkItemData target) { throw new NotImplementedException(); diff --git a/src/MigrationTools/Tools/FieldMappingTool.cs b/src/MigrationTools/Tools/FieldMappingTool.cs index d6d4eca8a..23b22ff10 100644 --- a/src/MigrationTools/Tools/FieldMappingTool.cs +++ b/src/MigrationTools/Tools/FieldMappingTool.cs @@ -1,16 +1,10 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using MigrationTools._EngineV1.Configuration; -using MigrationTools._EngineV1.Containers; using MigrationTools.DataContracts; -using MigrationTools.Enrichers; -using MigrationTools.Processors; using MigrationTools.Tools.Infrastructure; using MigrationTools.Tools.Interfaces; @@ -22,7 +16,7 @@ namespace MigrationTools.Tools public class FieldMappingTool : Tool, IFieldMappingTool { - private Dictionary> fieldMapps = new Dictionary>(); + private readonly Dictionary> _fieldMaps = new Dictionary>(StringComparer.OrdinalIgnoreCase); /// /// Initializes a new instance of the FieldMappingTool class. @@ -37,7 +31,7 @@ public FieldMappingTool(IOptions options, IServiceProvi { foreach (IFieldMapOptions fieldmapConfig in Options.FieldMaps) { - Log.LogInformation("FieldMappingTool: Adding FieldMap {FieldMapName} for {WorkItemTypeName}", fieldmapConfig.ConfigurationMetadata.OptionFor, fieldmapConfig.ApplyTo.Count == 0? "*ApplyTo is missing*" : string.Join(", ", fieldmapConfig.ApplyTo)); + Log.LogInformation("FieldMappingTool: Adding FieldMap {FieldMapName} for {WorkItemTypeName}", fieldmapConfig.ConfigurationMetadata.OptionFor, fieldmapConfig.ApplyTo.Count == 0 ? "*ApplyTo is missing*" : string.Join(", ", fieldmapConfig.ApplyTo)); string typePattern = $"MigrationTools.Sinks.*.FieldMaps.{fieldmapConfig.ConfigurationMetadata.OptionFor}"; Type type = AppDomain.CurrentDomain.GetAssemblies() @@ -60,13 +54,9 @@ public FieldMappingTool(IOptions options, IServiceProvi } } - public int Count { get { return fieldMapps.Count; } } - - public Dictionary> Items - { - get { return fieldMapps; } - } + public int Count => _fieldMaps.Count; + public Dictionary> Items => _fieldMaps; public void AddFieldMap(string workItemTypeName, IFieldMap fieldToTagFieldMap) { @@ -74,36 +64,31 @@ public void AddFieldMap(string workItemTypeName, IFieldMap fieldToTagFieldMap) { throw new IndexOutOfRangeException("workItemTypeName on all fieldmaps must be set to at least '*'."); } - if (!fieldMapps.ContainsKey(workItemTypeName)) + if (!_fieldMaps.ContainsKey(workItemTypeName)) { - fieldMapps.Add(workItemTypeName, new List()); + _fieldMaps.Add(workItemTypeName, new List()); } - fieldMapps[workItemTypeName].Add(fieldToTagFieldMap); + _fieldMaps[workItemTypeName].Add(fieldToTagFieldMap); } - public void ApplyFieldMappings(WorkItemData source, WorkItemData target) + public List GetFieldMappings(string witName) { - if (fieldMapps.ContainsKey("*")) + if (_fieldMaps.TryGetValue("*", out List fieldMaps)) { - ProcessFieldMapList(source, target, fieldMapps["*"]); + return fieldMaps; } - if (fieldMapps.ContainsKey(source.Fields["System.WorkItemType"].Value.ToString())) + else if (_fieldMaps.TryGetValue(witName, out fieldMaps)) { - ProcessFieldMapList(source, target, fieldMapps[source.Fields["System.WorkItemType"].Value.ToString()]); + return fieldMaps; } + return []; } + public void ApplyFieldMappings(WorkItemData source, WorkItemData target) + => ProcessFieldMapList(source, target, GetFieldMappings(source.Fields["System.WorkItemType"].Value.ToString())); + public void ApplyFieldMappings(WorkItemData target) - { - if (fieldMapps.ContainsKey("*")) - { - ProcessFieldMapList(target, target, fieldMapps["*"]); - } - if (fieldMapps.ContainsKey(target.Fields["System.WorkItemType"].Value.ToString())) - { - ProcessFieldMapList(target, target, fieldMapps[target.Fields["System.WorkItemType"].Value.ToString()]); - } - } + => ApplyFieldMappings(target, target); private void ProcessFieldMapList(WorkItemData source, WorkItemData target, List list) { diff --git a/src/MigrationTools/Tools/Interfaces/IFieldMappingTool.cs b/src/MigrationTools/Tools/Interfaces/IFieldMappingTool.cs index d7645bc3a..d198da393 100644 --- a/src/MigrationTools/Tools/Interfaces/IFieldMappingTool.cs +++ b/src/MigrationTools/Tools/Interfaces/IFieldMappingTool.cs @@ -1,6 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; +using System.Collections.Generic; using MigrationTools.DataContracts; using MigrationTools.Tools.Infrastructure; @@ -11,6 +9,7 @@ public interface IFieldMappingTool Dictionary> Items { get; } void AddFieldMap(string workItemTypeName, IFieldMap fieldToTagFieldMap); + List GetFieldMappings(string witName); void ApplyFieldMappings(WorkItemData source, WorkItemData target); void ApplyFieldMappings(WorkItemData target); }