Merge branch 'master' into dependabot/npm_and_yarn/simple-git-3.16.0

.azure-pipelines/NonAsciiValidations.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "NonAsciiValidations"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   variables:
     buildConfiguration: 'Release'
     dotnetSdkVersion: '3.1.401'

.azure-pipelines/contentValidations.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "ContentValidations"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.azure-pipelines/dataConnectorValidations.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "DataConnectorValidations"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.azure-pipelines/detectionTemplateSchemaValidation.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "DetectionTemplateSchemaValidation"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   variables:
     buildConfiguration: 'Release'
     dotnetSdkVersion: '3.1.401'

.azure-pipelines/detectionsValidations.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "DetectionsValidations"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.azure-pipelines/documentsLinkValidator.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "DocumentsLinkValidation"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.azure-pipelines/hyperLinkValidations.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "HyperLinkValidations"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.azure-pipelines/jsonFileValidator.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "JsonFileValidation"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.azure-pipelines/logoValidator.yaml

@@ -1,7 +1,7 @@
 jobs:
   - job: "logoValidator"
     pool:
-      vmImage: "ubuntu-18.04"
+      vmImage: "ubuntu-latest"
     steps:
       - script: 'npm install -g [email protected];which npm;npm -v'
       - task: Npm@1

.azure-pipelines/playbooksValidations.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "PlaybooksValidations"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.azure-pipelines/sampleDataValidator.yaml

@@ -1,7 +1,7 @@
 jobs:
   - job: "sampleDataValidator"
     pool:
-      vmImage: "ubuntu-18.04"
+      vmImage: "ubuntu-latest"
     steps:
       - script: 'npm install -g [email protected];which npm;npm -v'
       - task: Npm@1

.azure-pipelines/solutionValidator.yaml

@@ -0,0 +1,13 @@
+jobs:
+- job: "SolutionValidations"
+  pool:
+    vmImage: 'ubuntu-latest'
+  steps:
+  - script: 'npm install -g [email protected];which npm;npm -v'
+  - task: Npm@1
+    displayName: 'npm install'
+    inputs:
+      verbose: false
+      command: 'install'
+  - script: 'npm run tsc && node .script/SolutionValidations/solutionValidator.js'
+    displayName: 'Solution Validations'

.azure-pipelines/workbooksTemplateValidations.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "WorkbooksTemplateValidations"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.azure-pipelines/workbooksValidations.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "WorkbooksValidations"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.azure-pipelines/yamlFileValidator.yaml

@@ -1,7 +1,7 @@
 jobs:
 - job: "YamlFileValidation"
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: 'npm install -g [email protected];which npm;npm -v'
   - task: Npm@1

.github/workflows/arm-ttk-validations.yaml

@@ -9,11 +9,6 @@ jobs:
       solutionName: ${{ steps.step1.outputs.solutionName }}
       mainTemplateChanged: ${{ steps.step1.outputs.mainTemplateChanged }}
       createUiChanged: ${{ steps.step1.outputs.createUiChanged }}
-      hasPlaybooksChanged: ${{ steps.step1.outputs.hasPlaybooksChanged }}
-      playbookFilesList: ${{ steps.step1.outputs.playbookFilesList}}
-      dataConnectorFileName: ${{ steps.step1.outputs.dataConnectorFileName }}
-      isDataConnectorFolderNameWithSpace: ${{ steps.step1.outputs.isDataConnectorFolderNameWithSpace}}
-      hasDataConnectorFileChanged: ${{ steps.step1.outputs.hasDataConnectorFileChanged}}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -25,128 +20,60 @@ jobs:
           $diff = git diff --diff-filter=d --name-only HEAD^ HEAD
           Write-Host "List of files in PR: $diff"
           
-          $hasmainTemplateOrCreateUiDefinitionTemplateChanged = $false
-          $hasPlaybooksChanged = $false
-          $hasDataConnectorFileChanged = $false
-          $isDataConnectorFolderNameWithSpace = $false
-          $playbookFilesList = @()
-          $dataConnectorFileNames = @()
+          $hasmainTemplateChanged = $false
+          $hasCreateUiDefinitionTemplateChanged = $false
+          
+          $isChangeInSolutionsFolder = [bool]($diff | Where-Object {$_ -like 'Solutions/*'})
+          if (!$isChangeInSolutionsFolder)
+          {
+            Write-Host "Skipping as change is not in Solutions folder!"
+            exit 0
+          }
 
-          $SourceDiff = $diff | ForEach-Object {
-            $currentFile = Get-ChildItem -Path $_ -File
-            $currentFileStr = $currentFile.ToString()
-            $mainTemplateChanged = $currentFileStr.ToLower().Contains("maintemplate.json")
-            $createUiDefinitionTemplateChanged = $currentFileStr.ToLower().Contains("createuidefinition.json")
+          $requiredFiles = @("mainTemplate.json", "createUiDefinition.json")
+          $filteredFiles = $diff | Where-Object {$_ -match ($requiredFiles -Join "|")}
+          Write-Host "Filtered Files $filteredFiles"
 
-            if ($mainTemplateChanged -eq $true)
+          if ($filteredFiles.Count -gt 0)
+          {
+            $mainTemplateValue = $filteredFiles -match "mainTemplate.json" 
+            $createUiValue = $filteredFiles -match "createUiDefinition.json"
+
+            if ($mainTemplateValue)
             {
               $hasmainTemplateChanged = $true
             }
 
-            if ($createUiDefinitionTemplateChanged -eq $true)
+            if ($createUiValue)
             {
               $hasCreateUiDefinitionTemplateChanged = $true
             }
-
-            $fileValue = (($currentFile.DirectoryName).Split("\")[-1])
-            $playbooksFolderIndex = $fileValue.IndexOf("Playbooks/")
-            $playbooksWithSingleFileFolderIndex = $fileValue.IndexOf("Playbooks")
-            if ($playbooksFolderIndex -gt 0 -or $playbooksWithSingleFileFolderIndex -gt 0)
-            {
-              if ($currentFileStr.Contains(".json"))
-              {
-                $hasPlaybooksChanged = $true
-                $playbookFilesList += $currentFileStr + ","
-              }
-            }
-
-            $dataConnectorFolderWithSpaceIndex = $fileValue.IndexOf("Data Connectors")
-            $dataConnectorFolderWithoutSpaceIndex = $fileValue.IndexOf("DataConnectors")
-            
-            if ($dataConnectorFolderWithSpaceIndex -gt 0 -or $dataConnectorFolderWithoutSpaceIndex -gt 0)
-            {
-              # CHANGES ARE MADE IN DATA CONNECTOR FILE
-              if ($currentFileStr.Contains("azuredeploy"))
-              {
-                $dataConnectorFileNames += $currentFileStr
-                if ($dataConnectorFolderWithSpaceIndex -gt 0)
-                {
-                  $isDataConnectorFolderNameWithSpace = $true
-                }
-                $hasDataConnectorFileChanged = $true
-              }
-            }
             
-            $solutionIndex = $fileValue.IndexOf("Solutions/")
-            if ($solutionIndex -gt -1 -and $null -ne $solutionIndex  -and $solutionName -ne '')
+            if ($filteredFiles.Count -eq 1)
             {
-              $solutionNameWithOtherPath = $fileValue.SubString($solutionIndex + 10)
-              if ($solutionNameWithOtherPath -ne '' -and $solutionNameWithOtherPath -ne $null)
-              {
-                $headers = $solutionNameWithOtherPath -split '/', 3
-                if ($null -ne $headers[0])
-                {
-                  $sName = $headers[0]
-                }
-              }
-            }
-          }
-
-          if ($hasPlaybooksChanged -eq $true)
-          {
-            $playbooksBasePath = "/home/runner/work/Azure-Sentinel/Azure-Sentinel/Solutions/$sName/Playbooks/"
-            $filteredPlaybookFiles = $playbookFilesList.Split(",")
-            $finalizedPlaybookFiles = @()
-            foreach($playbookFile in $filteredPlaybookFiles)
-            {
-              if ($playbookFile.Contains(".json"))
-              {
-                $fileName = $playbookFile.Replace($playbooksBasePath, '')
-                $finalizedPlaybookFiles += $fileName
-              }
-            }
-          }
-
-          if ($hasDataConnectorFileChanged -eq $true)
-          {
-            if ($isDataConnectorFolderNameWithSpace -eq $true)
-            {
-              $dataConnectorBasePath = "/home/runner/work/Azure-Sentinel/Azure-Sentinel/Solutions/$sName/Data Connectors/"
+              $packageIndex = $filteredFiles.IndexOf("/Package")
+              $sName = $filteredFiles.SubString(10, $packageIndex - 10)
             }
             else
             {
-              $dataConnectorBasePath = "/home/runner/work/Azure-Sentinel/Azure-Sentinel/Solutions/$sName/DataConnectors/"
-            }
-            
-            $finalizedDataConnectorFilesList = @()
-            foreach($dataConnectorFileItem in $dataConnectorFileNames)
-            {
-              $finalizedDataConnectorFilesList += $dataConnectorFileItem.Replace($dataConnectorBasePath, '')
+              $packageIndex = $filteredFiles[0].IndexOf("/Package")
+              $sName = $filteredFiles[0].SubString(10, $packageIndex - 10)
             }
+            Write-Host "SolutionName: $sName"
           }
 
           Write-Output "::set-output name=solutionName::$sName"
           Write-Output "::set-output name=mainTemplateChanged::$hasmainTemplateChanged"
           Write-Output "::set-output name=createUiChanged::$hasCreateUiDefinitionTemplateChanged"
-          Write-Output "::set-output name=hasPlaybooksChanged::$hasPlaybooksChanged"
-          Write-Output "::set-output name=playbookFilesList::$finalizedPlaybookFiles"
-          Write-Output "::set-output name=hasDataConnectorFileChanged::$hasDataConnectorFileChanged"
-          Write-Output "::set-output name=dataConnectorFileNames::$finalizedDataConnectorFilesList"
-          Write-Output "::set-output name=isDataConnectorFolderNameWithSpace::$isDataConnectorFolderNameWithSpace"
 
       - uses: docker/build-push-action@v2
         id: publishGithubPackage
         name: Run ARM-TTK
-        if: ${{ success() && steps.step1.outcome == 'success' && steps.step1.outputs.solutionName != '' && (steps.step1.outputs.hasPlaybooksChanged == 'true' || steps.step1.outputs.mainTemplateChanged == 'true' || steps.step1.outputs.createUiChanged == 'true' || steps.step1.outputs.hasDataConnectorFileChanged == 'true') }}
+        if: ${{ success() && steps.step1.outcome == 'success' && steps.step1.outputs.solutionName != '' && (steps.step1.outputs.mainTemplateChanged == 'true' || steps.step1.outputs.createUiChanged == 'true') }}
         env: 
           SolutionName: ${{ steps.step1.outputs.solutionName }}
           mainTemplateChanged: ${{ steps.step1.outputs.mainTemplateChanged }}
           createUiChanged: ${{ steps.step1.outputs.createUiChanged }}
-          hasPlaybooksChanged: ${{ steps.step1.outputs.hasPlaybooksChanged }}
-          playbookFilesList: ${{ steps.step1.outputs.playbookFilesList }}
-          dataConnectorFileNames: ${{ steps.step1.outputs.dataConnectorFileNames }}
-          isDataConnectorFolderNameWithSpace: ${{ steps.step1.outputs.isDataConnectorFolderNameWithSpace }}
-          hasDataConnectorFileChanged: ${{ steps.step1.outputs.hasDataConnectorFileChanged }}
         with:
           context: .
           file: ./.github/actions/Dockerfile
@@ -155,8 +82,3 @@ jobs:
             SolutionName
             mainTemplateChanged
             createUiChanged
-            hasPlaybooksChanged
-            playbookFilesList
-            dataConnectorFileNames
-            isDataConnectorFolderNameWithSpace
-            hasDataConnectorFileChanged

.github/workflows/convertKqlFunctionYamlToArmTemplate.yaml

@@ -10,6 +10,9 @@ on:
        - 'Parsers/ASimWebSession/Parsers/**'
        - 'Parsers/ASimProcessEvent/Parsers/**'
        - 'Parsers/ASimAuditEvent/Parsers/**'
+       - 'Parsers/ASimAuthentication/Parsers/**'
+       - 'Parsers/ASimFileEvent/Parsers/**'
+       - 'Parsers/ASimRegistryEvent/Parsers/**'
 jobs:
   kqlFuncYaml2Arm:
     # The workflow should not run on forked repositories for security reasons

.script/ReadMe.md

@@ -38,7 +38,7 @@ By default, the logs color is white. In case you want another color you can use
    - Create an async validation function that gets the file path.  
    In case the validation pass the function returns success, otherwise throws an exception (don't return an error, this will handle by the infrastructure).
    - Create CheckOptions object with 3 properties:  
-     - onCheckFile: the validation validetion function that will run on each file (from step one)  
+     - onCheckFile: the validation validation function that will run on each file (from step one)  
      - onExecError: error behavior in case that the file validation failed  
      - onFinalFailed:  error behavior at the end of the validation, execute in case one of the validation failed
    - Call runCheckOverChangedFiles with properties:

.script/SolutionValidations/ValidDomainsVerticals.json

@@ -0,0 +1,35 @@
+{
+  "validDomains": [
+    "Application",
+    "Cloud Provider",
+    "Compliance",
+    "DevOps",
+    "Identity",
+    "Internet of Things (IoT)",
+    "IT Operations",
+    "Migration",
+    "Networking",
+    "Platform",
+    "Security - Others",
+    "Security - Threat Intelligence",
+    "Security - Threat Protection",
+    "Security - 0-day Vulnerability",
+    "Security - Automation (SOAR)",
+    "Security - Cloud Security",
+    "Security - Information Protection",
+    "Security - Insider Threat",
+    "Security - Network",
+    "Security - Vulnerability Management",
+    "Storage",
+    "Training and Tutorials",
+    "User Behavior (UEBA)"
+  ],
+  "validVerticals": [
+    "Aeronautics",
+    "Education",
+    "Finance",
+    "Healthcare",
+    "Manufacturing",
+    "Retail"
+  ]
+}