Setup crank for performance tests (#2091)

* Configure crank benchmarks

* Add benchmark instructions to docs

* Update docs

* Update project and solution to fix builds in crank agent

* Run benchmarks against other branches

* Fix PerformanceBuild.ps1

* Fix issues

* Remove obsolete log statement

* Address review comments

* Fix whitespace

PerformanceBuild.ps1

@@ -14,39 +14,31 @@ If("Debug","Release" -notcontains $Config)
 Function ExecuteTests
 {
     Param(
-        [string]$projectRoot,
+        [string]$projectPath,
         [string]$config,
-        [string]$servicePath,
-        [string]$baseName  
+        [string]$servicePath 
     )
 
-
-    $exeName = "$baseName.exe"
-    $exePath = "$projectRoot\bin\$config\$exeName";
-
-    Write-Host "*** Building tests project $projectRoot***"
-    dotnet build -c $config $projectRoot
+    Write-Host "*** Running tests project $projectPath***"
+    dotnet run -c $config --project $projectPath -- --filter *
 
     if($LASTEXITCODE -eq 0)
     {
-        Write-Host "Build $projectRoot SUCCESS" -ForegroundColor Green
+        Write-Host "$projectPath SUCCESS" -ForegroundColor Green
         write-host "`n"
     }
     else
     {
-        Write-Host "Build FAILED" -ForegroundColor Red
+        Write-Host "FAILED" -ForegroundColor Red
         exit
     }
 
-    Write-Host "Running $exePath tests...";
-    &$exePath --filter * --envVars ServicePath:"$servicePath"
-
 }
 
 
 $location = Get-Location
 $testServicePath = "$location\test\PerformanceTests\Framework\TestService"
 $perfTestsRoot = "$location\test\PerformanceTests"
 
-ExecuteTests "$perfTestsRoot\ComponentTests" $Config $testServicePath "Microsoft.OData.Performance.Component.Tests"
-ExecuteTests "$perfTestsRoot\ServiceTests" $Config $testServicePath "Microsoft.OData.Performance.Service.Tests"
+ExecuteTests "$perfTestsRoot\ComponentTests\Microsoft.OData.Performance.ComponentTests.csproj" $Config $testServicePath
+ExecuteTests "$perfTestsRoot\ServiceTests\Microsoft.OData.Performance.ServiceTests.csproj" $Config $testServicePath

README.md

@@ -115,6 +115,92 @@ You can query the latest nightly NuGet packages using this query: [MAGIC OData q
 
 The release of the component binaries is carried out regularly through [Nuget](http://www.nuget.org/).
 
+### 3.6 Performance benchmarks
+
+#### Installation
+
+The easiest way to run the perf benchmarks is to use the [Microsoft.Crank](https://github.com/dotnet/crank) toolset.
+
+- Install the [Crank controller](https://www.nuget.org/packages/Microsoft.Crank.Controller), the CLI used to run benchmarks:
+
+    ```text
+    dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*"
+    ```
+- Install the [Crank agent](https://www.nuget.org/packages/Microsoft.Crank.Agent), service that executes benchmark jobs. This should be installed on the server(s) where the benchmarks will run. Install locally if you intend to run benchmarks locally.
+
+    ```text
+    dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*" 
+    ```
+- Verify installation was complete by running:
+
+    ```text
+    crank
+    ```
+
+#### Start the agent
+
+- Start the agent by running the following command.:
+    
+    ```text
+    crank-agent
+    ```
+
+    Once the agent has started, you should see output like:
+
+    ```text
+    Now listening on: http://[::]:5010
+    ...
+    Agent ready, waiting for jobs...
+    ```
+
+#### Run benchmarks locally
+
+- Run benchmarks for different components (reader, writer, batch, URI parser, etc.) using in-memory payloads:
+    
+    ```text
+    crank --config benchmarks.yml --scenario Components --profile local
+    ```
+
+- Run benchmarks for end-to-end scenarios against a local OData service:
+    
+    ```text
+    crank --config benchmarks.yml --scenario Service --profile local
+    ```
+
+- Run only ODataReader tests:
+
+    ```text
+    crank --config benchmarks.yml --scenario Reader --profile local
+    ```
+- Run only ODataWriter tests:
+
+    ```text
+    crank --config benchmarks.yml --scenario Writer --profile local
+    ```
+
+- Run only UriParser tests:
+
+    ```text
+    crank --config benchmarks.yml --scenario UriParser --profile local
+    ```
+
+#### Run benchmarks against the official repo
+
+To run benchmarks against the official repo instead of your local repo, pass
+the `base=true` variable to the command, e.g.:
+
+```text
+crank --config benchmarks.yml --scenario Service --profile local --variable base=true
+```
+
+This will cause the crank agent to clone the official repo and run the tests against the `master` branch.
+
+You can specify a different branch, commit or tag using the `baseBranch` variable:
+
+```text
+crank --config benchmarks.yml --scenario Service --profile local --variable base=true --variable baseBranch=v7.6.4
+```
+
 ## 4. Documentation
 
 Please visit the [ODataLib docs](https://docs.microsoft.com/en-us/odata/) on docs.microsoft.com. It has detailed descriptions on each feature provided by OData lib, how to use the OData .Net Client to consume OData service etc.

benchmarks.yml

@@ -0,0 +1,82 @@
+
+variables:
+  # set to true to run benchmarks against master branch of main/official repo
+  base: false
+  # the branch of the official repo against which to run the benchmarks
+  baseBranch: master
+  # this script sets the source for the benchmarks to either the official repo
+  # or the local directory based on the base variable
+  setSourceScript: |
+    if (job.variables && job.variables.base && job.variables.base.toString() === 'true') {
+      const branch = (job.variables && job.variables.baseBranch && job.variables.baseBranch.toString()) || 'master';
+      console.log('Run benchmarks against official repository branch', branch);
+      job.source.repository = 'https://github.com/OData/odata.net';
+      job.source.branchOrCommit = branch;
+    }
+    else {
+      console.log('Run benchmarks against local repository');
+      job.source.localFolder = '.';
+    }
+
+jobs:
+  components:
+    source:
+      project: test/PerformanceTests/ComponentTests/Microsoft.OData.Performance.ComponentTests.csproj
+    variables:
+      filter: "*"
+      jobType: short
+      base: "{{base}}"
+      baseBranch: "{{baseBranch}}"
+    arguments: "--job {{jobType}} --filter {{filter}} --memory"
+    options:
+      benchmarkDotNet: true
+    onConfigure:
+      - "{{setSourceScript}}"
+  service:
+    source:
+      project: test/PerformanceTests/ServiceTests/Microsoft.OData.Performance.ServiceTests.csproj
+    variables:
+      filter: "*"
+      jobType: short
+      base: "{{base}}"
+      baseBranch: "{{baseBranch}}"
+    framework: netcoreapp3.1
+    arguments: "--job {{jobType}} --filter {{filter}} --memory"
+    options:
+      benchmarkDotNet: true
+    onConfigure:
+     - "{{setSourceScript}}"
+
+scenarios:
+  Reader:
+    application:
+      job: components
+      variables:
+        filter: "*ODataReader*"
+  Writer:
+    application:
+      job: components
+      variables:
+        filter: "*ODataWriter*"
+  UriParser:
+    application:
+      job: component
+      variables:
+        filter: "*UriParser*"
+  Components:
+    application:
+      job: components
+      variables:
+        filter: "*"
+  Service:
+    application:
+      job: service
+      variables:
+        filter: "*"
+
+profiles:
+  local:
+    jobs:
+      application:
+        endpoints:
+          - http://localhost:5010