Skip to content

Commit a58a77b

Browse files
bitfasterbitfaster
authored
Automate plots (#506)
1 parent dbefa46 commit a58a77b

File tree

17 files changed

+428
-11
lines changed

17 files changed

+428
-11
lines changed

‎.github/workflows/benchpr.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@ jobs:
3636
- name: Build split asm
3737
run: dotnet build splitasm --configuration Release
3838
- name: Benchmark
39-
run: dotnet run --project "BitFaster.Caching.Benchmarks" -f net6.0 -c Release --filter *Lru*
39+
run: dotnet run --project "BitFaster.Caching.Benchmarks" -f net6.0 -c Release --filter '*'
40+
- name: Plot results
41+
run: dotnet run --project "Tools\BenchPlot\Benchplot.csproj" --configuration Release "BenchmarkDotNet.Artifacts"
4042
- name: Post process disassembly
4143
run: splitasm\splitasm\bin\Release\net6.0\splitasm.exe %GITHUB_WORKSPACE%\BenchmarkDotNet.Artifacts\results
4244
shell: cmd
@@ -62,6 +64,8 @@ jobs:
6264
run: dotnet build --configuration Release --no-restore
6365
- name: Benchmark
6466
run: dotnet run --project "BitFaster.Caching.Benchmarks" -f net6.0 -c Release --filter '*'
67+
- name: Plot results
68+
run: dotnet run --project Tools/BenchPlot/BenchPlot.csproj --configuration Release "BenchmarkDotNet.Artifacts"
6569
- name: Publish Results
6670
uses: actions/upload-artifact@v3
6771
with:
@@ -84,6 +88,8 @@ jobs:
8488
run: dotnet build --configuration Release --no-restore
8589
- name: Benchmark
8690
run: dotnet run --project "BitFaster.Caching.Benchmarks" -f net6.0 -c Release --filter '*'
91+
- name: Plot results
92+
run: dotnet run --project "Tools\BenchPlot\BenchPlot.csproj" --configuration Release "BenchmarkDotNet.Artifacts"
8793
- name: Publish Results
8894
uses: actions/upload-artifact@v3
8995
with:

‎BitFaster.Caching.HitRateAnalysis/Analysis.cs

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Drawing;
34
using System.Globalization;
45
using System.IO;
6+
using System.Linq;
57
using BitFaster.Caching.Lfu;
68
using BitFaster.Caching.Lru;
79
using BitFaster.Caching.Scheduler;
810
using CsvHelper;
9-
11+
using Plotly.NET.CSharp;
12+
using Plotly.NET.ImageExport;
13+
1014
namespace BitFaster.Caching.HitRateAnalysis
1115
{
1216
public class Analysis<K>
@@ -44,11 +48,29 @@ public void TestKey(K key)
4448

4549
public static void WriteToFile(string path, IEnumerable<Analysis<K>> results)
4650
{
47-
using (var writer = new StreamWriter(path))
51+
using (var writer = new StreamWriter(path))
4852
using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
4953
{
5054
csv.WriteRecords(results);
5155
}
5256
}
57+
58+
public static void Plot(string path, string title, IEnumerable<Analysis<K>> results)
59+
{
60+
var xAxis = results.Select(x => x.CacheSize).ToArray();
61+
62+
var classic = Chart.Line<int, double, string>(xAxis, results.Select(x => x.ClassicLruHitRate), Name: "LRU", MarkerColor: Plotly.NET.Color.fromKeyword(Plotly.NET.ColorKeyword.Limegreen));
63+
var lru = Chart.Line<int, double, string>(xAxis, results.Select(x => x.ConcurrentLruHitRate), Name: "ConcurrentLru", MarkerColor: Plotly.NET.Color.fromKeyword(Plotly.NET.ColorKeyword.RoyalBlue));
64+
var lfu = Chart.Line<int, double, string>(xAxis, results.Select(x => x.ConcurrentLfuHitRate), Name: "ConcurrentLfu", MarkerColor: Plotly.NET.Color.fromRGB(255, 192, 0));
65+
var memory = Chart.Line<int, double, string>(xAxis, results.Select(x => x.MemoryCacheHitRate), Name: "MemoryCache", MarkerColor: Plotly.NET.Color.fromKeyword(Plotly.NET.ColorKeyword.FireBrick));
66+
67+
var combined = Chart.Combine(new[] { classic, lru, lfu, memory });
68+
69+
combined
70+
.WithLayout(title)
71+
.WithoutVerticalGridlines()
72+
.WithAxisTitles("Cache Size", "Hit Rate (%)")
73+
.SaveSVG(Path.GetFileNameWithoutExtension(path), Width: 1000, Height: 600);
74+
}
5375
}
5476
}

‎BitFaster.Caching.HitRateAnalysis/Arc/Runner.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Linq;
55
using System.Text;
66
using System.Threading.Tasks;
7+
using BitFaster.Caching.HitRateAnalysis.Zipfian;
78

89
namespace BitFaster.Caching.HitRateAnalysis.Arc
910
{
@@ -30,6 +31,7 @@ public async Task Run()
3031

3132
this.config.Analysis.WriteToConsole();
3233
Analysis<long>.WriteToFile(this.config.Name, this.config.Analysis);
34+
Analysis<long>.Plot(this.config.Title, this.config.Name, this.config.Analysis);
3335
}
3436

3537
private int AnalyzeSmall()

‎BitFaster.Caching.HitRateAnalysis/Arc/RunnerConfig.cs

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,24 +9,28 @@ namespace BitFaster.Caching.HitRateAnalysis.Arc
99
public class RunnerConfig
1010
{
1111
private readonly string name;
12+
private readonly string title;
1213
private readonly List<Analysis<long>> analysis;
1314
private readonly ArcDataFile file;
1415

15-
public RunnerConfig(string name, int[] cacheSizes, Uri dataUri)
16+
public RunnerConfig(string name, string title, int[] cacheSizes, Uri dataUri)
1617
{
1718
this.name = name;
19+
this.title = title;
1820
this.analysis = cacheSizes.Select(s => new Analysis<long>(s)).ToList();
1921
this.file = new ArcDataFile(dataUri);
2022
}
2123

2224
public string Name => this.name;
2325

26+
public string Title => this.title;
27+
2428
public IEnumerable<Analysis<long>> Analysis => this.analysis;
2529

2630
public ArcDataFile File => this.file;
2731

28-
public static RunnerConfig Database = new RunnerConfig("results.arc.database.csv", new[] { 1_000_000, 2_000_000, 3_000_000, 4_000_000, 5_000_000, 6_000_000, 7_000_000, 8_000_000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/DS1.lis.gz"));
29-
public static RunnerConfig Search = new RunnerConfig("results.arc.search.csv", new[] { 100_000, 200_000, 300_000, 400_000, 500_000, 600_000, 700_000, 800_000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/S3.lis.gz"));
30-
public static RunnerConfig Oltp = new RunnerConfig("results.arc.oltp.csv", new[] { 250, 500, 750, 1000, 1250, 1500, 1750, 2000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/OLTP.lis.gz"));
32+
public static RunnerConfig Database = new RunnerConfig("results.arc.database.csv", "Arc Database", new[] { 1_000_000, 2_000_000, 3_000_000, 4_000_000, 5_000_000, 6_000_000, 7_000_000, 8_000_000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/DS1.lis.gz"));
33+
public static RunnerConfig Search = new RunnerConfig("results.arc.search.csv", "Arc Search (S3)", new[] { 100_000, 200_000, 300_000, 400_000, 500_000, 600_000, 700_000, 800_000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/S3.lis.gz"));
34+
public static RunnerConfig Oltp = new RunnerConfig("results.arc.oltp.csv", "Arc OLTP", new[] { 250, 500, 750, 1000, 1250, 1500, 1750, 2000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/OLTP.lis.gz"));
3135
}
3236
}

‎BitFaster.Caching.HitRateAnalysis/BitFaster.Caching.HitRateAnalysis.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
</PackageReference>
2626
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
2727
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
28+
<PackageReference Include="Plotly.NET.CSharp" Version="0.11.1" />
29+
<PackageReference Include="Plotly.NET.ImageExport" Version="5.0.1" />
2830
</ItemGroup>
2931

3032
<ItemGroup>

‎BitFaster.Caching.HitRateAnalysis/Glimpse/Runner.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public static async Task Run()
3636
Console.WriteLine($"Tested {count} keys in {sw.Elapsed}");
3737
analysis.WriteToConsole();
3838
Analysis<long>.WriteToFile("results.glimpse.csv", analysis);
39+
Analysis<long>.Plot("results.glimpse.csv", "Glimpse", analysis);
3940
}
4041
}
4142
}

‎BitFaster.Caching.HitRateAnalysis/PlotExt.cs

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
using System;
2+
using Microsoft.FSharp.Core;
3+
using Plotly.NET;
4+
using Plotly.NET.CSharp;
5+
using Plotly.NET.LayoutObjects;
6+
7+
namespace BitFaster.Caching.HitRateAnalysis
8+
{
9+
public static class PlotExt
10+
{
11+
public static GenericChart.GenericChart WithAxisTitles(this GenericChart.GenericChart chart, string xTitle, string yTitle)
12+
{
13+
var font = new FSharpOption<Font>(Font.init(Size: new FSharpOption<double>(16)));
14+
FSharpOption<string> xt = new FSharpOption<string>(xTitle);
15+
FSharpOption<string> yt = new FSharpOption<string>(yTitle);
16+
return chart.WithXAxisStyle(Title.init(xt, Font: font)).WithYAxisStyle(Title.init(yt, Font: font));
17+
}
18+
19+
public static GenericChart.GenericChart WithoutVerticalGridlines(this GenericChart.GenericChart chart)
20+
{
21+
var gridColor = new FSharpOption<Color>(Color.fromKeyword(ColorKeyword.Gainsboro));
22+
var yaxis = LinearAxis.init<IConvertible, IConvertible, IConvertible, IConvertible, IConvertible, IConvertible>(
23+
GridColor: gridColor,
24+
ZeroLineColor: gridColor);
25+
26+
var axis = LinearAxis.init<IConvertible, IConvertible, IConvertible, IConvertible, IConvertible, IConvertible>(ShowGrid: new FSharpOption<bool>(false));
27+
return chart.WithXAxis(axis).WithYAxis(yaxis);
28+
}
29+
30+
public static GenericChart.GenericChart WithLayout(this GenericChart.GenericChart chart, string title)
31+
{
32+
var font = new FSharpOption<Font>(Font.init(Size: new FSharpOption<double>(24)));
33+
FSharpOption<Title> t = Title.init(Text: title, X: 0.5, Font: font);
34+
FSharpOption <Color> plotBGColor = new FSharpOption<Color>(Color.fromKeyword(ColorKeyword.WhiteSmoke));
35+
Layout layout = Layout.init<IConvertible>(PaperBGColor: plotBGColor, PlotBGColor: plotBGColor, Title: t);
36+
return chart.WithLayout(layout);
37+
}
38+
}
39+
}

‎BitFaster.Caching.HitRateAnalysis/Wikibench/Runner.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public static async Task Run()
4747
Console.WriteLine($"Tested {count} URIs in {sw.Elapsed}");
4848
analysis.WriteToConsole();
4949
Analysis<Uri>.WriteToFile("results.wikibench.csv", analysis);
50+
Analysis<Uri>.Plot("results.wiki.csv", "Wikipedia, 45 million request URIs", analysis);
5051
}
5152
}
5253
}

‎BitFaster.Caching.ThroughputAnalysis/BitFaster.Caching.ThroughputAnalysis.csproj

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
@@ -22,6 +22,8 @@
2222
<NoWarn>NU1701</NoWarn>
2323
</PackageReference>
2424
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
25+
<PackageReference Include="Plotly.NET.CSharp" Version="0.11.1" />
26+
<PackageReference Include="Plotly.NET.ImageExport" Version="5.0.1" />
2527
</ItemGroup>
2628

2729
<ItemGroup>

‎BitFaster.Caching.ThroughputAnalysis/Exporter.cs

Lines changed: 105 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@
44
using System.Globalization;
55
using System.IO;
66
using System.Linq;
7-
using System.Text;
8-
using System.Threading.Tasks;
97
using CsvHelper;
8+
using Microsoft.FSharp.Core;
9+
using Plotly.NET;
10+
using Plotly.NET.ImageExport;
11+
using Plotly.NET.LayoutObjects;
12+
using Chart = Plotly.NET.CSharp.Chart;
1013

1114
namespace BitFaster.Caching.ThroughputAnalysis
1215
{
@@ -67,5 +70,105 @@ public void ExportCsv(Mode mode, int cacheSize)
6770
}
6871
}
6972
}
73+
74+
public void ExportPlot(Mode mode, int cacheSize)
75+
{
76+
var columns = new List<string>();
77+
78+
for(int i = 1; i < resultTable.Columns.Count; i++)
79+
{
80+
columns.Add(resultTable.Columns[i].ColumnName);
81+
}
82+
83+
List<GenericChart.GenericChart> charts = new List<GenericChart.GenericChart>();
84+
85+
foreach (DataRow row in resultTable.Rows)
86+
{
87+
var rowData = new List<double>();
88+
string name = row[0].ToString();
89+
for (var i = 1; i < resultTable.Columns.Count; i++)
90+
{
91+
rowData.Add(double.Parse(row[i].ToString()) * 1_000_000);
92+
}
93+
94+
var chart = Chart.Line<string, double, string>(columns, rowData, Name: name, MarkerColor: MapColor(name));
95+
charts.Add(chart);
96+
97+
var combined = Chart.Combine(charts);
98+
99+
combined
100+
.WithLayout(MapTitle(mode, cacheSize))
101+
.WithoutVerticalGridlines()
102+
.WithAxisTitles("Number of threads", "Ops/sec")
103+
.SaveSVG($"Results_{mode}_{cacheSize}", Width: 1000, Height: 600);
104+
}
105+
}
106+
107+
public string MapTitle(Mode mode, int cacheSize)
108+
{
109+
switch (mode)
110+
{
111+
case Mode.Read:
112+
return $"Read throughput (100% cache hit) for size {cacheSize}";
113+
case Mode.ReadWrite:
114+
return $"Read + Write throughput for size {cacheSize}";
115+
case Mode.Update:
116+
return $"Update throughput for size {cacheSize}";
117+
case Mode.Evict:
118+
return $"Eviction throughput (100% cache miss) for size {cacheSize}";
119+
default:
120+
return $"{mode} {cacheSize}";
121+
}
122+
}
123+
124+
public Color MapColor(string name)
125+
{
126+
switch (name)
127+
{
128+
case "ClassicLru":
129+
return Plotly.NET.Color.fromKeyword(Plotly.NET.ColorKeyword.Limegreen);
130+
case "MemryCache":
131+
return Plotly.NET.Color.fromKeyword(Plotly.NET.ColorKeyword.FireBrick);
132+
case "FsTConcLRU":
133+
return Plotly.NET.Color.fromKeyword(Plotly.NET.ColorKeyword.Silver);
134+
case "ConcurrLRU":
135+
return Plotly.NET.Color.fromKeyword(Plotly.NET.ColorKeyword.RoyalBlue);
136+
case "ConcurrLFU":
137+
return Plotly.NET.Color.fromRGB(255, 192, 0);
138+
default:
139+
return Plotly.NET.Color.fromKeyword(Plotly.NET.ColorKeyword.FireBrick);
140+
}
141+
}
142+
}
143+
144+
public static class PlotExt
145+
{
146+
public static GenericChart.GenericChart WithAxisTitles(this GenericChart.GenericChart chart, string xTitle, string yTitle)
147+
{
148+
var font = new FSharpOption<Font>(Font.init(Size: new FSharpOption<double>(16)));
149+
FSharpOption<string> xt = new FSharpOption<string>(xTitle);
150+
FSharpOption<string> yt = new FSharpOption<string>(yTitle);
151+
return chart.WithXAxisStyle(Title.init(xt, Font: font)).WithYAxisStyle(Title.init(yt, Font: font));
152+
}
153+
154+
public static GenericChart.GenericChart WithoutVerticalGridlines(this GenericChart.GenericChart chart)
155+
{
156+
var gridColor = new FSharpOption<Color>(Color.fromKeyword(ColorKeyword.Gainsboro));
157+
var yaxis = LinearAxis.init<IConvertible, IConvertible, IConvertible, IConvertible, IConvertible, IConvertible>(
158+
GridColor: gridColor,
159+
ZeroLineColor: gridColor);
160+
161+
var axis = LinearAxis.init<IConvertible, IConvertible, IConvertible, IConvertible, IConvertible, IConvertible>(ShowGrid: new FSharpOption<bool>(false));
162+
return chart.WithXAxis(axis).WithYAxis(yaxis);
163+
}
164+
165+
public static GenericChart.GenericChart WithLayout(this GenericChart.GenericChart chart, string title)
166+
{
167+
var font = new FSharpOption<Font>(Font.init(Size: new FSharpOption<double>(24)));
168+
FSharpOption<Title> t = Title.init(Text: title, X: 0.5, Font: font);
169+
FSharpOption<Color> plotBGColor = new FSharpOption<Color>(Color.fromKeyword(ColorKeyword.WhiteSmoke));
170+
Layout layout = Layout.init<IConvertible>(PaperBGColor: plotBGColor, PlotBGColor: plotBGColor, Title: t);
171+
return chart.WithLayout(layout);
172+
}
70173
}
71174
}

0 commit comments

Comments
 (0)