Skip to content

Commit d6b3f65

Browse files
Raimo33Raimo33
committed
Add dev scripts [skip ci]
1 parent 5ac009c commit d6b3f65

File tree

4 files changed

+233
-0
lines changed

4 files changed

+233
-0
lines changed

‎benchmark_analysis.ipynb‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"id": "b49ae6d6",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"import pandas as pd\n",
11+
"import numpy as np\n",
12+
"import matplotlib.pyplot as plt\n",
13+
"import seaborn as sns\n",
14+
"\n",
15+
"plt.rcParams['figure.figsize'] = (16, 10)\n",
16+
"plt.rcParams['font.size'] = 11"
17+
]
18+
},
19+
{
20+
"cell_type": "code",
21+
"execution_count": 2,
22+
"id": "d236980d",
23+
"metadata": {},
24+
"outputs": [
25+
{
26+
"ename": "FileNotFoundError",
27+
"evalue": "[Errno 2] No such file or directory: 'BASELINE_bench.csv'",
28+
"output_type": "error",
29+
"traceback": [
30+
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
31+
"\u001b[31mFileNotFoundError\u001b[39m Traceback (most recent call last)",
32+
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 17\u001b[39m\n\u001b[32m 14\u001b[39m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[32m 15\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m pd.DataFrame(data)\n\u001b[32m---> \u001b[39m\u001b[32m17\u001b[39m baseline = \u001b[43mparse_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mBASELINE_bench.csv\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m 18\u001b[39m custom = parse_csv(\u001b[33m'\u001b[39m\u001b[33mCUSTOM_SIMD_bench.csv\u001b[39m\u001b[33m'\u001b[39m)\n\u001b[32m 20\u001b[39m merged = baseline.merge(custom, on=\u001b[33m'\u001b[39m\u001b[33mBenchmark\u001b[39m\u001b[33m'\u001b[39m, suffixes=(\u001b[33m'\u001b[39m\u001b[33m_baseline\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33m_custom\u001b[39m\u001b[33m'\u001b[39m))\n",
33+
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 2\u001b[39m, in \u001b[36mparse_csv\u001b[39m\u001b[34m(filepath)\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mparse_csv\u001b[39m(filepath):\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mfilepath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mr\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[32m 3\u001b[39m lines = f.readlines()[\u001b[32m1\u001b[39m:]\n\u001b[32m 5\u001b[39m data = []\n",
34+
"\u001b[36mFile \u001b[39m\u001b[32m~/Desktop/secp256k1/.venv/lib/python3.12/site-packages/IPython/core/interactiveshell.py:343\u001b[39m, in \u001b[36m_modified_open\u001b[39m\u001b[34m(file, *args, **kwargs)\u001b[39m\n\u001b[32m 336\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m {\u001b[32m0\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m2\u001b[39m}:\n\u001b[32m 337\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 338\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mIPython won\u001b[39m\u001b[33m'\u001b[39m\u001b[33mt let you open fd=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m by default \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 339\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 340\u001b[39m \u001b[33m\"\u001b[39m\u001b[33myou can use builtins\u001b[39m\u001b[33m'\u001b[39m\u001b[33m open.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 341\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m343\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mio_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
35+
"\u001b[31mFileNotFoundError\u001b[39m: [Errno 2] No such file or directory: 'BASELINE_bench.csv'"
36+
]
37+
}
38+
],
39+
"source": [
40+
"def parse_csv(filepath):\n",
41+
" with open(filepath, 'r') as f:\n",
42+
" lines = f.readlines()[1:]\n",
43+
" \n",
44+
" data = []\n",
45+
" for line in lines:\n",
46+
" line = line.strip()\n",
47+
" if line and ',' in line and not line.endswith(','):\n",
48+
" parts = line.split(',')\n",
49+
" if len(parts) >= 3:\n",
50+
" try:\n",
51+
" data.append({'Benchmark': parts[0].strip(), 'Time': float(parts[2])})\n",
52+
" except:\n",
53+
" continue\n",
54+
" return pd.DataFrame(data)\n",
55+
"\n",
56+
"baseline = parse_csv('BASELINE_bench.csv')\n",
57+
"custom = parse_csv('CUSTOM_SIMD_bench.csv')\n",
58+
"\n",
59+
"merged = baseline.merge(custom, on='Benchmark', suffixes=('_baseline', '_custom'))\n",
60+
"merged['improvement'] = ((merged['Time_baseline'] - merged['Time_custom']) / merged['Time_baseline']) * 100"
61+
]
62+
},
63+
{
64+
"cell_type": "code",
65+
"execution_count": null,
66+
"id": "8442b12d",
67+
"metadata": {},
68+
"outputs": [],
69+
"source": [
70+
"sorted_data = merged.sort_values('improvement', ascending=False)\n",
71+
"top10 = sorted_data.head(10)\n",
72+
"bottom10 = sorted_data.tail(10)\n",
73+
"filtered = pd.concat([top10, bottom10])"
74+
]
75+
},
76+
{
77+
"cell_type": "code",
78+
"execution_count": null,
79+
"id": "aa07550a",
80+
"metadata": {},
81+
"outputs": [],
82+
"source": [
83+
"heatmap_data = filtered.set_index('Benchmark')[['improvement']]\n",
84+
"\n",
85+
"plt.figure(figsize=(8, 12))\n",
86+
"sns.heatmap(heatmap_data, annot=True, fmt='.1f', cmap='RdYlGn', center=0, \n",
87+
" cbar_kws={'label': 'Performance Improvement (%)'})\n",
88+
"plt.title('CUSTOM_SIMD vs BASELINE Performance (Top/Bottom 10)', fontsize=14, fontweight='bold')\n",
89+
"plt.ylabel('')\n",
90+
"plt.tight_layout()\n",
91+
"plt.show()"
92+
]
93+
}
94+
],
95+
"metadata": {
96+
"kernelspec": {
97+
"display_name": ".venv",
98+
"language": "python",
99+
"name": "python3"
100+
},
101+
"language_info": {
102+
"codemirror_mode": {
103+
"name": "ipython",
104+
"version": 3
105+
},
106+
"file_extension": ".py",
107+
"mimetype": "text/x-python",
108+
"name": "python",
109+
"nbconvert_exporter": "python",
110+
"pygments_lexer": "ipython3",
111+
"version": "3.12.3"
112+
}
113+
},
114+
"nbformat": 4,
115+
"nbformat_minor": 5
116+
}

‎simd-bench.sh‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
set -e
3+
4+
options=("OFF" "ON")
5+
BENCH_ITERS=${SECP256K1_BENCH_ITERS:-20000}
6+
7+
GREEN='\033[0;32m'
8+
RED='\033[0;31m'
9+
YELLOW='\033[1;33m'
10+
NC='\033[0m'
11+
12+
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo > /dev/null
13+
sudo cpupower -c 0 frequency-set -g performance > /dev/null
14+
command -v taskset > /dev/null && TASKSET_CMD="taskset -c 0"
15+
16+
run_bench() {
17+
local dir=$1 bin=$2 log=$3
18+
(
19+
cd "$dir"
20+
$TASKSET_CMD env SECP256K1_BENCH_ITERS=$BENCH_ITERS nice -n 0 ./bin/$bin >> "../../$log" 2>&1
21+
echo "" >> "../../$log"
22+
)
23+
}
24+
25+
bench_all() {
26+
local config="$1"
27+
local dir="build/$config"
28+
local log="${config}_bench.csv"
29+
30+
if [[ ! -d "$dir" ]]; then
31+
echo -e "${RED}✖ $config${NC} (no dir)"
32+
return 1
33+
fi
34+
35+
{
36+
echo "Benchmark results for $config"
37+
echo "Generated on $(date)"
38+
echo "Iterations: $BENCH_ITERS"
39+
echo ""
40+
} > "$log"
41+
42+
for bin in bench bench_ecmult bench_internal; do
43+
if run_bench "$dir" "$bin" "$log"; then
44+
echo -e " ${GREEN}✔ $bin${NC}"
45+
else
46+
echo -e " ${RED}✖ $bin${NC}"
47+
return 1
48+
fi
49+
done
50+
51+
echo -e "${GREEN}✔ $config${NC} (log: $log)"
52+
}
53+
54+
bench_all "BASELINE"
55+
bench_all "CUSTOM_SIMD"
56+
57+
echo -e "\n${YELLOW}All benchmarks successful. Logs in project root${NC}"

‎simd-build.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
set -e
3+
4+
mkdir -p build
5+
6+
GREEN='\033[0;32m'
7+
RED='\033[0;31m'
8+
YELLOW='\033[1;33m'
9+
NC='\033[0m'
10+
11+
run_build() {
12+
local config="$1"
13+
local flags="-O3 -mavx -mavx2 $2"
14+
local dir="build/$config"
15+
local log="${config}_build.log"
16+
17+
mkdir -p "$dir"
18+
19+
if (cd "$dir" && cmake ../.. -G Ninja -DCMAKE_BUILD_TYPE=Release -DSECP256K1_APPEND_CFLAGS="$flags" >"../../$log" 2>&1 && ninja >>"../../$log" 2>&1); then
20+
echo -e "${GREEN}✔ $config${NC}"
21+
else
22+
echo -e "${RED}✖ $config failed${NC}"
23+
return 1
24+
fi
25+
}
26+
27+
run_build "BASELINE" "-U__AVX__ -U__AVX2__"
28+
run_build "CUSTOM_SIMD" "-D__AVX__ -D__AVX2__"
29+
30+
echo -e "\n${YELLOW}All builds done. Logs in project root${NC}"

‎simd-test.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
set -e
3+
4+
GREEN='\033[0;32m'
5+
RED='\033[0;31m'
6+
YELLOW='\033[1;33m'
7+
NC='\033[0m'
8+
9+
run_test() {
10+
local config="$1"
11+
local dir="build/$config"
12+
local log="${config}_test.log"
13+
14+
if [[ ! -d "$dir" ]]; then
15+
echo -e "${RED}✖ $config${NC} (no dir)"
16+
return 1
17+
fi
18+
19+
if (cd "$dir" && ctest --output-on-failure -j"$(nproc)" &> "../../$log"); then
20+
echo -e "${GREEN}✔ $config${NC} (log: $log)"
21+
else
22+
echo -e "${RED}✖ $config${NC} (log: $log)"
23+
return 1
24+
fi
25+
}
26+
27+
run_test "BASELINE"
28+
run_test "CUSTOM_SIMD"
29+
30+
echo -e "\n${YELLOW}All tests passed. Logs in project root${NC}"

0 commit comments

Comments
 (0)