diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..438f0f8 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +scripts/.ipynb_checkpoints/ diff --git a/scripts/deep_analysis_plotter.ipynb b/scripts/deep_analysis_plotter.ipynb new file mode 100644 index 0000000..d9165a9 --- /dev/null +++ b/scripts/deep_analysis_plotter.ipynb @@ -0,0 +1,200 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "aadf8bdc-2b8a-4737-992f-0ebf7e66cd8f", + "metadata": {}, + "outputs": [], + "source": [ + "%%cmd\n", + "python -m pip install flatten_json\n", + "python -m pip install pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbdc923a-44fd-4250-990b-80771446bbf0", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import json\n", + "from collections import defaultdict\n", + "from flatten_json import flatten" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59b6a968-ad86-48a0-8948-d1f42bf0c3c6", + "metadata": {}, + "outputs": [], + "source": [ + "files = [\n", + " 'a_inline_deep.json',\n", + " 'b_inline_deep.json',\n", + " 'a_pipeline_deep.json',\n", + " 'b_pipeline_deep.json',\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a60a4c63-b884-4558-98bd-ee3d14bbc569", + "metadata": {}, + "outputs": [], + "source": [ + "def aggregate_loops_passes(json):\n", + " results_per_frame = []\n", + " num_loops = len(json)\n", + " for loop_results in json:\n", + " for frame_index, frame_results in enumerate(loop_results[\"per_frame_results\"]):\n", + " if frame_index >= len(results_per_frame):\n", + " results_per_frame.append(defaultdict(int))\n", + " results_per_frame[frame_index]['sequence_time_ns'] = frame_results['sequence_time_ns']\n", + " for command_buffer_timings in frame_results[\n", + " \"command_buffer_timings\"\n", + " ].values():\n", + " for scope_name, scope_timings in command_buffer_timings[\n", + " \"scope_timings\"\n", + " ].items():\n", + " for scope_timing in scope_timings:\n", + " results_per_frame[frame_index][scope_name] += (\n", + " scope_timing[\"end\"] - scope_timing[\"start\"]\n", + " ) / num_loops / 1_000_000 # in ms\n", + " for metric_name, metric in frame_results[\"metrics\"].items():\n", + " # TODO: Flatten this in rust to fan_speed_rpm\n", + " if metric_name == \"fan_speed\":\n", + " value = metric[\"Percent\"] if \"Percent\" in metric else metric[\"Rpm\"]\n", + " results_per_frame[frame_index][\"fan_speed_rpm\"] += (\n", + " value / num_loops\n", + " )\n", + " # Filter out unavailable data and the timestamp\n", + " elif metric is not None and metric_name != \"timestamp\":\n", + " results_per_frame[frame_index][metric_name] += metric / num_loops\n", + " # TODO: Aggregate CPU timings\n", + " return pd.DataFrame([flatten(x) for x in results_per_frame])\n", + "\n", + "results = {}\n", + "\n", + "# Load all files into one large dataframe\n", + "for path in files:\n", + " with open(path, \"r\") as json_file:\n", + " # We aggregate passes within each frame, so we get one number per pass per frame per input file\n", + " json_data = aggregate_loops_passes(json.load(json_file))\n", + " results[path] = json_data\n", + "# Concat into input file + sequence time per row, metric per column\n", + "full_dataset = pd.concat(results)\n", + "full_dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7de3e2b9-0470-40a2-9ed7-c2d931aaf153", + "metadata": {}, + "outputs": [], + "source": [ + "# Print all possible metrics\n", + "full_dataset.columns.tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6046a029-f421-4a9e-a9d6-9889488ff874", + "metadata": {}, + "outputs": [], + "source": [ + "relevant_metrics = [\n", + " 'reflection-hits-shading',\n", + " 'water-compositing',\n", + " 'blur',\n", + " 'diffuse-spatial-filter',\n", + " 'spd',\n", + " 'sun-direct-lighting',\n", + " 'reflection-ray-tracing-inline',\n", + " 'trace-diffuse-nee-rays',\n", + " 'render pass',\n", + " 'shadow-ray-tracing-pipeline',\n", + " 'compositing',\n", + " 'build-gbuffers_0',\n", + " 'scale-raster',\n", + " 'Batch refit bottom level',\n", + " 'clock_speed_in_mhz',\n", + " 'board_power_usage_in_w',\n", + " 'vram_usage_in_mb',\n", + " 'edge_temperature_in_c'\n", + "]\n", + "# We want all relevant metrics with the sequence time, to properly plot on the x axis\n", + "relevant_metrics_with_time = relevant_metrics + [\"sequence_time_ns\"]\n", + "metrics = full_dataset[relevant_metrics_with_time]\n", + "\n", + "# Reshape into sequence time + metric type per row, input file per column\n", + "metrics = metrics.reset_index().set_index(['sequence_time_ns', 'level_0']).drop('level_1', axis=1)\n", + "metrics = metrics.stack().unstack(1).reset_index()\n", + "\n", + "# From ns to s\n", + "metrics['sequence_time_s'] = metrics['sequence_time_ns'] / 1_000_000_000\n", + "metrics = metrics.drop('sequence_time_ns', axis=1)\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6740136-668a-44a6-9e35-53f80d586ea5", + "metadata": {}, + "outputs": [], + "source": [ + "for graph_name in metrics['level_1'].unique():\n", + " # Grab the metric we want to plot\n", + " selected_metric = metrics[metrics['level_1'] == graph_name]\n", + " selected_metric = selected_metric.drop('level_1', axis=1)\n", + "\n", + " # Filter outliers out of view\n", + " max_mean = selected_metric[files].mean().mean()\n", + " max_mean = 0 if pd.isna(max_mean) else max_mean\n", + " max_std = selected_metric[files].std(axis=1).max()\n", + " max_std = selected_metric[files].max() / 3.0 if pd.isna(max_std) else max_std\n", + "\n", + " # Plot results \n", + " selected_metric.infer_objects(copy=False).interpolate(method='linear').plot(\n", + " x='sequence_time_s', \n", + " ylabel='shader execution time in ms',\n", + " xlabel='benchmark timeline in seconds', \n", + " ylim= (max(0, max_mean - max_std * 3), max_mean + max_std * 3),\n", + " figsize=(20,10), \n", + " colormap='Dark2', \n", + " grid=True, \n", + " legend=True,\n", + " title=graph_name\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/per_frame_plotter.ipynb b/scripts/per_frame_plotter.ipynb new file mode 100644 index 0000000..62bc696 --- /dev/null +++ b/scripts/per_frame_plotter.ipynb @@ -0,0 +1,122 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "1d9d9451-25fe-40d7-a67e-2fcaf6a9f54a", + "metadata": {}, + "outputs": [], + "source": [ + "%%cmd\n", + "python -m pip install pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a5f8126-43bc-4cba-b65b-136cbf8971ab", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "959dce8e-4f96-4505-ba35-0300c3aee0a1", + "metadata": {}, + "outputs": [], + "source": [ + "files = [\n", + " 'a_inline_perframe.csv',\n", + " 'a_pipeline_perframe.csv',\n", + " 'b_inline_perframe.csv',\n", + " 'b_pipeline_perframe.csv',\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5f9ea81-7bd2-4f70-a0b8-45164afe8f76", + "metadata": {}, + "outputs": [], + "source": [ + "scores = {}\n", + "\n", + "for file in files:\n", + " # Read csv\n", + " file_scores = pd.read_csv(file)\n", + " # Remove empty results\n", + " file_scores = file_scores.loc[:, (file_scores != 0).any(axis=0)]\n", + " scores[file] = file_scores\n", + "# Reshape into row per input file + sequence time, column per metric\n", + "scores = pd.concat(scores).reset_index().set_index(['level_0', 'Sequence Time (ns)']).drop(['level_1', 'Loop Index', 'Frame'], axis=1)\n", + "scores" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4bbc8ad-fbf4-4c96-8aa4-afb02ff305a0", + "metadata": {}, + "outputs": [], + "source": [ + "# Reshape into row per sequence time + metric, column per input file\n", + "new_scores = scores.stack().unstack(0).reset_index()\n", + "# ns to s for the sequence time\n", + "new_scores['Sequence Time (s)'] = new_scores['Sequence Time (ns)'] / 1_000_000_000\n", + "new_scores = new_scores.drop('Sequence Time (ns)', axis=1)\n", + "new_scores" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d6b6f75-251b-4bb2-aa44-e6393cf8140a", + "metadata": {}, + "outputs": [], + "source": [ + "for graph_name in new_scores['level_1'].unique():\n", + " graph_data = new_scores[new_scores['level_1'] == graph_name]\n", + " graph_data = graph_data.drop('level_1', axis=1)\n", + " graph_data = graph_data.apply(pd.to_numeric, errors='coerce').interpolate(method='linear')\n", + " \n", + " graph_data.plot(\n", + " x='Sequence Time (s)', \n", + " ylabel='value',\n", + " xlabel='benchmark timeline in seconds', \n", + " kind=\"line\", \n", + " logy=True,\n", + " figsize=(20,5), \n", + " colormap='Dark2', \n", + " grid=True, \n", + " rot=0,\n", + " title=graph_name\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/scores_plotter.ipynb b/scripts/scores_plotter.ipynb new file mode 100644 index 0000000..ec4ced0 --- /dev/null +++ b/scripts/scores_plotter.ipynb @@ -0,0 +1,95 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "c7a3c3bd-54d5-4979-8502-ac69a32a13fb", + "metadata": {}, + "outputs": [], + "source": [ + "%%cmd\n", + "python -m pip install pandas\n", + "python -m pip install matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21658370-184d-4ab0-b210-d8ab93f0818b", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07bb93e8-5d29-4bbf-8f44-be5db0af70c7", + "metadata": {}, + "outputs": [], + "source": [ + "files = [\n", + " 'scores/5070ti_inline.csv',\n", + " 'scores/5070ti_pipeline.csv',\n", + " 'scores/9070xt_inline.csv',\n", + " 'scores/9070xt_pipeline.csv',\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9088c71c-0fe8-4c87-852e-265b662dcacc", + "metadata": {}, + "outputs": [], + "source": [ + "scores = {}\n", + "\n", + "for file in files:\n", + " # Read csv\n", + " file_scores = pd.read_csv(file)\n", + " # Remove empty results\n", + " file_scores = file_scores.loc[:, (file_scores != 0).any(axis=0)]\n", + " # Use os.path.basename() to get only the file name\n", + " file_name = os.path.splitext(os.path.basename(file))[0]\n", + " scores[file_name] = file_scores\n", + "# Reshape into row per metric, column per input file\n", + "scores = pd.concat(scores).groupby(level=0, sort=False).mean().T.drop('Loop', errors=\"ignore\")\n", + "scores" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d89bab16-7553-4fd2-953b-05a054362999", + "metadata": {}, + "outputs": [], + "source": [ + "scores.plot(kind=\"bar\", figsize=(20,5), colormap='Dark2', grid=True, rot=0, title=\"Evolve scores (higher is better)\", ylabel=\"score\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}