Skip to content

Commit

Permalink
update plotly figures
Browse files Browse the repository at this point in the history
  • Loading branch information
biphasic committed Oct 25, 2023
1 parent 742762b commit 6809371
Show file tree
Hide file tree
Showing 7 changed files with 53 additions and 87 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ So how can we store such data efficiently?
A straightforward idea is to resort to formats such as hdf5 and numpy and store the arrays of events directly. But without exploiting any structure in the recorded data, those uncompressed formats end up having the largest file footprint. For our example automotive dataset, this would result in some 7-8 TB of data, which is undesirable. Event camera manufacturers have come up with ways to encode event streams more efficiently. Not only are we concerned about the size of event files on disk, but we also want to be able to read them back to memory as fast as possible!
In the following figure we plot the results of our benchmark of different file type encodings and software frameworks that can decode files.

{{< chart data="file_read_benchmark" alt="Comparison among file size and read speed of different encodings and software tools." caption="Comparison among file size and read speed of different encodings and software tools.">}}
{{< chart data="file_read_benchmark" mobile="file_read_benchmark.png" alt="Comparison among file size and read speed of different encodings and software tools." caption="Comparison among file size and read speed of different encodings and software tools.">}}

Ideally, we want to be close to the origin where we read fast and compression is high. The file size depends on the encoding, whereas the reading speed depends on the particular implementation/framework of how files are read. In terms of file size, we can see that numpy doesn't use any compression whatsoever, resulting in some 1.7GB file for our sample recording. Prophesee's [evt3](https://docs.prophesee.ai/stable/data/encoding_formats/evt3.html) and the generic lossless [brotli](https://github.com/google/brotli) formats achieve the best compression. In terms of reading speed, numpy is the fastest as it doesn't deal with any compression on disk. Unzipping the compressed events from disk on the other hand using h5py is by far the slowest. Using [Expelliarmus](https://github.com/open-neuromorphic/expelliarmus) and the [evt2](https://docs.prophesee.ai/stable/data/encoding_formats/evt2.html) file format, we get very close to numpy reading speeds while at the same time only using a fourth of the disk space. For more information about Prophesee event encoding formats, check out [this blog post](https://fabrizio-ottati.dev/blog/file-reading-benchmark/).

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -42,7 +42,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -59,6 +59,8 @@
"figure.update_layout(\n",
" # legend=dict(orientation=\"v\", yanchor=\"top\", y=0, xanchor=\"right\", x=0.99, entrywidth=0.8, entrywidthmode=\"fraction\"),\n",
" margin=dict(l=10, r=10, t=80, b=10),\n",
" # hovermode='x unified', \n",
" # hoverlabel=dict(bgcolor='rgba(255,255,255,0.75)')\n",
")\n",
"figure.update_traces(marker_size=13)\n",
"figure.write_json(\"file_read_benchmark.json\")\n",
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
"metadata": {},
"outputs": [],
"source": [
"frameworks = df['framework'].unique()"
"df['framework'] = df['framework'].str.replace('<br>', ' ')\n",
"df['framework'] = df['framework'].str.replace('SpikingJelly CuPy v0.0.0.0.15', 'SpikingJelly v0.0.15')"
]
},
{
Expand All @@ -31,36 +32,49 @@
"import plotly.express as px\n",
"import plotly.graph_objects as go\n",
"\n",
"frameworks = df['framework'].unique()\n",
"\n",
"df8k = df[df['neurons'] == 8192]\n",
"\n",
"totals8k = df8k[\"time [s]\"][df8k[\"pass\"] == \"forward\"].to_numpy() + df8k[\"time [s]\"][df8k[\"pass\"] == \"backward\"].to_numpy()\n",
"def get_figure(df, rounding=2, title=\"\"):\n",
" totals = df[\"time [s]\"][df[\"pass\"] == \"forward\"].to_numpy() + df[\"time [s]\"][df[\"pass\"] == \"backward\"].to_numpy()\n",
" fig = px.bar(\n",
" df,\n",
" y=\"framework\",\n",
" x=\"time [s]\",\n",
" color=\"pass\",\n",
" log_x=True,\n",
" text_auto=f'.{rounding}f',\n",
" orientation='h',\n",
" ).add_trace(go.Scatter(\n",
" y=frameworks, x=totals*1.05, \n",
" mode='text',\n",
" text=totals.round(rounding),\n",
" textposition='middle right',\n",
" showlegend=False\n",
" ))\n",
" fig.data[0]['textposition'] = 'inside'\n",
" fig.data[1]['textposition'] = 'inside'\n",
"\n",
"fig = px.bar(\n",
" df8k,\n",
" x=\"framework\",\n",
" y=\"time [s]\",\n",
" color=\"pass\",\n",
" log_y=True,\n",
" text_auto='.2f',\n",
").add_trace(go.Scatter(\n",
" x=frameworks, y=totals8k*1.05, \n",
" mode='text',\n",
" text=totals8k.round(2),\n",
" textposition='top center',\n",
" showlegend=False\n",
"))\n",
"fig.data[0]['textposition'] = 'inside'\n",
"fig.data[1]['textposition'] = 'inside'\n",
" fig.update_layout(\n",
" title=title,\n",
" yaxis={'categoryorder':'total descending'},\n",
" legend=dict(orientation=\"h\", yanchor=\"bottom\", y=1.01, xanchor=\"right\", x=1),\n",
" margin=dict(l=10, r=10, t=80, b=10),\n",
" yaxis_title=\"\",\n",
" )\n",
" # increase size of facet titles\n",
" fig.update_annotations(font_size=14)\n",
" return fig\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df8k = df[df['neurons'] == 8192]\n",
"\n",
"fig.update_layout(\n",
" title=f\"8k neurons, 1 Linear + 1 LIF layer, {n_steps} time steps, batch size {batch_size}\",\n",
" xaxis={'categoryorder':'total ascending'},\n",
" legend=dict(orientation=\"h\", yanchor=\"bottom\", y=1.01, xanchor=\"right\", x=1),\n",
" margin=dict(l=10, r=10, t=80, b=10),\n",
")\n",
"# increase size of facet titles\n",
"fig.update_annotations(font_size=14)\n",
"fig = get_figure(df8k, title=\"Latency for 8k neurons, lower is better\")\n",
"\n",
"fig.write_json(\"framework-benchmarking-8k.json\")\n",
"# fig.write_image(\"framework-benchmarking-8k.png\", width=1024)\n",
Expand All @@ -76,32 +90,8 @@
"source": [
"df4k = df[df['neurons'] == 4096]\n",
"\n",
"totals4k = df4k[\"time [s]\"][df4k[\"pass\"] == \"forward\"].to_numpy() + df4k[\"time [s]\"][df4k[\"pass\"] == \"backward\"].to_numpy()\n",
"fig = get_figure(df4k, title=\"Latency for 4k neurons, lower is better\")\n",
"\n",
"fig = px.bar(\n",
" df4k,\n",
" x=\"framework\",\n",
" y=\"time [s]\",\n",
" color=\"pass\",\n",
" log_y=True,\n",
" text_auto='.2f',\n",
").add_trace(go.Scatter(\n",
" x=frameworks, y=totals4k*1.05, \n",
" mode='text',\n",
" text=totals4k.round(2),\n",
" textposition='top center',\n",
" showlegend=False\n",
"))\n",
"fig.data[0]['textposition'] = 'inside'\n",
"fig.data[1]['textposition'] = 'inside'\n",
"fig.update_layout(\n",
" title=f\"4k neurons, 1 Linear + 1 LIF layer, {n_steps} time steps, batch size {batch_size}\",\n",
" xaxis={'categoryorder':'total ascending'},\n",
" legend=dict(orientation=\"h\", yanchor=\"bottom\", y=1.01, xanchor=\"right\", x=1),\n",
" margin=dict(l=10, r=10, t=80, b=10),\n",
")\n",
"# increase size of facet titles\n",
"fig.update_annotations(font_size=14)\n",
"fig.write_json(\"framework-benchmarking-4k.json\")\n",
"# fig.write_image(\"framework-benchmarking-4k.png\", width=1024)# scale=2)\n",
"fig.write_image(\"framework-benchmarking-4k-header.png\", width=1024, height=570)\n",
Expand All @@ -115,38 +105,12 @@
"outputs": [],
"source": [
"df512 = df[df['neurons'] == 512]\n",
"# df512[\"time [s]\"] *= 1000\n",
"\n",
"totals512 = df512[\"time [s]\"][df512[\"pass\"] == \"forward\"].to_numpy() + df512[\"time [s]\"][df512[\"pass\"] == \"backward\"].to_numpy()\n",
"\n",
"fig = px.bar(\n",
" df512,\n",
" x=\"framework\",\n",
" y=\"time [s]\",\n",
" color=\"pass\",\n",
" log_y=True,\n",
" text_auto=\".3f\",\n",
").add_trace(go.Scatter(\n",
" x=frameworks, y=totals512*1.05, \n",
" mode='text',\n",
" text=totals512.round(3),\n",
" textposition='top center',\n",
" showlegend=False\n",
"))\n",
"fig.data[0]['textposition'] = 'inside'\n",
"fig.data[1]['textposition'] = 'inside'\n",
"fig.update_layout(\n",
" title=f\"512 neurons, 1 Linear + 1 LIF layer, {n_steps} time steps, batch size {batch_size}\",\n",
" xaxis={'categoryorder':'total ascending'},\n",
" legend=dict(orientation=\"h\", yanchor=\"bottom\", y=1.01, xanchor=\"right\", x=1),\n",
" margin=dict(l=10, r=10, t=80, b=10),\n",
")\n",
"# increase size of facet titles\n",
"fig.update_annotations(font_size=14)\n",
"fig = get_figure(df512, rounding=3, title=\"Latency for 512 neurons, lower is better\")\n",
"\n",
"fig.write_json(\"framework-benchmarking-512.json\")\n",
"# fig.write_image(\"framework-benchmarking-512.png\", width=1024)# scale=2)\n",
"# fig.show()"
"fig.show()"
]
},
{
Expand Down

0 comments on commit 6809371

Please sign in to comment.