Kch/fix fee history

.github/workflows/docker.yml

@@ -0,0 +1,25 @@
+name: Docker
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  docker:
+    strategy:
+      matrix:
+        test: ["eth_getBalance", "eth_getBlockByNumber"]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v2
+      - run: docker build --tag paradigmxyz/flood:latest .
+      - run: docker run --rm paradigmxyz/flood:latest version
+      - name: ${{ matrix.test }}
+        run: |
+          docker run --rm paradigmxyz/flood:latest \
+          ${{ matrix.test }} \
+          node1=https://eth.llamarpc.com \
+          --duration 3 --rate 1

.gitignore

@@ -21,3 +21,7 @@ dist/*
 notebooks/*
 roadmap/*
 todo/*
+graveyard.py
+
+*.ipynb
+*.html

Dockerfile

@@ -1,32 +1,39 @@
+# uploaded to GitHub Container Registry
+# https://github.com/paradigmxyz/flood/pkgs/container/flood
 
-FROM python:3.11.3
-
-# create user
+# Stage 1: Install flood
+FROM python:3.11-slim AS flood-builder
 ENV USERNAME="flood"
+ENV PATH="${PATH}:/home/${USERNAME}/.local/bin"
 RUN adduser $USERNAME
 USER $USERNAME
 
-# install flood
-RUN mkdir /home/$USERNAME/repos
 COPY ./ /home/$USERNAME/repos/flood/
 WORKDIR /home/$USERNAME/repos/flood
-RUN pip install ./
+RUN pip install --user --no-cache-dir ./
+
+# Stage 2: Install vegeta
+FROM debian:stable-slim AS vegeta-builder
+ENV VEGETA_VERSION=12.8.4
+RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates \
+ && rm -rf /var/lib/apt/lists/* \
+ && mkdir /vegeta
+WORKDIR /vegeta
+RUN wget https://github.com/tsenart/vegeta/releases/download/v${VEGETA_VERSION}/vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz \
+ && tar xzf vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz \
+ && rm vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz
 
-# install ctc
-WORKDIR /home/$USERNAME/repos
-RUN git clone https://github.com/checkthechain/checkthechain
-WORKDIR /home/$USERNAME/repos/checkthechain
-RUN pip install ./
+# Final stage: Combine flood and vegeta
+FROM python:3.11-slim
+ENV USERNAME="flood"
+ENV PATH="${PATH}:/home/${USERNAME}/.local/bin"
+RUN adduser $USERNAME
 
-# # install vegeta
-RUN mkdir -p /home/$USERNAME/bin/vegeta
-WORKDIR /home/$USERNAME/bin/vegeta_files
-RUN wget https://github.com/tsenart/vegeta/releases/download/v12.8.4/vegeta_12.8.4_linux_amd64.tar.gz
-RUN tar xzf vegeta_12.8.4_linux_amd64.tar.gz
-WORKDIR /home/$USERNAME/bin/
-RUN ln -s /home/$USERNAME/bin/vegeta_files/vegeta /home/$USERNAME/bin/vegeta
+RUN apt-get update && apt-get install -y --no-install-recommends curl \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=flood-builder /home/$USERNAME/.local /home/$USERNAME/.local
+COPY --from=vegeta-builder /vegeta/vegeta /home/$USERNAME/.local/bin/vegeta
 
-# run flood
+USER $USERNAME
 WORKDIR /home/$USERNAME
 ENTRYPOINT ["python", "-m", "flood"]
-

README.md

@@ -6,9 +6,11 @@
 ![](./assets/cover.png)
 
 [![CI status](https://github.com/paradigmxyz/flood/workflows/Pytest/badge.svg)][gh-ci]
+[![Docker](https://github.com/paradigmxyz/flood/actions/workflows/docker.yml/badge.svg)][gh-docker]
 [![Telegram Chat][tg-badge]][tg-url]
 
 [gh-ci]: https://github.com/paradigmxyz/flood/actions/workflows/ci.yml
+[gh-docker]: https://github.com/paradigmxyz/flood/actions/workflows/docker.yml
 [tg-badge]: https://img.shields.io/endpoint?color=neon&logo=telegram&label=chat&url=https%3A%2F%2Ftg.sumanjay.workers.dev%2Fparadigm%5Fflood
 [tg-url]: https://t.me/paradigm_flood
 
@@ -24,6 +26,21 @@ For each RPC method, `flood` measures how load affects metrics such as:
 
 `flood` can generate tables, figures, and reports for easy sharing of results (example report [here](https://datasets.paradigm.xyz/notebooks/flood/example_report.html))
 
+## Contents
+1. ### [Installation](#installation)
+    1. [Prerequisites](#prerequisites)
+    2. [Installing Flood](#installing-flood)
+    3. [Docker](#docker)
+2. ### [Usage Guide](#usage-guide)
+    1. [Basic Load Tests](#basic-load-tests)
+    2. [Remote Load Tests](#remote-load-tests)
+    3. [Printing Test Results](#printing-test-results)
+    4. [Report Generation](#report-generation)
+    5. [Differential Tests](#differential-tests)
+    6. [From Python](#from-python)
+    7. [Performing Deep Checks](#performing-deep-checks)
+3. ### [Contributing](#contributing)
+
 
 ## Installation
 
@@ -45,29 +62,75 @@ pip install paradigm-flood
 
 Typing `flood help` in your terminal should show help output. If it does not, you probably have not set up `pip` to install items to your `$PATH`. You may need to add something like `export PATH=$PATH:~/.local/bin` to your terminal config file (e.g. `~/.profile`). Alternatively, you can avoid setting up your `$PATH` and just type `python3 -m flood` instead of `flood`.
 
-## Usage
+#### Docker
 
-#### run test
-```
-flood eth_getBlockByNumber NODE1_NAME=NODE1_URL NODE2_NAME=NODE2_URL --rates 10 100 1000 --duration 30
-```
+Alternatively, flood can be used as a [Docker image](https://github.com/paradigmxyz/flood/pkgs/container/flood).
 
-#### orchestrate tests on remote nodes
-```
-flood eth_getBlockByNumber NODE1_NAME=NODE1_URL:localhost:8545 NODE2_NAME=NODE2_URL:localhost:8545
-```
+## Usage guide
 
-#### create report
-```
-flood report tests/test1_output tests/test2_output
-````
+`flood` works by bombarding an RPC endpoint with different patterns of RPC calls. Measurements of the RPC endpoint's performance under different controlled loads are then used to paint a detailed view of the node's performance.
+
+Every time flood runs, it saves its parameters and test results to an output directory. You can specify this output directory with the `--output` parameter, otherwise a temporary directory will be created. Running a test will populate the folder with the following files:
+- `figures/`: directory containing PNG's summarizing node performance
+- `results.json`: results of the test including performance metrics
+- `summary.txt`: printed summary of test that was output to the console
+- `test.json`: metadata and parameters used to create and run the test
+
+### Basic load tests
+
+Here is an example of a basic test with `flood`. It will benchmark block retrieval from two different nodes. It will test at 3 different rates (10, 100, and 1000 requests per second) and it will test them for 30 seconds each.
+
+`flood eth_getBlockByNumber NODE1_NAME=NODE1_URL NODE2_NAME=NODE2_URL --rates 10 100 1000 --duration 30`
+
+To see all of the parameters available for controlling `flood` tests use `flood --help`
+
+### Remote load tests
+
+Instead of broadcasting RPC calls from whatever machine running the `flood` CLI command, `flood` can broadcast the calls from a remote process on a remote machine. In particular, `flood` can broadcast the calls from the same machine that is running the EVM node in order to eliminate any noise or bottlenecks associated with networking.
+
+This can be accomplished by installing flood on the remote machine and then providing `flood` with login credentials and routing details using the following syntax:
+
+`flood <test> [node_name=][username@]hostname:[test_url] ...`
+
+For example, the following command will test a reth node and an erigon node remotely:
+
+`flood eth_call [email protected]:localhost:8545 [email protected]:localhost:8545`
+
+If there are multiple remote tests, these tests will be run in parallel. After the tests are complete, `flood` will retrieve the results and summarize using the same methodology as a local test.
+
+### Printing test results
+
+By default `flood` produces verbose output of each test as it runs. This can be disabled with the `--quiet` parameter. To re-print the results of an old test, use `flood print <TEST_DIR>`. To print a summary of multiple tests, use `flood print <test_1_dir> <test_2_dir>`.
+
+### Report generation
+
+After running tests, you can generate an HTML + Jupyter report similar to [this](https://datasets.paradigm.xyz/notebooks/flood/example_report.html) one. This is done by running `flood report <TEST_DIR>`. Multiple tests can be combined into one report with `flood repos <TEST_DIR_1> <TEST_DIR_2> ...`.
+
+### Differential tests
+
+Instead of testing the raw performance of an RPC node, `flood` can be used to test the correctness of a node's responses using a differential testing approach. This works by using two nodes and making sure that their responses match under a variety of RPC calls. This is done using the `--equality` parameter. For example:
+
+`flood all reth=91.91.91.91 erigon=92.92.92.92 --equality`
+
+### From python
+
+All of `flood`'s functionality can be used from python instead of the CLI. Some functions:
+
+|description|python|
+|-|-|
+|Import flood|`import flood`|
+|Run tests|`flood.run(...)`|
+|Load the parameters of a test|`flood.load_single_run_results_payload(output_dir)`|
+|Load the results of a test|`flood.load_single_run_results_payload(output_dir)`|
+|Run a live version of a results notebook|`jupyter notebook <TEST_DIR>`|
+
+### Performing deep checks
+
+Under normal operation `flood` relies on vegeta to compute performance summaries of each test. This works well, but sometimes it is desirable to implement custom introspection not available in `vegeta`.
 
-## Code Layout
-- `flood/cli`: command line interface
-- `flood/generators`: utilities for generating rpc calls with parameterized distributions
-- `flood/tests`: implementations of load tests and equality tests
-- `flood/user_io`: utiltiies for parsing user io
+In particular, `vegeta` counts any status-200 response as a success, even if the contents of the response is an RPC error. Running with the `--deep-check` command will check every response to make sure that it returns well-formed JSON with no RPC errors. With `--deep-check`, `flood` also computes separate performance statistics successful vs failed calls.
 
+If you want to save the timing information and raw contents of every single response from the test to the `results.json` output, use the `--save-raw-output` argument. This allows for performing own custom analyses on the raw data.
 
 ## Contributing
 

flood/__init__.py

@@ -1,24 +1,38 @@
 """tool for benchmarking RPC endpoints"""
+# ruff: noqa: F401
 
-from .generators import *
-from .runners import *
 from .spec import *
-from .tests import *
-from .user_io import *
+import flood.generators
+import flood.ops
+import flood.runners
+import flood.tests
+import flood.user_io
 
+from flood.generators import generate_test
+from flood.ops import get_flood_version
+from flood.ops import get_local_installation
+from flood.ops import get_remote_installation
+from flood.ops import get_dependency_versions
+from flood.runners import load_single_run_test_payload
+from flood.runners import load_single_run_results_payload
+from flood.runners import run
+from flood.tests.equality_tests import run_equality_test
+from flood.tests.load_tests import run_load_test
+from flood.tests.load_tests import run_load_tests
 
-__version__ = '0.2.5'
+
+__version__ = '0.3.1'
 
 
 def _clean_package_imports() -> None:
     """remove deep nested modules from flood namespace"""
 
     import sys
 
-    flood = sys.modules['flood']
+    flood_module = sys.modules['flood']
     moduletype = type(flood)
     delattr(flood, 'annotations')
-    for key, value in tuple(vars(flood).items()):
+    for key, value in tuple(vars(flood_module).items()):
         if isinstance(value, moduletype):
             name = value.__name__
             if not name.startswith('flood') or name.count('.') > 1:

flood/cli/cli_run.py

@@ -11,7 +11,7 @@
 
 def cd_dir_getter(dirname: str) -> str:
     if dirname == 'samples':
-        return flood.get_flood_samples_dir()
+        return flood.generators.get_flood_samples_dir()
     else:
         raise Exception('unknown path: ' + str(dirname))
 
@@ -26,10 +26,9 @@ def run_cli(raw_command: str | None = None) -> None:
         ('samples', 'collect'): 'flood.cli.samples_collect_command',
         ('samples', 'download'): 'flood.cli.samples_download_command',
         ('samples', 'ls'): 'flood.cli.samples_ls_command',
-        (
-            'version',
-        ): 'toolcli.command_utils.standard_subcommands.version_command',
+        ('version',): 'flood.cli.version_command',
         ('cd',): 'toolcli.command_utils.standard_subcommands.cd_command',
+        ('update',): 'flood.cli.update_command',
     }
 
     config: toolcli.CLIConfig = {
@@ -40,7 +39,7 @@ def run_cli(raw_command: str | None = None) -> None:
         'root_help_arguments': True,
         # 'root_help_subcommands': False,
         'include_debug_arg': True,
-        'style_theme': flood.styles,
+        'style_theme': flood.user_io.styles,
         'cd_dir_help': cd_dir_help,
         'cd_dir_getter': cd_dir_getter,
     }

flood/cli/ls_command.py

@@ -15,24 +15,26 @@ def get_command_spec() -> toolcli.CommandSpec:
 def ls_command() -> None:
     import toolstr
 
+    styles = flood.user_io.styles
+
     toolstr.print_text_box(
         'Available tests',
-        text_style=flood.styles['metavar'],
-        style=flood.styles['content'],
+        text_style=styles['metavar'],
+        style=styles['content'],
     )
 
     print()
     toolstr.print_header(
         'Single Load Tests',
-        style=flood.styles['content'],
-        text_style=flood.styles['metavar'],
+        style=styles['content'],
+        text_style=styles['metavar'],
     )
-    for test in flood.get_single_test_generators():
+    for test in flood.generators.get_single_test_generators():
         toolstr.print_bullet(key=test, value='', colon_str='')
     print()
     toolstr.print_header(
         'Multi Load Tests',
-        style=flood.styles['content'],
-        text_style=flood.styles['metavar'],
+        style=styles['content'],
+        text_style=styles['metavar'],
     )
     toolstr.print_bullet(key='all', value='', colon_str='')