From 0ad05a9f7ac5520a3d4c6683f0f45161a8b263f4 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Fri, 31 May 2024 10:14:39 +0200 Subject: [PATCH] devtools/bench: introduce RPC Request Per Second benchmark tool --- devtools/benchmarks-tools/bench_RPS/README.md | 55 +++++++++++++++++++ devtools/benchmarks-tools/bench_RPS/rps.sh | 54 ++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 devtools/benchmarks-tools/bench_RPS/README.md create mode 100755 devtools/benchmarks-tools/bench_RPS/rps.sh diff --git a/devtools/benchmarks-tools/bench_RPS/README.md b/devtools/benchmarks-tools/bench_RPS/README.md new file mode 100644 index 000000000000..0fa098066a36 --- /dev/null +++ b/devtools/benchmarks-tools/bench_RPS/README.md @@ -0,0 +1,55 @@ +# RPC benchmarking + +This benchmark utility aims to benchmark the Requests per second that +a node is able to handle. + +This benchmark relies on the +[cassowary](https://github.com/rogerwelin/cassowary) tool. + +In the `rps.sh` script, we rely on `cassowary` through a `docker` +image. Feel free to manually update the script if you have a `cassowary` +binary on your machine. + +To run the benchmarks you need to: + +- set up a node with populated storage (like a recent `mainnet`) + - For example: `./octez-node snapshot import my_recent_snapshot.rolling --data-dir ./mainnet --no-check` +- run the node with an RPC server listening on localhost:8732 (default port) + - For example: `./octez-node run --data-dir ./mainnet --connections 0 --rpc-addr 127.0.0.1:8732` +- run `./rps.sh output_directory number_of_runs` + +The current version of the benchmark aims to define a simple set of RPCs to test: + +- `/health/ready` +- `/chains/main/blocks/header` +- `/chains/main/blocks/head/helpers/baking_rights` +- `/chains/main/blocks/head/helpers/attestation_rights` +The set of RPCs is run twice, first without concurrency (1 thread), +then with a concurrency of 10. + +Note that the benchmark of block injection or mempool operation could +be performed as well. However, as the RPC process will always forward +such requests to the node, the benchmark is not valuable. Only the +latency of the forwarding will impact the result (this latency ranges from +80ms on high-end hardware to 320ms on the GCP `n2-standard-4` +recommended hardware). + +After the benchmarks are finished, the results can be found in +[output_directory], as several `.json` files, one for each RPC bench. + +Here is a small (dirty) script to aggregate the RPS value and compute +a mean value for each kind: + +```bash! +#!/bin/bash + +for kind in health header baking attestation; do + for th in 1-th 10-th; do + echo -n "${kind}-${th}: " + for i in $(find $1 -type f | grep $kind | grep $th); do + jq .requests_per_second <"$i" + done | awk '{sum+=$0; count+=1} END {print sum/count}' + done +done + +``` diff --git a/devtools/benchmarks-tools/bench_RPS/rps.sh b/devtools/benchmarks-tools/bench_RPS/rps.sh new file mode 100755 index 000000000000..3cc37141a843 --- /dev/null +++ b/devtools/benchmarks-tools/bench_RPS/rps.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# +# SPDX-License-Identifier: MIT +# Copyright (c) 2024 Nomadic Labs, +# + +# One might run this to purge caches +#echo 3 > /proc/sys/vm/drop_caches + +if [ "$#" -ne 2 ]; then + echo "usage: ./rps [output_folder_name] [number_of_runs]" + echo "[output_folder_name]: location of the .json output files + containing the results." + echo "[number_of_runs]: additional runs to reduce the bench + noise/variance." + exit +fi + +cassowary="docker run --network=host -w $(pwd) -v $(pwd):$(pwd) rogerw/cassowary:v0.14.1" + +timeout=15 +concurrency=1 +header="Accept: application/json" + +for id in $(seq 1 1 "$2"); do + + metrics_output="$1_$id" + + mkdir -p "$metrics_output" + + for concurrency in 1 10; do + echo "-------------" + echo "Concurrency=$concurrency" + + echo "Health:" + $cassowary -t "$timeout" -u http://127.0.0.1:8732/health/ready --json-metrics --json-metrics-file="$metrics_output/health-$concurrency-th.json" -c "$concurrency" -n 100000 -H "$header" + + echo "Header:" + $cassowary -t "$timeout" -u http://127.0.0.1:8732/chains/main/blocks/header --json-metrics --json-metrics-file="$metrics_output/header-$concurrency-th.json" -c "$concurrency" -n 100000 -H "$header" + + echo "Baking rights:" + $cassowary -t "$timeout" -u http://127.0.0.1:8732/chains/main/blocks/head/helpers/baking_rights --json-metrics --json-metrics-file="$metrics_output/baking-$concurrency-th.json" -c "$concurrency" -n 10000 -H "$header" + + echo "Attestation rights:" + $cassowary -t "$timeout" -u http://127.0.0.1:8732/chains/main/blocks/head/helpers/attestation_rights --json-metrics --json-metrics-file="$metrics_output/attestation-$concurrency-th.json" -c "$concurrency" -n 1000 -H "$header" + + done + + echo "-------------" + echo -n "RPS: " + cat "$metrics_output"/*.json | jq .requests_per_second | awk '{sum+=$0;count+=1} END{print sum/count}' + +done -- GitLab