diff --git a/devtools/benchmarks-tools/profiler_viewer.sh b/devtools/benchmarks-tools/profiler_viewer.sh new file mode 100755 index 0000000000000000000000000000000000000000..83318ef71b2ee942d9b43ead2ce3a27269ebdab5 --- /dev/null +++ b/devtools/benchmarks-tools/profiler_viewer.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash + +############################################################################## +# # +# SPDX-License-Identifier: MIT # +# SPDX-FileCopyrightText: 2025 Nomadic Labs # +# # +############################################################################## + +# Translate a profiler text output to json + +# From: +# 2024-12-16T22:40:45.015-00:00 +# prepare_data ................................................................... 1 1368.105ms 100% +# 2024-12-16T22:40:46.383-00:00 +# lwt_worker_lwt_handlers ........................................................ 1 2327.835ms 100% +# BLS .......................................................................... 1 1677.783ms 100% +# Ed255519 ..................................................................... 1 87.102ms 99% +# P256 ......................................................................... 1 507.328ms 100% +# Secp256k1 .................................................................... 1 55.509ms 100% +# 2024-12-16T22:40:48.711-00:00 +# eio_worker_lwt_handlers ........................................................ 1 2545.143ms 102% +# BLS .......................................................................... 1 1719.494ms 101% +# Ed255519 ..................................................................... 1 144.398ms 108% +# P256 ......................................................................... 1 563.150ms 102% +# Secp256k1 .................................................................... 1 117.906ms 108% +# 2024-12-16T22:40:51.260-00:00 +# eio_worker_eio_handler_1_domains ............................................... 1 2272.277ms 102% +# BLS .......................................................................... 1 1620.084ms 101% +# Ed255519 ..................................................................... 1 87.296ms 113% +# P256 ......................................................................... 1 505.604ms 102% +# Secp256k1 .................................................................... 1 59.059ms 121% +# 2024-12-16T22:40:53.532-00:00 +# eio_worker_eio_handler_2_domains ............................................... 1 1153.309ms 204% +# BLS .......................................................................... 1 823.329ms 202% +# Ed255519 ..................................................................... 1 44.293ms 220% +# P256 ......................................................................... 1 255.167ms 204% +# Secp256k1 .................................................................... 1 30.097ms 234% +# 2024-12-16T22:40:54.685-00:00 +# eio_worker_eio_handler_4_domains ............................................... 1 580.687ms 403% +# BLS .......................................................................... 1 415.767ms 400% +# Ed255519 ..................................................................... 1 22.650ms 414% +# P256 ......................................................................... 1 126.214ms 407% +# Secp256k1 .................................................................... 1 15.149ms 452% +# 2024-12-16T22:40:55.266-00:00 +# eio_worker_eio_handler_8_domains ............................................... 1 299.904ms 798% +# BLS .......................................................................... 1 211.680ms 801% +# Ed255519 ..................................................................... 1 12.684ms 774% +# P256 ......................................................................... 1 65.167ms 805% +# Secp256k1 .................................................................... 1 8.375ms 802% +# 2024-12-16T22:40:55.566-00:00 +# eio_worker_eio_handler_16_domains .............................................. 1 161.059ms 1541% +# BLS .......................................................................... 1 108.325ms 1588% +# Ed255519 ..................................................................... 1 7.336ms 1490% +# P256 ......................................................................... 1 35.012ms 1557% +# Secp256k1 .................................................................... 1 5.665ms 1426% + +# To: +# [{ "section" : "prepare_data", "count": 1, "duration":1368.105, "cpu":100, "sub":[]}, +# { "section":"lwt_worker_lwt_handlers", "count": 1, "duration":2327.853, "cpu":100" "sub":[ +# { "section":"BLS", "count": 1, "duration":1677.783, "cpu":100}, +# { "section":"Ed255519 ", "count": 1, "duration":87.102, "cpu":99}, +# { "section":"P256", "count": 1, "duration":507.328, "cpu":100}, +# { "section":"Secp256k1", "count": 1, "duration":55.509, "cpu":100} +# ]}, +# ... +# ] + +# The input is a text file with the profiler output +# The output is a json file with the profiler output + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +input=$1 +output=$2 + +echo "[" > "$output" + +# sections starts with a timestamp +# sub-sections are indented +# all lines have the same format : "section_name ... count duration cpu%" + +section="" +sub_section="" + +while IFS= read -r line; do + # is it a section? + if [[ $line =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}-[0-9]{2}:[0-9]{2}$ ]]; then + # if we have a section, close it + read -r line + if [ -n "$section" ]; then + # if no sub-section, copy the section + if [ -z "$sub_section" ]; then + echo " { \"subsection\" : \"$section\", \"count\": $count, \"duration\":\"$duration\", \"cpu\":$cpu }" >> "$output" + fi + echo " ]}," >> "$output" + fi + # start a new section + sub_section="" + + # shellcheck disable=SC2086 + { + section=$(echo $line | cut -d' ' -f1) + count=$(echo $line | cut -d' ' -f3) + duration=$(echo $line | cut -d' ' -f4 | sed 's/ms//') + cpu=$(echo $line | cut -d' ' -f5 | sed 's/%//') + } + echo " { \"section\" : \"$section\", \"count\": $count, \"duration\":\"$duration\", \"cpu\":$cpu, \"sub\":[" >> "$output" + else + # is it a sub-section? + if [[ $line =~ ^\ +[A-Za-z0-9_]+ ]]; then + # if we have a sub-section, close it + if [ -n "$sub_section" ]; then + echo " ," >> "$output" + fi + # start a new sub-section + # shellcheck disable=SC2086 + { + sub_section=$(echo $line | cut -d' ' -f1) + count=$(echo $line | cut -d' ' -f3) + duration=$(echo $line | cut -d' ' -f4 | sed 's/ms//') + cpu=$(echo $line | cut -d' ' -f5 | sed 's/%//') + } + echo " { \"subsection\" : \"$sub_section\", \"count\": $count, \"duration\":\"$duration\", \"cpu\":$cpu }" >> "$output" + fi + + fi + +done < "$input" + +# close the last section +echo " ]}" >> "$output" + +echo "]" >> "$output" + +# using this json produce a vegalite graph with each section as a bar and each sub-section as a stacked bar +# the duration is the size of the bar, color is the sub-section +# the x-axis is the section, the y-axis is the duration + +# shellcheck disable=SC2154 +cat > "$3" << EOF +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "width":600, + "height":400, + "data": { + "values": $(cat "$output") + }, + "transform" : [ + { + "flatten": ["sub"] + } + ], + "params": [ + + {"name": "select", + "select": {"type": "point", "fields": ["sub.subsection"]}} + + ], + "mark": { + "type": "bar", + "stroke": "black", + "cursor": "pointer", + "tooltip": true + }, + "encoding": { + "y": { + "aggregate": "sum", + "field": "sub.duration", + "type": "quantitative", + "axis": { + "title": "Duration (ms)" + }, + "sort": "y" + }, + "x": { + "field": "section", + "type": "nominal", + + "axis": { + "title": "Profiling section", + "labelAngle": -30, + "labelAlign": "right", + "labelfonSize":25 + }, + "sort": "y" + }, + "tooltip": [ + {"field": "section", "type": "nominal", "title": "Section"}, + {"field": "sub.subsection", "type": "nominal", "title": "Profiling function"}, + {"field": "sub.duration", "type": "quantitative", "title": "Duration (ms)"}, + {"field": "sub.cpu", "type": "quantitative", "title": "CPU %"}, + {"field": "sub.count", "type": "quantitative", "title": "Count"} + ], + "fillOpacity": { + "condition": {"param": "select", "value": 1}, + "value": 0.3 + }, + "opacity": { + "condition": {"param": "select", "value": 1}, + "value": 0.2 + }, + "color": { + "field": "sub.subsection", + "type": "nominal", + "legend": { + "title": "Profiling function" + }, + "scale": { + "scheme": "set3" + }, + "strokeWidth": { + "condition": [ + { + "param": "select", + "value": 1 + } + ], + "value": 0 + } + } + } + } + +EOF + +# if vl-convert is available, convert the vega-lite to a url + +if command -v vl-convert &> /dev/null; then + echo -e "\033[33mTo display and edit the graph, open in your local browser:\033[0m" + printf "%s/view\n" "$(vl-convert vl2url --input "$3")" + echo "Generating out.svg file" + vl-convert vl2svg --input "$3" --output ./out.svg +else + echo "Install vl-convert using \"cargo install vl-convert\" to generate fancy outputs" +fi + +exit 0 diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index 2ded41146a4db0a22d4b72ed907213f51c8034fd..68b70f68341a855885b1ef7fe4945f7f065b08b9 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -9179,6 +9179,27 @@ let _tezt_manual_tests = yes_wallet_lib; ] +let _tezt_eio_benchmarks = + let (PPX {preprocess; preprocessor_deps}) = ppx_profiler in + private_exe + "test" + ~opam:"" + ~path:"tezt/manual_tests/eio_benchmarks" + ~bisect_ppx:No + ~with_macos_security_framework:true + ~preprocess + ~preprocessor_deps + ~linkall:true + ~deps: + [ + bls12_381_archive; + bls12_381; + octez_profiler_backends; + octez_crypto; + octez_bees; + octez_client_base_unix; + ] + let _tezt_remote_tests = private_exe "main" diff --git a/tezt/manual_tests/eio_benchmarks/dune b/tezt/manual_tests/eio_benchmarks/dune new file mode 100644 index 0000000000000000000000000000000000000000..50b5f57066fd4710bc314efc1e94e41947351a30 --- /dev/null +++ b/tezt/manual_tests/eio_benchmarks/dune @@ -0,0 +1,19 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(executable + (name test) + (libraries + octez-rust-deps + bls12-381.archive + bls12-381 + octez-libs.octez-profiler.backends + octez-libs.crypto + octez-libs.tezos-bees + octez-shell-libs.client-base-unix) + (preprocess (pps octez-libs.ppx_profiler)) + (preprocessor_deps (env_var TEZOS_PPX_PROFILER)) + (link_flags + (:standard) + (:include %{workspace_root}/macos-link-flags.sexp) + (-linkall))) diff --git a/tezt/manual_tests/eio_benchmarks/eio_handlers.ml b/tezt/manual_tests/eio_benchmarks/eio_handlers.ml new file mode 100644 index 0000000000000000000000000000000000000000..e76d15602fe2783cb95dfb393da908f024ea229d --- /dev/null +++ b/tezt/manual_tests/eio_benchmarks/eio_handlers.ml @@ -0,0 +1,42 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2018-2025 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +open Helpers +open Tezos_base +open TzPervasives + +module MAKE (Worker : sig + type infinite + + type 'a queue + + type 'a t +end) = +struct + type self = Worker.infinite Worker.queue Worker.t + + type launch_error = error list + + let on_launch _ _ _ = Ok () + + let on_close _ = () + + let on_error _ _ _ _ = Error [] + + let on_completion (type a b) _w (_req : (a, b) Request.t) (_res : a) _status = + () + + let on_no_request _ = () + + let on_request : type a b. self -> (a, b) Request.t -> (a, b) result = + fun _w req -> + match req with + | Request.Check_signature (pk, signature, msg) -> + Ok (Signature.check pk signature msg) + + let on_request w req = on_request w req +end diff --git a/tezt/manual_tests/eio_benchmarks/handlers.ml b/tezt/manual_tests/eio_benchmarks/handlers.ml new file mode 100644 index 0000000000000000000000000000000000000000..0235c55212b909ab08a50fcc87dad3bdc90bb5b7 --- /dev/null +++ b/tezt/manual_tests/eio_benchmarks/handlers.ml @@ -0,0 +1,42 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2018-2025 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +open Helpers +open Tezos_base +open TzPervasives + +module MAKE (Worker : sig + type infinite + + type 'a queue + + type 'a t +end) = +struct + type self = Worker.infinite Worker.queue Worker.t + + type launch_error = error list + + let on_launch _ _ _ = Lwt.return @@ Ok () + + let on_close _ = Lwt.return @@ () + + let on_error _ _ _ _ = Lwt.return @@ Error [] + + let on_completion (type a b) _w (_req : (a, b) Request.t) (_res : a) _status = + Lwt.return @@ () + + let on_no_request _ = Lwt.return @@ () + + let on_request : type a b. self -> (a, b) Request.t -> (a, b) result = + fun _w req -> + match req with + | Request.Check_signature (pk, signature, msg) -> + Ok (Signature.check pk signature msg) + + let on_request w req = Lwt.return @@ on_request w req +end diff --git a/tezt/manual_tests/eio_benchmarks/helpers.ml b/tezt/manual_tests/eio_benchmarks/helpers.ml new file mode 100644 index 0000000000000000000000000000000000000000..014cd4cab400f02ea0364ecc5c9375665a638eb4 --- /dev/null +++ b/tezt/manual_tests/eio_benchmarks/helpers.ml @@ -0,0 +1,51 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2018-2025 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +open Tezos_base +open TzPervasives + +module Name = struct + type t = string + + let encoding = Data_encoding.string + + let base = ["test"; "main"] + + let pp = Format.pp_print_string + + let equal = String.equal +end + +module Request = struct + type (_, _) t = + | Check_signature : + Signature.public_key * Signature.t * Bytes.t + -> (bool, exn) t + + type view = Signature.public_key + + let encoding = Signature.Public_key.encoding + + let pp = Signature.Public_key.pp + + let view (type a b) (req : (a, b) t) : view = + match req with Check_signature (pk, _signature, _msg) -> pk +end + +module Types = struct + type parameters = unit + + type state = unit +end + +module Worker = Tezos_bees.Worker.MakeSingle (Name) (Request) (Types) + +type t = Worker.infinite Worker.queue Worker.t + +module Lwt_worker = Tezos_workers.Worker.MakeSingle (Name) (Request) (Types) + +let table = Lwt_worker.create_table Lwt_worker.Queue diff --git a/tezt/manual_tests/eio_benchmarks/test.ml b/tezt/manual_tests/eio_benchmarks/test.ml new file mode 100644 index 0000000000000000000000000000000000000000..8604028af69483bcf063f5a193e56403ab6bb742 --- /dev/null +++ b/tezt/manual_tests/eio_benchmarks/test.ml @@ -0,0 +1,206 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2018-2025 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +(* You can run the benchmark using: + TEZOS_PPX_PROFILER=t dune build tezt/manual_tests/eio_benchmarks && PROFILING="main->debug" PROFILING_BACKEND=txt _build/default/tezt/manual_tests/eio_benchmarks/test.exe 32 +*) + +open Tezos_base +open TzPervasives +open Helpers + +let () = + match Tezos_profiler_unix.Profiler_instance.selected_backend () with + | Some {instance_maker; _} -> ( + let profiler_maker = instance_maker ~directory:"/tmp" ~name:"main" in + match profiler_maker with + | Some instance -> Tezos_profiler.Profiler.(plug main) instance + | None -> ()) + | None -> () + +module Profiler = + (val Tezos_profiler.Profiler.wrap Tezos_profiler.Profiler.main) + +let msg_size = 138 + 4 + +let number = 2000 + +let data = Stdlib.Hashtbl.create 5 + +let prepare_data label algo = + match Stdlib.Hashtbl.find_opt data label with + | Some v -> v + | None -> + let mk_msg i = + let buf = Bytes.create msg_size in + Bytes.set_int64_be buf 0 (Int64.of_int i) ; + buf + in + let keys = + List.map + (fun i -> + let _pkh, pk, sk = Signature.generate_key ~algo () in + let msg = mk_msg i in + (i, (pk, sk, msg))) + (1 -- number) + in + let keys_and_signatures = + List.map + (fun (_i, (pk, sk, msg)) -> (pk, sk, msg, Signature.sign sk msg)) + keys + in + Stdlib.Hashtbl.add data label keys_and_signatures ; + keys_and_signatures + +let label_and_algos = + [ + ("Secp256k1", Signature.Secp256k1); + ("Ed255519", Signature.Ed25519); + ("P256", Signature.P256); + ("BLS", Signature.Bls); + ] + +let () = + let () = (() [@profiler.record {verbosity = Notice} "prepare_data"]) in + List.iter + (fun (label, algo) -> + Eio.traceln "Preparing data for %s@." label ; + let _ = prepare_data label algo in + ()) + label_and_algos ; + () [@profiler.stop] + +let compute worker label algo f = + let label = Printf.sprintf "%s" label in + ignore label ; + let keys_and_signatures = prepare_data label algo in + f worker keys_and_signatures + +let eio_compute worker label algo = + (fun worker keys_and_signatures -> + (Eio.Fiber.all + (Stdlib.List.map + (fun (pk, _sk, msg, signature) () -> + let res = + Eio.Promise.await + (Worker.Queue.push_request_and_wait_eio + (* ~label:"check_signature" *) + worker + (Request.Check_signature (pk, signature, msg))) + in + match res with Ok b -> assert b | Error _ -> assert false) + keys_and_signatures) + [@profiler.aggregate_f {verbosity = Notice} (Printf.sprintf "%s" label)])) + |> compute worker label algo + +let eio_lwt_compute worker label algo = + (fun worker keys_and_signatures -> + let open Lwt_syntax in + assert ( + Lwt_eio.run_lwt @@ fun () -> + (Lwt_list.for_all_p + (fun (pk, _sk, msg, signature) -> + let* r = + Worker.Queue.push_request_and_wait + worker + (Request.Check_signature (pk, signature, msg)) + in + match r with Ok b -> Lwt.return b | Error _ -> Lwt.return_false) + keys_and_signatures + [@profiler.aggregate_s {verbosity = Notice} (Printf.sprintf "%s" label)]))) + |> compute worker label algo + +let compute_lwt worker label algo = + (fun worker keys_and_signatures -> + let open Lwt_syntax in + assert ( + Lwt_eio.run_lwt @@ fun () -> + (Lwt_list.for_all_p + (fun (pk, _sk, msg, signature) -> + let* r = + Lwt_worker.Queue.push_request_and_wait + worker + (Request.Check_signature (pk, signature, msg)) + in + match r with Ok b -> Lwt.return b | Error _ -> Lwt.return_false) + keys_and_signatures + [@profiler.aggregate_s {verbosity = Notice} (Printf.sprintf "%s" label)]))) + |> compute worker label algo + +let run compute worker = + List.iter (fun (label, algo) -> compute worker label algo) label_and_algos + +let () = + Tezos_base_unix.Event_loop.main_run ~eio:true @@ fun _env -> + ((* Lwt worker running with Lwt handlers *) + let () = + (() [@profiler.record {verbosity = Notice} "lwt_worker_lwt_handlers"]) + in + let worker = + Lwt_eio.run_lwt @@ fun () -> + Lwt_worker.launch table "lwt_worker" () (module Handlers.MAKE (Lwt_worker)) + in + + Eio.traceln "Lwt Worker running with lwt handlers@." ; + let worker = match worker with Ok w -> w | Error _ -> assert false in + + run compute_lwt worker ; + + let _ = Lwt_worker.shutdown worker in + let () = (() [@profiler.stop]) in + + (* Eio worker running with Lwt handlers *) + let () = + (() [@profiler.record {verbosity = Notice} "eio_worker_lwt_handlers"]) + in + let table = Worker.create_table Worker.Queue in + + let worker = + Lwt_eio.run_lwt @@ fun () -> + Worker.launch table "eio_lwt_worker" () (module Handlers.MAKE (Worker)) + in + Eio.traceln "Eio Worker running with lwt handlers@." ; + let worker = match worker with Ok w -> w | Error _ -> assert false in + + run eio_lwt_compute worker ; + let () = Lwt_eio.run_lwt @@ fun () -> Worker.shutdown worker in + + let () = (() [@profiler.stop]) in + ()) ; + + (* Eio Worker running with Eio handler *) + let rec bench_all domains = + if domains > int_of_string Sys.argv.(1) then Lwt.return_unit + else ( + Eio.traceln + "Eio Worker running with eio handlers and %d domains@." + domains ; + let () = + (() + [@profiler.record + {verbosity = Notice} + (Format.sprintf "eio_worker_eio_handler_%d_domains" domains)]) + in + let table = Worker.create_table Worker.Queue in + let worker = + Worker.launch_eio + table + ~domains + ~name:"eio_eio_worker" + () + (module Eio_handlers.MAKE (Worker)) + in + + let worker = match worker with Ok w -> w | Error _ -> assert false in + run eio_compute worker ; + let _ = Worker.shutdown_eio worker in + Format.printf "stop@." ; + let () = (() [@profiler.stop]) in + + bench_all (domains * 2)) + in + bench_all 1