From 5bb2acd895f99297675fdd0e3396881899112ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Fri, 5 Jul 2024 01:13:10 +0200 Subject: [PATCH 01/20] Tezt/Cloud: Big refactoring --- dune-project | 1 - etherlink/tezt/lib/dune | 6 +- etherlink/tezt/lib/evm_node.mli | 4 +- manifest/product_etherlink.ml | 1 + manifest/product_octez.ml | 30 +- opam/tezt-cloud.opam | 21 - opam/tezt-tests-cloud.opam | 1 - opam/tezt-tezos.opam | 2 +- tezt/{lib_tezos => lib_cloud}/agent.ml | 27 +- tezt/{lib_tezos => lib_cloud}/agent.mli | 16 +- tezt/lib_cloud/cli.ml | 64 ++- tezt/lib_cloud/cli.mli | 24 +- tezt/lib_cloud/cloud.ml | 113 +++--- tezt/lib_cloud/cloud.mli | 7 +- tezt/lib_cloud/configuration.ml | 45 +-- tezt/lib_cloud/configuration.mli | 12 +- tezt/lib_cloud/deployement.ml | 364 +++++++----------- tezt/lib_cloud/deployement.mli | 20 +- tezt/lib_cloud/docker.ml | 108 +++--- tezt/lib_cloud/docker.mli | 36 +- tezt/lib_cloud/dockerfiles/octez.Dockerfile | 22 ++ tezt/lib_cloud/dune | 7 +- tezt/lib_cloud/env.ml | 176 +++++++-- tezt/lib_cloud/env.mli | 86 ++++- tezt/lib_cloud/gcloud.ml | 42 +- tezt/lib_cloud/gcloud.mli | 14 +- tezt/lib_cloud/jobs.ml | 240 +++++------- tezt/lib_cloud/jobs.mli | 9 +- tezt/lib_cloud/monitoring.ml | 17 +- tezt/lib_cloud/monitoring.mli | 7 +- tezt/lib_cloud/path.ml | 3 +- tezt/lib_cloud/path.mli | 2 +- tezt/lib_cloud/prometheus.ml | 104 ++--- tezt/lib_cloud/prometheus.mli | 12 +- tezt/lib_cloud/ssh.ml | 2 +- tezt/lib_cloud/ssh.mli | 4 +- tezt/lib_cloud/terraform.ml | 79 ++-- tezt/lib_cloud/terraform.mli | 22 +- .../terraform/docker-registry/main.tf | 11 + tezt/lib_cloud/tezt_cloud.ml | 82 +++- tezt/lib_cloud/tezt_cloud.mli | 45 ++- tezt/lib_cloud/web.ml | 124 +++++- tezt/lib_cloud/web.mli | 5 +- tezt/lib_tezos/dune | 4 +- tezt/tests/cloud/basic.ml | 2 +- tezt/tests/cloud/dal.ml | 13 +- tezt/tests/cloud/dune | 4 +- 47 files changed, 1131 insertions(+), 909 deletions(-) delete mode 100644 opam/tezt-cloud.opam rename tezt/{lib_tezos => lib_cloud}/agent.ml (81%) rename tezt/{lib_tezos => lib_cloud}/agent.mli (80%) create mode 100644 tezt/lib_cloud/dockerfiles/octez.Dockerfile diff --git a/dune-project b/dune-project index 400dbcf4457c..57dc3a669710 100644 --- a/dune-project +++ b/dune-project @@ -153,7 +153,6 @@ (package (name tezos-tree-encoding-test)(allow_empty)) (package (name tezos_internal_brassaia_tests)(allow_empty)) (package (name tezos_internal_irmin_tests)(allow_empty)) -(package (name tezt-cloud)(allow_empty)) (package (name tezt-etherlink)(allow_empty)) (package (name tezt-riscv-sandbox)(allow_empty)) (package (name tezt-tests-cloud)(allow_empty)) diff --git a/etherlink/tezt/lib/dune b/etherlink/tezt/lib/dune index 2c4a0631ac5c..117cde38f96e 100644 --- a/etherlink/tezt/lib/dune +++ b/etherlink/tezt/lib/dune @@ -8,11 +8,13 @@ octez-libs.tezt-wrapper tezt-tezos.tezt-performance-regression octez-libs.crypto - tezt-tezos) + tezt-tezos + tezt-tezos.tezt-cloud) (flags (:standard) -open Tezt_wrapper -open Tezt_wrapper.Base -open Tezt_tezos_tezt_performance_regression -open Tezt_tezos - -open Tezt_tezos.Runnable.Syntax)) + -open Tezt_tezos.Runnable.Syntax + -open Tezt_tezos_tezt_cloud)) diff --git a/etherlink/tezt/lib/evm_node.mli b/etherlink/tezt/lib/evm_node.mli index d60096cf2fe9..d707e23adfab 100644 --- a/etherlink/tezt/lib/evm_node.mli +++ b/etherlink/tezt/lib/evm_node.mli @@ -416,7 +416,7 @@ module Agent : sig ?data_dir:string -> ?mode:mode -> string -> - Tezt_tezos.Agent.t -> + Agent.t -> t Lwt.t val init : @@ -425,7 +425,7 @@ module Agent : sig ?mode:mode -> ?data_dir:string -> string -> - Tezt_tezos.Agent.t -> + Agent.t -> t Lwt.t val rpc_port : t -> int diff --git a/manifest/product_etherlink.ml b/manifest/product_etherlink.ml index 81f9058eb4e3..a6d48cceb2dc 100644 --- a/manifest/product_etherlink.ml +++ b/manifest/product_etherlink.ml @@ -31,6 +31,7 @@ let tezt_etherlink = tezt_performance_regression |> open_; octez_crypto; tezt_tezos |> open_ |> open_ ~m:"Runnable.Syntax"; + tezt_cloud |> open_; ] ~release_status:Unreleased diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index af5c33d81e10..e868e2aa2948 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -2316,6 +2316,18 @@ let tezt_performance_regression = ~bisect_ppx:No ~deps:[tezt_wrapper |> open_ |> open_ ~m:"Base"; uri; cohttp_lwt_unix] +let tezt_cloud = + public_lib + "tezt-tezos.tezt-cloud" + ~path:"tezt/lib_cloud" + ~opam:"tezt-tezos" + ~bisect_ppx:No + ~deps: + [ + tezt_lib |> open_ |> open_ ~m:"Base"; + tezt_performance_regression |> open_; + ] + let tezt_tezos = public_lib "tezt-tezos" @@ -2327,6 +2339,7 @@ let tezt_tezos = [ tezt_wrapper |> open_ |> open_ ~m:"Base"; tezt_performance_regression |> open_; + tezt_cloud |> open_; uri; hex; octez_crypto_dal; @@ -2337,23 +2350,6 @@ let tezt_tezos = ~cram:true ~release_status:Released -let tezt_cloud = - private_lib - "tezt_cloud" - ~path:"tezt/lib_cloud" - ~opam:"tezt-cloud" - ~synopsis: - "A library to ease the writing of test using machines onto cloud \ - providers (GCP, AWS, ...)" - ~bisect_ppx:No - ~deps: - [ - tezt_lib |> open_ |> open_ ~m:"Base"; - tezt_tezos |> open_ |> open_ ~m:"Runnable.Syntax"; - tezt_performance_regression |> open_; - ] - ~release_status:Unreleased - let octez_p2p_test_common = octez_shell_lib "p2p_test_common" diff --git a/opam/tezt-cloud.opam b/opam/tezt-cloud.opam deleted file mode 100644 index d592297b88ba..000000000000 --- a/opam/tezt-cloud.opam +++ /dev/null @@ -1,21 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file manifest/main.ml instead. -opam-version: "2.0" -maintainer: "contact@tezos.com" -authors: ["Tezos devteam"] -homepage: "https://www.tezos.com/" -bug-reports: "https://gitlab.com/tezos/tezos/issues" -dev-repo: "git+https://gitlab.com/tezos/tezos.git" -license: "MIT" -depends: [ - "dune" { >= "3.11.1" } - "ocaml" { >= "4.14" } - "tezt" { >= "4.1.0" & < "5.0.0" } - "tezt-tezos" { = version } -] -build: [ - ["rm" "-r" "vendors" "contrib"] - ["dune" "build" "-p" name "-j" jobs] - ["dune" "runtest" "-p" name "-j" jobs] {with-test} -] -synopsis: "A library to ease the writing of test using machines onto cloud providers (GCP, AWS, ...)" diff --git a/opam/tezt-tests-cloud.opam b/opam/tezt-tests-cloud.opam index e51cb0446a43..3e67a6b299a5 100644 --- a/opam/tezt-tests-cloud.opam +++ b/opam/tezt-tests-cloud.opam @@ -12,7 +12,6 @@ depends: [ "ocaml" { >= "4.14" } "octez-libs" "tezt-tezos" - "tezt-cloud" "tezt-etherlink" { = version } ] build: [ diff --git a/opam/tezt-tezos.opam b/opam/tezt-tezos.opam index aac3c416373f..bd2d08e5ad21 100644 --- a/opam/tezt-tezos.opam +++ b/opam/tezt-tezos.opam @@ -13,8 +13,8 @@ depends: [ "octez-libs" { = version } "uri" { >= "3.1.0" } "cohttp-lwt-unix" { >= "5.2.0" } + "tezt" { >= "4.1.0" & < "5.0.0" } "hex" { >= "1.3.0" } - "tezt" { with-test & >= "4.1.0" & < "5.0.0" } ] build: [ ["rm" "-r" "vendors" "contrib"] diff --git a/tezt/lib_tezos/agent.ml b/tezt/lib_cloud/agent.ml similarity index 81% rename from tezt/lib_tezos/agent.ml rename to tezt/lib_cloud/agent.ml index f5366cd09445..a7727f7280d0 100644 --- a/tezt/lib_tezos/agent.ml +++ b/tezt/lib_cloud/agent.ml @@ -7,22 +7,37 @@ type t = { mutable name : string; + vm_name : string; + cmd_wrapper : Gcloud.cmd_wrapper option; point : string * int; runner : Runner.t; next_available_port : unit -> int; - binaries_path : string; + configuration : Configuration.t; } -let make ?(ssh_user = "root") ~ssh_id ~point:((address, ssh_port) as point) - ~binaries_path ~next_available_port ~name () = +let make ?cmd_wrapper ~ssh_id ~point:((address, ssh_port) as point) + ~configuration ~next_available_port ~name () = + let ssh_user = "root" in let runner = Runner.create ~ssh_user ~ssh_id ~ssh_port ~address () in - {point; runner; name; next_available_port; binaries_path} + { + point; + runner; + name; + vm_name = name; + next_available_port; + configuration; + cmd_wrapper; + } let name {name; _} = name +let point {point; _} = point + +let cmd_wrapper agent = agent.cmd_wrapper + let set_name agent name = agent.name <- name -let path_of agent binary = agent.binaries_path // binary +let path_of agent binary = agent.configuration.binaries_path // binary let copy agent ~source ~destination = let runner = agent.runner in @@ -85,4 +100,4 @@ let next_available_port t = t.next_available_port () let runner {runner; _} = runner -let binaries_path {binaries_path; _} = binaries_path +let configuration {configuration; _} = configuration diff --git a/tezt/lib_tezos/agent.mli b/tezt/lib_cloud/agent.mli similarity index 80% rename from tezt/lib_tezos/agent.mli rename to tezt/lib_cloud/agent.mli index 816e615a977b..438679a90c12 100644 --- a/tezt/lib_tezos/agent.mli +++ b/tezt/lib_cloud/agent.mli @@ -10,17 +10,17 @@ (** Datatype for an agent *) type t -(** [make ?binaries_path ?ssh_user ~ssh_id ~point ~next_available_port ~name] +(** [make ?binaries_path ~ssh_id ~point ~next_available_port ~name] creates an [agent] from the given parameters. [point] is the point on which the VM is reachabled. [next_available_port] should always provide an available port or raise [Not_found] otherwise. [name] is the name of the agent. [ssh_id] is a path to the private key that will be used for the ssh connection. *) val make : - ?ssh_user:string -> + ?cmd_wrapper:Gcloud.cmd_wrapper -> ssh_id:string -> point:string * int -> - binaries_path:string -> + configuration:Configuration.t -> next_available_port:(unit -> int) -> name:string -> unit -> @@ -45,5 +45,11 @@ val next_available_port : t -> int (** [runner agent] returns the runner associated with the agent. *) val runner : t -> Runner.t -(** [binaries_path t] the path where binaries should be stored by the agent. *) -val binaries_path : t -> string +(** [point agent] returns the point asociated with the agent. *) +val point : t -> string * int + +(** [configuration t] the configuration of the agent. *) +val configuration : t -> Configuration.t + +(** A wrapper to run a command on the VM of the agent. *) +val cmd_wrapper : t -> Gcloud.cmd_wrapper option diff --git a/tezt/lib_cloud/cli.ml b/tezt/lib_cloud/cli.ml index d1f90b14a023..a67b1e7a87fb 100644 --- a/tezt/lib_cloud/cli.ml +++ b/tezt/lib_cloud/cli.ml @@ -17,6 +17,7 @@ let localhost = Clap.flag ~section ~set_long:"localhost" + ~unset_long:"cloud" ~description:"If set, the test is run locally" false @@ -61,6 +62,13 @@ let vms = ~description:"Number of VMs running for the test." () +let vm_base_port = + Clap.default_int + ~section + ~long:"vm-base-port" + ~description:"The first available port on the VM" + 30_000 + let ports_per_vm = Clap.default_int ~section @@ -73,7 +81,7 @@ let grafana = ~section ~set_long:"grafana" ~description:"Flag to set whether to run grafana" - false + true let prometheus = Clap.flag @@ -82,6 +90,36 @@ let prometheus = ~description:"Flag to set whether metrics are exported into prometheus" grafana +let prometheus_export = + Clap.flag + ~section + ~set_long:"prometheus-export" + ~description:"Export a Prometheus snapshot at the end of the scenario" + true + +let prometheus_port = + Clap.default_int + ~section + ~long:"prometheus-port" + ~description: + "Set the port on which the prometheus instance will run (default: 9090)." + 9090 + +let prometheus_snapshot_filename = + Clap.optional_string + ~section + ~long:"prometheus-snapshot-filename" + ~description:"Name of the prometheus snapshot file" + () + +let prometheus_scrape_interval = + Clap.default_int + ~section + ~long:"prometheus-scrape-interval" + ~description: + "Set the scraping interval of the prometheus instance (default: 5)" + 5 + let website = Clap.flag ~section @@ -98,22 +136,6 @@ let website_port = ~description:"Set the port used for the website. Default is 8080" 8080 -let prometheus_snapshot_directory = - Clap.default_string - ~section - ~long:"prometheus-snapshot-directory" - ~description: - "The directory where the prometheus snapshot can be exported (the \ - default is /tmp)" - "/tmp" - -let prometheus_snapshot = - Clap.optional_string - ~section - ~long:"prometheus-snapshot" - ~description:"Name of the prometheus snapshot file" - () - let machine_type = Clap.default_string ~section @@ -123,13 +145,13 @@ let machine_type = https://cloud.google.com/compute/docs/general-purpose-machines#c3d_series)" "n1-standard-2" -let dockerfile = +let dockerfile_alias = Clap.optional_string ~section - ~long:"dockerfile" + ~long:"dockerfile-alias" ~description: - "Specify the name of the dockerfile to use (default is given by the \ - environment variable `TEZT_CLOUD`)" + "Specify the name of the dockerfile alias to use (default is given by \ + the environment variable `TEZT_CLOUD`)" () let max_run_duration = diff --git a/tezt/lib_cloud/cli.mli b/tezt/lib_cloud/cli.mli index 994c2c86dc54..f4b5a7619643 100644 --- a/tezt/lib_cloud/cli.mli +++ b/tezt/lib_cloud/cli.mli @@ -31,6 +31,10 @@ val keep_alive : bool (** [vms] enables to set the number of vms that will be spawned. *) val vms : int option +(** First port opened on the VM. *) +val vm_base_port : int + +(** Number of consecutive ports opened on the VM from the base port. *) val ports_per_vm : int (** When [website] is [true] (default) a website is up for summarizing various @@ -41,16 +45,24 @@ val website : bool [8080]. *) val website_port : int -(** When [prometheus_export] is [true] a Prometheus instance is run locally and +(** When [prometheus] is [true] a Prometheus instance is run locally and metrics of the experiments are exported to prometheus. At the end of the test, the database is snapshotted so that it can be imported later on. *) val prometheus : bool -val grafana : bool +(** If [true], exports a Promtheus snapshot at the end of the scenario. *) +val prometheus_export : bool + +(** Specify the name of the prometheus snapshot. *) +val prometheus_snapshot_filename : string option -val prometheus_snapshot_directory : string +(** Specify the port of the Prometheus instance on the host machine. *) +val prometheus_port : int -val prometheus_snapshot : string option +(** Specify the scraping interval of Prometheus. *) +val prometheus_scrape_interval : int + +val grafana : bool (** Enable to specify a machine type. The string must be a machine description compliant with GCP (ex: "n1-standard-2"). See the different type of machines @@ -61,9 +73,9 @@ val prometheus_snapshot : string option *) val machine_type : string -(** Specify the dockerfile image to use. If not specified, the image +(** Specify the dockerfile alias to use. If not specified, the image name will be given by the value of the variable `TEZT_CLOUD`. *) -val dockerfile : string option +val dockerfile_alias : string option (** Maximum running time of a VM. *) val max_run_duration : int diff --git a/tezt/lib_cloud/cloud.ml b/tezt/lib_cloud/cloud.ml index ca5d7d5c5947..573d932adea6 100644 --- a/tezt/lib_cloud/cloud.ml +++ b/tezt/lib_cloud/cloud.ml @@ -5,8 +5,6 @@ (* *) (*****************************************************************************) -module Configuration = Configuration - (* Tezt-cloud requires to bypass the clean-up process of Tezt. Hence, when the user press Ctrl+C, tezt-cloud needs to catch-up the signal before Tezt. @@ -42,22 +40,45 @@ type t = { let shutdown ?exn t = let* () = - if Cli.keep_alive then ( + if Env.keep_alive then ( Log.info "Please press to terminate the scenario." ; let* _ = Lwt_io.read_line Lwt_io.stdin in Lwt.return_unit) else Lwt.return_unit in Log.info "Shutting down processes..." ; - let* () = Process.clean_up () in let* () = - Option.fold - ~none:Lwt.return_unit - ~some:Prometheus.export_snapshot - t.prometheus + Lwt.catch + (fun () -> Process.clean_up ()) + (fun exn -> + Log.warn + "Tezt failed to clean up processes: %s" + (Printexc.to_string exn) ; + Log.warn + "in case you did not destroy VMs, you should execute the 'clean up' \ + job" ; + Lwt.return_unit) + in + let* () = + Lwt.catch + (fun () -> + Option.fold + ~none:Lwt.return_unit + ~some:Prometheus.export_snapshot + t.prometheus) + (fun exn -> + Log.warn "Prometheus snapshot export fails: %s" (Printexc.to_string exn) ; + Lwt.return_unit) in let* () = - Option.fold ~none:Lwt.return_unit ~some:Prometheus.shutdown t.prometheus + Lwt.catch + (fun () -> + Option.fold ~none:Lwt.return_unit ~some:Prometheus.shutdown t.prometheus) + (fun exn -> + Log.warn + "unable to shutdown Prometheus properly: %s" + (Printexc.to_string exn) ; + Lwt.return_unit) in let* () = Option.fold ~none:Lwt.return_unit ~some:Grafana.shutdown t.grafana @@ -76,35 +97,24 @@ let shutdown ?exn t = | Some exn -> (* The exception is raised to Tezt. *) Lwt.reraise exn (* This function is used to ensure we can connect to the docker image on the VM. *) -let rec wait_ssh_server_running runner = - let cmd, args = - Runner.wrap_with_ssh runner (Runner.Shell.cmd [] "echo" ["-n"; "check"]) - in - let* status = +let wait_ssh_server_running runner = + let is_ready _output = true in + let run () = + let cmd, args = + Runner.wrap_with_ssh runner (Runner.Shell.cmd [] "echo" ["-n"; "check"]) + in Process.spawn cmd (["-o"; "StrictHostKeyChecking=no"] @ args) - |> Process.wait in - match status with - | Unix.WEXITED 0 -> Lwt.return_unit - | _ -> - Log.info - "SSH server is not running, let's wait 2 seconds and check again..." ; - let* () = Lwt_unix.sleep 2. in - wait_ssh_server_running runner + let* _ = Env.wait_process ~is_ready ~run () in + Lwt.return_unit -let register ?(docker_push = not Cli.localhost) ?vms ~__FILE__ ~title ~tags - ?seed f = +let register ?vms ~__FILE__ ~title ~tags ?seed f = Test.register ~__FILE__ ~title ~tags ?seed @@ fun () -> - let* () = - if docker_push then - let* () = Jobs.deploy_docker_registry () in - Jobs.docker_build ~push:true () - else Jobs.docker_build ~push:false () - in + let* () = Env.init () in let vms = (* The Cli arguments by-pass the argument given here. This enable the user to always have decide precisely the number of vms to be run. *) - match (vms, Cli.vms) with + match (vms, Env.vms) with | None, None -> None | None, Some i | Some _, Some i -> let vms = List.init i (fun _ -> Configuration.make ()) in @@ -123,48 +133,34 @@ let register ?(docker_push = not Cli.localhost) ?vms ~__FILE__ ~title ~tags deployement = None; } | Some configurations -> - let ports_per_vm = Cli.ports_per_vm in - let* deployement = - Deployement.deploy - ~ports_per_vm - ~configurations - ~localhost:Cli.localhost - () + let* () = + match Env.mode with + | `Localhost -> Jobs.docker_build ~push:false () + | `Cloud -> + let* () = Jobs.deploy_docker_registry () in + Jobs.docker_build ~push:true () in + let* deployement = Deployement.deploy ~configurations in let agents = Deployement.agents deployement in let* () = agents |> List.map (fun agent -> Agent.runner agent |> wait_ssh_server_running) |> Lwt.join in - if Cli.monitoring then ( - Log.info "Monitoring enabled" ; - agents - |> List.iter (fun agent -> - let address = - Agent.runner agent |> fun runner -> - Runner.address (Some runner) - in - Log.info - "Monitoring of agent %s can be accessed at 'http://%s:19999'" - (Agent.name agent) - address)) ; let* website = - if Cli.website then - let* website = - Web.start ~port:Cli.website_port ~deployement ~agents - in + if Env.website then + let* website = Web.start ~agents in Lwt.return_some website else Lwt.return_none in let* prometheus = - if Cli.prometheus then + if Env.prometheus then let* prometheus = Prometheus.start agents in Lwt.return_some prometheus else Lwt.return_none in let* grafana = - if Cli.grafana then + if Env.grafana then let* grafana = Grafana.run () in Lwt.return_some grafana else Lwt.return_none @@ -198,10 +194,7 @@ let register ?(docker_push = not Cli.localhost) ?vms ~__FILE__ ~title ~tags let agents t = t.agents -let get_configuration t agent = - match t.deployement with - | None -> Configuration.make () - | Some deployement -> Deployement.get_configuration deployement agent +let get_configuration = Agent.configuration let set_agent_name t agent name = Agent.set_name agent name ; diff --git a/tezt/lib_cloud/cloud.mli b/tezt/lib_cloud/cloud.mli index 6c00367087ae..0626449101f7 100644 --- a/tezt/lib_cloud/cloud.mli +++ b/tezt/lib_cloud/cloud.mli @@ -5,17 +5,12 @@ (* *) (*****************************************************************************) -module Configuration : sig - include module type of Configuration -end - type t (** [register ?docker_push ?vms] is a wrapper around [Test.register]. It enables to run a test that can use machines deployed onto the cloud. *) val register : - ?docker_push:bool -> ?vms:Configuration.t list -> __FILE__:string -> title:string -> @@ -26,7 +21,7 @@ val register : val agents : t -> Agent.t list -val get_configuration : t -> Agent.t -> Configuration.t +val get_configuration : Agent.t -> Configuration.t val push_metric : t -> ?labels:(string * string) list -> name:string -> float -> unit diff --git a/tezt/lib_cloud/configuration.ml b/tezt/lib_cloud/configuration.ml index e2a27d55961d..424dc0aa9ca2 100644 --- a/tezt/lib_cloud/configuration.ml +++ b/tezt/lib_cloud/configuration.ml @@ -5,41 +5,30 @@ (* *) (*****************************************************************************) -type docker_image = - | Custom of {tezt_cloud : string} - | Image of {docker_image : string} - -let string_of_docker_image ~project_id = function - | Custom {tezt_cloud} -> - Env.custom_docker_image ~docker_image_name:tezt_cloud ~project_id () - | Image {docker_image} -> docker_image - type t = { machine_type : string; - docker_image : docker_image; + docker_image : Env.docker_image; max_run_duration : int option; + binaries_path : string; } -let default_docker_image = - match (Cli.dockerfile, Cli.localhost) with - | None, false -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - Custom {tezt_cloud} - | None, true -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - let docker_image = Format.asprintf "%s:latest" tezt_cloud in - Image {docker_image} - | Some tezt_cloud, false -> Custom {tezt_cloud} - | Some tezt_cloud, true -> - let docker_image = Format.asprintf "%s:latest" tezt_cloud in - Image {docker_image} - -let make ?max_run_duration ?(machine_type = Cli.machine_type) ?docker_image () = - let docker_image = Option.value ~default:default_docker_image docker_image in +let make ?binaries_path ?max_run_duration ?machine_type ?docker_image () = + (* We don't use [env] because of dependency issue: We want the user of the + library to use this value function the [env] value is created. *) + let docker_image = Option.value ~default:Env.docker_image docker_image in + let machine_type = Option.value ~default:Env.machine_type machine_type in + let default_binaries_path = + match docker_image with + | Env.Gcp _ -> Path.default_binaries_path () + | Octez_latest_release -> "/usr/local/bin" + in + let binaries_path = + Option.value ~default:default_binaries_path binaries_path + in let max_run_duration = match max_run_duration with | None -> - if Cli.no_max_run_duration then None else Some Cli.max_run_duration + if Env.no_max_run_duration then None else Some Env.max_run_duration | Some max_run_duration -> Some max_run_duration in - {machine_type; docker_image; max_run_duration} + {machine_type; docker_image; max_run_duration; binaries_path} diff --git a/tezt/lib_cloud/configuration.mli b/tezt/lib_cloud/configuration.mli index d55a3a96b087..bf0f894528f3 100644 --- a/tezt/lib_cloud/configuration.mli +++ b/tezt/lib_cloud/configuration.mli @@ -5,21 +5,17 @@ (* *) (*****************************************************************************) -type docker_image = - | Custom of {tezt_cloud : string} - | Image of {docker_image : string} - -val string_of_docker_image : project_id:string -> docker_image -> string - type t = private { machine_type : string; - docker_image : docker_image; + docker_image : Env.docker_image; max_run_duration : int option; + binaries_path : string; } val make : + ?binaries_path:string -> ?max_run_duration:int -> ?machine_type:string -> - ?docker_image:docker_image -> + ?docker_image:Env.docker_image -> unit -> t diff --git a/tezt/lib_cloud/deployement.ml b/tezt/lib_cloud/deployement.ml index 70b33c428b29..0095c5c8a2c4 100644 --- a/tezt/lib_cloud/deployement.ml +++ b/tezt/lib_cloud/deployement.ml @@ -6,77 +6,54 @@ (*****************************************************************************) module Remote = struct - type workspace_info = {configuration : Configuration.t; number_of_vms : int} - type point_info = {workspace_name : string; gcp_name : string} - type address = string - - type t = { - base_port : int; - ports_per_vm : int; - agents_info : (address, point_info) Hashtbl.t; - agents : Agent.t list; - workspaces_info : (string, workspace_info) Hashtbl.t; - zone : string; - } + type t = {agents : Agent.t list} - let rec wait_docker_running ~zone ~vm_name = - let*? process = + let wait_docker_running ~vm_name () = + let ssh_private_key_filename = Env.ssh_private_key_filename in + let* zone = Env.zone () in + let is_ready _output = true in + let run () = + (* Try to get the docker images up. *) Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["ps"; "--format"; "{{.Names}}"] in - let* status = Process.wait process in - match status with - | Unix.WEXITED 0 -> - let* images_name = Process.check_and_read_stdout process in - String.split_on_char '\n' images_name - |> List.filter (fun s -> s <> "") - |> List.map (fun image_name -> - let*? process = - Gcloud.compute_ssh - ~zone - ~vm_name - "docker" - ["inspect"; "--format"; "{{.State.Running}}"; image_name] - in - let* status = Process.wait process in - match status with - | Unix.WEXITED 0 -> - let* is_running = - let* status = Process.check_and_read_stdout process in - String.trim status |> Stdlib.bool_of_string |> Lwt.return - in - if is_running then Lwt.return_unit - else ( - Log.info - "Docker image is not ready, let's wait 2 seconds and \ - check again..." ; - wait_docker_running ~zone ~vm_name) - | _ -> - Log.info - "Docker image is not ready, let's wait 2 seconds and \ - check again..." ; - let* () = Lwt_unix.sleep 2. in - wait_docker_running ~zone ~vm_name) - |> Lwt.join - | _ -> - Log.info - "Docker image is not ready, let's wait 2 seconds and check again..." ; - let* () = Lwt_unix.sleep 2. in - wait_docker_running ~zone ~vm_name + let* output = Env.wait_process ~is_ready ~run () in + let images_name = + output |> String.split_on_char '\n' |> List.filter (fun s -> s <> "") + in + let is_ready output = String.trim output |> Stdlib.bool_of_string in + let run image_name () = + (* Try to get the docker images that are actually running. *) + Gcloud.compute_ssh + ~zone + ~vm_name + ~ssh_private_key_filename + "docker" + ["inspect"; "--format"; "{{.State.Running}}"; image_name] + in + let* _ = + images_name + |> List.map (fun image_name -> + Env.wait_process ~is_ready ~run:(run image_name) ()) + |> Lwt.all + in + unit - let workspace_deploy ?(base_port = 30_000) ?(ports_per_vm = 50) - ~max_run_duration ~workspace_name ~machine_type ~number_of_vms - ~docker_image () = + let workspace_deploy ~workspace_name ~number_of_vms ~configuration = let* () = Terraform.VM.Workspace.select workspace_name in - let* project_id = Gcloud.project_id () in - let docker_image = - Configuration.string_of_docker_image ~project_id docker_image + let* docker_image = + Env.uri_of_docker_image configuration.Configuration.docker_image in + let machine_type = configuration.machine_type in + let max_run_duration = configuration.max_run_duration in + let ports_per_vm = Env.ports_per_vm in + let base_port = Env.vm_base_port in let* () = Terraform.VM.deploy ~max_run_duration @@ -90,26 +67,14 @@ module Remote = struct List.init number_of_vms (fun i -> Format.asprintf "%s-%03d" workspace_name (i + 1)) in - let* zone = Terraform.VM.zone () in + let* zone = Env.zone () in let* () = - List.map (fun vm_name -> wait_docker_running ~zone ~vm_name) names + List.map (fun vm_name -> wait_docker_running ~vm_name ()) names |> Lwt.join in - let* () = - let run_command ~vm_name cmd args = - Gcloud.compute_ssh ~zone ~vm_name cmd args - in - if Cli.monitoring then - List.map - (fun vm_name -> Monitoring.run ~run_command:(run_command ~vm_name)) - names - |> Lwt.join - else Lwt.return_unit - in - let ssh_id = Lazy.force Env.ssh_private_key in - let binaries_path = Path.default_binaries_path () in - let agent_of_name name = - let* ip = Gcloud.get_ip_address_from_name ~zone name in + let ssh_private_key_filename = Env.ssh_private_key_filename in + let make_agent vm_name = + let* ip = Gcloud.get_ip_address_from_name ~zone vm_name in let point = (ip, base_port) in let next_available_port = let port = ref base_port in @@ -117,25 +82,31 @@ module Remote = struct incr port ; !port in - Agent.make ~ssh_id ~point ~binaries_path ~next_available_port ~name () + let cmd_wrapper = + Gcloud.cmd_wrapper ~zone ~vm_name ~ssh_private_key_filename + in + let* () = + if Env.monitoring then Monitoring.run ~cmd_wrapper () + else Lwt.return_unit + in + Agent.make + ~ssh_id:ssh_private_key_filename + ~cmd_wrapper + ~point + ~configuration + ~next_available_port + ~name:vm_name + () |> Lwt.return in - let* agents = names |> Lwt_list.map_p agent_of_name in + let* agents = names |> Lwt_list.map_p make_agent in Lwt.return agents - let get_configuration agents_info workspaces_info agent = - let address = agent |> Agent.runner |> Option.some |> Runner.address in - let {workspace_name; _} = Hashtbl.find agents_info address in - let {configuration; _} = Hashtbl.find workspaces_info workspace_name in - configuration - - let order_agents agents_info workspaces_info agents configurations = + let order_agents agents configurations = let bindings = agents |> List.map (fun agent -> - let configuration = - get_configuration agents_info workspaces_info agent - in + let configuration = Agent.configuration agent in (configuration, agent)) in let rec order configurations bindings = @@ -148,48 +119,38 @@ module Remote = struct in order configurations bindings - let deploy ?(base_port = 30_000) ?(ports_per_vm = 50) ~configurations () = - let tezt_cloud = Lazy.force Env.tezt_cloud in - let workspaces_info = Hashtbl.create 11 in + (* + Deployement requires to create new VMs and organizing them per group of + configuration. Each configuration leads to one terraform workspace. + *) + let deploy ~configurations = let agents_info = Hashtbl.create 11 in - let () = + let workspaces_info = + (* VMs are grouped per group of configuration. Each group leads to one workspace. *) List.to_seq configurations |> Seq.group ( = ) - |> Seq.iteri (fun i seq -> + |> Seq.mapi (fun i seq -> let configuration = Seq.uncons seq |> Option.get |> fst in - let name = Format.asprintf "%s-%d" tezt_cloud i in - Hashtbl.add - workspaces_info - name - {configuration; number_of_vms = Seq.length seq}) + let workspace_name = Format.asprintf "%s-%d" Env.tezt_cloud i in + (workspace_name, (configuration, Seq.length seq))) in let* () = Terraform.Docker_registry.init () in let* () = Terraform.VM.init () in - let workspaces_names = - workspaces_info |> Hashtbl.to_seq_keys |> List.of_seq - in - let* () = Terraform.VM.Workspace.init workspaces_names in + let workspaces_names = workspaces_info |> Seq.map fst |> List.of_seq in + let tezt_cloud = Env.tezt_cloud in + let* () = Terraform.VM.Workspace.init ~tezt_cloud workspaces_names in let* agents = - workspaces_info |> Hashtbl.to_seq |> List.of_seq - |> Lwt_list.map_s - (fun - ( workspace_name, - { - configuration = {machine_type; docker_image; max_run_duration}; - number_of_vms; - } ) - -> + workspaces_info |> List.of_seq + |> Lwt_list.map_s (fun (workspace_name, (configuration, number_of_vms)) -> + let* () = + Jobs.docker_build + ~docker_image:configuration.Configuration.docker_image + ~push:true + () + in let* () = Terraform.VM.Workspace.select workspace_name in let* () = Terraform.VM.init () in let* agents = - workspace_deploy - ~base_port - ~ports_per_vm - ~workspace_name - ~max_run_duration - ~machine_type - ~number_of_vms - ~docker_image - () + workspace_deploy ~workspace_name ~number_of_vms ~configuration in agents |> List.iter (fun agent -> @@ -204,27 +165,15 @@ module Remote = struct Lwt.return agents) in let agents = - order_agents - agents_info - workspaces_info - (List.concat agents) - configurations + (* We want to ensure agents are given in the same order than the + configurations. *) + order_agents (List.concat agents) configurations in - let* zone = Terraform.VM.zone () in - Lwt.return - {base_port; ports_per_vm; zone; agents_info; agents; workspaces_info} - - let get_configuration t agent = - get_configuration t.agents_info t.workspaces_info agent - - let run_vm_command {agents_info; zone; _} agent cmd args = - let address = agent |> Agent.runner |> Option.some |> Runner.address in - let {gcp_name = vm_name; _} = Hashtbl.find agents_info address in - Gcloud.compute_ssh ~zone ~vm_name cmd args + Lwt.return {agents} let agents t = t.agents - let terminate ?exn t = + let terminate ?exn _t = (match exn with | None -> Log.report ~color:Log.Color.FG.green "Scenario ended successfully." @@ -233,11 +182,13 @@ module Remote = struct ~color:Log.Color.FG.red "Scenario terminated unexpectedly:'%s'" (Printexc.to_string exn)) ; - if Cli.destroy then ( + if Env.destroy then ( Log.report "Destroying VMs, this may take a while..." ; - let workspaces = Hashtbl.to_seq_keys t.workspaces_info |> List.of_seq in - let* () = Terraform.VM.destroy workspaces in - Terraform.VM.Workspace.destroy ()) + let tezt_cloud = Env.tezt_cloud in + let* workspaces = Terraform.VM.Workspace.list ~tezt_cloud in + let* project_id = Gcloud.project_id () in + let* () = Terraform.VM.destroy workspaces ~project_id in + Terraform.VM.Workspace.destroy ~tezt_cloud) else ( Log.report "No VM destroyed! Don't forget to destroy them when you are done with \ @@ -251,35 +202,34 @@ module Localhost = struct processes : Process.t list; base_port : int; ports_per_vm : int; - names : (string, string) Hashtbl.t; agents : Agent.t list; } - let deploy ?(base_port = 30_000) ?(ports_per_vm = 50) ~configurations () = - (* We need to intialize the docker registry even on localhost to fetch the - docker image. *) - let* () = Terraform.Docker_registry.init () in - let* project_id = Gcloud.project_id () in + let vm_name i = Format.asprintf "%s-%03d" Env.tezt_cloud (i + 1) + + let deploy ~configurations () = let number_of_vms = List.length configurations in - let tezt_cloud = Lazy.force Env.tezt_cloud in - let names = Hashtbl.create 11 in - (* The current configuration is actually unused in localhost. Only the - number of VMs matters. *) - let processes = + let base_port = Env.vm_base_port in + let ports_per_vm = Env.ports_per_vm in + let* processes = List.to_seq configurations |> Seq.mapi (fun i configuration -> - let name = Format.asprintf "%s-%03d" tezt_cloud (i + 1) in + let name = vm_name i in let start = base_port + (i * ports_per_vm) |> string_of_int in let stop = base_port + ((i + 1) * ports_per_vm) - 1 |> string_of_int in let publish_ports = (start, stop, start, stop) in - let docker_image = - Configuration.string_of_docker_image - ~project_id - configuration.Configuration.docker_image + let* () = + Jobs.docker_build + ~docker_image:configuration.Configuration.docker_image + ~push:false + () + in + let* docker_image = + Env.uri_of_docker_image configuration.docker_image in - let*? process = + let process = Docker.run ~rm:true ~name @@ -288,8 +238,8 @@ module Localhost = struct docker_image ["-D"; "-p"; start; "-e"] in - process) - |> List.of_seq + Lwt.return process) + |> List.of_seq |> Lwt.all in (* We need to wait a little the machine is up. As for the remote case, we could be more robust to handle that. *) @@ -299,15 +249,7 @@ module Localhost = struct |> Seq.iter (fun i -> let port = base_port + (i * ports_per_vm) in Hashtbl.replace next_port ("localhost", port) (port + 1)) ; - let* () = - let run_command cmd args = - let value : Process.t = Process.spawn cmd args in - let run process = Process.check_and_read_stdout process in - {value; run} - in - if Cli.monitoring then Monitoring.run ~run_command else Lwt.return_unit - in - let ssh_id = Lazy.force Env.ssh_private_key in + let ssh_id = Env.ssh_private_key_filename in let get_point i = let port = base_port + (i * ports_per_vm) in ("localhost", port) @@ -317,36 +259,25 @@ module Localhost = struct Hashtbl.replace next_port point (port + 1) ; port in - let binaries_path = Path.default_binaries_path () in + let* () = if Env.monitoring then Monitoring.run () else Lwt.return_unit in let agents = - List.init number_of_vms (fun i -> - let name = Format.asprintf "localhost_docker_%d" i in - let point = get_point i in - Agent.make - ~ssh_id - ~point - ~binaries_path - ~next_available_port:(fun () -> next_port point) - ~name - ()) + configurations + |> List.mapi (fun i configuration -> + let name = Format.asprintf "localhost_docker_%d" i in + let point = get_point i in + Agent.make + ~ssh_id + ~point + ~configuration + ~next_available_port:(fun () -> next_port point) + ~name + ()) in - Lwt.return - {number_of_vms; processes; base_port; ports_per_vm; names; agents} - - (* Since in [localhost] mode, the VM is the host machine, this comes back - to just run a command on the host machine. *) - let run_vm_command cmd args = - let value : Process.t = Process.spawn cmd args in - let run process = Process.check_and_read_stdout process in - {value; run} + Lwt.return {number_of_vms; processes; base_port; ports_per_vm; agents} let agents t = t.agents - let get_configuration _t _agent = - (* The configuration is not used in localhost. *) - Configuration.make () - - let terminate ?exn _t = + let terminate ?exn t = (match exn with | None -> Log.report ~color:Log.Color.FG.green "Scenario ended successfully." @@ -357,45 +288,34 @@ module Localhost = struct (Printexc.to_string exn)) ; Log.report "Terminate test: tearing down docker containers..." ; let* () = - (* The netdata docker is run in the background. It must be shut - explicitely. *) - if Cli.monitoring then - let*! () = Docker.kill "netdata" in - let*! () = Docker.rm "netdata" in - Lwt.return_unit - else Lwt.return_unit + t.agents + |> List.mapi (fun i _agent -> + let* _ = Docker.kill (vm_name i) |> Process.wait in + Lwt.return_unit) + |> Lwt.join in - Lwt.return_unit + if Env.monitoring then + let* () = Docker.kill "netdata" |> Process.check in + Lwt.return_unit + else Lwt.return_unit end type t = Remote of Remote.t | Localhost of Localhost.t -let deploy ?(base_port = 30_000) ?(ports_per_vm = 50) ~configurations ~localhost - () = - if localhost then - let* localhost = - Localhost.deploy ~base_port ~ports_per_vm ~configurations () - in - Lwt.return (Localhost localhost) - else - let* remote = Remote.deploy ~base_port ~ports_per_vm ~configurations () in - Lwt.return (Remote remote) - -let run_vm_command t agent cmd args = - match t with - | Remote remote -> Remote.run_vm_command remote agent cmd args - | Localhost _localhost -> Localhost.run_vm_command cmd args +let deploy ~configurations = + match Env.mode with + | `Localhost -> + let* localhost = Localhost.deploy ~configurations () in + Lwt.return (Localhost localhost) + | `Cloud -> + let* remote = Remote.deploy ~configurations in + Lwt.return (Remote remote) let agents t = match t with | Remote remote -> Remote.agents remote | Localhost localhost -> Localhost.agents localhost -let get_configuration t = - match t with - | Remote remote -> Remote.get_configuration remote - | Localhost localhost -> Localhost.get_configuration localhost - let terminate ?exn t = match t with | Remote remote -> Remote.terminate ?exn remote diff --git a/tezt/lib_cloud/deployement.mli b/tezt/lib_cloud/deployement.mli index 08f2e301d1fc..022c765fdcee 100644 --- a/tezt/lib_cloud/deployement.mli +++ b/tezt/lib_cloud/deployement.mli @@ -8,32 +8,18 @@ (** Type for resources managed by a deployment. *) type t -(** [deploy ?base_port ?ports_per_vm ~max_run_duration ~configuration ~localhost ()] deploys the +(** [deploy ~configurations] deploys the expected number of vms. For each vm, we can specify a [base_port] which is the first port to be opened and [ports_per_vm] specify the number of opened port from the [base_port]. The promise returned by this function is fulfilled when all the vms are deployed. Consequently, it can take some times. *) -val deploy : - ?base_port:int -> - ?ports_per_vm:int -> - configurations:Configuration.t list -> - localhost:bool -> - unit -> - t Lwt.t +val deploy : configurations:Configuration.t list -> t Lwt.t (** [get_agents t] returns the list of agents deployed. *) val agents : t -> Agent.t list -val get_configuration : t -> Agent.t -> Configuration.t - -(** [run_vm_command t ~address cmd args] can run a command on the vm located at - [address]. This is different from running a commend on the agent directly - since the agent runs on a docker image. *) -val run_vm_command : - t -> Agent.t -> string -> string list -> (Process.t, string) Runnable.t - -(** [terminate ?exn t] should be called to tear down the machine. Do note +(** [terminate ?exn env t] should be called to tear down the machine. Do note that this call may or may not destroy the machine depending on what the user has chosen (see [Cli.destroy].*) val terminate : ?exn:exn -> t -> unit Lwt.t diff --git a/tezt/lib_cloud/docker.ml b/tezt/lib_cloud/docker.ml index c904dec0cf27..a2bf45398f0e 100644 --- a/tezt/lib_cloud/docker.ml +++ b/tezt/lib_cloud/docker.ml @@ -4,56 +4,48 @@ (* SPDX-FileCopyrightText: 2024 Nomadic Labs *) (* *) (*****************************************************************************) + let name = "docker" let color = Log.Color.FG.yellow -let build ?(tag = "latest") ?dockerfile ~args () = +let build ?image_name ?alias ?(tag = "latest") ?dockerfile ~args () = let build_args = args |> List.map (fun (key, value) -> ["--build-arg"; Format.asprintf "%s=%s" key value]) |> List.concat in - let tezt_cloud = Lazy.force Env.tezt_cloud in - let dockerfile_path = Lazy.force Env.dockerfile in - let dockerfile = Option.value ~default:dockerfile_path dockerfile in - let tag = ["-t"; Format.asprintf "%s:%s" tezt_cloud tag] in + let alias = Option.value ~default:Env.dockerfile_alias alias in + let dockerfile = Option.value ~default:(Path.dockerfile ~alias) dockerfile in + let image_name = Option.value ~default:alias image_name in + let tag = ["-t"; Format.asprintf "%s:%s" image_name tag] in let args = ["build"; "-f"; dockerfile] @ build_args @ tag @ ["."] in - let value = Process.spawn ~name ~color "docker" args in - let run = Process.check in - {value; run} + Process.spawn ~name ~color "docker" args -let tag ?(tag = "latest") docker_registry = - let tezt_cloud = Lazy.force Env.tezt_cloud in +let tag ?image_name ?alias ?(tag = "latest") ~registry_uri () = + let alias = Option.value ~default:Env.dockerfile_alias alias in + let image_name = Option.value ~default:alias image_name in let args = [ "tag"; - Format.asprintf "%s:%s" tezt_cloud tag; - Format.asprintf "%s/%s:%s" docker_registry tezt_cloud tag; + Format.asprintf "%s:%s" image_name tag; + Format.asprintf "%s/%s:%s" registry_uri image_name tag; ] in - let value = Process.spawn ~name ~color "docker" args in - let run = Process.check in - {value; run} + Process.spawn ~name ~color "docker" args -let push ?(tag = "latest") docker_registry = - let tezt_cloud = Lazy.force Env.tezt_cloud in - let args = - ["push"; Format.asprintf "%s/%s:%s" docker_registry tezt_cloud tag] - in - let value = Process.spawn ~name ~color "docker" args in - let run = Process.check in - {value; run} +let push ?image_name ?alias ?(tag = "latest") ~registry_uri () = + let alias = Option.value ~default:Env.dockerfile_alias alias in + let image_name = Option.value ~default:alias image_name in + let args = ["push"; Format.asprintf "%s/%s:%s" registry_uri image_name tag] in + Process.spawn ~name ~color "docker" args -let pull ?(tag = "latest") docker_registry = - let tezt_cloud = Lazy.force Env.tezt_cloud in - let args = - ["pull"; Format.asprintf "%s/%s:%s" docker_registry tezt_cloud tag] - in - let value = Process.spawn ~name ~color "docker" args in - let run = Process.check in - {value; run} +let pull ?image_name ?alias ?(tag = "latest") ~registry_uri () = + let alias = Option.value ~default:Env.dockerfile_alias alias in + let image_name = Option.value ~default:alias image_name in + let args = ["pull"; Format.asprintf "%s/%s:%s" registry_uri image_name tag] in + Process.spawn ~name ~color "docker" args let run ?(rm = false) ?name ?network ?publish_ports image args = let publish_ports = @@ -67,40 +59,28 @@ let run ?(rm = false) ?name ?network ?publish_ports image args = in let name = match name with None -> [] | Some name -> ["--name"; name] in let rm = if rm then ["--rm"] else [] in - let value = - Process.spawn - ~color - "docker" - (["run"] @ rm @ name @ network @ publish_ports - @ [Format.asprintf "%s" image] - @ args) - in - let run = Process.check in - {value; run} + (* [init] can be used to ensure signals are forwarded properly to + the entrypoint run by the container. *) + Process.spawn + ~color + "docker" + (["run"] @ rm @ name @ network @ publish_ports + @ [Format.asprintf "%s" image] + @ args) -let kill container_name = - let value = Process.spawn ~color "docker" ["kill"; container_name] in - let run = Process.check in - {value; run} +let kill container_name = Process.spawn ~color "docker" ["kill"; container_name] -let rm container_name = - let value = Process.spawn ~color "docker" ["rm"; container_name] in - let run = Process.check in - {value; run} +let rm container_name = Process.spawn ~color "docker" ["rm"; container_name] let cp container_name ~kind ~source ~destination = - let value = - match kind with - | `From_host -> - Process.spawn - ~color - "docker" - ["cp"; source; Format.asprintf "%s:%s" container_name destination] - | `To_host -> - Process.spawn - ~color - "docker" - ["cp"; Format.asprintf "%s:%s" container_name source; destination] - in - let run = Process.check in - {value; run} + match kind with + | `From_host -> + Process.spawn + ~color + "docker" + ["cp"; source; Format.asprintf "%s:%s" container_name destination] + | `To_host -> + Process.spawn + ~color + "docker" + ["cp"; Format.asprintf "%s:%s" container_name source; destination] diff --git a/tezt/lib_cloud/docker.mli b/tezt/lib_cloud/docker.mli index d26c5cb8265a..693817cdf196 100644 --- a/tezt/lib_cloud/docker.mli +++ b/tezt/lib_cloud/docker.mli @@ -9,20 +9,40 @@ (** [build] is an alias for [docker build]. *) val build : + ?image_name:string -> + ?alias:string -> ?tag:string -> ?dockerfile:string -> args:(string * string) list -> unit -> - (Process.t, unit) runnable + Process.t (** [tag] is an alias for [docker tag]. *) -val tag : ?tag:string -> string -> (Process.t, unit) runnable +val tag : + ?image_name:string -> + ?alias:string -> + ?tag:string -> + registry_uri:string -> + unit -> + Process.t (** [push] is an alias for [docker push]. *) -val push : ?tag:string -> string -> (Process.t, unit) runnable +val push : + ?image_name:string -> + ?alias:string -> + ?tag:string -> + registry_uri:string -> + unit -> + Process.t (** [pull] is an alias for [docker pull]. *) -val pull : ?tag:string -> string -> (Process.t, unit) runnable +val pull : + ?image_name:string -> + ?alias:string -> + ?tag:string -> + registry_uri:string -> + unit -> + Process.t (** [run] is an alias for [docker run]. *) val run : @@ -32,13 +52,13 @@ val run : ?publish_ports:string * string * string * string -> string -> string list -> - (Process.t, unit) runnable + Process.t (** [kill] is an alias for [docker kill]. *) -val kill : string -> (Process.t, unit) runnable +val kill : string -> Process.t (** [rm] is an alias for [docker rm]. *) -val rm : string -> (Process.t, unit) runnable +val rm : string -> Process.t (** [cp] is an alias for [docker cp]. *) val cp : @@ -46,4 +66,4 @@ val cp : kind:[`From_host | `To_host] -> source:string -> destination:string -> - (Process.t, unit) runnable + Process.t diff --git a/tezt/lib_cloud/dockerfiles/octez.Dockerfile b/tezt/lib_cloud/dockerfiles/octez.Dockerfile new file mode 100644 index 000000000000..34a77a8040fc --- /dev/null +++ b/tezt/lib_cloud/dockerfiles/octez.Dockerfile @@ -0,0 +1,22 @@ +# Start from last known release +FROM tezos/tezos:latest + +# Set back the user for executing the following commands +USER root + +# SSH key that will be used for the SSH server +ARG SSH_PUBLIC_KEY + +# This is extracted from the link below +# https://dev.to/yakovlev_alexey/running-ssh-in-an-alpine-docker-container-3lop +RUN mkdir -p /root/.ssh \ + && chmod 0700 /root/.ssh \ + && echo "$SSH_PUBLIC_KEY" > /root/.ssh/authorized_keys \ + && apk add openrc openssh \ + && ssh-keygen -A \ + && echo -e "PasswordAuthentication no" >> /etc/ssh/sshd_config \ + && mkdir -p /run/openrc \ + && touch /run/openrc/softlevel + +CMD ["-D", "-p", "30000", "-e"] +ENTRYPOINT ["/usr/sbin/sshd"] diff --git a/tezt/lib_cloud/dune b/tezt/lib_cloud/dune index 8c5855675b5b..667fd94082ef 100644 --- a/tezt/lib_cloud/dune +++ b/tezt/lib_cloud/dune @@ -2,16 +2,13 @@ ; Edit file manifest/main.ml instead. (library - (name tezt_cloud) - (package tezt-cloud) + (name tezt_tezos_tezt_cloud) + (public_name tezt-tezos.tezt-cloud) (libraries tezt - tezt-tezos tezt-tezos.tezt-performance-regression) (flags (:standard) -open Tezt -open Tezt.Base - -open Tezt_tezos - -open Tezt_tezos.Runnable.Syntax -open Tezt_tezos_tezt_performance_regression)) diff --git a/tezt/lib_cloud/env.ml b/tezt/lib_cloud/env.ml index b396286cf59b..88cce8e74c89 100644 --- a/tezt/lib_cloud/env.ml +++ b/tezt/lib_cloud/env.ml @@ -5,47 +5,151 @@ (* *) (*****************************************************************************) +type docker_image = Gcp of {alias : string} | Octez_latest_release + let tezt_cloud = (* This is a lazy value to be sure that this is evaluated only inside a Tezt test. *) - Lazy.from_fun @@ fun () -> match Sys.getenv_opt "TEZT_CLOUD" with | None -> - Test.fail + Stdlib.failwith "The environment variable 'TEZT_CLOUD' is not defined. See README for \ more information why this variable must be defined." | Some value -> value -let ssh_private_key = - Lazy.from_fun (fun () -> - let home = Sys.getenv "HOME" in - let tezt_cloud = Lazy.force tezt_cloud in - home // ".ssh" // Format.asprintf "%s-tf" tezt_cloud) - -let ssh_public_key = - Lazy.from_fun (fun () -> - let ssh_key = Lazy.force ssh_private_key in - Format.asprintf "%s.pub" ssh_key) - -let dockerfile = - Lazy.from_fun (fun () -> - let tezt_cloud = Lazy.force tezt_cloud in - let basename = Option.value ~default:tezt_cloud Cli.dockerfile in - Path.docker // Format.asprintf "%s.Dockerfile" basename) - -let docker_registry = - Lazy.from_fun (fun () -> - let tezt_cloud = Lazy.force tezt_cloud in - Format.asprintf "%s-docker-registry" tezt_cloud) - -let custom_docker_image ?docker_image_name ~project_id () = - let artifact_registry = "europe-west1-docker.pkg.dev" in - let docker_registry = Lazy.force docker_registry in - let docker_image_name = - Option.value ~default:(Lazy.force tezt_cloud) docker_image_name - in - Format.asprintf - "%s/%s/%s/%s" - artifact_registry - project_id - docker_registry - docker_image_name +let ssh_private_key_filename = + let home = Sys.getenv "HOME" in + home // ".ssh" // Format.asprintf "%s-tf" tezt_cloud + +let ssh_public_key_filename = + let ssh_key = ssh_private_key_filename in + Format.asprintf "%s.pub" ssh_key + +let docker_registry = Format.asprintf "%s-docker-registry" tezt_cloud + +let mode = if Cli.localhost then `Localhost else `Cloud + +let prometheus = Cli.prometheus + +let prometheus_export = Cli.prometheus_export + +let prometheus_port = Cli.prometheus_port + +let prometheus_snapshot_filename = Cli.prometheus_snapshot_filename + +let prometheus_scrape_interval = Cli.prometheus_scrape_interval + +let grafana = Cli.grafana + +let website = Cli.website + +let website_port = Cli.website_port + +let destroy = Cli.destroy + +let monitoring = Cli.monitoring + +let keep_alive = Cli.keep_alive + +let vms = Cli.vms + +let vm_base_port = Cli.vm_base_port + +let ports_per_vm = Cli.ports_per_vm + +let machine_type = Cli.machine_type + +let max_run_duration = Cli.max_run_duration + +let no_max_run_duration = Cli.no_max_run_duration + +let docker_image = + (* In localhost mode, we don't want to interact with GCP. The image is taken + locally. *) + match Cli.dockerfile_alias with + | None -> Gcp {alias = tezt_cloud} + | Some alias -> Gcp {alias} + +let dockerfile_alias = Option.value ~default:tezt_cloud Cli.dockerfile_alias + +let dockerfile = Path.dockerfile ~alias:dockerfile_alias + +let project_id = Gcloud.project_id + +let init () = + if mode = `Localhost then unit + else + let tezt_cloud = tezt_cloud in + let* project_id = project_id () in + Log.info "Initializing docker registry..." ; + let* () = Terraform.Docker_registry.init () in + Log.info "Deploying docker registry..." ; + let* () = Terraform.Docker_registry.deploy ~tezt_cloud ~project_id in + Lwt.return_unit + +(* Even though we could get this information locally, it is interesting to fetch + it through terraform to get the correct value if a scenario was launch with + different parameters. *) +let hostname = + let hostname = ref "" in + fun () -> + if !hostname = "" then ( + let tezt_cloud = tezt_cloud in + let* hostname' = Terraform.Docker_registry.get_hostname ~tezt_cloud in + hostname := hostname' ; + Log.info "Authenticate docker hostname..." ; + let* () = Gcloud.auth_configure_docker ~hostname:!hostname in + return !hostname) + else Lwt.return !hostname + +let zone = + let zone = ref "" in + fun () -> + (* To ensure the docker registry is deployed. *) + let* zone' = Terraform.VM.zone () in + zone := zone' ; + Lwt.return !zone + +let registry_uri () = + let* hostname = hostname () in + let* project_id = project_id () in + let uri = Format.asprintf "%s/%s/%s" hostname project_id docker_registry in + Lwt.return uri + +let uri_of_docker_image docker_image = + match (docker_image, mode) with + | Gcp {alias}, `Cloud -> + let* registry_uri = registry_uri () in + Lwt.return (Format.asprintf "%s/%s" registry_uri alias) + | Gcp {alias}, `Localhost -> Lwt.return alias + | Octez_latest_release, `Cloud -> + let* registry_uri = registry_uri () in + Lwt.return (Format.asprintf "%s/octez" registry_uri) + | Octez_latest_release, `Localhost -> Lwt.return "octez" + +let rec wait_process ?(sleep = 4) ~is_ready ~run () = + let process = run () in + let* status = Process.wait process in + match status with + | Unix.WEXITED 0 -> + let* output = Process.check_and_read_stdout process in + if is_ready output then Lwt.return output + else ( + Log.info + "Process '%s' is not ready. Let's wait %d seconds" + (Process.name process) + sleep ; + let* () = Lwt_unix.sleep (float_of_int sleep) in + wait_process ~sleep ~is_ready ~run ()) + | _ -> + Log.info + "Process '%s' failed. Let's wait %d seconds" + (Process.name process) + sleep ; + let* () = Lwt_unix.sleep (float_of_int sleep) in + wait_process ~sleep ~is_ready ~run () + +let run_command ?cmd_wrapper cmd args = + match cmd_wrapper with + | None -> Process.spawn cmd args + | Some cmd_wrapper -> + Process.spawn cmd_wrapper.Gcloud.cmd (cmd_wrapper.args @ [cmd] @ args) diff --git a/tezt/lib_cloud/env.mli b/tezt/lib_cloud/env.mli index da0405ead206..edd38bf377a1 100644 --- a/tezt/lib_cloud/env.mli +++ b/tezt/lib_cloud/env.mli @@ -5,25 +5,77 @@ (* *) (*****************************************************************************) -(** This module contains values that depend on environment - variables. Those values are lazy so that they are evaluated within - a test and fail if one environment variable is missing. *) +(* This module aims to encapsulate static values from the CLI and several + functions that are used by the library. *) -(** Value of the environment variable [TEZT_CLOUD_BASENAME]. *) -val tezt_cloud : string Lazy.t +type docker_image = Gcp of {alias : string} | Octez_latest_release -(** Path to the ssh private key that will be used with the docker - image. Depends on [workspace]. *) -val ssh_private_key : string Lazy.t +val tezt_cloud : string -(** Path to the ssh public key that will be used with the docker - image. Depends on [workspace]. *) -val ssh_public_key : string Lazy.t +val mode : [`Localhost | `Cloud] -(** Path to the docker image that will be used when running - VMs. Depends on [workspace]. *) -val dockerfile : string Lazy.t +val prometheus : bool -(** URI where to find the custom docker image based on the [tezt_cloud] value. *) -val custom_docker_image : - ?docker_image_name:string -> project_id:string -> unit -> string +val prometheus_port : int + +val prometheus_export : bool + +val prometheus_snapshot_filename : string option + +val prometheus_scrape_interval : int + +val grafana : bool + +val website : bool + +val website_port : int + +val destroy : bool + +val monitoring : bool + +val keep_alive : bool + +val ssh_private_key_filename : string + +val ssh_public_key_filename : string + +val dockerfile_alias : string + +val dockerfile : string + +val docker_registry : string + +val docker_image : docker_image + +val vms : int option + +val vm_base_port : int + +val ports_per_vm : int + +val machine_type : string + +val max_run_duration : int + +val no_max_run_duration : bool + +val init : unit -> unit Lwt.t + +val project_id : unit -> string Lwt.t + +val registry_uri : unit -> string Lwt.t + +val uri_of_docker_image : docker_image -> string Lwt.t + +val zone : unit -> string Lwt.t + +val wait_process : + ?sleep:int -> + is_ready:(string -> bool) -> + run:(unit -> Process.t) -> + unit -> + string Lwt.t + +val run_command : + ?cmd_wrapper:Gcloud.cmd_wrapper -> string -> string list -> Process.t diff --git a/tezt/lib_cloud/gcloud.ml b/tezt/lib_cloud/gcloud.ml index 86d537782d83..e05a9ef9dff2 100644 --- a/tezt/lib_cloud/gcloud.ml +++ b/tezt/lib_cloud/gcloud.ml @@ -32,27 +32,27 @@ let project_id = Lwt.return !project_id) else Lwt.return !project_id -let compute_ssh ~zone ~vm_name cmd args = - let ssh_private_key = Lazy.force Env.ssh_private_key in - let value = - Process.spawn - ~name - ~color - "gcloud" - ([ - "compute"; - "ssh"; - "--ssh-key-file"; - ssh_private_key; - vm_name; - "--zone"; - zone; - "--"; - ] - @ [cmd] @ args) - in - let run = Process.check_and_read_stdout in - {value; run} +type cmd_wrapper = {cmd : string; args : string list} + +let cmd_wrapper ~zone ~vm_name ~ssh_private_key_filename = + { + cmd = "gcloud"; + args = + [ + "compute"; + "ssh"; + "--ssh-key-file"; + ssh_private_key_filename; + vm_name; + "--zone"; + zone; + "--"; + ]; + } + +let compute_ssh ~zone ~vm_name ~ssh_private_key_filename cmd args = + let wrapper = cmd_wrapper ~zone ~vm_name ~ssh_private_key_filename in + Process.spawn ~name ~color wrapper.cmd (wrapper.args @ [cmd] @ args) let get_ip_address_from_name ~zone name = let* output = diff --git a/tezt/lib_cloud/gcloud.mli b/tezt/lib_cloud/gcloud.mli index ed6455b59e9e..1b7b7e5b042f 100644 --- a/tezt/lib_cloud/gcloud.mli +++ b/tezt/lib_cloud/gcloud.mli @@ -22,9 +22,21 @@ val project_id : unit -> string Lwt.t val compute_ssh : zone:string -> vm_name:string -> + ssh_private_key_filename:string -> string -> string list -> - (Process.t, string) runnable + Process.t + +type cmd_wrapper = {cmd : string; args : string list} + +(** [cmd_wrapper ~zone ~vm_name cmd args] is the wrapper to run a command on the + vm [vm_name]. + *) +val cmd_wrapper : + zone:string -> + vm_name:string -> + ssh_private_key_filename:string -> + cmd_wrapper (** [get_ip_address_from_name ~zone name] allows to get the external IP address of a VM from its name [name]. *) diff --git a/tezt/lib_cloud/jobs.ml b/tezt/lib_cloud/jobs.ml index cf09e9aea2a6..a81500a0324d 100644 --- a/tezt/lib_cloud/jobs.ml +++ b/tezt/lib_cloud/jobs.ml @@ -10,133 +10,90 @@ module C = Cli open Tezt module Cli = C -let docker_build ~push () = - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "TEZT_CLOUD environment variable found with value: %s" tezt_cloud ; - let ssh_public_key = Lazy.force Env.ssh_public_key in - Log.info "Checking the existence of ssh public key '%s'..." ssh_public_key ; - let* ssh_public_key = - let ssh_public_key_file = Lazy.force Env.ssh_public_key in - let* () = - if not (Sys.file_exists ssh_public_key_file) then Ssh.generate_key () - else Lwt.return_unit - in - Process.run_and_read_stdout ~name:"cat" "cat" [ssh_public_key_file] - in - let dockerfile = Lazy.force Env.dockerfile in - Log.info - "Checking the existence of the docker file %s.Dockerfile..." - dockerfile ; - if not (Sys.file_exists dockerfile) then - Test.fail - "Could not find the dockerfile %s. See README for more information why \ - this dockerfile is necessary." - dockerfile ; - Log.info "Checking the existence of the zcash parameters..." ; - if not (Sys.file_exists Path.zcash_params) then - Test.fail - "Could not find the zcash parameters in %s. See README for more \ - information why such files are necessary." - Path.zcash_params ; - Log.info "Initializing terraform docker state..." ; - let* () = Terraform.Docker_registry.init () in - Log.info "Fetching or creating the remote docker registry..." ; - let* hostname = Terraform.Docker_registry.get_hostname () in - let* docker_registry = Terraform.Docker_registry.get_docker_registry () in - Log.info "Authenticate docker hostname..." ; - let* () = Gcloud.auth_configure_docker ~hostname in - let args = - [ - ("SSH_PUBLIC_KEY", ssh_public_key); - ("ZCASH_PARAMS_PATH", Path.zcash_params); - ("DAL_TRUSTED_SETUP_PATH", Path.dal_trusted_setup); - ("BINARIES_DESTINATION_PATH", Path.default_binaries_path ()); - ] - in - Log.info "Building image from %s.Dockerfile..." tezt_cloud ; - let*! () = Docker.build ~dockerfile ~args () in - Log.info "Tagging the image..." ; - let*! () = Docker.tag docker_registry in - Log.info "Push the image..." ; - let* () = - if push then - let*! () = Docker.push docker_registry in - Lwt.return_unit - else Lwt.return_unit - in - unit - -let register_docker_push ~tags = - Test.register - ~__FILE__ - ~title:"Push the dockerfile to the registry" - ~tags:("docker" :: "push" :: tags) - @@ fun () -> docker_build ~push:true () +let docker_build = + let cache = Hashtbl.create 11 in + fun ?(docker_image = Env.Gcp {alias = Env.dockerfile_alias}) ~push () -> + if Hashtbl.mem cache docker_image then ( + Log.info "Docker image is already built. Nothing to do" ; + Lwt.return_unit) + else ( + Hashtbl.replace cache docker_image () ; + Log.info + "Checking the existence of ssh public key '%s'..." + Env.ssh_public_key_filename ; + let* ssh_public_key = + let ssh_public_key_file = Env.ssh_public_key_filename in + let* () = + if not (Sys.file_exists ssh_public_key_file) then ( + Log.info "SSh public key not found, creating it..." ; + Ssh.generate_key ()) + else Lwt.return_unit + in + Process.run_and_read_stdout ~name:"cat" "cat" [ssh_public_key_file] + in + let alias = + match docker_image with + | Gcp {alias} -> alias + | Octez_latest_release -> "octez" + in + let dockerfile = Path.dockerfile ~alias in + Log.info "Checking the existence of the docker file %s..." dockerfile ; + if not (Sys.file_exists dockerfile) then + Test.fail + "Could not find the dockerfile %s. See README for more information \ + why this dockerfile is necessary." + dockerfile ; + Log.info "Checking the existence of the zcash parameters..." ; + if (not (Sys.file_exists Path.zcash_params)) && alias = Env.tezt_cloud + then + Test.fail + "Could not find the zcash parameters in %s. See README for more \ + information why such files are necessary." + Path.zcash_params ; + if + (not (Sys.file_exists Path.dal_trusted_setup)) && alias = Env.tezt_cloud + then + Test.fail + "Could not find the dal_trusted_setup in %s. See README for more \ + information why such files are necessary." + Path.dal_trusted_setup ; + let args = + match docker_image with + | Gcp _ -> + [ + ("SSH_PUBLIC_KEY", ssh_public_key); + ("ZCASH_PARAMS_PATH", Path.zcash_params); + ("DAL_TRUSTED_SETUP_PATH", Path.dal_trusted_setup); + ("BINARIES_DESTINATION_PATH", Path.default_binaries_path ()); + ] + | Octez_latest_release -> [("SSH_PUBLIC_KEY", ssh_public_key)] + in + Log.info "Building image from %s.Dockerfile..." Env.tezt_cloud ; + let* () = Docker.build ~alias ~args () |> Process.check in + let* () = + if push then ( + Log.info "Compute the registry uri..." ; + let* registry_uri = Env.registry_uri () in + Log.info "Tagging the image..." ; + let* () = Docker.tag ~alias ~registry_uri () |> Process.check in + Log.info "Push the image..." ; + let* () = Docker.push ~alias ~registry_uri () |> Process.check in + Lwt.return_unit) + else Lwt.return_unit + in + unit) let deploy_docker_registry () = - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "Tezt_Cloud found with value: %s" tezt_cloud ; + Log.info "Tezt_Cloud found with value: %s" Env.tezt_cloud ; let* () = Terraform.Docker_registry.init () in - Terraform.Docker_registry.deploy () - -let register_deploy_docker_registry ~tags = - Test.register - ~__FILE__ - ~title:"Deploy docker registry" - ~tags:("docker" :: "registry" :: "deploy" :: tags) - deploy_docker_registry + let tezt_cloud = Env.tezt_cloud in + let* project_id = Gcloud.project_id () in + Terraform.Docker_registry.deploy ~tezt_cloud ~project_id -let deploy_terraform_state_bucket ~tags = - Test.register - ~__FILE__ - ~title:"Deploy terraform state bucket" - ~tags:("terraform" :: "state" :: "bucket" :: "deploy" :: tags) - @@ fun () -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "Tezt_Cloud found with value: %s" tezt_cloud ; - let* () = Terraform.State_bucket.init () in - Terraform.State_bucket.deploy () - -let destroy_vms ~tags = - Test.register - ~__FILE__ - ~title:"Destroy terraform VMs" - ~tags:("terraform" :: "destroy" :: "vms" :: tags) - @@ fun () -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "Tezt_Cloud found with value: %s" tezt_cloud ; - let* () = Terraform.VM.Workspace.select "default" in - let* workspaces = Terraform.VM.Workspace.get () in - let* () = Terraform.VM.destroy workspaces in - Terraform.VM.Workspace.destroy () - -let prometheus_import ~tags = - Test.register - ~__FILE__ - ~title:"Import a snapshot into a prometheus container" - ~tags:("prometheus" :: "import" :: tags) - @@ fun () -> - match Cli.prometheus_snapshot with - | None -> - Test.fail "Use --prometheus-snapshot to provide the snapshot filename" - | Some filename -> - let* prometheus = - Prometheus.run_with_snapshot - ~snapshot:(Cli.prometheus_snapshot_directory // filename) - ~port:9090 - in - Log.info "You can find the prometheus instance at http://localhost:9090" ; - Log.info - "Use Ctrl+C to end the scenario and kill the prometheus instance." ; - Prometheus.shutdown prometheus - -let clean_up_vms ~tags = - Test.register - ~__FILE__ - ~title:"Clean ups VMs manually" - ~tags:("clean" :: "up" :: tags) - @@ fun () -> - let* workspaces = Terraform.VM.Workspace.get () in +let clean_up_vms () = + let tezt_cloud = Env.tezt_cloud in + let* workspaces = Terraform.VM.Workspace.list ~tezt_cloud in + let ssh_private_key_filename = Env.ssh_private_key_filename in workspaces |> Lwt_list.iter_s (fun workspace -> let* () = Terraform.VM.Workspace.select workspace in @@ -153,12 +110,14 @@ let clean_up_vms ~tags = let* () = names |> Lwt_list.iter_p (fun vm_name -> - let*! output = + let* output = Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["ps"; "--format"; "{{.Names}}"] + |> Process.check_and_read_stdout in let images_name = String.split_on_char '\n' output @@ -174,56 +133,47 @@ let clean_up_vms ~tags = in this list" (String.concat ";" images_name) ; let main_image = List.hd main_image in - let*! _ = + let* _ = Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["stop"; main_image] + |> Process.check in - let*! _ = + let* _ = Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["start"; main_image] + |> Process.check in let* () = other_images |> Lwt_list.iter_p (fun image -> - let*! _ = + let* _ = Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["kill"; image] + |> Process.check in - let*! _ = + let* _ = Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["rm"; image] + |> Process.check in Lwt.return_unit) in Lwt.return_unit) in unit) - -let list_vms ~tags = - Test.register ~__FILE__ ~title:"List VMs" ~tags:("list" :: "vms" :: tags) - @@ fun () -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "TEZT_CLOUD environment variable found with value: %s" tezt_cloud ; - let* _ = Gcloud.list_vms ~prefix:tezt_cloud in - Lwt.return_unit - -let register ~tags = - register_docker_push ~tags ; - register_deploy_docker_registry ~tags ; - deploy_terraform_state_bucket ~tags ; - destroy_vms ~tags ; - prometheus_import ~tags ; - clean_up_vms ~tags ; - list_vms ~tags diff --git a/tezt/lib_cloud/jobs.mli b/tezt/lib_cloud/jobs.mli index 45780b8476a3..8592b4ef52d3 100644 --- a/tezt/lib_cloud/jobs.mli +++ b/tezt/lib_cloud/jobs.mli @@ -9,9 +9,8 @@ val deploy_docker_registry : unit -> unit Lwt.t (* A job for building and pushing docker images on the registry. *) -val docker_build : push:bool -> unit -> unit Lwt.t +val docker_build : + ?docker_image:Env.docker_image -> push:bool -> unit -> unit Lwt.t -(** [register ~tags] register a set of jobs that can be used for setting - requirements related to cloud scenarios. Some tags can be given for all the - registered jobs. *) -val register : tags:string list -> unit +(* The docker containers are killed and restarted from scratch. *) +val clean_up_vms : unit -> unit Lwt.t diff --git a/tezt/lib_cloud/monitoring.ml b/tezt/lib_cloud/monitoring.ml index 165e18a02c21..dd77374c31de 100644 --- a/tezt/lib_cloud/monitoring.ml +++ b/tezt/lib_cloud/monitoring.ml @@ -5,9 +5,11 @@ (* *) (*****************************************************************************) -let run ~run_command = +let run ?cmd_wrapper () = let* is_already_running = - let*? process = run_command "docker" ["ps"; "--format"; "{{.Names}}"] in + let process = + Env.run_command ?cmd_wrapper "docker" ["ps"; "--format"; "{{.Names}}"] + in let* status = Process.wait process in match status with | Unix.WEXITED 0 -> @@ -26,18 +28,17 @@ let run ~run_command = (* Those arguments are the ones given by netdata on their online documentation. We could decide to put it in the eDSL module of docker. *) let args = - "run -d --name=netdata --pid=host --network=host -v \ + "run -d --rm --name=netdata --pid=host --network=host -v \ netdataconfig:/etc/netdata -v netdatalib:/var/lib/netdata -v \ netdatacache:/var/cache/netdata -v /etc/passwd:/host/etc/passwd:ro \ -v /etc/group:/host/etc/group:ro -v \ /etc/localtime:/etc/localtime:ro -v /proc:/host/proc:ro -v \ /sys:/host/sys:ro -v /etc/os-release:/host/etc/os-release:ro -v \ /var/log:/host/var/log:ro -v \ - /var/run/docker.sock:/var/run/docker.sock:ro --restart \ - unless-stopped --cap-add SYS_PTRACE --cap-add SYS_ADMIN \ - --security-opt apparmor=unconfined netdata/netdata" - |> String.split_on_char ' ' + /var/run/docker.sock:/var/run/docker.sock:ro --cap-add SYS_PTRACE \ + --cap-add SYS_ADMIN --security-opt apparmor=unconfined \ + netdata/netdata" |> String.split_on_char ' ' |> List.filter (fun s -> s <> "") in - let*! _ = run_command cmd args in + let* _ = Env.run_command ?cmd_wrapper cmd args |> Process.check in Lwt.return_unit diff --git a/tezt/lib_cloud/monitoring.mli b/tezt/lib_cloud/monitoring.mli index c7ff51b95395..53594f012b66 100644 --- a/tezt/lib_cloud/monitoring.mli +++ b/tezt/lib_cloud/monitoring.mli @@ -5,8 +5,5 @@ (* *) (*****************************************************************************) -(** [run ~run_command] runs netdata on the VM. [run_command] is a callback to - run a command on the VM. *) -val run : - run_command:(string -> string list -> (Process.t, string) Runnable.t) -> - unit Lwt.t +(** [run ?cmd_wrapper ()] runs netdata on the VM. *) +val run : ?cmd_wrapper:Gcloud.cmd_wrapper -> unit -> unit Lwt.t diff --git a/tezt/lib_cloud/path.ml b/tezt/lib_cloud/path.ml index b059333a0f3b..71ec07ffc906 100644 --- a/tezt/lib_cloud/path.ml +++ b/tezt/lib_cloud/path.ml @@ -19,7 +19,8 @@ let terraform_state_bucket = terraform // "state-bucket" let terraform_vm = terraform // "vm" -let docker = project // "dockerfiles" +let dockerfile ~alias = + project // "dockerfiles" // Format.asprintf "%s.Dockerfile" alias let zcash_params = "_opam" // "share" // "zcash-params" diff --git a/tezt/lib_cloud/path.mli b/tezt/lib_cloud/path.mli index d6800b8eeef1..1280864eff9a 100644 --- a/tezt/lib_cloud/path.mli +++ b/tezt/lib_cloud/path.mli @@ -18,7 +18,7 @@ val terraform_state_bucket : string val terraform_vm : string (** Path to the directory containing dockerfiles for each user. *) -val docker : string +val dockerfile : alias:string -> string (** Path to the zcash_params directory that can be used for the dockerfiles. *) diff --git a/tezt/lib_cloud/prometheus.ml b/tezt/lib_cloud/prometheus.ml index e29849659641..6c02940ce971 100644 --- a/tezt/lib_cloud/prometheus.ml +++ b/tezt/lib_cloud/prometheus.ml @@ -13,6 +13,8 @@ type t = { configuration_file : string; mutable sources : source list; scrape_interval : int; + snapshot_filename : string option; + port : int; } let netdata_source_of_agents agents = @@ -71,7 +73,7 @@ let config ~scrape_interval sources = let sources = List.map str_of_source sources |> String.concat "" in prefix ~scrape_interval () ^ sources -let write_configuration_file {scrape_interval; configuration_file; sources} = +let write_configuration_file {scrape_interval; configuration_file; sources; _} = let config = config ~scrape_interval sources in with_open_out configuration_file (fun oc -> Stdlib.seek_out oc 0 ; @@ -89,13 +91,18 @@ let add_source t ?(metric_path = "/metrics") ~job_name targets = write_configuration_file t ; reload t -let start ?(scrape_interval = 5) agents = +let start agents = let sources = - if Cli.monitoring then [tezt_source; netdata_source_of_agents agents] + if Env.monitoring then [tezt_source; netdata_source_of_agents agents] else [tezt_source] in let configuration_file = Temp.file "prometheus.yml" in - let t = {configuration_file; sources; scrape_interval} in + let snapshot_filename = Env.prometheus_snapshot_filename in + let port = Env.prometheus_port in + let scrape_interval = Env.prometheus_scrape_interval in + let t = + {configuration_file; sources; scrape_interval; snapshot_filename; port} + in write_configuration_file t ; let process = Process.spawn @@ -108,8 +115,8 @@ let start ?(scrape_interval = 5) agents = "prometheus"; "--network"; "host"; - "-p"; - "9090:9090"; + (* We use the host mode so that in [localhost], prometheus can see the + metrics endpoint run by other docker containers. *) "-v"; Format.asprintf "%s:/etc/prometheus/prometheus.yml" configuration_file; "prom/prometheus"; @@ -118,6 +125,8 @@ let start ?(scrape_interval = 5) agents = (* To export a snapshot. *) "--web.enable-lifecycle"; (* To reload the configuration while prometheus is running.*) + Format.asprintf "--web.listen-address=:%d" port; + (* Specify the port on which the prometheus instance runs. *) ] in let* status = Process.wait process in @@ -133,43 +142,25 @@ let start ?(scrape_interval = 5) agents = (* Fail if something unexpected happens. *) Process.check process in - let rec wait_for_ready () = - let process = - Process.spawn - "curl" - [ - "-s"; - "-o"; - "/dev/null"; - "-w"; - "%{http_code}"; - "http://localhost:9090/-/ready"; - ] - in - let* status = Process.wait process in - match status with - | Unix.WEXITED 0 -> - let* status = Process.check_and_read_stdout process in - if String.trim status = "200" then Lwt.return_unit - else ( - Log.info - "Prometheus container is not ready, let's wait 2 seconds and check \ - again..." ; - let* () = Lwt_unix.sleep 2. in - wait_for_ready ()) - | _ -> - Log.info - "Prometheus container is not ready, let's wait 2 seconds and check \ - again..." ; - let* () = Lwt_unix.sleep 2. in - wait_for_ready () + let is_ready output = String.trim output = "200" in + let run () = + Process.spawn + "curl" + [ + "-s"; + "-o"; + "/dev/null"; + "-w"; + "%{http_code}"; + "http://localhost:9090/-/ready"; + ] in - let* () = wait_for_ready () in + let* _ = Env.wait_process ~is_ready ~run () in Lwt.return t let snapshots_path = "/prometheus" // "data" // "snapshots" -let export_snapshot {configuration_file = _; _} = +let export_snapshot {snapshot_filename; _} = Log.info "Exporting snapshot..." ; let* stdout = Process.run_and_read_stdout @@ -179,27 +170,41 @@ let export_snapshot {configuration_file = _; _} = let json = JSON.parse ~origin:"Prometheus.export" stdout in let snapshot_name = JSON.(json |-> "data" |-> "name" |> as_string) in let destination = - match Cli.prometheus_snapshot with - | None -> Cli.prometheus_snapshot_directory - | Some file -> Cli.prometheus_snapshot_directory // file + Option.value + ~default:(Filename.get_temp_dir_name () // snapshot_name) + snapshot_filename in - let*! () = + let* () = Docker.cp "prometheus" ~kind:`To_host ~source:(snapshots_path // snapshot_name) ~destination + |> Process.check in Log.info "You can find the prometheus snapshot at %s" destination ; Lwt.return_unit let shutdown {configuration_file = _; _} = - let*! () = Docker.kill "prometheus" in + let* () = Docker.kill "prometheus" |> Process.check in Lwt.return_unit -let run_with_snapshot ~snapshot ~port = +let run_with_snapshot () = (* No need for a configuration file here. *) let configuration_file = "" in + let port = Env.prometheus_port in + let snapshot_filename = + match Env.prometheus_snapshot_filename with + | None -> + Test.fail + "You must specify the snapshot filename via \ + --prometheus-snapshot-filename" + | Some file -> file + in + Log.info + "You can find the prometheus instance at http://localhost:%d" + Env.prometheus_port ; + Log.info "Use Ctrl+C to end the scenario and kill the prometheus instance." ; let* () = Process.run "docker" @@ -207,7 +212,7 @@ let run_with_snapshot ~snapshot ~port = "run"; "-uroot"; "-v"; - Format.asprintf "%s:/prometheus" snapshot; + Format.asprintf "%s:/prometheus" snapshot_filename; "-p"; Format.asprintf "%d:9090" port; "prom/prometheus"; @@ -215,4 +220,11 @@ let run_with_snapshot ~snapshot ~port = "--storage.tsdb.path=/prometheus"; ] in - Lwt.return {configuration_file; sources = []; scrape_interval = 0} + Lwt.return + { + configuration_file; + sources = []; + scrape_interval = 0; + snapshot_filename = Some snapshot_filename; + port; + } diff --git a/tezt/lib_cloud/prometheus.mli b/tezt/lib_cloud/prometheus.mli index e12fe452debe..047b9beb82ab 100644 --- a/tezt/lib_cloud/prometheus.mli +++ b/tezt/lib_cloud/prometheus.mli @@ -9,9 +9,8 @@ type t type target = {address : string; port : int; app_name : string} -(** [start ?scrape_interval agents] starts the prometheus server. - [scrape_interval] is set to [5] seconds by default. *) -val start : ?scrape_interval:int -> Agent.t List.t -> t Lwt.t +(** [start agents] starts the prometheus server. *) +val start : Agent.t List.t -> t Lwt.t (** [shutdown prometheus] shutdowns the prometheus server. *) val shutdown : t -> unit Lwt.t @@ -19,9 +18,10 @@ val shutdown : t -> unit Lwt.t (** [export_snapshot prometheus] exports a prometheus snapshot. *) val export_snapshot : t -> unit Lwt.t -(** [run_wtih_snapshot ~snapshot ~port] allows to run the prometheus with the - given snapshot on port [port]. *) -val run_with_snapshot : snapshot:string -> port:int -> t Lwt.t +(** [run_wtih_snapshot ()] allows to run the prometheus with the given + snapshot. The snapshot must have been exported by running a tezt-cloud + scenario beforehand. *) +val run_with_snapshot : unit -> t Lwt.t (** [reload prometheus] reloads the prometheus configuration. This can be used to take into account a change such a different agent name or a new source. *) diff --git a/tezt/lib_cloud/ssh.ml b/tezt/lib_cloud/ssh.ml index 68ff9cced734..ed6146204692 100644 --- a/tezt/lib_cloud/ssh.ml +++ b/tezt/lib_cloud/ssh.ml @@ -6,7 +6,7 @@ (*****************************************************************************) let generate_key () = - let path = Format.asprintf "%s" (Lazy.force Env.ssh_private_key) in + let path = Format.asprintf "%s" Env.ssh_private_key_filename in (* -C is for comment; -N is for the passphrase (no passphrase here) *) diff --git a/tezt/lib_cloud/ssh.mli b/tezt/lib_cloud/ssh.mli index 8bb8a58b1b66..c17679024736 100644 --- a/tezt/lib_cloud/ssh.mli +++ b/tezt/lib_cloud/ssh.mli @@ -5,6 +5,6 @@ (* *) (*****************************************************************************) -(* [generate_key ()] generates an ssh key based on the - [Env.tezt_cloud] variable environment. *) +(** [generate_key ()] generates an ssh key based on the [Env.tezt_cloud] + variable environment. *) val generate_key : unit -> unit Lwt.t diff --git a/tezt/lib_cloud/terraform.ml b/tezt/lib_cloud/terraform.ml index 81bc98662b23..48d0e6c7e7e0 100644 --- a/tezt/lib_cloud/terraform.ml +++ b/tezt/lib_cloud/terraform.ml @@ -21,9 +21,7 @@ module Docker_registry = struct "terraform" (chdir Path.terraform_docker_registry @ ["init"]) - let deploy () = - let* project_id = Gcloud.project_id () in - let tezt_cloud = Lazy.force Env.tezt_cloud in + let deploy ~tezt_cloud ~project_id = Process.run ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) ~name @@ -37,8 +35,7 @@ module Docker_registry = struct Format.asprintf "project_id=%s" project_id; ]) - let get_docker_registry () = - let tezt_cloud = Lazy.force Env.tezt_cloud in + let get_docker_registry ~tezt_cloud = let* output = Process.run_and_read_stdout ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) @@ -47,14 +44,13 @@ module Docker_registry = struct "terraform" (chdir Path.terraform_docker_registry @ ["output"; "-json"]) in - let json = JSON.parse ~origin:"docker_registry" output in + let json = JSON.parse ~origin:"get_docker_registry" output in let registry_name = JSON.(json |-> "docker_registry" |-> "value" |> as_string) in Lwt.return registry_name - let get_hostname () = - let tezt_cloud = Lazy.force Env.tezt_cloud in + let get_hostname ~tezt_cloud = let* output = Process.run_and_read_stdout ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) @@ -63,9 +59,22 @@ module Docker_registry = struct "terraform" (chdir Path.terraform_docker_registry @ ["output"; "-json"]) in - let json = JSON.parse ~origin:"docker_registry_hostname" output in + let json = JSON.parse ~origin:"get_hostname" output in let registry_name = JSON.(json |-> "hostname" |-> "value" |> as_string) in Lwt.return registry_name + + let get_zone ~tezt_cloud = + let* output = + Process.run_and_read_stdout + ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) + ~name + ~color + "terraform" + (chdir Path.terraform_docker_registry @ ["output"; "-json"]) + in + let json = JSON.parse ~origin:"get_zone" output in + let registry_name = JSON.(json |-> "zone" |-> "value" |> as_string) in + Lwt.return registry_name end module VM = struct @@ -81,8 +90,7 @@ module VM = struct (* Return all the workspaces associated with the current tezt cloud environment. *) - let get () = - let tezt_cloud = Lazy.force Env.tezt_cloud in + let list ~tezt_cloud = (* We select the default workspace to be sure we can parse correctly the output. *) let* () = select "default" in @@ -96,14 +104,17 @@ module VM = struct String.split_on_char '\n' output |> List.map String.trim |> List.filter (fun workspace -> - String.starts_with ~prefix:tezt_cloud workspace) + String.starts_with ~prefix:tezt_cloud workspace + (* The workspace named "tezt-cloud" is used for the docker + registry, it does not need to be listed. *) + && workspace <> tezt_cloud) |> Lwt.return - (* Create workspaces that will be used for the experiment. Delete the ones - that won't be used. *) - let init workspaces = - let* existing_workspaces = get () in + (* Create workspaces that will be used for the experiment. *) + let init ~tezt_cloud workspaces = + let* existing_workspaces = list ~tezt_cloud in let to_create = + (* Create only new workspaces. *) List.filter (fun workspace -> not @@ List.mem workspace existing_workspaces) workspaces @@ -124,8 +135,10 @@ module VM = struct let* () = select "default" in unit - let destroy () = - let* workspaces = get () in + let destroy ~tezt_cloud = + (* We ensure we are not using a workspace we want to delete. *) + let* () = select "default" in + let* workspaces = list ~tezt_cloud in workspaces |> List.map (fun workspace -> Process.run @@ -210,9 +223,7 @@ module VM = struct in Lwt.return machine_type - let destroy workspaces = - let* project_id = Gcloud.project_id () in - (* let docker_image = Env.custom_docker_image ~project_id in *) + let destroy workspaces ~project_id = workspaces |> Lwt_list.iter_s (fun workspace -> let* () = Workspace.select workspace in @@ -241,29 +252,3 @@ module VM = struct "terraform" (chdir Path.terraform_vm @ ["destroy"; "--auto-approve"] @ vars)) end - -module State_bucket = struct - let init () = - let* () = Docker_registry.init () in - Process.run - ~name - ~color - "terraform" - (chdir Path.terraform_state_bucket @ ["init"]) - - let deploy () = - let tezt_cloud = Lazy.force Env.tezt_cloud in - let* project_id = Gcloud.project_id () in - Process.run - ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) - ~name - ~color - "terraform" - (chdir Path.terraform_state_bucket - @ [ - "apply"; - "--auto-approve"; - "--var"; - Format.asprintf "project_id=%s" project_id; - ]) -end diff --git a/tezt/lib_cloud/terraform.mli b/tezt/lib_cloud/terraform.mli index 6707c97933ad..49026d7b3e1b 100644 --- a/tezt/lib_cloud/terraform.mli +++ b/tezt/lib_cloud/terraform.mli @@ -8,22 +8,24 @@ module Docker_registry : sig val init : unit -> unit Lwt.t - val deploy : unit -> unit Lwt.t + val deploy : tezt_cloud:string -> project_id:string -> unit Lwt.t - val get_docker_registry : unit -> string Lwt.t + val get_docker_registry : tezt_cloud:string -> string Lwt.t - val get_hostname : unit -> string Lwt.t + val get_hostname : tezt_cloud:string -> string Lwt.t + + val get_zone : tezt_cloud:string -> string Lwt.t end module VM : sig module Workspace : sig - val init : string list -> unit Lwt.t + val init : tezt_cloud:string -> string list -> unit Lwt.t - val get : unit -> string list Lwt.t + val list : tezt_cloud:string -> string list Lwt.t val select : string -> unit Lwt.t - val destroy : unit -> unit Lwt.t + val destroy : tezt_cloud:string -> unit Lwt.t end val init : unit -> unit Lwt.t @@ -41,11 +43,5 @@ module VM : sig val zone : unit -> string Lwt.t - val destroy : string list -> unit Lwt.t -end - -module State_bucket : sig - val init : unit -> unit Lwt.t - - val deploy : unit -> unit Lwt.t + val destroy : string list -> project_id:string -> unit Lwt.t end diff --git a/tezt/lib_cloud/terraform/docker-registry/main.tf b/tezt/lib_cloud/terraform/docker-registry/main.tf index a01cd88040b7..cb976243caa7 100644 --- a/tezt/lib_cloud/terraform/docker-registry/main.tf +++ b/tezt/lib_cloud/terraform/docker-registry/main.tf @@ -32,6 +32,12 @@ terraform { version = "~> 5.0" } } + + backend "gcs" { + # Generic bucket state that contains all the states for all the + # workspaces + bucket = "dal-tfstate" + } } resource "google_artifact_registry_repository" "docker-registry" { @@ -61,3 +67,8 @@ output "docker_registry" { value = local.registry description = "URL of the GCP docker registry for this region" } + +output "zone" { + value = var.zone + description = "Zone of the docker registry" +} diff --git a/tezt/lib_cloud/tezt_cloud.ml b/tezt/lib_cloud/tezt_cloud.ml index b2440bd430f7..a3435e140039 100644 --- a/tezt/lib_cloud/tezt_cloud.ml +++ b/tezt/lib_cloud/tezt_cloud.ml @@ -5,6 +5,86 @@ (* *) (*****************************************************************************) +module Agent = Agent + +module Configuration = struct + include Env + include Configuration +end + module Cloud = Cloud -let register ~tags = Jobs.register ~tags +let register_docker_push ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Push the dockerfile to the GCP registry" + ~tags:("docker" :: "push" :: tags) + @@ fun _cloud -> Jobs.docker_build ~push:true () + +let register_docker_build ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Build the dockerfile" + ~tags:("docker" :: "build" :: tags) + @@ fun _cloud -> Jobs.docker_build ~push:false () + +let register_deploy_docker_registry ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Deploy docker registry" + ~tags:("docker" :: "registry" :: "deploy" :: tags) + @@ fun _cloud -> Jobs.deploy_docker_registry () + +let register_destroy_vms ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Destroy terraform VMs" + ~tags:("terraform" :: "destroy" :: "vms" :: tags) + @@ fun _cloud -> + let tezt_cloud = Env.tezt_cloud in + let* project_id = Gcloud.project_id () in + let* workspaces = Terraform.VM.Workspace.list ~tezt_cloud in + let* () = Terraform.VM.destroy workspaces ~project_id in + Terraform.VM.Workspace.destroy ~tezt_cloud + +let register_prometheus_import ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Import a snapshot into a prometheus container" + ~tags:("prometheus" :: "import" :: tags) + @@ fun _cloud -> + let* prometheus = Prometheus.run_with_snapshot () in + Prometheus.shutdown prometheus + +let register_clean_up_vms ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Clean ups VMs manually" + ~tags:("clean" :: "up" :: tags) + @@ fun _cloud -> Jobs.clean_up_vms () + +let register_list_vms ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"List VMs" + ~tags:("list" :: "vms" :: tags) + @@ fun _cloud -> + Log.info "TEZT_CLOUD environment variable found with value: %s" Env.tezt_cloud ; + let* _ = Gcloud.list_vms ~prefix:Env.tezt_cloud in + Lwt.return_unit + +let register ~tags = + register_docker_push ~tags ; + register_docker_build ~tags ; + register_deploy_docker_registry ~tags ; + register_destroy_vms ~tags ; + register_prometheus_import ~tags ; + register_clean_up_vms ~tags ; + register_list_vms ~tags diff --git a/tezt/lib_cloud/tezt_cloud.mli b/tezt/lib_cloud/tezt_cloud.mli index 895f14cc92bb..8419e9a9caa4 100644 --- a/tezt/lib_cloud/tezt_cloud.mli +++ b/tezt/lib_cloud/tezt_cloud.mli @@ -5,21 +5,23 @@ (* *) (*****************************************************************************) -module Cloud : sig +module Agent : sig type t - module Configuration : sig - type docker_image = - | Custom of {tezt_cloud : string} - | Image of {docker_image : string} + val configuration : Agent.t -> Configuration.t +end + +module Configuration : sig + type docker_image = Gcp of {alias : string} | Octez_latest_release - type t = private { - machine_type : string; - docker_image : docker_image; - max_run_duration : int option; - } + type t = private { + machine_type : string; + docker_image : docker_image; + max_run_duration : int option; + binaries_path : string; + } - (** [make ?max_run_duration ?machine_type ?docker_image ()] is a smart-constructor to make a VM + (** [make ?machine_type ()] is a smart-constructor to make a VM configuration. Default value for [max_run_duration] is [7200]. @@ -29,19 +31,22 @@ module Cloud : sig Default value for [docker_image] is [Custom {tezt_cloud}] where [tezt_cloud] is the value provided by the environement variable [$TEZT_CLOUD]. *) - val make : - ?max_run_duration:int -> - ?machine_type:string -> - ?docker_image:docker_image -> - unit -> - t - end + val make : + ?binaries_path:string -> + ?max_run_duration:int -> + ?machine_type:string -> + ?docker_image:docker_image -> + unit -> + t +end + +module Cloud : sig + type t (** A wrapper around [Test.register] that can be used to register new tests using VMs provided as a map indexed by name. Each VM is abstracted via the [Agent] module. *) val register : - ?docker_push:bool -> ?vms:Configuration.t list -> __FILE__:string -> title:string -> @@ -72,8 +77,6 @@ module Cloud : sig points to scrap. Each point can have a name defined by [app_name]. *) val add_prometheus_source : t -> ?metric_path:string -> job_name:string -> target list -> unit Lwt.t - - val get_configuration : t -> Agent.t -> Configuration.t end (** [register ~tags] register a set of jobs that can be used for setting diff --git a/tezt/lib_cloud/web.ml b/tezt/lib_cloud/web.ml index 274a3f3d2edc..d6631e30569a 100644 --- a/tezt/lib_cloud/web.ml +++ b/tezt/lib_cloud/web.ml @@ -5,25 +5,98 @@ (* *) (*****************************************************************************) -type t = {process : Process.t; dir : string; deployement : Deployement.t} +type t = { + process : Process.t; + dir : string; + monitoring : bool; + prometheus : bool; +} -let configuration ~deployement ~agents = +let pp_docker_image fmt = function + | Env.Gcp {alias} -> Format.fprintf fmt "%s" alias + | Octez_latest_release -> Format.fprintf fmt "Octez latest release" + +let configuration ~agents = let str = agents |> List.map (fun agent -> - let configuration = - Deployement.get_configuration deployement agent - in + let configuration = Agent.configuration agent in Format.asprintf - "- %s: %s" + {| +## %s + +- **Machine type**: %s +- **Docker_image**: %a + + |} (Agent.name agent) - configuration.machine_type) + configuration.machine_type + pp_docker_image + configuration.docker_image) |> String.concat "\n" in Format.asprintf "# Configurations@.%s\n" str +let string_docker_command agent = + let point = Agent.point agent in + let ssh_id = (Agent.runner agent).Runner.ssh_id in + String.concat + " " + [ + "ssh"; + Format.asprintf "root@%s" (fst point); + "-p"; + string_of_int (snd point); + "-o"; + "StrictHostKeyChecking=no"; + "-i"; + ssh_id |> Option.get; + ] + +let string_vm_command agent = + match Agent.cmd_wrapper agent with + | None -> "# Just run the command, the VM is the host machine" + | Some cmd_wrapper -> + String.concat " " (cmd_wrapper.Gcloud.cmd :: cmd_wrapper.args) + +let debugging ~agents = + let str = + agents + |> List.map (fun agent -> + let host_run_command = + Printf.sprintf + {| +```bash +%s +``` + |} + (string_vm_command agent) + in + let docker_command = + Printf.sprintf {| +```bash +%s +``` +|} (string_docker_command agent) + in + Printf.sprintf + {| +## %s +Connect on the VM: +%s + +Connect on the Docker: +%s +|} + (Agent.name agent) + host_run_command + docker_command) + |> String.concat "\n" + in + Format.asprintf "# Debugging@.%s\n" str + let monitoring ~agents = - if Cli.monitoring then + if Env.monitoring then let str = agents |> List.map (fun agent -> @@ -40,13 +113,23 @@ let monitoring ~agents = else "# Monitoring\n Monitoring disabled. Use `--monitoring` to activate it.\n" -let prometheus () = - if Cli.prometheus then - "# Prometheus\n [Prometheus dashboard](http://localhost:9090)" - else "Prometheus disabled. Use `--prometheus` to activate it." +let prometheus = + if Env.prometheus then + "# Prometheus\n [Prometheus dashboard](http://localhost:9090)\n" + else "Prometheus disabled. Use `--prometheus` to activate it.\n" + +let grafana = + if Env.grafana then "# Grafana\n [Grafana dashboard](http://localhost:3000)\n" + else "Grafana disabled. Use `--grafana` to activate it.\n" -let markdown_content ~deployement ~agents = - [configuration ~deployement ~agents; monitoring ~agents; prometheus ()] +let markdown_content ~agents = + [ + configuration ~agents; + grafana; + prometheus; + monitoring ~agents; + debugging ~agents; + ] |> String.concat "\n" let index dir = dir // "index.md" @@ -54,7 +137,7 @@ let index dir = dir // "index.md" let write t ~agents = (* The content is formatted in markdown and will be rendered in html via pandoc. *) - let content = markdown_content ~deployement:t.deployement ~agents in + let content = markdown_content ~agents in let dir = t.dir in let index = index dir in Base.with_open_out index (fun oc -> output_string oc content) ; @@ -72,9 +155,12 @@ let write t ~agents = "-s"; ] -let run ~port ~deployement = +let run () = let dir = Temp.dir "website" in let index = index dir in + let port = Env.website_port in + let prometheus = Env.prometheus in + let monitoring = Env.monitoring in let process = Process.spawn "python3" @@ -86,10 +172,10 @@ let run ~port ~deployement = Filename.dirname index; ] in - Lwt.return {process; dir; deployement} + {process; dir; monitoring; prometheus} -let start ~port ~deployement ~agents = - let* t = run ~port ~deployement in +let start ~agents = + let t = run () in let* () = write t ~agents in Lwt.return t diff --git a/tezt/lib_cloud/web.mli b/tezt/lib_cloud/web.mli index a22637d05e2c..7aaea7e4c614 100644 --- a/tezt/lib_cloud/web.mli +++ b/tezt/lib_cloud/web.mli @@ -7,10 +7,9 @@ type t -(** [start ~port ~agents] starts a webpage with experimentations information +(** [start ~agents env] starts a webpage with experimentations information if [Cli.website] is [true]. *) -val start : - port:int -> deployement:Deployement.t -> agents:Agent.t List.t -> t Lwt.t +val start : agents:Agent.t List.t -> t Lwt.t (** [shutdown website] shutdowns the website. *) val shutdown : t -> unit Lwt.t diff --git a/tezt/lib_tezos/dune b/tezt/lib_tezos/dune index 379c0aac7bfd..e406f0557820 100644 --- a/tezt/lib_tezos/dune +++ b/tezt/lib_tezos/dune @@ -7,6 +7,7 @@ (libraries octez-libs.tezt-wrapper tezt-tezos.tezt-performance-regression + tezt-tezos.tezt-cloud uri hex octez-libs.crypto-dal @@ -17,4 +18,5 @@ (:standard) -open Tezt_wrapper -open Tezt_wrapper.Base - -open Tezt_tezos_tezt_performance_regression)) + -open Tezt_tezos_tezt_performance_regression + -open Tezt_tezos_tezt_cloud)) diff --git a/tezt/tests/cloud/basic.ml b/tezt/tests/cloud/basic.ml index 8754c29bbad2..f3ce0cdc7622 100644 --- a/tezt/tests/cloud/basic.ml +++ b/tezt/tests/cloud/basic.ml @@ -7,7 +7,7 @@ let simple () = Cloud.register - ~vms:[Cloud.Configuration.make (); Cloud.Configuration.make ()] + ~vms:[Configuration.make (); Configuration.make ()] ~__FILE__ ~tags:["simple"; "health"; Tag.cloud] ~title:"Simple health check to check local configuration" diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index c40cb02f17cc..2a2d82715c3a 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -1451,7 +1451,9 @@ let etherlink_loop (etherlink : etherlink) = List.nth etherlink.producers (i mod List.length etherlink.producers) in let runner = Node.runner producer.node |> Option.get in - let firehose = Agent.binaries_path producer.agent // "firehose" in + let firehose = + (Agent.configuration producer.agent).binaries_path // "firehose" + in let* () = Process.spawn ~runner @@ -1511,23 +1513,24 @@ let benchmark () = ] |> List.concat in - let default_vm_configuration = Cloud.Configuration.make () in + let default_vm_configuration = Configuration.make () in let vms = vms |> List.map (function - | `Bootstrap -> default_vm_configuration + | `Bootstrap -> + Configuration.make ~docker_image:Octez_latest_release () | `Baker i -> ( match configuration.stake_machine_type with | None -> default_vm_configuration | Some list -> ( try let machine_type = List.nth list i in - Cloud.Configuration.make ~machine_type () + Configuration.make ~machine_type () with _ -> default_vm_configuration)) | `Producer | `Observer -> ( match configuration.producer_machine_type with | None -> default_vm_configuration - | Some machine_type -> Cloud.Configuration.make ~machine_type ()) + | Some machine_type -> Configuration.make ~machine_type ()) | `Etherlink_operator -> default_vm_configuration | `Etherlink_producer _ -> default_vm_configuration) in diff --git a/tezt/tests/cloud/dune b/tezt/tests/cloud/dune index 5164063aa60b..9b1145aaf882 100644 --- a/tezt/tests/cloud/dune +++ b/tezt/tests/cloud/dune @@ -7,7 +7,7 @@ octez-libs.test-helpers octez-libs.tezt-wrapper tezt-tezos - tezt_cloud + tezt-tezos.tezt-cloud tezt_etherlink) (link_flags (:standard) @@ -19,4 +19,4 @@ -open Tezt_wrapper.Base -open Tezt_tezos -open Tezt_tezos.Runnable.Syntax - -open Tezt_cloud)) + -open Tezt_tezos_tezt_cloud)) -- GitLab From 289578ff8806a1d9fed9539c38462f5a689fddb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Fri, 5 Jul 2024 14:36:35 +0200 Subject: [PATCH 02/20] Tezt/Cloud: Add a password (just in case) --- tezt/lib_cloud/grafana.ml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tezt/lib_cloud/grafana.ml b/tezt/lib_cloud/grafana.ml index 39b0056d1a08..f2b973304f9d 100644 --- a/tezt/lib_cloud/grafana.ml +++ b/tezt/lib_cloud/grafana.ml @@ -7,7 +7,13 @@ include Tezt_tezos_tezt_performance_regression.Grafana -type t = {provisioning_file : string; dashboard_directory : string} +type t = { + provisioning_file : string; + dashboard_directory : string; + password : string; +} + +let generate_password () = "saucisse" let provisioning_file () = let provisioning_file = @@ -55,6 +61,10 @@ let run () = Process.run "mkdir" ["-p"; dashboard_directory |> Filename.dirname] in let* provisioning_file = provisioning_file () in + (* We generate a password to use admin features. This is not completely + secured but this should prevent easy attacks if the grafana port is opened. *) + let password = generate_password () in + Log.info "Grafana admin password: %s" password ; let args = [ "run"; @@ -70,6 +80,8 @@ let run () = "GF_AUTH_ANONYMOUS_ENABLED=true"; "-e"; "GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer"; + "-e"; + Format.asprintf "GF_SECURITY_ADMIN_PASSWORD=%s" password; "-v"; Format.asprintf "%s:/etc/grafana/provisioning" @@ -88,4 +100,4 @@ let run () = let* () = shutdown () in Process.run cmd args in - Lwt.return {provisioning_file; dashboard_directory} + Lwt.return {provisioning_file; dashboard_directory; password} -- GitLab From 1f63492f1c4b264831bd37ad427f67d39da19725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Fri, 5 Jul 2024 14:41:37 +0200 Subject: [PATCH 03/20] Tezt/Cloud: Ensure grafana starts well --- tezt/lib_cloud/grafana.ml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tezt/lib_cloud/grafana.ml b/tezt/lib_cloud/grafana.ml index f2b973304f9d..044e4cab25e6 100644 --- a/tezt/lib_cloud/grafana.ml +++ b/tezt/lib_cloud/grafana.ml @@ -100,4 +100,18 @@ let run () = let* () = shutdown () in Process.run cmd args in + let is_ready output = String.trim output = "200" in + let run () = + Process.spawn + "curl" + [ + "-s"; + "-o"; + "/dev/null"; + "-w"; + "%{http_code}"; + "http://localhost:3000/api/health"; + ] + in + let* _ = Env.wait_process ~is_ready ~run () in Lwt.return {provisioning_file; dashboard_directory; password} -- GitLab From c161984bd3432f1c65ca38b3644b8adc187a321c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Fri, 5 Jul 2024 14:45:20 +0200 Subject: [PATCH 04/20] Tezt/Cloud: Add a few grafana dashboards --- tezt/lib_cloud/grafana/dashboards/README.md | 12 + .../grafana/dashboards/dal-basic.json | 1951 +++++ .../grafana/dashboards/evm-node.json | 1093 +++ .../grafana/dashboards/octez-basic.json | 5139 ++++++++++++++ .../grafana/dashboards/octez-compact.json | 1941 +++++ .../grafana/dashboards/octez-full.json | 6287 +++++++++++++++++ .../grafana/dashboards/octez-with-logs.json | 5309 ++++++++++++++ tezt/lib_cloud/grafana/dashboards/rollup.json | 1213 ++++ 8 files changed, 22945 insertions(+) create mode 100644 tezt/lib_cloud/grafana/dashboards/README.md create mode 100644 tezt/lib_cloud/grafana/dashboards/dal-basic.json create mode 100644 tezt/lib_cloud/grafana/dashboards/evm-node.json create mode 100644 tezt/lib_cloud/grafana/dashboards/octez-basic.json create mode 100644 tezt/lib_cloud/grafana/dashboards/octez-compact.json create mode 100644 tezt/lib_cloud/grafana/dashboards/octez-full.json create mode 100644 tezt/lib_cloud/grafana/dashboards/octez-with-logs.json create mode 100644 tezt/lib_cloud/grafana/dashboards/rollup.json diff --git a/tezt/lib_cloud/grafana/dashboards/README.md b/tezt/lib_cloud/grafana/dashboards/README.md new file mode 100644 index 000000000000..83a4de69b93e --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/README.md @@ -0,0 +1,12 @@ +# Grafana dashboards + +Those dashboards where generated via the +[grafazos](https://gitlab.com/nomadic-labs/grafazos/-/tree/master?ref_type=heads) +project using the following command: + +``` +$ make all NODE_INSTANCE_LABEL=app +``` + +The result where copied in this directory. Feel free to update those +dashboards in consequence. diff --git a/tezt/lib_cloud/grafana/dashboards/dal-basic.json b/tezt/lib_cloud/grafana/dashboards/dal-basic.json new file mode 100644 index 000000000000..4bbaa226d320 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/dal-basic.json @@ -0,0 +1,1951 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "DAL node's core stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Finalized L1 blocks": "green", + "Seen L1 heads": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_node_new_layer1_head{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 1, + "legendFormat": "Seen L1 heads", + "refId": "A" + }, + { + "expr": "dal_node_layer1_block_finalized{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 1, + "legendFormat": "Finalized L1 blocks", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Layer 1 heads & finalized blocks seen by the DAL node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Finalized L1 blocks' rounds": "green", + "Seen L1 heads' rounds": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_node_new_layer1_head_round{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 1, + "legendFormat": "Seen L1 heads' rounds", + "refId": "A" + }, + { + "expr": "dal_node_layer1_block_finalized_round{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 1, + "legendFormat": "Finalized L1 blocks' rounds", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rounds of layer 1 heads & finalized blocks seen by the DAL node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Stored shards": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(dal_node_number_of_stored_shards{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Stored shards", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "The shards stored by this node (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Number of attested slots": "green", + "Number of slots waiting for attestation": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(dal_node_slots_waiting_for_attestaion{app=\"$node_instance\", slot_index=~\"$slot_index\"})", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Number of slots waiting for attestation", + "refId": "A" + }, + { + "expr": "sum(dal_node_slots_attested{app=\"$node_instance\", slot_index=~\"$slot_index\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Number of attested slots", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Number of slots waiting for attesatation and of attested slots", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_node_slots_waiting_for_attestaion{app=\"$node_instance\", slot_index=~\"$slot_index\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Slot index: {{ slot_index }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Indexes of slots waiting for attestation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_node_slots_attested{app=\"$node_instance\", slot_index=~\"$slot_index\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Slot index: {{ slot_index }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Indexes of attested slots", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 9, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Gossipsub worker's stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_scores_of_peers{app=\"$node_instance\", peer=\"$peer\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "{{ peer }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Score of the peers connected to this node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_count_peers_per_topic{app=\"$node_instance\", slot_index=~\"$slot_index\", pkh=\"$pkh\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Topic: {{ slot_index }}, {{ pkh }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Number of peers in the mesh for each subscribed topic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Node's topics": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 25 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_count_topics{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node's topics", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "The number of topics the node is subscribed to", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "All connections": "green", + "Bootstrap connections": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 25 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_count_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "All connections", + "refId": "A" + }, + { + "expr": "dal_gs_count_bootstrap_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bootstrap connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "The number of (bootstrap) connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Application output stream": "yellow", + "Input stream": "green", + "P2P output stream": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 25 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_input_events_stream_length{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Input stream", + "refId": "A" + }, + { + "expr": "dal_gs_p2p_output_stream_length{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "P2P output stream", + "refId": "B" + }, + { + "expr": "dal_gs_app_output_stream_length{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Application output stream", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "The size of different worker streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Received & invalid": "red", + "Received & unchecked validity": "orange", + "Received & valid": "green", + "Sent": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(dal_gs_count_received_valid_messages{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Received & valid", + "refId": "A" + }, + { + "expr": "deriv(dal_gs_count_received_invalid_messages{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Received & invalid", + "refId": "B" + }, + { + "expr": "deriv(dal_gs_count_received_unknown_validity_messages{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Received & unchecked validity", + "refId": "C" + }, + { + "expr": "deriv(dal_gs_count_sent_messages{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Average sent & received app messages (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Graft": "blue", + "IHave": "purple", + "IWant": "magenta", + "Prune": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(dal_gs_count_sent_grafts{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Graft", + "refId": "A" + }, + { + "expr": "deriv(dal_gs_count_sent_prunes{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Prune", + "refId": "B" + }, + { + "expr": "deriv(dal_gs_count_sent_ihaves{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "IHave", + "refId": "C" + }, + { + "expr": "deriv(dal_gs_count_sent_iwants{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "IWant", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Average sent control & metadata messages (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Graft": "blue", + "IHave": "purple", + "IWant": "magenta", + "Prune": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(dal_gs_count_received_grafts{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Graft", + "refId": "A" + }, + { + "expr": "deriv(dal_gs_count_received_prunes{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Prune", + "refId": "B" + }, + { + "expr": "deriv(dal_gs_count_received_ihaves{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "IHave", + "refId": "C" + }, + { + "expr": "deriv(dal_gs_count_received_iwants{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "IWant", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Average received control & metadata messages (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 18, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "P2P stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Received data (KB)": "blue", + "Sent data (KB)": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 19, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(octez_p2p_io_scheduler_total_recv{app=\"$node_instance\"}[1m]) / 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Received data (KB)", + "refId": "A" + }, + { + "expr": "deriv(octez_p2p_io_scheduler_total_sent{app=\"$node_instance\"}[1m]) / 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sent data (KB)", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Average data exchange (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected peers": "light-yellow", + "Running peers": "light-green", + "Waiting to reconnect": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_peers_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected peers", + "refId": "A" + }, + { + "expr": "octez_p2p_peers_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running peers", + "refId": "B" + }, + { + "expr": "octez_p2p_points_greylisted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Waiting to reconnect", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P peers connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected points": "light-red", + "Running points": "light-green", + "Trusted points": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_points_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected points", + "refId": "A" + }, + { + "expr": "octez_p2p_points_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running points", + "refId": "B" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P points connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez", + "dal" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(dal_node_new_layer1_head,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "0", + "value": "0" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Slot index", + "multi": true, + "name": "slot_index", + "options": [ ], + "query": "label_values(dal_gs_count_peers_per_topic, slot_index)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Pkh", + "multi": false, + "name": "pkh", + "options": [ ], + "query": "label_values(dal_gs_count_peers_per_topic, pkh)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Peer", + "multi": false, + "name": "peer", + "options": [ ], + "query": "label_values(dal_gs_scores_of_peers, peer)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez DAL Node Dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/evm-node.json b/tezt/lib_cloud/grafana/dashboards/evm-node.json new file mode 100644 index 000000000000..601e0f94a6b6 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/evm-node.json @@ -0,0 +1,1093 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_info{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "{{commit_hash}}", + "range": false, + "refId": "A" + } + ], + "title": "Commit", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 3, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_info{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "{{commit_date}}", + "range": false, + "refId": "A" + } + ], + "title": "Commit Date", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_info{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "{{devmode}}", + "range": false, + "refId": "A" + } + ], + "title": "Devmode", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 12, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_info{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "{{mode}}", + "range": false, + "refId": "A" + } + ], + "title": "Mode", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(octez_evm_node_calls_count{app=\"$node_instance\"})", + "instant": true, + "legendFormat": "{{commit_hash}}", + "range": false, + "refId": "A" + } + ], + "title": "RPC Calls", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(octez_evm_node_calls_sum{app=\"$node_instance\"})/sum(octez_evm_node_calls_count{app=\"$node_instance\"})", + "instant": true, + "legendFormat": "{{commit_hash}}", + "range": false, + "refId": "A" + } + ], + "title": "RPC Average duration", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 0, + "y": 3 + }, + "id": 5, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_head{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "Node head", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_confirmed_head{app=\"$node_instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Rollup node head", + "range": true, + "refId": "B" + } + ], + "title": "Head", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 2, + "y": 3 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_confirmed_head{app=\"$node_instance\"}", + "instant": false, + "legendFormat": "Rollup-node's level", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_head{app=\"$node_instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Node's level", + "range": true, + "refId": "B" + } + ], + "title": "Levels", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "greater", + "options": { + "value": 0 + } + }, + "fieldName": "Rollup-node's level" + } + ], + "match": "any", + "type": "include" + } + } + ], + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "The average is computed over 15 seconds.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "yellow", + "mode": "shades", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 10, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "E" + }, + "properties": [ + { + "id": "displayName", + "value": "Observed time processed" + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Observed time processed" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 13, + "y": 3 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_time_processed{app=\"$node_instance\"}", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "Processing Time", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_time_processed{app=\"$node_instance\"} offset 15s", + "hide": true, + "instant": false, + "legendFormat": "Offset Time Processed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_head{app=\"$node_instance\"}", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_head{app=\"$node_instance\"} offset 15s", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "($A-$B)/($C-$D)", + "hide": false, + "refId": "E", + "type": "math" + } + ], + "title": "Average processing time", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "greater", + "options": { + "value": 0 + } + }, + "fieldName": "E" + } + ], + "match": "any", + "type": "include" + } + } + ], + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 13, + "x": 0, + "y": 13 + }, + "id": 7, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "octez_evm_node_block_process_time_histogram_bucket{app=\"$node_instance\"}", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Processing time distribution", + "type": "bargauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 21 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_tx_pool{kind=\"number_of_addresses\", app=\"$node_instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Transaction pool addresses count", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 21 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_tx_pool{kind=\"number_of_transactions\", app=\"$node_instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Transaction pool transactions count", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { }, + "datasource": "Prometheus", + "definition": "label_values(octez_evm_node_info,app)", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(octez_evm_node_info,app)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Etherlink", + "version": 0, + "weekStart": "" +} diff --git a/tezt/lib_cloud/grafana/dashboards/octez-basic.json b/tezt/lib_cloud/grafana/dashboards/octez-basic.json new file mode 100644 index 000000000000..9b254c0d67ae --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/octez-basic.json @@ -0,0 +1,5139 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Node stats", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bootstrapping" + }, + "1": { + "color": "green", + "index": 0, + "text": "Bootstrapped" + }, + "null": { + "color": "yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 3, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_is_bootstrapped{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bootstrap status", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Unsync" + }, + "1": { + "color": "green", + "index": 1, + "text": "Sync" + }, + "2": { + "color": "red", + "index": 2, + "text": "Stuck" + }, + "null": { + "color": "yellow", + "index": 3, + "text": "Unknow" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 4, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_synchronisation_status{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sync status", + "transparent": false, + "type": "stat" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ chain_name }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Chain name", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 8, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 11, + "y": 1 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_version{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Reflects the uptime of the monitoring of the job, not the uptime of the process.", + "format": "dtdhms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time()-(process_start_time_seconds{app=\"$node_instance\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "node uptime", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 7 + }, + "id": 9, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current head level", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current head level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 10 + }, + "id": 10, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ p2p_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "P2p version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 10 + }, + "id": 11, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ distributed_db_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Distributed db version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "value" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 13 + }, + "id": 12, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_savepoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current savepoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current savepoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 13 + }, + "id": 13, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_checkpoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current checkpoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current checkpoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 16 + }, + "id": 14, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_caboose_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current caboose", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current caboose level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 16 + }, + "id": 15, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_cycle{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current cycle", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current cycle", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 19 + }, + "id": 16, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trusted points", + "refId": "A" + } + ], + "thresholds": "", + "title": "Trusted points", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 19 + }, + "id": 17, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_connections_private{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private Connections", + "refId": "A" + } + ], + "thresholds": "", + "title": "Private Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "aliasColors": { + "Head level": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 4, + "y": 4 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head level", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head level history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Validation time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 14 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validation time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block validation time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 22 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"0\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consensus", + "refId": "A" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"1\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Vote", + "refId": "B" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"2\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anonymous", + "refId": "C" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"3\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Manager", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Gas consumed": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 30 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_consumed_gas{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Gas consumed", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Gas consumed history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Invalid blocks": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 10, + "x": 14, + "y": 1 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_invalid_blocks{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Invalid blocks", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Invalid blocks history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Round": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 10, + "x": 14, + "y": 10 + }, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Round", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Round history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Written block size": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 14, + "y": 19 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_written_block_size{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Written block size", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Last written block size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Merge time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 10, + "x": 14, + "y": 29 + }, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_merge_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Merge time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Store merge time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 26, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "P2P stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Branch delayed": "light-red", + "Branch refused": "light-orange", + "Outdated": "blue", + "Refused": "red", + "Unprocessed": "light-yellow", + "Validated": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_mempool_pending_validated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validated", + "refId": "A" + }, + { + "expr": "octez_mempool_pending_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Refused", + "refId": "B" + }, + { + "expr": "octez_mempool_pending_outdated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outdated", + "refId": "C" + }, + { + "expr": "octez_mempool_pending_branch_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch refused", + "refId": "D" + }, + { + "expr": "octez_mempool_pending_branch_delayed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch delayed", + "refId": "E" + }, + { + "expr": "octez_mempool_pending_unprocessed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unprocessed", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Mempool status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected peers": "light-yellow", + "Running peers": "light-green", + "Waiting to reconnect": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_peers_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected peers", + "refId": "A" + }, + { + "expr": "octez_p2p_peers_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running peers", + "refId": "B" + }, + { + "expr": "octez_p2p_points_greylisted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Waiting to reconnect", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P peers connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected points": "light-red", + "Running points": "light-green", + "Trusted points": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_points_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected points", + "refId": "A" + }, + { + "expr": "octez_p2p_points_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running points", + "refId": "B" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P points connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current head broadcasted": "light-green", + "current head received": "light-blue", + "current head sent": "light-green", + "get current head broadcasted": "green", + "get current head received": "blue", + "get current head sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 55 + }, + "id": 31, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head broadcasted", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head sent", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head received", + "refId": "D" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head sent", + "refId": "E" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head broadcasted", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current head messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current branch received": "light-blue", + "current branch sent": "light-green", + "get current branch received": "blue", + "get current branch sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 55 + }, + "id": 32, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block header received": "light-blue", + "block header sent": "light-green", + "get block headers received": "blue", + "get block headers sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 55 + }, + "id": 33, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB block headers messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get predecessor header received": "blue", + "get predecessor header sent": "green", + "predecessor header received": "light-blue", + "predecessor header sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 55 + }, + "id": 34, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB predecessor header messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get operations received": "blue", + "get operations sent": "green", + "operation received": "light-blue", + "operation sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 63 + }, + "id": 35, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get ops for blocks received": "blue", + "get ops for blocks sent": "green", + "ops for block received": "light-blue", + "ops for block sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 63 + }, + "id": 36, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations for blocks messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocols received": "blue", + "get protocols sent": "green", + "protocol received": "light-blue", + "protocol sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 63 + }, + "id": 37, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocols messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocol branch received": "blue", + "get protocol branch sent": "green", + "protocol branch received": "light-blue", + "protocol branch sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 63 + }, + "id": 38, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocol branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "checkpoint received": "light-blue", + "checkpoint sent": "light-green", + "get checkpoint received": "blue", + "get checkpoint sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 63 + }, + "id": 39, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB checkpoint messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "deactivate received": "light-blue", + "deactivate sent": "light-green", + "get deactivate received": "blue", + "get deactivate sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 63 + }, + "id": 40, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB deactivate messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 72 + }, + "id": 41, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Workers stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator request count", + "refId": "A" + }, + { + "expr": "octez_validator_block_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator completion count", + "refId": "B" + }, + { + "expr": "octez_validator_block_worker_request_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator error count", + "refId": "C" + }, + { + "expr": "octez_validator_chain_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator request count", + "refId": "D" + }, + { + "expr": "octez_validator_chain_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator completion count", + "refId": "E" + }, + { + "expr": "octez_validator_chain_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator error count", + "refId": "F" + }, + { + "expr": "octez_mempool_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator request count", + "refId": "G" + }, + { + "expr": "octez_mempool_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator completion count", + "refId": "H" + }, + { + "expr": "octez_mempool_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator error count", + "refId": "I" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validation workers requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers table", + "refId": "A" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers scheduler", + "refId": "B" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations table", + "refId": "C" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations scheduler", + "refId": "D" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation table", + "refId": "E" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation scheduler", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB Workers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Treatments", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 88 + }, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Completion", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "connections": "light-green", + "fetching canceled knew unknown head": "light-orange", + "fetching canceled new branch": "green", + "fetching canceled new known valid head": "light-yellow", + "invalid blocks": "light-red", + "invalid locator": "light-orange", + "new branch completed": "light-blue", + "new head completed": "blue", + "on no request count": "white", + "system error": "orange", + "too short locator": "brown", + "unavailable protocol": "light-red", + "unknown ancestor": "yellow", + "unknown error": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "id": 46, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_peer_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "connections", + "refId": "A" + }, + { + "expr": "octez_validator_peer_invalid_block{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid blocks", + "refId": "B" + }, + { + "expr": "octez_validator_peer_invalid_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid locator", + "refId": "C" + }, + { + "expr": "octez_validator_peer_new_branch_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new branch completed", + "refId": "D" + }, + { + "expr": "octez_validator_peer_new_head_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new head completed", + "refId": "E" + }, + { + "expr": "octez_validator_peer_on_no_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "on no request count", + "refId": "F" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_branch{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new branch", + "refId": "G" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_known_valid_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new known valid head", + "refId": "H" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_unknown_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled knew unknown head", + "refId": "I" + }, + { + "expr": "octez_validator_peer_system_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "system error", + "refId": "J" + }, + { + "expr": "octez_validator_peer_too_short_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "too short locator", + "refId": "K" + }, + { + "expr": "octez_validator_peer_unavailable_protocol{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unavailable protocol", + "refId": "L" + }, + { + "expr": "octez_validator_peer_unknown_ancestor{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown ancestor", + "refId": "M" + }, + { + "expr": "octez_validator_peer_unknown_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown error", + "refId": "N" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Peer validators", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 16, + "w": 8, + "x": 0, + "y": 97 + }, + "id": 47, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "RPC", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC calls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/version.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/config.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/network.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/workers.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/injection.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/protocols.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/private.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 97 + }, + "id": 50, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total calls", + "refId": "A" + } + ], + "title": "Total of RPC calls", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 97 + }, + "id": 51, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(octez_rpc_calls_count{endpoint=~\".*\",app=\"$node_instance\"}[1h]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "calls rate", + "refId": "A" + } + ], + "title": "RPC calls rate per hour", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 105 + }, + "id": 52, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average duration", + "refId": "A" + } + ], + "title": "Average of calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 105 + }, + "id": 53, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"} / octez_rpc_calls_count{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max average duration", + "refId": "A" + } + ], + "title": "Max of average calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 105 + }, + "id": 54, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max total duration", + "refId": "A" + } + ], + "title": "Max of total calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 105 + }, + "id": 55, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total duration", + "refId": "A" + } + ], + "title": "Total of RPC calls duration", + "transparent": false, + "type": "stat" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 114 + }, + "id": 56, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Miscellaneous", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Forced major collections": "light-blue", + "Heap compactions": "light-red", + "Major collections": "light-yellow", + "Minor collections": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 57, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_minor_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Minor collections", + "refId": "A" + }, + { + "expr": "ocaml_gc_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major collections", + "refId": "B" + }, + { + "expr": "ocaml_gc_forced_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Forced major collections", + "refId": "C" + }, + { + "expr": "ocaml_gc_compactions{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Heap compactions", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC maintenance operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Major heap": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 58, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_heap_words{app=\"$node_instance\"}* 8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major heap", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC major word sizes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "5s", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_version,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez basic dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/octez-compact.json b/tezt/lib_cloud/grafana/dashboards/octez-compact.json new file mode 100644 index 000000000000..ea52382d6611 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/octez-compact.json @@ -0,0 +1,1941 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bootstrapping" + }, + "1": { + "color": "green", + "index": 0, + "text": "Bootstrapped" + }, + "null": { + "color": "yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_is_bootstrapped{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bootstrap status", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Unsync" + }, + "1": { + "color": "green", + "index": 1, + "text": "Sync" + }, + "2": { + "color": "red", + "index": 2, + "text": "Stuck" + }, + "null": { + "color": "yellow", + "index": 3, + "text": "Unknow" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 3, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_synchronisation_status{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sync status", + "transparent": false, + "type": "stat" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 0 + }, + "id": 4, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ chain_name }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Chain name", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 10, + "y": 0 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 13, + "y": 0 + }, + "id": 6, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_version{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Reflects the uptime of the monitoring of the job, not the uptime of the process.", + "format": "dtdhms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time()-(process_start_time_seconds{app=\"$node_instance\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "node uptime", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current head level", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current head level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [ ], + "datasource": "Prometheus", + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 9, + "links": [ ], + "styles": [ + { + "alias": "Time", + "pattern": "Time", + "type": "hidden" + } + ], + "targets": [ + { + "expr": "octez_validator_chain_head_cycle{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current cycle", + "refId": "A" + }, + { + "expr": "octez_store_caboose_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current caboose", + "refId": "B" + }, + { + "expr": "octez_store_checkpoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current checkpoint", + "refId": "C" + }, + { + "expr": "octez_store_savepoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current savepoint", + "refId": "D" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Chain levels", + "transform": "timeseries_to_rows", + "type": "table" + }, + { + "columns": [ ], + "datasource": "Prometheus", + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 10, + "links": [ ], + "styles": [ + { + "alias": "Time", + "pattern": "Time", + "type": "hidden" + } + ], + "targets": [ + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Incomming connections", + "refId": "A" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "B" + }, + { + "expr": "octez_p2p_connections_private{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Private points", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Connections", + "transform": "timeseries_to_rows", + "type": "table" + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 20 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 4, + "y": 3 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"0\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consensus", + "refId": "A" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"1\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Vote", + "refId": "B" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"2\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anonymous", + "refId": "C" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"3\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Manager", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Head level": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 6, + "x": 10, + "y": 11 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head level", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head level history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Validation time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 6, + "x": 4, + "y": 11 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validation time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block validation time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Gas consumed": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_consumed_gas{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Gas consumed", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Gas consumed history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Round": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Round", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Round history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Written block size": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 4, + "y": 20 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_written_block_size{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Written block size", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Last written block size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Alternate heads count": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 10, + "y": 20 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_alternate_heads_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alternate heads count", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Alternate heads count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 16, + "y": 0 + }, + "id": 19, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_store_invalid_blocks{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Invalid blocks mean", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 18, + "y": 0 + }, + "id": 20, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_branch_switch_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Branch switch count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 20, + "y": 5 + }, + "id": 21, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_store_last_merge_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Merge time", + "refId": "A" + } + ], + "title": "Store merge time", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 22, + "y": 0 + }, + "id": 22, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max round", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 16, + "y": 5 + }, + "id": 23, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "blocks validation mean", + "refId": "A" + } + ], + "title": "Blocks validation mean", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 18, + "y": 5 + }, + "id": 24, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Chain validator request completion", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 20, + "y": 5 + }, + "id": 25, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_peer_system_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "octez_validator_peer_too_short_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "B" + }, + { + "expr": "octez_validator_peer_unavailable_protocol{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "C" + }, + { + "expr": "octez_validator_peer_unknown_ancestor{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "D" + }, + { + "expr": "octez_validator_peer_unknown_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "E" + } + ], + "title": "Peer validators errors", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reducer": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 22, + "y": 5 + }, + "id": 26, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Incoming connections mean", + "transparent": false, + "type": "stat" + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_version,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez compact dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/octez-full.json b/tezt/lib_cloud/grafana/dashboards/octez-full.json new file mode 100644 index 000000000000..99793dc50bfc --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/octez-full.json @@ -0,0 +1,6287 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Node stats", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bootstrapping" + }, + "1": { + "color": "green", + "index": 0, + "text": "Bootstrapped" + }, + "null": { + "color": "yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 3, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_is_bootstrapped{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bootstrap status", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Unsync" + }, + "1": { + "color": "green", + "index": 1, + "text": "Sync" + }, + "2": { + "color": "red", + "index": 2, + "text": "Stuck" + }, + "null": { + "color": "yellow", + "index": 3, + "text": "Unknow" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 4, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_synchronisation_status{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sync status", + "transparent": false, + "type": "stat" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ chain_name }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Chain name", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 8, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 11, + "y": 1 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_version{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Reflects the uptime of the monitoring of the job, not the uptime of the process.", + "format": "dtdhms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time()-(process_start_time_seconds{app=\"$node_instance\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "node uptime", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 7 + }, + "id": 9, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current head level", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current head level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 10 + }, + "id": 10, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ p2p_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "P2p version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 10 + }, + "id": 11, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ distributed_db_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Distributed db version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "value" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 13 + }, + "id": 12, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_savepoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current savepoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current savepoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 13 + }, + "id": 13, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_checkpoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current checkpoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current checkpoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 16 + }, + "id": 14, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_caboose_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current caboose", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current caboose level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 16 + }, + "id": 15, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_cycle{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current cycle", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current cycle", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 19 + }, + "id": 16, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trusted points", + "refId": "A" + } + ], + "thresholds": "", + "title": "Trusted points", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 19 + }, + "id": 17, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_connections_private{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private Connections", + "refId": "A" + } + ], + "thresholds": "", + "title": "Private Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "aliasColors": { + "Head level": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 4, + "y": 4 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head level", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head level history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Validation time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 14 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validation time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block validation time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Loki", + "gridPos": { + "h": 21, + "w": 10, + "x": 14, + "y": 0 + }, + "id": 20, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node logs", + "type": "logs" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 22 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"0\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consensus", + "refId": "A" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"1\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Vote", + "refId": "B" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"2\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anonymous", + "refId": "C" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"3\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Manager", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Invalid blocks": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 22 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_invalid_blocks{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Invalid blocks", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Invalid blocks history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Gas consumed": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 30 + }, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_consumed_gas{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Gas consumed", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Gas consumed history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Round": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 30 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Round", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Round history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Merge time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 38 + }, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_merge_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Merge time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Store merge time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Written block size": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 38 + }, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_written_block_size{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Written block size", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Last written block size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 47 + }, + "id": 27, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Node Hardware stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Cpu load": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(netdata_app_cpu_utilization_percentage_average{app_group=\"octez\",app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cpu load", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Cpu actitvity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Memory usage": "light-green", + "Swap usage": "light-orange" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_app_mem_usage_MiB_average{app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory usage", + "refId": "A" + }, + { + "expr": "netdata_app_swap_usage_MiB_average{app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Swap usage", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "mbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "mbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "decbytes" + } + }, + "gridPos": { + "h": 8, + "w": 2, + "x": 0, + "y": 55 + }, + "id": 30, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "node_filesystem_free_bytes{mountpoint=\"/\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Available bytes on disk.", + "refId": "A" + } + ], + "title": "Disk free space", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 11, + "x": 2, + "y": 55 + }, + "id": 31, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_disk_space_GiB_average{chart=\"disk_space._\",dimension=\"used\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{dimension}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Storage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "reads": "light-green", + "writes": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 11, + "x": 13, + "y": 55 + }, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_app_disk_logical_io_KiB_persec_average{app_group=\"octez\",dimension=\"reads\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "reads", + "refId": "A" + }, + { + "expr": "netdata_app_disk_logical_io_KiB_persec_average{app_group=\"octez\",dimension=\"writes\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "writes", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "IOs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "kbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "kbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/.*received/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total[5m]) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bytes received", + "refId": "A" + }, + { + "expr": "irate(node_network_transmit_bytes_total[5m]) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bytes transmitted", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Network traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "All fds": "light-green", + "Files": "light-blue", + "Pipes": "light-orange", + "Sockets": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 34, + "legend": { + "alignAsTable": "true", + "avg": "true", + "current": "true", + "max": "true", + "min": "true", + "rightSide": "true", + "show": "true", + "sideWidth": null, + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(netdata_app_fds_open_fds_average{dimension=\"sockets\",app_group=\"octez\",app=\"$node_instance\"}) + sum(netdata_app_fds_open_fds_average{dimension=\"files\",app_group=\"octez\",app=\"$node_instance\"}) + sum(netdata_app_fds_open_fds_average{dimension=\"pipes\",app_group=\"octez\",app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "All fds", + "refId": "A" + }, + { + "expr": "netdata_app_fds_open_fds_average{dimension=\"sockets\",app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sockets", + "refId": "B" + }, + { + "expr": "netdata_app_fds_open_fds_average{dimension=\"files\",app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Files", + "refId": "C" + }, + { + "expr": "netdata_app_fds_open_fds_average{dimension=\"pipes\",app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pipes", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "File descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 72 + }, + "id": 35, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "P2P stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Branch delayed": "light-red", + "Branch refused": "light-orange", + "Outdated": "blue", + "Refused": "red", + "Unprocessed": "light-yellow", + "Validated": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_mempool_pending_validated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validated", + "refId": "A" + }, + { + "expr": "octez_mempool_pending_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Refused", + "refId": "B" + }, + { + "expr": "octez_mempool_pending_outdated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outdated", + "refId": "C" + }, + { + "expr": "octez_mempool_pending_branch_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch refused", + "refId": "D" + }, + { + "expr": "octez_mempool_pending_branch_delayed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch delayed", + "refId": "E" + }, + { + "expr": "octez_mempool_pending_unprocessed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unprocessed", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Mempool status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 37, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected peers": "light-yellow", + "Running peers": "light-green", + "Waiting to reconnect": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_peers_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected peers", + "refId": "A" + }, + { + "expr": "octez_p2p_peers_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running peers", + "refId": "B" + }, + { + "expr": "octez_p2p_points_greylisted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Waiting to reconnect", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P peers connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected points": "light-red", + "Running points": "light-green", + "Trusted points": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_points_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected points", + "refId": "A" + }, + { + "expr": "octez_p2p_points_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running points", + "refId": "B" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P points connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current head broadcasted": "light-green", + "current head received": "light-blue", + "current head sent": "light-green", + "get current head broadcasted": "green", + "get current head received": "blue", + "get current head sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 88 + }, + "id": 40, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head broadcasted", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head sent", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head received", + "refId": "D" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head sent", + "refId": "E" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head broadcasted", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current head messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current branch received": "light-blue", + "current branch sent": "light-green", + "get current branch received": "blue", + "get current branch sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 88 + }, + "id": 41, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block header received": "light-blue", + "block header sent": "light-green", + "get block headers received": "blue", + "get block headers sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 88 + }, + "id": 42, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB block headers messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get predecessor header received": "blue", + "get predecessor header sent": "green", + "predecessor header received": "light-blue", + "predecessor header sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 88 + }, + "id": 43, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB predecessor header messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get operations received": "blue", + "get operations sent": "green", + "operation received": "light-blue", + "operation sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 96 + }, + "id": 44, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get ops for blocks received": "blue", + "get ops for blocks sent": "green", + "ops for block received": "light-blue", + "ops for block sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 96 + }, + "id": 45, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations for blocks messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocols received": "blue", + "get protocols sent": "green", + "protocol received": "light-blue", + "protocol sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 96 + }, + "id": 46, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocols messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocol branch received": "blue", + "get protocol branch sent": "green", + "protocol branch received": "light-blue", + "protocol branch sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 96 + }, + "id": 47, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocol branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "checkpoint received": "light-blue", + "checkpoint sent": "light-green", + "get checkpoint received": "blue", + "get checkpoint sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 96 + }, + "id": 48, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB checkpoint messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "deactivate received": "light-blue", + "deactivate sent": "light-green", + "get deactivate received": "blue", + "get deactivate sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 96 + }, + "id": 49, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB deactivate messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 105 + }, + "id": 50, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Workers stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator request count", + "refId": "A" + }, + { + "expr": "octez_validator_block_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator completion count", + "refId": "B" + }, + { + "expr": "octez_validator_block_worker_request_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator error count", + "refId": "C" + }, + { + "expr": "octez_validator_chain_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator request count", + "refId": "D" + }, + { + "expr": "octez_validator_chain_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator completion count", + "refId": "E" + }, + { + "expr": "octez_validator_chain_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator error count", + "refId": "F" + }, + { + "expr": "octez_mempool_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator request count", + "refId": "G" + }, + { + "expr": "octez_mempool_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator completion count", + "refId": "H" + }, + { + "expr": "octez_mempool_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator error count", + "refId": "I" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validation workers requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 105 + }, + "id": 52, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers table", + "refId": "A" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers scheduler", + "refId": "B" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations table", + "refId": "C" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations scheduler", + "refId": "D" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation table", + "refId": "E" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation scheduler", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB Workers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 121 + }, + "id": 53, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Treatments", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 121 + }, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Completion", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "connections": "light-green", + "fetching canceled knew unknown head": "light-orange", + "fetching canceled new branch": "green", + "fetching canceled new known valid head": "light-yellow", + "invalid blocks": "light-red", + "invalid locator": "light-orange", + "new branch completed": "light-blue", + "new head completed": "blue", + "on no request count": "white", + "system error": "orange", + "too short locator": "brown", + "unavailable protocol": "light-red", + "unknown ancestor": "yellow", + "unknown error": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 129 + }, + "id": 55, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_peer_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "connections", + "refId": "A" + }, + { + "expr": "octez_validator_peer_invalid_block{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid blocks", + "refId": "B" + }, + { + "expr": "octez_validator_peer_invalid_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid locator", + "refId": "C" + }, + { + "expr": "octez_validator_peer_new_branch_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new branch completed", + "refId": "D" + }, + { + "expr": "octez_validator_peer_new_head_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new head completed", + "refId": "E" + }, + { + "expr": "octez_validator_peer_on_no_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "on no request count", + "refId": "F" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_branch{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new branch", + "refId": "G" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_known_valid_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new known valid head", + "refId": "H" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_unknown_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled knew unknown head", + "refId": "I" + }, + { + "expr": "octez_validator_peer_system_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "system error", + "refId": "J" + }, + { + "expr": "octez_validator_peer_too_short_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "too short locator", + "refId": "K" + }, + { + "expr": "octez_validator_peer_unavailable_protocol{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unavailable protocol", + "refId": "L" + }, + { + "expr": "octez_validator_peer_unknown_ancestor{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown ancestor", + "refId": "M" + }, + { + "expr": "octez_validator_peer_unknown_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown error", + "refId": "N" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Peer validators", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 16, + "w": 8, + "x": 0, + "y": 130 + }, + "id": 56, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "RPC", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 130 + }, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC calls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 138 + }, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/version.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/config.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/network.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/workers.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/injection.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/protocols.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/private.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 130 + }, + "id": 59, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total calls", + "refId": "A" + } + ], + "title": "Total of RPC calls", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 130 + }, + "id": 60, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(octez_rpc_calls_count{endpoint=~\".*\",app=\"$node_instance\"}[1h]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "calls rate", + "refId": "A" + } + ], + "title": "RPC calls rate per hour", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 138 + }, + "id": 61, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average duration", + "refId": "A" + } + ], + "title": "Average of calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 138 + }, + "id": 62, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"} / octez_rpc_calls_count{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max average duration", + "refId": "A" + } + ], + "title": "Max of average calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 138 + }, + "id": 63, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max total duration", + "refId": "A" + } + ], + "title": "Max of total calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 138 + }, + "id": 64, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total duration", + "refId": "A" + } + ], + "title": "Total of RPC calls duration", + "transparent": false, + "type": "stat" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 147 + }, + "id": 65, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Miscellaneous", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Forced major collections": "light-blue", + "Heap compactions": "light-red", + "Major collections": "light-yellow", + "Minor collections": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 147 + }, + "id": 66, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_minor_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Minor collections", + "refId": "A" + }, + { + "expr": "ocaml_gc_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major collections", + "refId": "B" + }, + { + "expr": "ocaml_gc_forced_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Forced major collections", + "refId": "C" + }, + { + "expr": "ocaml_gc_compactions{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Heap compactions", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC maintenance operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Major heap": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 147 + }, + "id": 67, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_heap_words{app=\"$node_instance\"}* 8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major heap", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC major word sizes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 156 + }, + "id": 68, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Delegates Hardware stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Cpu load accuser": "light-yellow", + "Cpu load baker": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 156 + }, + "id": 69, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_apps_cpu_percentage_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cpu load accuser", + "refId": "A" + }, + { + "expr": "netdata_apps_cpu_percentage_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cpu load baker", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Cpu actitvity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Memory usage accuser": "dark-yellow", + "Memory usage baker": "dark-red", + "Swap usage accuser": "light-yellow", + "Swap usage baker": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 156 + }, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_apps_mem_MiB_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory usage accuser", + "refId": "A" + }, + { + "expr": "netdata_apps_swap_MiB_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Swap usage accuser", + "refId": "B" + }, + { + "expr": "netdata_apps_mem_MiB_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory usage baker", + "refId": "C" + }, + { + "expr": "netdata_apps_swap_MiB_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Swap usage baker", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "mbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "mbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "reads accuser": "dark-yellow", + "reads baker": "dark-red", + "writes accuser": "light-yellow", + "writes baker": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 156 + }, + "id": 71, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_apps_lreads_KiB_persec_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "reads accuser", + "refId": "A" + }, + { + "expr": "netdata_apps_lwrites_KiB_persec_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "writes accuser", + "refId": "B" + }, + { + "expr": "netdata_apps_lreads_KiB_persec_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "reads baker", + "refId": "C" + }, + { + "expr": "netdata_apps_lweads_KiB_persec_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "writes baker", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "IOs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "kbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "kbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 165 + }, + "id": 72, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Logs", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 165 + }, + "id": 73, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 165 + }, + "id": 74, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Baker logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Baker logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 165 + }, + "id": 75, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Accuser logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Accuser logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 175 + }, + "id": 76, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"varlogs\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "System logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System logs", + "type": "logs" + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez", + "hardware", + "logs" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_version,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez full dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/octez-with-logs.json b/tezt/lib_cloud/grafana/dashboards/octez-with-logs.json new file mode 100644 index 000000000000..45684421a8b1 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/octez-with-logs.json @@ -0,0 +1,5309 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Node stats", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bootstrapping" + }, + "1": { + "color": "green", + "index": 0, + "text": "Bootstrapped" + }, + "null": { + "color": "yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 3, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_is_bootstrapped{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bootstrap status", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Unsync" + }, + "1": { + "color": "green", + "index": 1, + "text": "Sync" + }, + "2": { + "color": "red", + "index": 2, + "text": "Stuck" + }, + "null": { + "color": "yellow", + "index": 3, + "text": "Unknow" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 4, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_synchronisation_status{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sync status", + "transparent": false, + "type": "stat" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ chain_name }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Chain name", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 8, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 11, + "y": 1 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_version{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Reflects the uptime of the monitoring of the job, not the uptime of the process.", + "format": "dtdhms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time()-(process_start_time_seconds{app=\"$node_instance\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "node uptime", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 7 + }, + "id": 9, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current head level", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current head level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 10 + }, + "id": 10, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ p2p_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "P2p version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 10 + }, + "id": 11, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ distributed_db_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Distributed db version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "value" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 13 + }, + "id": 12, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_savepoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current savepoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current savepoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 13 + }, + "id": 13, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_checkpoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current checkpoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current checkpoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 16 + }, + "id": 14, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_caboose_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current caboose", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current caboose level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 16 + }, + "id": 15, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_cycle{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current cycle", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current cycle", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 19 + }, + "id": 16, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trusted points", + "refId": "A" + } + ], + "thresholds": "", + "title": "Trusted points", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 19 + }, + "id": 17, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_connections_private{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private Connections", + "refId": "A" + } + ], + "thresholds": "", + "title": "Private Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "aliasColors": { + "Head level": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 4, + "y": 4 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head level", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head level history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Validation time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 14 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validation time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block validation time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Loki", + "gridPos": { + "h": 21, + "w": 10, + "x": 14, + "y": 0 + }, + "id": 20, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node logs", + "type": "logs" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 22 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"0\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consensus", + "refId": "A" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"1\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Vote", + "refId": "B" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"2\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anonymous", + "refId": "C" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"3\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Manager", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Invalid blocks": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 22 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_invalid_blocks{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Invalid blocks", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Invalid blocks history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Gas consumed": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 30 + }, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_consumed_gas{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Gas consumed", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Gas consumed history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Round": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 30 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Round", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Round history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Merge time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 38 + }, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_merge_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Merge time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Store merge time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Written block size": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 38 + }, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_written_block_size{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Written block size", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Last written block size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 47 + }, + "id": 27, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "P2P stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Branch delayed": "light-red", + "Branch refused": "light-orange", + "Outdated": "blue", + "Refused": "red", + "Unprocessed": "light-yellow", + "Validated": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_mempool_pending_validated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validated", + "refId": "A" + }, + { + "expr": "octez_mempool_pending_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Refused", + "refId": "B" + }, + { + "expr": "octez_mempool_pending_outdated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outdated", + "refId": "C" + }, + { + "expr": "octez_mempool_pending_branch_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch refused", + "refId": "D" + }, + { + "expr": "octez_mempool_pending_branch_delayed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch delayed", + "refId": "E" + }, + { + "expr": "octez_mempool_pending_unprocessed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unprocessed", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Mempool status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected peers": "light-yellow", + "Running peers": "light-green", + "Waiting to reconnect": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_peers_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected peers", + "refId": "A" + }, + { + "expr": "octez_p2p_peers_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running peers", + "refId": "B" + }, + { + "expr": "octez_p2p_points_greylisted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Waiting to reconnect", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P peers connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected points": "light-red", + "Running points": "light-green", + "Trusted points": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 31, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_points_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected points", + "refId": "A" + }, + { + "expr": "octez_p2p_points_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running points", + "refId": "B" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P points connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current head broadcasted": "light-green", + "current head received": "light-blue", + "current head sent": "light-green", + "get current head broadcasted": "green", + "get current head received": "blue", + "get current head sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 63 + }, + "id": 32, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head broadcasted", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head sent", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head received", + "refId": "D" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head sent", + "refId": "E" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head broadcasted", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current head messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current branch received": "light-blue", + "current branch sent": "light-green", + "get current branch received": "blue", + "get current branch sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 63 + }, + "id": 33, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block header received": "light-blue", + "block header sent": "light-green", + "get block headers received": "blue", + "get block headers sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 63 + }, + "id": 34, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB block headers messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get predecessor header received": "blue", + "get predecessor header sent": "green", + "predecessor header received": "light-blue", + "predecessor header sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 63 + }, + "id": 35, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB predecessor header messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get operations received": "blue", + "get operations sent": "green", + "operation received": "light-blue", + "operation sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 71 + }, + "id": 36, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get ops for blocks received": "blue", + "get ops for blocks sent": "green", + "ops for block received": "light-blue", + "ops for block sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 71 + }, + "id": 37, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations for blocks messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocols received": "blue", + "get protocols sent": "green", + "protocol received": "light-blue", + "protocol sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 71 + }, + "id": 38, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocols messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocol branch received": "blue", + "get protocol branch sent": "green", + "protocol branch received": "light-blue", + "protocol branch sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 71 + }, + "id": 39, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocol branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "checkpoint received": "light-blue", + "checkpoint sent": "light-green", + "get checkpoint received": "blue", + "get checkpoint sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 71 + }, + "id": 40, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB checkpoint messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "deactivate received": "light-blue", + "deactivate sent": "light-green", + "get deactivate received": "blue", + "get deactivate sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 71 + }, + "id": 41, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB deactivate messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 80 + }, + "id": 42, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Workers stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator request count", + "refId": "A" + }, + { + "expr": "octez_validator_block_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator completion count", + "refId": "B" + }, + { + "expr": "octez_validator_block_worker_request_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator error count", + "refId": "C" + }, + { + "expr": "octez_validator_chain_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator request count", + "refId": "D" + }, + { + "expr": "octez_validator_chain_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator completion count", + "refId": "E" + }, + { + "expr": "octez_validator_chain_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator error count", + "refId": "F" + }, + { + "expr": "octez_mempool_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator request count", + "refId": "G" + }, + { + "expr": "octez_mempool_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator completion count", + "refId": "H" + }, + { + "expr": "octez_mempool_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator error count", + "refId": "I" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validation workers requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers table", + "refId": "A" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers scheduler", + "refId": "B" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations table", + "refId": "C" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations scheduler", + "refId": "D" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation table", + "refId": "E" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation scheduler", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB Workers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Treatments", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 96 + }, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Completion", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "connections": "light-green", + "fetching canceled knew unknown head": "light-orange", + "fetching canceled new branch": "green", + "fetching canceled new known valid head": "light-yellow", + "invalid blocks": "light-red", + "invalid locator": "light-orange", + "new branch completed": "light-blue", + "new head completed": "blue", + "on no request count": "white", + "system error": "orange", + "too short locator": "brown", + "unavailable protocol": "light-red", + "unknown ancestor": "yellow", + "unknown error": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 47, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_peer_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "connections", + "refId": "A" + }, + { + "expr": "octez_validator_peer_invalid_block{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid blocks", + "refId": "B" + }, + { + "expr": "octez_validator_peer_invalid_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid locator", + "refId": "C" + }, + { + "expr": "octez_validator_peer_new_branch_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new branch completed", + "refId": "D" + }, + { + "expr": "octez_validator_peer_new_head_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new head completed", + "refId": "E" + }, + { + "expr": "octez_validator_peer_on_no_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "on no request count", + "refId": "F" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_branch{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new branch", + "refId": "G" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_known_valid_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new known valid head", + "refId": "H" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_unknown_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled knew unknown head", + "refId": "I" + }, + { + "expr": "octez_validator_peer_system_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "system error", + "refId": "J" + }, + { + "expr": "octez_validator_peer_too_short_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "too short locator", + "refId": "K" + }, + { + "expr": "octez_validator_peer_unavailable_protocol{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unavailable protocol", + "refId": "L" + }, + { + "expr": "octez_validator_peer_unknown_ancestor{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown ancestor", + "refId": "M" + }, + { + "expr": "octez_validator_peer_unknown_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown error", + "refId": "N" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Peer validators", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 16, + "w": 8, + "x": 0, + "y": 105 + }, + "id": 48, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "RPC", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC calls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 113 + }, + "id": 50, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/version.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/config.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/network.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/workers.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/injection.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/protocols.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/private.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 105 + }, + "id": 51, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total calls", + "refId": "A" + } + ], + "title": "Total of RPC calls", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 105 + }, + "id": 52, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(octez_rpc_calls_count{endpoint=~\".*\",app=\"$node_instance\"}[1h]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "calls rate", + "refId": "A" + } + ], + "title": "RPC calls rate per hour", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 113 + }, + "id": 53, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average duration", + "refId": "A" + } + ], + "title": "Average of calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 113 + }, + "id": 54, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"} / octez_rpc_calls_count{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max average duration", + "refId": "A" + } + ], + "title": "Max of average calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 113 + }, + "id": 55, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max total duration", + "refId": "A" + } + ], + "title": "Max of total calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 113 + }, + "id": 56, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total duration", + "refId": "A" + } + ], + "title": "Total of RPC calls duration", + "transparent": false, + "type": "stat" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 122 + }, + "id": 57, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Miscellaneous", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Forced major collections": "light-blue", + "Heap compactions": "light-red", + "Major collections": "light-yellow", + "Minor collections": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 122 + }, + "id": 58, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_minor_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Minor collections", + "refId": "A" + }, + { + "expr": "ocaml_gc_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major collections", + "refId": "B" + }, + { + "expr": "ocaml_gc_forced_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Forced major collections", + "refId": "C" + }, + { + "expr": "ocaml_gc_compactions{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Heap compactions", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC maintenance operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Major heap": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 122 + }, + "id": 59, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_heap_words{app=\"$node_instance\"}* 8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major heap", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC major word sizes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 131 + }, + "id": 60, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Logs", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 131 + }, + "id": 61, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 131 + }, + "id": 62, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Baker logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Baker logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 131 + }, + "id": 63, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Accuser logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Accuser logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 141 + }, + "id": 64, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"varlogs\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "System logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System logs", + "type": "logs" + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez", + "logs" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_version,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez with logs dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/rollup.json b/tezt/lib_cloud/grafana/dashboards/rollup.json new file mode 100644 index 000000000000..dfbbcb6e01ab --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/rollup.json @@ -0,0 +1,1213 @@ +{ + "__inputs": [], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Version info", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 3, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 4, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_date }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Git Commit Date", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_sc_rollup_node_node_info{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Git Commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 12, + "w": 17, + "x": 0, + "y": 5 + }, + "id": 6, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Generic information", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ genesis_level }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Genesis Level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ pvm_kind }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Pvm Kind", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 9, + "x": 8, + "y": 5 + }, + "id": 9, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ genesis_hash }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Genesis Hash", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 9, + "x": 8, + "y": 7 + }, + "id": 10, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ rollup_address }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Rollup Address", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 12, + "w": 17, + "x": 0, + "y": 17 + }, + "id": 11, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Inbox", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_head_inbox_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Head Inbox Level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "rate(octez_sc_rollup_node_head_inbox_internal_messages_number{app=\"$node_instance\"}[15m])+ rate(octez_sc_rollup_node_head_inbox_external_messages_number{app=\"$node_instance\"}[15m])", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Genesis Level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 17 + }, + "id": 14, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "rate(octez_sc_rollup_node_head_inbox_internal_messages_number{app=\"$node_instance\"}[15m])", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Internal Messages Rate", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 17 + }, + "id": 15, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "rate(octez_sc_rollup_node_head_inbox_internal_messages_number{app=\"$node_instance\"}[15m])", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "External Messages Rate", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + "External Messages": "yellow", + "Internal messages": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 20 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_sc_rollup_node_head_inbox_internal_messages_number{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Internal messages", + "refId": "A" + }, + { + "expr": "octez_sc_rollup_node_head_inbox_external_messages_number{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "External Messages", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Inbox Messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 29 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_sc_rollup_node_head_inbox_process_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time (in s)", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head process time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez", + "rollups" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_sc_rollup_node_node_info,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez Rollup Node", + "version": 0 +} -- GitLab From 452ed8d422a173c27b782b32f44fc2366fc07497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Fri, 5 Jul 2024 15:01:08 +0200 Subject: [PATCH 05/20] Tezt/Performance: Add a new `update_dashboard_from_json` function --- tezt/lib_performance_regression/grafana.ml | 18 +++++++++++------- tezt/lib_performance_regression/grafana.mli | 5 +++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tezt/lib_performance_regression/grafana.ml b/tezt/lib_performance_regression/grafana.ml index 0aad67d7b369..c4d6a69dcd2c 100644 --- a/tezt/lib_performance_regression/grafana.ml +++ b/tezt/lib_performance_regression/grafana.ml @@ -246,12 +246,12 @@ let handle_http_error resp_status resp_body request = body req -let update_dashboard config dashboard = - if dashboard.uid =~! uid_rex then +let update_dashboard_from_json config ~json ~uid = + if uid =~! uid_rex then invalid_arg (sf "Grafana.update_dashboard: invalid UID: %s (must match: %s)" - dashboard.uid + uid (show_rex uid_rex)) ; let authorization = Option.map (fun t -> ("Authorization", "Bearer " ^ t)) config.api_token @@ -261,7 +261,7 @@ let update_dashboard config dashboard = let* () = let delete_request = { - uri = make_url config ("dashboards/uid/" ^ dashboard.uid); + uri = make_url config ("dashboards/uid/" ^ uid); meth = `DELETE; headers = Cohttp.Header.of_list authorization; body = None; @@ -274,9 +274,7 @@ let update_dashboard config dashboard = | status -> handle_http_error status body delete_request in (* (Re-)create dashboard. *) - let body = - `O [("dashboard", encode_dashboard config dashboard)] |> JSON.encode_u - in + let body = json in let create_request = { uri = make_url config "dashboards/db"; @@ -292,6 +290,12 @@ let update_dashboard config dashboard = | #Cohttp.Code.success_status -> Cohttp_lwt.Body.drain_body body | status -> handle_http_error status body create_request +let update_dashboard config dashboard = + let json = + `O [("dashboard", encode_dashboard config dashboard)] |> JSON.encode_u + in + update_dashboard_from_json config ~json ~uid:dashboard.uid + let where_clause_of_tag (tag_name, tag_label) = InfluxDB.Tag (tag_name, EQ, tag_label) diff --git a/tezt/lib_performance_regression/grafana.mli b/tezt/lib_performance_regression/grafana.mli index d637e35f0704..180e44f5ac0e 100644 --- a/tezt/lib_performance_regression/grafana.mli +++ b/tezt/lib_performance_regression/grafana.mli @@ -120,6 +120,11 @@ type dashboard = { @raise Invalid_arg if the dashboard UID is invalid. *) val update_dashboard : config -> dashboard -> unit Lwt.t +(** Similar to update_dashboard, except it is provided as a raw JSON + represented as a string. *) +val update_dashboard_from_json : + config -> json:string -> uid:string -> unit Lwt.t + (** Make a simple SELECT query for a graph panel. Usage: [simple_query -- GitLab From 940bb755e4d2e22ef4e94fba7a3cdaa3678ab2ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Fri, 5 Jul 2024 17:01:00 +0200 Subject: [PATCH 06/20] Tezt/Performance: Update API paths --- tezt/lib_performance_regression/grafana.ml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tezt/lib_performance_regression/grafana.ml b/tezt/lib_performance_regression/grafana.ml index c4d6a69dcd2c..f05f4d460830 100644 --- a/tezt/lib_performance_regression/grafana.ml +++ b/tezt/lib_performance_regression/grafana.ml @@ -261,7 +261,7 @@ let update_dashboard_from_json config ~json ~uid = let* () = let delete_request = { - uri = make_url config ("dashboards/uid/" ^ uid); + uri = make_url config ("api/dashboards/uid/" ^ uid); meth = `DELETE; headers = Cohttp.Header.of_list authorization; body = None; @@ -277,7 +277,7 @@ let update_dashboard_from_json config ~json ~uid = let body = json in let create_request = { - uri = make_url config "dashboards/db"; + uri = make_url config "api/dashboards/db"; meth = `POST; headers = Cohttp.Header.of_list -- GitLab From 08fe57fe6c42b9d2b37868111f3d94c42843ce7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Fri, 5 Jul 2024 17:01:11 +0200 Subject: [PATCH 07/20] Tezt/Cloud: Make grafana working --- tezt/lib_cloud/grafana.ml | 93 ++++++++++++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 20 deletions(-) diff --git a/tezt/lib_cloud/grafana.ml b/tezt/lib_cloud/grafana.ml index 044e4cab25e6..248e05e58c40 100644 --- a/tezt/lib_cloud/grafana.ml +++ b/tezt/lib_cloud/grafana.ml @@ -8,17 +8,51 @@ include Tezt_tezos_tezt_performance_regression.Grafana type t = { - provisioning_file : string; + provisioning_directory : string; dashboard_directory : string; password : string; } let generate_password () = "saucisse" -let provisioning_file () = +let generate_admin_api_key password = + let cmd = "curl" in + let args = + [ + "-X"; + "POST"; + "-H"; + "Content-Type: application/json"; + "-d"; + "\n\ + \ {\n\ + \ \"name\": \"admin_api_key\",\n\ + \ \"role\": \"Admin\"\n\ + \ }"; + "-u"; + Format.asprintf "admin:%s" password; + "http://localhost:3000/api/auth/keys"; + ] + in + let* output = Process.run_and_read_stdout cmd args in + let json = JSON.parse ~origin:"Grafana.generate_admin_api_key" output in + let key = JSON.(json |-> "key" |> as_string) in + Lwt.return key + +let configuration admin_api_key : config = + { + url = Uri.of_string "http://localhost:3000"; + api_token = Some admin_api_key; + data_source = "Prometheus"; + timeout = 2.0; + } + +let provisioning_directory () = + let provisioning_directory = + Filename.get_temp_dir_name () // "grafana" // "provisioning" + in let provisioning_file = - Filename.get_temp_dir_name () - // "grafana" // "provisioning" // "provisioning.yml" + provisioning_directory // "datasources" // "datasource.yml" in let* () = Process.run "mkdir" ["-p"; provisioning_file |> Filename.dirname] in let content = @@ -29,23 +63,14 @@ datasources: - name: Prometheus type: prometheus access: proxy - url: http://prometheus:9090 + url: http://localhost:9090 isDefault: true - -providers: - - name: 'default' - orgId: 1 - folder: '' - type: file - disableDeletion: true - options: - path: /var/lib/grafana/dashboards |} in with_open_out provisioning_file (fun oc -> Stdlib.seek_out oc 0 ; output_string oc content) ; - Lwt.return provisioning_file + Lwt.return provisioning_directory let shutdown _t = Process.run "docker" ["kill"; "grafana"] @@ -60,7 +85,7 @@ let run () = let* () = Process.run "mkdir" ["-p"; dashboard_directory |> Filename.dirname] in - let* provisioning_file = provisioning_file () in + let* provisioning_directory = provisioning_directory () in (* We generate a password to use admin features. This is not completely secured but this should prevent easy attacks if the grafana port is opened. *) let password = generate_password () in @@ -83,9 +108,7 @@ let run () = "-e"; Format.asprintf "GF_SECURITY_ADMIN_PASSWORD=%s" password; "-v"; - Format.asprintf - "%s:/etc/grafana/provisioning" - (Filename.dirname provisioning_file); + Format.asprintf "%s:/etc/grafana/provisioning" provisioning_directory; "-v"; Format.asprintf "%s:/var/lib/grafana/dashboards" dashboard_directory; "grafana/grafana"; @@ -114,4 +137,34 @@ let run () = ] in let* _ = Env.wait_process ~is_ready ~run () in - Lwt.return {provisioning_file; dashboard_directory; password} + let* admin_api_key = generate_admin_api_key password in + let configuration = configuration admin_api_key in + let basenames = + [ + "octez-basic"; + "dal-basic"; + "octez-compact"; + "octez-full"; + "octez-with-logs"; + "evm-node"; + "rollup"; + ] + in + let dashboard basename = + read_file + (Format.asprintf "./tezt/lib_cloud/grafana/dashboards/%s.json" basename) + |> Format.asprintf "{\"dashboard\": %s, \"overwrite\": true}" + in + let rec loop = function + | [] -> Lwt.return_unit + | name :: l -> + let* () = + update_dashboard_from_json + configuration + ~json:(dashboard name) + ~uid:name + in + loop l + in + let* () = loop basenames in + Lwt.return {provisioning_directory; dashboard_directory; password} -- GitLab From 1d8df60b1aa5e751c15b21018112d4f44c0c22a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sat, 6 Jul 2024 15:42:54 +0200 Subject: [PATCH 08/20] Tezt/Cloud: Cli option for the proxy mode --- tezt/lib_cloud/cli.ml | 12 +++++++++++- tezt/lib_cloud/cli.mli | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tezt/lib_cloud/cli.ml b/tezt/lib_cloud/cli.ml index a67b1e7a87fb..83e23043c8bf 100644 --- a/tezt/lib_cloud/cli.ml +++ b/tezt/lib_cloud/cli.ml @@ -154,6 +154,14 @@ let dockerfile_alias = the environment variable `TEZT_CLOUD`)" () +let proxy = + Clap.flag + ~section + ~set_long:"proxy" + ~description: + "Enables to run the orchestrator on a VM instead of the host machine" + false + let max_run_duration = Clap.default_int ~section @@ -168,4 +176,6 @@ let no_max_run_duration = ~section ~set_long:"no-max-run-duration" ~description:"Ensure the VM can only be destroyed manually." - false + (* If the proxy mode is active, we don't want to use [max_run_duration] + since it aims to run long running tests. *) + proxy diff --git a/tezt/lib_cloud/cli.mli b/tezt/lib_cloud/cli.mli index f4b5a7619643..edef459c7a87 100644 --- a/tezt/lib_cloud/cli.mli +++ b/tezt/lib_cloud/cli.mli @@ -82,3 +82,7 @@ val max_run_duration : int (** Deactivate the max run duration parameter. *) val no_max_run_duration : bool + +(** When [proxy] is [true], the scenario is run via a VM instead of the host + machine. *) +val proxy : bool -- GitLab From 55e99b27345753db998e9bafd8dc9984262d9207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sat, 6 Jul 2024 15:43:57 +0200 Subject: [PATCH 09/20] Tezt/Cloud: Open ports on the VM for proxy mode --- tezt/lib_cloud/terraform/vm/main.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tezt/lib_cloud/terraform/vm/main.tf b/tezt/lib_cloud/terraform/vm/main.tf index 4ff8b0fa328d..700b33a1072c 100644 --- a/tezt/lib_cloud/terraform/vm/main.tf +++ b/tezt/lib_cloud/terraform/vm/main.tf @@ -161,6 +161,24 @@ resource "google_compute_firewall" "default" { ports = ["19999"] } + # Rule to enable static page web access + allow { + protocol = "tcp" + ports = ["8080"] + } + + # Rule to enable prometheus access + allow { + protocol = "tcp" + ports = ["9090"] + } + + # Rule to enable grafana access + allow { + protocol = "tcp" + ports = ["3000"] + } + # Anybody can contact the machine on the open ports. source_ranges = ["0.0.0.0/0"] } -- GitLab From df21860f08e39c06202c6f33f3e0f2ad8c928496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sat, 6 Jul 2024 16:06:52 +0200 Subject: [PATCH 10/20] Tezt/Cloud: Update prometheus directory --- tezt/lib_cloud/prometheus.ml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tezt/lib_cloud/prometheus.ml b/tezt/lib_cloud/prometheus.ml index 6c02940ce971..27e78423209f 100644 --- a/tezt/lib_cloud/prometheus.ml +++ b/tezt/lib_cloud/prometheus.ml @@ -96,7 +96,14 @@ let start agents = if Env.monitoring then [tezt_source; netdata_source_of_agents agents] else [tezt_source] in - let configuration_file = Temp.file "prometheus.yml" in + let* () = + Process.run "mkdir" ["-p"; Filename.get_temp_dir_name () // "prometheus"] + in + (* We do not use the Temp.dir so that the base directory is predictable and + can be mounted by the proxy VM if [--proxy] is used. *) + let configuration_file = + Filename.get_temp_dir_name () // "prometheus" // "prometheus.yml" + in let snapshot_filename = Env.prometheus_snapshot_filename in let port = Env.prometheus_port in let scrape_interval = Env.prometheus_scrape_interval in -- GitLab From 2064ebaccb0eb7d30bb06449bc2a614376e572a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sat, 6 Jul 2024 16:05:34 +0200 Subject: [PATCH 11/20] Tezt/Cloud: Mount volumes for the proxy VM --- tezt/lib_cloud/terraform/vm/main.tf | 58 ++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/tezt/lib_cloud/terraform/vm/main.tf b/tezt/lib_cloud/terraform/vm/main.tf index 700b33a1072c..e0be5308111d 100644 --- a/tezt/lib_cloud/terraform/vm/main.tf +++ b/tezt/lib_cloud/terraform/vm/main.tf @@ -99,7 +99,63 @@ module "gce-container" { source = "terraform-google-modules/container-vm/google" version = "~> 3.0" - container = { image = "${var.docker_image}" } + container = { image = "${var.docker_image}" + + # Volume settings is only necessary for the proxy VM + volumeMounts = [ + { + # Using the proxy mode, this is necessary if the docker image runs another docker image + mountPath = "/var/run/docker.sock" + name = "docker-socket" + readOnly = false + }, + { + # Necessary to provide access from the image docker to the website + mountPath = "/tmp/website" + name = "website" + readOnly = false + }, + { + # Same for Prometheus + mountPath = "/tmp/prometheus" + name = "prometheus" + readOnly = false + }, + { + # Same for Grafana + mountPath = "/tmp/grafana" + name = "grafana" + readOnly = false + } + ] + } + + volumes = [ + { + name = "docker-socket" + hostPath = { + path = "/var/run/docker.sock" + } + }, + { + name = "website" + hostPath = { + path = "/tmp/website" + } + }, + { + name = "prometheus" + hostPath = { + path = "/tmp/prometheus" + } + }, + { + name = "grafana" + hostPath = { + path = "/tmp/grafana" + } + } + ] } # When running a VM, it must be associated with a Virtual Private -- GitLab From bfe57665ffcb1cb62d711e2c57692e21bc4f252a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sat, 6 Jul 2024 16:00:36 +0200 Subject: [PATCH 12/20] Tezt/Cloud: Update canonical dockerfile --- tezt/lib_cloud/dockerfiles/saroupille.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile b/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile index 3cb70ced4791..adec8fab32da 100644 --- a/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile +++ b/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile @@ -11,6 +11,8 @@ RUN apt-get update && apt-get install -y \ openssh-server \ # Tezos dependencies libgmp-dev curl libev-dev libhidapi-dev \ + # With the proxy mode we may want to run docker inside docker + docker.io \ # Necessary certificates for mirages dependencies ca-certificates \ # DL3015: Use --no-install-recommends -- GitLab From 3144d32d100037038b192e768e4450f5f8034013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sat, 6 Jul 2024 16:10:48 +0200 Subject: [PATCH 13/20] Tezt/Cloud: Add path to self --- tezt/lib_cloud/path.ml | 2 ++ tezt/lib_cloud/path.mli | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tezt/lib_cloud/path.ml b/tezt/lib_cloud/path.ml index 71ec07ffc906..9b0923177482 100644 --- a/tezt/lib_cloud/path.ml +++ b/tezt/lib_cloud/path.ml @@ -27,3 +27,5 @@ let zcash_params = "_opam" // "share" // "zcash-params" let dal_trusted_setup = "_opam" // "share" // "dal-trusted-setup" let default_binaries_path () = Filename.get_temp_dir_name () // "tezt-runners" + +let self = Sys.argv.(0) diff --git a/tezt/lib_cloud/path.mli b/tezt/lib_cloud/path.mli index 1280864eff9a..8c901fabeeb2 100644 --- a/tezt/lib_cloud/path.mli +++ b/tezt/lib_cloud/path.mli @@ -30,3 +30,6 @@ val dal_trusted_setup : string (** Default path where images can be found on the docker container. *) val default_binaries_path : unit -> string + +(** Path to the current binary. *) +val self : string -- GitLab From 7db95a4b856b8042280ba1363e304aa704f733f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sat, 6 Jul 2024 21:32:59 +0200 Subject: [PATCH 14/20] Tezt/Cloud: Add data-encoding as a dependency --- manifest/product_octez.ml | 1 + tezt/lib_cloud/dune | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index e868e2aa2948..7e1d0c23bddf 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -2326,6 +2326,7 @@ let tezt_cloud = [ tezt_lib |> open_ |> open_ ~m:"Base"; tezt_performance_regression |> open_; + data_encoding; ] let tezt_tezos = diff --git a/tezt/lib_cloud/dune b/tezt/lib_cloud/dune index 667fd94082ef..ecc8d09f296a 100644 --- a/tezt/lib_cloud/dune +++ b/tezt/lib_cloud/dune @@ -6,7 +6,8 @@ (public_name tezt-tezos.tezt-cloud) (libraries tezt - tezt-tezos.tezt-performance-regression) + tezt-tezos.tezt-performance-regression + octez-libs.data-encoding) (flags (:standard) -open Tezt -- GitLab From b450b4cd5c55446f697b57b11e08c5374aaf6dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sat, 6 Jul 2024 22:00:21 +0200 Subject: [PATCH 15/20] Tezt/Cloud: Define an encoding for an agent --- tezt/lib_cloud/agent.ml | 98 ++++++++++++++++++++++++++++++++++++++++ tezt/lib_cloud/agent.mli | 3 ++ 2 files changed, 101 insertions(+) diff --git a/tezt/lib_cloud/agent.ml b/tezt/lib_cloud/agent.ml index a7727f7280d0..d069edb68729 100644 --- a/tezt/lib_cloud/agent.ml +++ b/tezt/lib_cloud/agent.ml @@ -15,6 +15,104 @@ type t = { configuration : Configuration.t; } +let docker_image_encoding = + let open Data_encoding in + union + [ + case + ~title:"gcp" + Json_only + string + (function Env.Gcp {alias} -> Some alias | _ -> None) + (fun alias -> Gcp {alias}); + case + ~title:"octez_latest_release" + Json_only + (constant "octez-latest-release") + (function Env.Octez_latest_release -> Some () | _ -> None) + (fun () -> Octez_latest_release); + ] + +let configuration_encoding = + let open Data_encoding in + let open Configuration in + conv + (fun {machine_type; binaries_path; docker_image; max_run_duration = _} -> + (machine_type, binaries_path, docker_image)) + (fun (machine_type, binaries_path, docker_image) -> + Configuration.make ~machine_type ~binaries_path ~docker_image ()) + (obj3 + (req "machine_type" Data_encoding.string) + (req "binaries_path" Data_encoding.string) + (req "docker_image" docker_image_encoding)) + +let cmd_wrapper_encoding = + let open Data_encoding in + conv + (fun Gcloud.{cmd; args} -> (cmd, args)) + (fun (cmd, args) -> {cmd; args}) + (obj2 + (req "cmd" Data_encoding.string) + (req "args" (Data_encoding.list Data_encoding.string))) + +let encoding = + let open Data_encoding in + conv + (fun { + name; + vm_name; + cmd_wrapper; + point; + runner; + next_available_port; + configuration; + } -> + let ssh_id = runner.Runner.ssh_id |> Option.get in + ( name, + vm_name, + cmd_wrapper, + point, + ssh_id, + next_available_port (), + configuration )) + (fun ( name, + vm_name, + cmd_wrapper, + point, + ssh_id, + next_available_port, + configuration ) -> + let ssh_port = snd point in + let address = fst point in + let runner = + Runner.create ~ssh_user:"root" ~ssh_id ~ssh_port ~address () + in + let next_available_port = + let current_port = ref (next_available_port - 1) in + fun () -> + incr current_port ; + !current_port + in + { + name; + vm_name; + cmd_wrapper; + point; + runner; + next_available_port; + configuration; + }) + (obj7 + (req "name" Data_encoding.string) + (req "vm_name" Data_encoding.string) + (req "cmd_wrapper" (Data_encoding.option cmd_wrapper_encoding)) + (req + "point" + (Data_encoding.tup2 Data_encoding.string Data_encoding.int31)) + (req "ssh_id" Data_encoding.string) + (req "next_available_port" Data_encoding.int31) + (req "configuration" configuration_encoding)) + let make ?cmd_wrapper ~ssh_id ~point:((address, ssh_port) as point) ~configuration ~next_available_port ~name () = let ssh_user = "root" in diff --git a/tezt/lib_cloud/agent.mli b/tezt/lib_cloud/agent.mli index 438679a90c12..5bf2225ed18f 100644 --- a/tezt/lib_cloud/agent.mli +++ b/tezt/lib_cloud/agent.mli @@ -26,6 +26,9 @@ val make : unit -> t +(** Encode an agent configuration. *) +val encoding : t Data_encoding.t + (** [name agent] returns the name of the agent. *) val name : t -> string -- GitLab From 1e10d1b6b837e37cacedd349fb4a49acbd05fbeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sun, 14 Jul 2024 16:01:00 +0200 Subject: [PATCH 16/20] Tezt/Cloud: Implement the proxy mode --- tezt/lib_cloud/agent.ml | 139 ++++--- tezt/lib_cloud/agent.mli | 15 +- tezt/lib_cloud/cli.ml | 44 +- tezt/lib_cloud/cli.mli | 2 + tezt/lib_cloud/cloud.ml | 392 +++++++++++++++--- tezt/lib_cloud/cloud.mli | 1 + tezt/lib_cloud/deployement.ml | 35 +- tezt/lib_cloud/deployement.mli | 2 + .../dockerfiles/saroupille.Dockerfile | 2 +- tezt/lib_cloud/env.ml | 55 +-- tezt/lib_cloud/env.mli | 19 +- tezt/lib_cloud/grafana.ml | 34 +- tezt/lib_cloud/grafana.mli | 2 + tezt/lib_cloud/jobs.ml | 33 +- tezt/lib_cloud/path.ml | 6 + tezt/lib_cloud/path.mli | 6 + tezt/lib_cloud/proxy.ml | 102 +++++ tezt/lib_cloud/proxy.mli | 19 + tezt/lib_cloud/ssh.ml | 2 +- tezt/lib_cloud/terraform.ml | 3 +- tezt/lib_cloud/terraform/vm/main.tf | 4 +- tezt/lib_cloud/tezt_cloud.mli | 1 + tezt/lib_cloud/web.ml | 35 +- tezt/tests/cloud/dal.ml | 9 +- 24 files changed, 749 insertions(+), 213 deletions(-) create mode 100644 tezt/lib_cloud/proxy.ml create mode 100644 tezt/lib_cloud/proxy.mli diff --git a/tezt/lib_cloud/agent.ml b/tezt/lib_cloud/agent.ml index d069edb68729..f053ff68e3fc 100644 --- a/tezt/lib_cloud/agent.ml +++ b/tezt/lib_cloud/agent.ml @@ -6,15 +6,20 @@ (*****************************************************************************) type t = { + (* The name initially is the same as [vm_name] and can be changed dynamically by the scenario. *) mutable name : string; vm_name : string; - cmd_wrapper : Gcloud.cmd_wrapper option; + zone : string option; point : string * int; runner : Runner.t; next_available_port : unit -> int; configuration : Configuration.t; } +let ssh_id () = + if Env.mode = `Orchestrator then Env.ssh_private_key_filename ~home:"/root" () + else Env.ssh_private_key_filename () + let docker_image_encoding = let open Data_encoding in union @@ -46,44 +51,23 @@ let configuration_encoding = (req "binaries_path" Data_encoding.string) (req "docker_image" docker_image_encoding)) -let cmd_wrapper_encoding = - let open Data_encoding in - conv - (fun Gcloud.{cmd; args} -> (cmd, args)) - (fun (cmd, args) -> {cmd; args}) - (obj2 - (req "cmd" Data_encoding.string) - (req "args" (Data_encoding.list Data_encoding.string))) - let encoding = let open Data_encoding in conv (fun { name; vm_name; - cmd_wrapper; + zone; point; - runner; + runner = _; next_available_port; configuration; } -> - let ssh_id = runner.Runner.ssh_id |> Option.get in - ( name, - vm_name, - cmd_wrapper, - point, - ssh_id, - next_available_port (), - configuration )) - (fun ( name, - vm_name, - cmd_wrapper, - point, - ssh_id, - next_available_port, - configuration ) -> + (name, vm_name, zone, point, next_available_port (), configuration)) + (fun (name, vm_name, zone, point, next_available_port, configuration) -> let ssh_port = snd point in let address = fst point in + let ssh_id = ssh_id () in let runner = Runner.create ~ssh_user:"root" ~ssh_id ~ssh_port ~address () in @@ -93,28 +77,19 @@ let encoding = incr current_port ; !current_port in - { - name; - vm_name; - cmd_wrapper; - point; - runner; - next_available_port; - configuration; - }) - (obj7 + {name; vm_name; zone; point; runner; next_available_port; configuration}) + (obj6 (req "name" Data_encoding.string) (req "vm_name" Data_encoding.string) - (req "cmd_wrapper" (Data_encoding.option cmd_wrapper_encoding)) + (req "zone" (Data_encoding.option Data_encoding.string)) (req "point" (Data_encoding.tup2 Data_encoding.string Data_encoding.int31)) - (req "ssh_id" Data_encoding.string) (req "next_available_port" Data_encoding.int31) (req "configuration" configuration_encoding)) -let make ?cmd_wrapper ~ssh_id ~point:((address, ssh_port) as point) - ~configuration ~next_available_port ~name () = +let make ?zone ~ssh_id ~point:((address, ssh_port) as point) ~configuration + ~next_available_port ~name () = let ssh_user = "root" in let runner = Runner.create ~ssh_user ~ssh_id ~ssh_port ~address () in { @@ -124,19 +99,52 @@ let make ?cmd_wrapper ~ssh_id ~point:((address, ssh_port) as point) vm_name = name; next_available_port; configuration; - cmd_wrapper; + zone; } let name {name; _} = name +let vm_name {vm_name; _} = vm_name + let point {point; _} = point -let cmd_wrapper agent = agent.cmd_wrapper +let cmd_wrapper {zone; vm_name; _} = + match zone with + | None -> None + | Some zone -> + let ssh_private_key_filename = + if Env.mode = `Orchestrator then + Env.ssh_private_key_filename ~home:"/root" () + else Env.ssh_private_key_filename () + in + Some (Gcloud.cmd_wrapper ~zone ~vm_name ~ssh_private_key_filename) let set_name agent name = agent.name <- name let path_of agent binary = agent.configuration.binaries_path // binary +let host_run_command agent cmd args = + match cmd_wrapper agent with + | None -> Process.spawn cmd args + | Some cmd_wrapper -> + Process.spawn cmd_wrapper.Gcloud.cmd (cmd_wrapper.args @ [cmd] @ args) + +let docker_run_command agent cmd args = + let cmd, args = + Runner.wrap_with_ssh agent.runner (Runner.Shell.cmd [] cmd args) + in + (* The host check is very not convenient at all for this setting. This makes + tezt cloud sensible to man in the middle attacks. There are other options + like doing the check the first time only etc... But they all fail at some + point. I think the issue is that: + + - GCP may reuse IP addresses + - The docker image generates new key anytime it is generated + + I don't have a good proposition that keeps a nice UX and is secure at the moment. + *) + Process.spawn cmd (["-o"; "StrictHostKeyChecking=no"] @ args) + let copy agent ~source ~destination = let runner = agent.runner in let identity = @@ -151,7 +159,33 @@ let copy agent ~source ~destination = let* exists = let process = Process.spawn ~runner "ls" [destination] in let* status = process |> Process.wait in - match status with WEXITED 0 -> Lwt.return_true | _ -> Lwt.return_false + match status with + | WEXITED 0 -> + let hash_of_md5_output output = + String.trim output |> String.split_on_char ' ' |> List.hd + in + (* If the file already exists on the remote machine, we compare the + hashes to be sure they are the same. *) + Lwt.catch + (fun () -> + let* destination_hash = + let* output = + docker_run_command agent "md5sum" [destination] + |> Process.check_and_read_stdout + in + (* md5sum output is: ' '. We only take the hash. *) + Lwt.return (hash_of_md5_output output) + in + let process = Process.spawn "md5sum" [source] in + let* status = process |> Process.wait in + match status with + | WEXITED 0 -> + let* output = Process.check_and_read_stdout process in + let source_hash = hash_of_md5_output output in + Lwt.return (destination_hash = source_hash) + | _ -> Lwt.return_true) + (fun _ -> Lwt.return_false) + | _ -> Lwt.return_false in if exists then Lwt.return_unit else @@ -166,7 +200,12 @@ let copy agent ~source ~destination = destination in let* () = - Process.run "scp" (["-O"] @ identity @ port @ [source] @ [destination]) + (* FIXME: I forgot why we enforce [-0]. *) + Process.run + "scp" + (["-O"] + @ ["-o"; "StrictHostKeyChecking=no"] + @ identity @ port @ [source] @ [destination]) in Lwt.return_unit @@ -175,15 +214,17 @@ let copy = scenario. This optimisation ease the writing of scenario so that copy can always be called before using the file copied. *) let already_copied = Hashtbl.create 11 in - fun agent ~source -> - let destination = path_of agent source in + fun ?destination agent ~source -> + let destination = + Option.value ~default:(path_of agent source) destination + in match Hashtbl.find_opt already_copied (agent, destination) with | Some promise -> promise | None -> let p = let* () = - Process.spawn - ~runner:agent.runner + docker_run_command + agent "mkdir" ["-p"; Filename.dirname destination] |> Process.check diff --git a/tezt/lib_cloud/agent.mli b/tezt/lib_cloud/agent.mli index 5bf2225ed18f..5c545ed7a241 100644 --- a/tezt/lib_cloud/agent.mli +++ b/tezt/lib_cloud/agent.mli @@ -17,7 +17,7 @@ type t the agent. [ssh_id] is a path to the private key that will be used for the ssh connection. *) val make : - ?cmd_wrapper:Gcloud.cmd_wrapper -> + ?zone:string -> ssh_id:string -> point:string * int -> configuration:Configuration.t -> @@ -32,14 +32,17 @@ val encoding : t Data_encoding.t (** [name agent] returns the name of the agent. *) val name : t -> string +(** [vm_name agent] returns the vm name of the agent. *) +val vm_name : t -> string + (** [set_name agent name] sets the name of the agent to [name]. *) val set_name : t -> string -> unit -(** [copy agent ~source] copy the file into the [agent] directory and +(** [copy ?destination agent ~source] copy the file into the [agent] directory and returned the directory where the file can be found. It is assumed the [source] file does not exist on the agent machine. If the parent directory does not exist, it will be created. *) -val copy : t -> source:string -> string Lwt.t +val copy : ?destination:string -> t -> source:string -> string Lwt.t (** [next_available_port agent] returns the next available port for this agent. Raises [Not_found] if no port is available. *) @@ -56,3 +59,9 @@ val configuration : t -> Configuration.t (** A wrapper to run a command on the VM of the agent. *) val cmd_wrapper : t -> Gcloud.cmd_wrapper option + +(** Run a command on the host machine of the VM. *) +val host_run_command : t -> string -> string list -> Process.t + +(** Run a command on the docker image run by the agent. *) +val docker_run_command : t -> string -> string list -> Process.t diff --git a/tezt/lib_cloud/cli.ml b/tezt/lib_cloud/cli.ml index 83e23043c8bf..e6dc2da97bc0 100644 --- a/tezt/lib_cloud/cli.ml +++ b/tezt/lib_cloud/cli.ml @@ -25,6 +25,7 @@ let monitoring = Clap.flag ~section ~set_long:"monitoring" + ~unset_long:"no-monitoring" ~description: "If set, all the VMs are monitored with netdata. The monitoring \ dashboard is accessible on port 13999 of the monitored VM." @@ -34,6 +35,7 @@ let destroy = Clap.flag ~section ~set_long:"destroy" + ~unset_long:"no-destroy" ~description:"If set, the machines are destroyed at the end of the test" false @@ -76,17 +78,27 @@ let ports_per_vm = ~description:"Number of opened port per VM (default is 50)" 50 +let proxy = + Clap.flag + ~section + ~set_long:"proxy" + ~description: + "Enables to run the orchestrator on a VM instead of the host machine" + false + let grafana = Clap.flag ~section ~set_long:"grafana" + ~unset_long:"no-grafana" ~description:"Flag to set whether to run grafana" - true + ((not localhost) || proxy) let prometheus = Clap.flag ~section ~set_long:"prometheus" + ~unset_long:"no-prometheus" ~description:"Flag to set whether metrics are exported into prometheus" grafana @@ -94,6 +106,7 @@ let prometheus_export = Clap.flag ~section ~set_long:"prometheus-export" + ~unset_long:"no-prometheus-export" ~description:"Export a Prometheus snapshot at the end of the scenario" true @@ -124,18 +137,12 @@ let website = Clap.flag ~section ~set_long:"website" + ~unset_long:"no-website" ~description: "A webpage is up on localhost to summarize various informations related \ to the experiment" prometheus -let website_port = - Clap.default_int - ~section - ~long:"website-port" - ~description:"Set the port used for the website. Default is 8080" - 8080 - let machine_type = Clap.default_string ~section @@ -154,13 +161,15 @@ let dockerfile_alias = the environment variable `TEZT_CLOUD`)" () -let proxy = - Clap.flag +let website_port = + Clap.default_int ~section - ~set_long:"proxy" - ~description: - "Enables to run the orchestrator on a VM instead of the host machine" - false + ~long:"website-port" + ~description:"Set the port used for the website. Default is 8080" + (* When the website is run by the orchestrator, it is easier to + use port 80. We could decide in the future to use 80 by + default. *) + (if proxy && localhost then 80 else 8080) let max_run_duration = Clap.default_int @@ -179,3 +188,10 @@ let no_max_run_duration = (* If the proxy mode is active, we don't want to use [max_run_duration] since it aims to run long running tests. *) proxy + +let tezt_cloud = + Clap.optional_string + ~section + ~long:"tezt-cloud" + ~description:"Overwrite the TEZT_CLOUD variable" + () diff --git a/tezt/lib_cloud/cli.mli b/tezt/lib_cloud/cli.mli index edef459c7a87..97124643562e 100644 --- a/tezt/lib_cloud/cli.mli +++ b/tezt/lib_cloud/cli.mli @@ -86,3 +86,5 @@ val no_max_run_duration : bool (** When [proxy] is [true], the scenario is run via a VM instead of the host machine. *) val proxy : bool + +val tezt_cloud : string option diff --git a/tezt/lib_cloud/cloud.ml b/tezt/lib_cloud/cloud.ml index 573d932adea6..6e533c7fd507 100644 --- a/tezt/lib_cloud/cloud.ml +++ b/tezt/lib_cloud/cloud.ml @@ -27,6 +27,15 @@ let sigint = previous_behaviour := previous_handler ; promise +let eof = + let promise, resolver = Lwt.task () in + Lwt.dont_wait + (fun () -> + let* _ = Lwt_io.read_line Lwt_io.stdin in + Lwt.return_unit) + (fun _ -> Lwt.wakeup resolver ()) ; + promise + (* This exception is raised when the test is interrupted by Ctrl+C. *) exception Interrupted @@ -97,7 +106,8 @@ let shutdown ?exn t = | Some exn -> (* The exception is raised to Tezt. *) Lwt.reraise exn (* This function is used to ensure we can connect to the docker image on the VM. *) -let wait_ssh_server_running runner = +let wait_ssh_server_running agent = + let runner = Agent.runner agent in let is_ready _output = true in let run () = let cmd, args = @@ -108,7 +118,267 @@ let wait_ssh_server_running runner = let* _ = Env.wait_process ~is_ready ~run () in Lwt.return_unit -let register ?vms ~__FILE__ ~title ~tags ?seed f = +let orchestrator deployement f = + let agents = Deployement.agents deployement in + let* website = + if Env.website then + let* website = Web.start ~agents in + Lwt.return_some website + else Lwt.return_none + in + let* prometheus = + if Env.prometheus then + let* prometheus = Prometheus.start agents in + Lwt.return_some prometheus + else Lwt.return_none + in + let* grafana = + if Env.grafana then + let* grafana = Grafana.run () in + Lwt.return_some grafana + else Lwt.return_none + in + Log.info "Post prometheus" ; + let t = + {website; agents; prometheus; grafana; deployement = Some deployement} + in + let sigint = sigint () in + let main_promise = + (* We also catch error raised from the scenario directly. *) + Lwt.catch + (fun () -> + let* () = f t in + Lwt.return_none) + (fun exn -> Lwt.return_some exn) + in + (* This part is tricky! We want to catch Ctrl+C so that tezt does not + kill all the VMs directly before tezt-cloud termination tasks are + over. When the signal is caught, tezt-cloud takes over. Processes are + cleaned up manually via [Process.clean_up ()]. + *) + let* exn = + Lwt.pick + [ + (let* () = sigint in + Lwt.return_some Interrupted); + main_promise; + ] + in + shutdown ?exn t + +let attach agent = + let runner = Agent.runner agent in + let hooks = + Process. + { + on_log = + (fun str -> Log.debug ~color:Log.Color.reset "%s" (String.trim str)); + on_spawn = (fun _ _ -> ()); + } + in + let has_sigint = ref false in + let on_sigint = + let* () = sigint () in + has_sigint := true ; + Log.debug "Sigint triggered..." ; + let cmd, args = + Runner.wrap_with_ssh + runner + (Runner.Shell.cmd [] "screen" ["-S"; "tezt-cloud"; "-X"; "stuff"; "^C"]) + in + let* () = + Process.spawn ~hooks cmd (["-o"; "StrictHostKeyChecking=no"] @ args) + |> Process.check + in + let cmd, args = + Runner.wrap_with_ssh + runner + (Runner.Shell.cmd [] "stdbuf" ["-oL"; "tail"; "-f"; "screenlog.0"]) + in + let _p = + Process.spawn ~hooks cmd (["-o"; "StrictHostKeyChecking=no"] @ args) + |> Process.check + in + let* _ = eof in + let* () = + let process = Process.spawn ~runner "pkill" ["screen"] in + let* _ = Process.wait process in + Lwt.return_unit + in + let* () = + let process = Process.spawn ~runner "rm" ["-f"; "screenlog.0"] in + let* _ = Process.wait process in + Lwt.return_unit + in + let* () = + let process = + Process.spawn "rm" [Path.proxy_deployement ~tezt_cloud:Env.tezt_cloud] + in + let* _ = Process.wait process in + Lwt.return_unit + in + Lwt.return_unit + in + let on_eof = + let* () = eof in + Log.info "Detach from the proxy process." ; + if !has_sigint then on_sigint + else ( + Log.info + "Deployement website can be accessed here: http://%s:8080" + (Agent.point agent |> fst) ; + Lwt.return_unit) + in + Log.Style.set_prefix Log.Style.Hidden ; + Log.Style.set_timestamp Log.Style.Hidden ; + let cmd, args = + Runner.wrap_with_ssh + runner + (Runner.Shell.cmd [] "stdbuf" ["-oL"; "tail"; "-f"; "screenlog.0"]) + in + let logger = + Lwt.catch + (fun () -> + let* () = + Process.spawn ~hooks cmd (["-o"; "StrictHostKeyChecking=no"] @ args) + |> Process.check + in + Lwt.return_unit) + (fun exn -> + Log.debug "Interrupted: %s" (Printexc.to_string exn) ; + on_sigint) + in + Lwt.choose [logger; on_sigint; on_eof] + +let try_reattach () = + let tezt_cloud = Env.tezt_cloud in + if Sys.file_exists (Path.proxy_deployement ~tezt_cloud) then + let contents = Base.read_file (Path.proxy_deployement ~tezt_cloud) in + let json = Data_encoding.Json.from_string contents |> Result.get_ok in + let deployement = + Data_encoding.Json.destruct (Data_encoding.list Agent.encoding) json + |> Deployement.of_agents + in + let agents = Deployement.agents deployement in + let proxy_agent = + agents + |> List.find (fun agent -> + let proxy_agent_prefix = + Format.asprintf "%s-proxy" Env.tezt_cloud + in + String.starts_with ~prefix:proxy_agent_prefix (Agent.name agent)) + in + let* is_ssh_server_running = + Lwt.pick + [ + (let* () = Lwt_unix.sleep 5. in + Lwt.return_false); + (let* () = wait_ssh_server_running proxy_agent in + Lwt.return_true); + ] + in + if is_ssh_server_running then + let* is_running = + Lwt.catch + (fun () -> + let* status = + Process.spawn + ~runner:(Agent.runner proxy_agent) + "ls" + ["screenlog.0"] + |> Process.wait + in + match status with + | WEXITED 0 -> Lwt.return_true + | _ -> Lwt.return_false) + (fun _exn -> Lwt.return_false) + in + if is_running then + let* () = + Lwt.catch + (fun () -> attach proxy_agent) + (fun exn -> + Log.debug "INFO: %s" (Printexc.to_string exn) ; + Lwt.return_unit) + in + Lwt.return_true + else Lwt.return_false + else Lwt.return_false + else Lwt.return_false + +let init_proxy ?(proxy_files = []) deployement = + let agents = Deployement.agents deployement in + let proxy_agent = + agents + |> List.find (fun agent -> + let proxy_agent_prefix = Format.asprintf "%s-proxy" Env.tezt_cloud in + String.starts_with ~prefix:proxy_agent_prefix (Agent.name agent)) + in + let* () = wait_ssh_server_running proxy_agent in + let destination = + (Agent.configuration proxy_agent).binaries_path + // Filename.basename Path.self + in + let* self = Agent.copy ~destination proxy_agent ~source:Path.self in + let json = + Data_encoding.Json.construct (Data_encoding.list Agent.encoding) agents + in + let contents = Data_encoding.Json.to_string json in + let tezt_cloud = Env.tezt_cloud in + let proxy_deployement = Path.proxy_deployement ~tezt_cloud in + Base.write_file proxy_deployement ~contents ; + let* () = + Proxy.copy_files proxy_agent ~scenario_files:proxy_files ~proxy_deployement + in + let runner = Agent.runner proxy_agent in + let* () = + (* This should not be necessary, this is to ensure there is no [screen] leftover. *) + let process = Process.spawn ~runner "pkill" ["screen"] in + let* _ = Process.wait process in + Lwt.return_unit + in + let* () = + (* We start a screen session in detached mode. The orchestrator will run in this session. *) + Process.spawn ~runner "screen" ["-S"; "tezt-cloud"; "-d"; "-m"] + |> Process.check + in + let process = + let args = + let args = Sys.argv |> Array.to_list |> List.tl in + args @ ["--localhost"; "--tezt-cloud"; Env.tezt_cloud] + (* [--localhost] will be combined with --proxy, this enables to detect we want to run in [`Orchestrator]. + + [--tezt-cloud] is used so that the [`Orchestrator] mode knows this value. + *) + in + (* We execute a command in a screen session that will start the orchestrator. *) + Agent.docker_run_command + proxy_agent + "screen" + (["-S"; "tezt-cloud"; "-X"; "exec"] @ (self :: args)) + in + let* () = + Agent.docker_run_command + proxy_agent + "screen" + ["-S"; "tezt-cloud"; "-X"; "log"] + |> Process.check + in + let* () = + Lwt.catch + (fun () -> attach proxy_agent) + (fun exn -> + Test.fail "Unable to attach process: %s" (Printexc.to_string exn)) + in + let* () = + let* status = Process.wait process in + match status with + | WEXITED 0 -> Lwt.return_unit + | _ -> Test.fail "Proxy scenario has failed" + in + if Env.destroy then Deployement.terminate deployement else Lwt.return_unit + +let register ?proxy_files ?vms ~__FILE__ ~title ~tags ?seed f = Test.register ~__FILE__ ~title ~tags ?seed @@ fun () -> let* () = Env.init () in let vms = @@ -132,67 +402,67 @@ let register ?vms ~__FILE__ ~title ~tags ?seed f = prometheus = None; deployement = None; } - | Some configurations -> - let* () = - match Env.mode with - | `Localhost -> Jobs.docker_build ~push:false () - | `Cloud -> + | Some configurations -> ( + let tezt_cloud = Env.tezt_cloud in + match Env.mode with + | `Orchestrator -> + (* The scenario is executed locally on the proxy VM. *) + let contents = Base.read_file (Path.proxy_deployement ~tezt_cloud) in + let json = Data_encoding.Json.from_string contents |> Result.get_ok in + let deployement = + Data_encoding.Json.destruct (Data_encoding.list Agent.encoding) json + |> Deployement.of_agents + in + let* () = + Deployement.agents deployement + |> List.map wait_ssh_server_running + |> Lwt.join + in + orchestrator deployement f + | `Localhost -> + (* The scenario is executed locally and the VM are on the host machine. *) + let* () = Jobs.docker_build ~push:false () in + let* deployement = Deployement.deploy ~configurations in + let* () = + Deployement.agents deployement + |> List.map wait_ssh_server_running + |> Lwt.join + in + orchestrator deployement f + | `Cloud -> + (* The scenario is executed locally and the VMs are on the cloud. *) + let* () = Jobs.deploy_docker_registry () in + let* () = Jobs.docker_build ~push:true () in + let* deployement = Deployement.deploy ~configurations in + let* () = + Deployement.agents deployement + |> List.map wait_ssh_server_running + |> Lwt.join + in + orchestrator deployement f + | `Host -> + (* The scenario is executed remotely. *) + let* proxy_running = try_reattach () in + if not proxy_running then let* () = Jobs.deploy_docker_registry () in - Jobs.docker_build ~push:true () - in - let* deployement = Deployement.deploy ~configurations in - let agents = Deployement.agents deployement in - let* () = - agents - |> List.map (fun agent -> Agent.runner agent |> wait_ssh_server_running) - |> Lwt.join - in - let* website = - if Env.website then - let* website = Web.start ~agents in - Lwt.return_some website - else Lwt.return_none - in - let* prometheus = - if Env.prometheus then - let* prometheus = Prometheus.start agents in - Lwt.return_some prometheus - else Lwt.return_none - in - let* grafana = - if Env.grafana then - let* grafana = Grafana.run () in - Lwt.return_some grafana - else Lwt.return_none - in - let t = - {website; agents; prometheus; grafana; deployement = Some deployement} - in - let sigint = sigint () in - let main_promise = - (* We also catch error raised from the scenario directly. *) - Lwt.catch - (fun () -> - let* () = f t in - Lwt.return_none) - (fun exn -> Lwt.return_some exn) - in - (* This part is tricky! We want to catch Ctrl+C so that tezt does not - kill all the VMs directly before tezt-cloud termination tasks are - over. When the signal is caught, tezt-cloud takes over. Processes are - cleaned up manually via [Process.clean_up ()]. - *) - let* exn = - Lwt.pick - [ - (let* () = sigint in - Lwt.return_some Interrupted); - main_promise; - ] - in - shutdown ?exn t + let* () = Jobs.docker_build ~push:true () in + let* deployement = Deployement.deploy ~configurations in + let* () = + Deployement.agents deployement + |> List.map wait_ssh_server_running + |> Lwt.join + in + init_proxy ?proxy_files deployement + else Lwt.return_unit) -let agents t = t.agents +let agents t = + match Env.mode with + | `Orchestrator -> + let proxy_agent = Proxy.get_agent t.agents in + let proxy_vm_name = Agent.vm_name proxy_agent in + t.agents + |> List.filter (fun agent -> Agent.vm_name agent <> proxy_vm_name) + | `Host | `Cloud | `Localhost -> t.agents let get_configuration = Agent.configuration diff --git a/tezt/lib_cloud/cloud.mli b/tezt/lib_cloud/cloud.mli index 0626449101f7..01b3425513c3 100644 --- a/tezt/lib_cloud/cloud.mli +++ b/tezt/lib_cloud/cloud.mli @@ -11,6 +11,7 @@ type t enables to run a test that can use machines deployed onto the cloud. *) val register : + ?proxy_files:string list -> ?vms:Configuration.t list -> __FILE__:string -> title:string -> diff --git a/tezt/lib_cloud/deployement.ml b/tezt/lib_cloud/deployement.ml index 0095c5c8a2c4..0f4c55acf067 100644 --- a/tezt/lib_cloud/deployement.ml +++ b/tezt/lib_cloud/deployement.ml @@ -11,7 +11,7 @@ module Remote = struct type t = {agents : Agent.t list} let wait_docker_running ~vm_name () = - let ssh_private_key_filename = Env.ssh_private_key_filename in + let ssh_private_key_filename = Env.ssh_private_key_filename () in let* zone = Env.zone () in let is_ready _output = true in let run () = @@ -72,7 +72,7 @@ module Remote = struct List.map (fun vm_name -> wait_docker_running ~vm_name ()) names |> Lwt.join in - let ssh_private_key_filename = Env.ssh_private_key_filename in + let ssh_private_key_filename = Env.ssh_private_key_filename () in let make_agent vm_name = let* ip = Gcloud.get_ip_address_from_name ~zone vm_name in let point = (ip, base_port) in @@ -83,7 +83,7 @@ module Remote = struct !port in let cmd_wrapper = - Gcloud.cmd_wrapper ~zone ~vm_name ~ssh_private_key_filename + Gcloud.cmd_wrapper ~zone ~ssh_private_key_filename ~vm_name in let* () = if Env.monitoring then Monitoring.run ~cmd_wrapper () @@ -91,7 +91,7 @@ module Remote = struct in Agent.make ~ssh_id:ssh_private_key_filename - ~cmd_wrapper + ~zone ~point ~configuration ~next_available_port @@ -119,11 +119,21 @@ module Remote = struct in order configurations bindings + let deploy_proxy () = + let workspace_name = Format.asprintf "%s-proxy" Env.tezt_cloud in + let configuration = Configuration.make () in + let tezt_cloud = Env.tezt_cloud in + let* () = Terraform.VM.Workspace.init ~tezt_cloud [workspace_name] in + let* agents = + workspace_deploy ~workspace_name ~configuration ~number_of_vms:1 + in + match agents with [agent] -> Lwt.return agent | _ -> assert false + (* Deployement requires to create new VMs and organizing them per group of configuration. Each configuration leads to one terraform workspace. *) - let deploy ~configurations = + let deploy ~proxy ~configurations = let agents_info = Hashtbl.create 11 in let workspaces_info = (* VMs are grouped per group of configuration. Each group leads to one workspace. *) @@ -169,7 +179,10 @@ module Remote = struct configurations. *) order_agents (List.concat agents) configurations in - Lwt.return {agents} + if proxy then + let* agent = deploy_proxy () in + Lwt.return {agents = agent :: agents} + else Lwt.return {agents} let agents t = t.agents @@ -249,7 +262,7 @@ module Localhost = struct |> Seq.iter (fun i -> let port = base_port + (i * ports_per_vm) in Hashtbl.replace next_port ("localhost", port) (port + 1)) ; - let ssh_id = Env.ssh_private_key_filename in + let ssh_id = Env.ssh_private_key_filename () in let get_point i = let port = base_port + (i * ports_per_vm) in ("localhost", port) @@ -308,8 +321,12 @@ let deploy ~configurations = let* localhost = Localhost.deploy ~configurations () in Lwt.return (Localhost localhost) | `Cloud -> - let* remote = Remote.deploy ~configurations in + let* remote = Remote.deploy ~proxy:false ~configurations in + Lwt.return (Remote remote) + | `Host -> + let* remote = Remote.deploy ~proxy:true ~configurations in Lwt.return (Remote remote) + | `Orchestrator -> assert false let agents t = match t with @@ -320,3 +337,5 @@ let terminate ?exn t = match t with | Remote remote -> Remote.terminate ?exn remote | Localhost localhost -> Localhost.terminate ?exn localhost + +let of_agents agents = Remote {agents} diff --git a/tezt/lib_cloud/deployement.mli b/tezt/lib_cloud/deployement.mli index 022c765fdcee..4de1bb13ffff 100644 --- a/tezt/lib_cloud/deployement.mli +++ b/tezt/lib_cloud/deployement.mli @@ -23,3 +23,5 @@ val agents : t -> Agent.t list that this call may or may not destroy the machine depending on what the user has chosen (see [Cli.destroy].*) val terminate : ?exn:exn -> t -> unit Lwt.t + +val of_agents : Agent.t list -> t diff --git a/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile b/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile index adec8fab32da..cf17cc89dc7a 100644 --- a/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile +++ b/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile @@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y \ # Tezos dependencies libgmp-dev curl libev-dev libhidapi-dev \ # With the proxy mode we may want to run docker inside docker - docker.io \ + docker.io screen \ # Necessary certificates for mirages dependencies ca-certificates \ # DL3015: Use --no-install-recommends diff --git a/tezt/lib_cloud/env.ml b/tezt/lib_cloud/env.ml index 88cce8e74c89..49bdff650f9a 100644 --- a/tezt/lib_cloud/env.ml +++ b/tezt/lib_cloud/env.ml @@ -8,25 +8,27 @@ type docker_image = Gcp of {alias : string} | Octez_latest_release let tezt_cloud = - (* This is a lazy value to be sure that this is evaluated only inside a Tezt test. *) - match Sys.getenv_opt "TEZT_CLOUD" with - | None -> - Stdlib.failwith - "The environment variable 'TEZT_CLOUD' is not defined. See README for \ - more information why this variable must be defined." - | Some value -> value - -let ssh_private_key_filename = - let home = Sys.getenv "HOME" in + match Cli.tezt_cloud with + | Some tezt_cloud -> tezt_cloud + | None -> ( + (* This is a lazy value to be sure that this is evaluated only inside a Tezt test. *) + match Sys.getenv_opt "TEZT_CLOUD" with None -> "" | Some value -> value) + +let ssh_private_key_filename ?(home = Sys.getenv "HOME") () = home // ".ssh" // Format.asprintf "%s-tf" tezt_cloud -let ssh_public_key_filename = - let ssh_key = ssh_private_key_filename in +let ssh_public_key_filename ?home () = + let ssh_key = ssh_private_key_filename ?home () in Format.asprintf "%s.pub" ssh_key let docker_registry = Format.asprintf "%s-docker-registry" tezt_cloud -let mode = if Cli.localhost then `Localhost else `Cloud +let mode = + match (Cli.localhost, Cli.proxy) with + | true, true -> `Orchestrator + | true, false -> `Localhost + | false, true -> `Host + | false, false -> `Cloud let prometheus = Cli.prometheus @@ -76,15 +78,20 @@ let dockerfile = Path.dockerfile ~alias:dockerfile_alias let project_id = Gcloud.project_id let init () = - if mode = `Localhost then unit - else - let tezt_cloud = tezt_cloud in - let* project_id = project_id () in - Log.info "Initializing docker registry..." ; - let* () = Terraform.Docker_registry.init () in - Log.info "Deploying docker registry..." ; - let* () = Terraform.Docker_registry.deploy ~tezt_cloud ~project_id in - Lwt.return_unit + if tezt_cloud = "" then + Test.fail + "The tezt-cloud value should be set. Either via the CLI or via the \ + environement variable 'TEZT_CLOUD'" ; + match mode with + | `Localhost | `Orchestrator -> Lwt.return_unit + | `Host | `Cloud -> + let tezt_cloud = tezt_cloud in + let* project_id = project_id () in + Log.info "Initializing docker registry..." ; + let* () = Terraform.Docker_registry.init () in + Log.info "Deploying docker registry..." ; + let* () = Terraform.Docker_registry.deploy ~tezt_cloud ~project_id in + Lwt.return_unit (* Even though we could get this information locally, it is interesting to fetch it through terraform to get the correct value if a scenario was launch with @@ -117,11 +124,11 @@ let registry_uri () = let uri_of_docker_image docker_image = match (docker_image, mode) with - | Gcp {alias}, `Cloud -> + | Gcp {alias}, (`Cloud | `Host | `Orchestrator) -> let* registry_uri = registry_uri () in Lwt.return (Format.asprintf "%s/%s" registry_uri alias) | Gcp {alias}, `Localhost -> Lwt.return alias - | Octez_latest_release, `Cloud -> + | Octez_latest_release, (`Cloud | `Host | `Orchestrator) -> let* registry_uri = registry_uri () in Lwt.return (Format.asprintf "%s/octez" registry_uri) | Octez_latest_release, `Localhost -> Lwt.return "octez" diff --git a/tezt/lib_cloud/env.mli b/tezt/lib_cloud/env.mli index edd38bf377a1..8f259aa416a7 100644 --- a/tezt/lib_cloud/env.mli +++ b/tezt/lib_cloud/env.mli @@ -12,7 +12,20 @@ type docker_image = Gcp of {alias : string} | Octez_latest_release val tezt_cloud : string -val mode : [`Localhost | `Cloud] +(** + + - [Localhost]: Agents and the orchestrator are run on the host machine + + - [Cloud]: The orchestrator is on the host machine. The agents are run onto + the cloud + + - [`Orchestrator]: The orchestrator is run on a VM. This is mode used by the + proxy from the orchestrator point of view. + + - [`Host]: This mode is run by the host machine that initializes the + orchestrator running an a VM. +*) +val mode : [`Localhost | `Cloud | `Orchestrator | `Host] val prometheus : bool @@ -36,9 +49,9 @@ val monitoring : bool val keep_alive : bool -val ssh_private_key_filename : string +val ssh_private_key_filename : ?home:string -> unit -> string -val ssh_public_key_filename : string +val ssh_public_key_filename : ?home:string -> unit -> string val dockerfile_alias : string diff --git a/tezt/lib_cloud/grafana.ml b/tezt/lib_cloud/grafana.ml index 248e05e58c40..39cbfed6aaee 100644 --- a/tezt/lib_cloud/grafana.ml +++ b/tezt/lib_cloud/grafana.ml @@ -74,6 +74,19 @@ datasources: let shutdown _t = Process.run "docker" ["kill"; "grafana"] +let dashboards_filepaths () = + let path = + if Env.mode = `Orchestrator then + Filename.get_temp_dir_name () // "grafana" // "dashboards" + else Path.grafana_dashboards + in + let* output = Process.run_and_read_stdout "ls" [path] in + let basenames = + String.trim output |> String.split_on_char '\n' + |> List.filter (fun file -> String.ends_with ~suffix:".json" file) + in + List.map (fun basename -> path // basename) basenames |> Lwt.return + let run () = let cmd = "docker" in let* () = @@ -99,8 +112,6 @@ let run () = "grafana"; "--network"; "host"; - "-p"; - "3000:3000"; "-e"; "GF_AUTH_ANONYMOUS_ENABLED=true"; "-e"; @@ -139,20 +150,9 @@ let run () = let* _ = Env.wait_process ~is_ready ~run () in let* admin_api_key = generate_admin_api_key password in let configuration = configuration admin_api_key in - let basenames = - [ - "octez-basic"; - "dal-basic"; - "octez-compact"; - "octez-full"; - "octez-with-logs"; - "evm-node"; - "rollup"; - ] - in - let dashboard basename = - read_file - (Format.asprintf "./tezt/lib_cloud/grafana/dashboards/%s.json" basename) + let* dashboards_filepaths = dashboards_filepaths () in + let dashboard dashboard_filepath = + read_file dashboard_filepath |> Format.asprintf "{\"dashboard\": %s, \"overwrite\": true}" in let rec loop = function @@ -166,5 +166,5 @@ let run () = in loop l in - let* () = loop basenames in + let* () = loop dashboards_filepaths in Lwt.return {provisioning_directory; dashboard_directory; password} diff --git a/tezt/lib_cloud/grafana.mli b/tezt/lib_cloud/grafana.mli index ee3b5694eca9..84628e457df2 100644 --- a/tezt/lib_cloud/grafana.mli +++ b/tezt/lib_cloud/grafana.mli @@ -12,3 +12,5 @@ type t val run : unit -> t Lwt.t val shutdown : t -> unit Lwt.t + +val dashboards_filepaths : unit -> string list Lwt.t diff --git a/tezt/lib_cloud/jobs.ml b/tezt/lib_cloud/jobs.ml index a81500a0324d..c24b605b671b 100644 --- a/tezt/lib_cloud/jobs.ml +++ b/tezt/lib_cloud/jobs.ml @@ -16,20 +16,20 @@ let docker_build = if Hashtbl.mem cache docker_image then ( Log.info "Docker image is already built. Nothing to do" ; Lwt.return_unit) - else ( + else + let ssh_public_key_filename = Env.ssh_public_key_filename () in Hashtbl.replace cache docker_image () ; Log.info "Checking the existence of ssh public key '%s'..." - Env.ssh_public_key_filename ; + ssh_public_key_filename ; let* ssh_public_key = - let ssh_public_key_file = Env.ssh_public_key_filename in let* () = - if not (Sys.file_exists ssh_public_key_file) then ( - Log.info "SSh public key not found, creating it..." ; + if not (Sys.file_exists ssh_public_key_filename) then ( + Log.info "SSH public key not found, creating it..." ; Ssh.generate_key ()) else Lwt.return_unit in - Process.run_and_read_stdout ~name:"cat" "cat" [ssh_public_key_file] + Process.run_and_read_stdout ~name:"cat" "cat" [ssh_public_key_filename] in let alias = match docker_image with @@ -68,7 +68,7 @@ let docker_build = ] | Octez_latest_release -> [("SSH_PUBLIC_KEY", ssh_public_key)] in - Log.info "Building image from %s.Dockerfile..." Env.tezt_cloud ; + Log.info "Building image from %s..." Env.tezt_cloud ; let* () = Docker.build ~alias ~args () |> Process.check in let* () = if push then ( @@ -81,7 +81,7 @@ let docker_build = Lwt.return_unit) else Lwt.return_unit in - unit) + unit let deploy_docker_registry () = Log.info "Tezt_Cloud found with value: %s" Env.tezt_cloud ; @@ -93,7 +93,7 @@ let deploy_docker_registry () = let clean_up_vms () = let tezt_cloud = Env.tezt_cloud in let* workspaces = Terraform.VM.Workspace.list ~tezt_cloud in - let ssh_private_key_filename = Env.ssh_private_key_filename in + let ssh_private_key_filename = Env.ssh_private_key_filename () in workspaces |> Lwt_list.iter_s (fun workspace -> let* () = Terraform.VM.Workspace.select workspace in @@ -124,7 +124,11 @@ let clean_up_vms () = |> List.filter (fun str -> str <> "") in let main_image, other_images = - List.partition (fun str -> str <> "netdata") images_name + List.partition + (fun str -> + str <> "netdata" && str <> "grafana" + && str <> "prometheus") + images_name in if List.length main_image <> 1 then Test.fail @@ -163,15 +167,6 @@ let clean_up_vms () = ["kill"; image] |> Process.check in - let* _ = - Gcloud.compute_ssh - ~zone - ~vm_name - ~ssh_private_key_filename - "docker" - ["rm"; image] - |> Process.check - in Lwt.return_unit) in Lwt.return_unit) diff --git a/tezt/lib_cloud/path.ml b/tezt/lib_cloud/path.ml index 9b0923177482..a289484aa395 100644 --- a/tezt/lib_cloud/path.ml +++ b/tezt/lib_cloud/path.ml @@ -29,3 +29,9 @@ let dal_trusted_setup = "_opam" // "share" // "dal-trusted-setup" let default_binaries_path () = Filename.get_temp_dir_name () // "tezt-runners" let self = Sys.argv.(0) + +let proxy_deployement ~tezt_cloud = + Filename.get_temp_dir_name () + // Format.asprintf "%s-tezt-cloud-deployement" tezt_cloud + +let grafana_dashboards = project // "grafana" // "dashboards" diff --git a/tezt/lib_cloud/path.mli b/tezt/lib_cloud/path.mli index 8c901fabeeb2..7d39ac1f1169 100644 --- a/tezt/lib_cloud/path.mli +++ b/tezt/lib_cloud/path.mli @@ -33,3 +33,9 @@ val default_binaries_path : unit -> string (** Path to the current binary. *) val self : string + +(** Path where to find the encoding of tezt cloud deployement. *) +val proxy_deployement : tezt_cloud:string -> string + +(** Path where are stored grafana dashboards. *) +val grafana_dashboards : string diff --git a/tezt/lib_cloud/proxy.ml b/tezt/lib_cloud/proxy.ml new file mode 100644 index 000000000000..90b0900a990f --- /dev/null +++ b/tezt/lib_cloud/proxy.ml @@ -0,0 +1,102 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +let find_agent agents = + agents + |> List.find_opt (fun agent -> + let proxy_agent_prefix = Format.asprintf "%s-proxy" Env.tezt_cloud in + String.starts_with ~prefix:proxy_agent_prefix (Agent.name agent)) + +let get_agent agents = find_agent agents |> Option.get + +let copy_files proxy_agent ~scenario_files ~proxy_deployement = + (* This file is necessary to get the agents configurations. *) + let* _ = + Agent.copy + proxy_agent + ~source:proxy_deployement + ~destination:proxy_deployement + in + (* Copying the ssh key is necessary for the proxy agent to connect with the + other VMs. *) + let ssh_public_key_filename = Env.ssh_public_key_filename () in + let ssh_private_key_filename = Env.ssh_private_key_filename () in + let* _ = + Agent.copy + proxy_agent + ~source:ssh_public_key_filename + ~destination: + ("/root" // ".ssh" // Filename.basename ssh_public_key_filename) + in + let* _ = + Agent.copy + proxy_agent + ~source:ssh_private_key_filename + ~destination: + ("/root" // ".ssh" // Filename.basename ssh_private_key_filename) + in + (* If the Proxy agent uses grafana, it needs some dashboards. We copy them to + the proxy VM and then import them. + + Another way to do it, would be to push them from the host machine to the + grafana instance directly. This is doable and slightly better in theory. + *) + let* _ = + if Env.grafana then + let* grafana_dashboards = Grafana.dashboards_filepaths () in + grafana_dashboards + |> List.map (fun dashboard -> + let* _ = + Agent.copy + proxy_agent + ~source:dashboard + ~destination: + (Filename.get_temp_dir_name () + // "grafana" // "dashboards" + // Filename.basename dashboard) + in + Lwt.return_unit) + |> Lwt.join + else Lwt.return_unit + in + (* The proxy agent requires the daemons to be in the current working + directory. We add symbolink links to all the known binaries. + + This requires the docker image to contain all the binaries used by the + proxy agent. + *) + let* output = + Process.spawn + ~runner:(Agent.runner proxy_agent) + "ls" + [Path.default_binaries_path ()] + |> Process.check_and_read_stdout + in + let files = String.trim output |> String.split_on_char '\n' in + let* () = + files + |> List.map (fun file -> + Process.spawn + ~runner:(Agent.runner proxy_agent) + "ln" + ["-s"; Path.default_binaries_path () // file; file; "-f"] + |> Process.check) + |> Lwt.join + in + (* The scenario itself may need some files that exist on the host machine but + are not present by default on the proxy machine. Since it may be difficult to + know ahead which files are needed, instead we require the scenario to provide + them manually. *) + let* () = + List.map + (fun file -> + let* _ = Agent.copy proxy_agent ~source:file ~destination:file in + Lwt.return_unit) + scenario_files + |> Lwt.join + in + unit diff --git a/tezt/lib_cloud/proxy.mli b/tezt/lib_cloud/proxy.mli new file mode 100644 index 000000000000..ce6a3ba220e0 --- /dev/null +++ b/tezt/lib_cloud/proxy.mli @@ -0,0 +1,19 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(** [get_agent agents] returns the proxy agent. It raises [Not_found] if the + proxy agent was not found. This function should be safe to call when + [Env.mode] is [`Orchestrator] or [`Host]. *) +val get_agent : Agent.t list -> Agent.t + +(** [copy_files agent] copies all the necessary files for the proxy orchestrator + to run correctly. *) +val copy_files : + Agent.t -> + scenario_files:string list -> + proxy_deployement:string -> + unit Lwt.t diff --git a/tezt/lib_cloud/ssh.ml b/tezt/lib_cloud/ssh.ml index ed6146204692..86d7f429106d 100644 --- a/tezt/lib_cloud/ssh.ml +++ b/tezt/lib_cloud/ssh.ml @@ -6,7 +6,7 @@ (*****************************************************************************) let generate_key () = - let path = Format.asprintf "%s" Env.ssh_private_key_filename in + let path = Format.asprintf "%s" (Env.ssh_private_key_filename ()) in (* -C is for comment; -N is for the passphrase (no passphrase here) *) diff --git a/tezt/lib_cloud/terraform.ml b/tezt/lib_cloud/terraform.ml index 48d0e6c7e7e0..c2e859d75d1a 100644 --- a/tezt/lib_cloud/terraform.ml +++ b/tezt/lib_cloud/terraform.ml @@ -104,7 +104,8 @@ module VM = struct String.split_on_char '\n' output |> List.map String.trim |> List.filter (fun workspace -> - String.starts_with ~prefix:tezt_cloud workspace + let prefix = Format.asprintf "%s-" tezt_cloud in + String.starts_with ~prefix workspace (* The workspace named "tezt-cloud" is used for the docker registry, it does not need to be listed. *) && workspace <> tezt_cloud) diff --git a/tezt/lib_cloud/terraform/vm/main.tf b/tezt/lib_cloud/terraform/vm/main.tf index e0be5308111d..2a22acc8c5f8 100644 --- a/tezt/lib_cloud/terraform/vm/main.tf +++ b/tezt/lib_cloud/terraform/vm/main.tf @@ -220,7 +220,7 @@ resource "google_compute_firewall" "default" { # Rule to enable static page web access allow { protocol = "tcp" - ports = ["8080"] + ports = ["80", "8080"] } # Rule to enable prometheus access @@ -252,7 +252,7 @@ resource "google_compute_instance_template" "default" { project = var.project_id - name = "instance-template" + name_prefix = "instance-template-${terraform.workspace}" service_account { # Google recommends custom service accounts that have cloud-platform scope and permissions granted via IAM Roles. diff --git a/tezt/lib_cloud/tezt_cloud.mli b/tezt/lib_cloud/tezt_cloud.mli index 8419e9a9caa4..fbe6364cdb80 100644 --- a/tezt/lib_cloud/tezt_cloud.mli +++ b/tezt/lib_cloud/tezt_cloud.mli @@ -47,6 +47,7 @@ module Cloud : sig using VMs provided as a map indexed by name. Each VM is abstracted via the [Agent] module. *) val register : + ?proxy_files:string list -> ?vms:Configuration.t list -> __FILE__:string -> title:string -> diff --git a/tezt/lib_cloud/web.ml b/tezt/lib_cloud/web.ml index d6631e30569a..b986b00eb8c4 100644 --- a/tezt/lib_cloud/web.ml +++ b/tezt/lib_cloud/web.ml @@ -113,20 +113,34 @@ let monitoring ~agents = else "# Monitoring\n Monitoring disabled. Use `--monitoring` to activate it.\n" -let prometheus = +let prometheus ~agents = + let domain = + match Env.mode with + | `Orchestrator -> Proxy.get_agent agents |> Agent.point |> fst + | `Host | `Localhost | `Cloud -> "localhost" + in if Env.prometheus then - "# Prometheus\n [Prometheus dashboard](http://localhost:9090)\n" + Format.asprintf + "# Prometheus\n [Prometheus dashboard](http://%s:%d)\n" + domain + Env.prometheus_port else "Prometheus disabled. Use `--prometheus` to activate it.\n" -let grafana = - if Env.grafana then "# Grafana\n [Grafana dashboard](http://localhost:3000)\n" +let grafana ~agents = + let domain = + match Env.mode with + | `Orchestrator -> Proxy.get_agent agents |> Agent.point |> fst + | `Host | `Localhost | `Cloud -> "localhost" + in + if Env.grafana then + Format.asprintf "# Grafana\n [Grafana dashboard](http://%s:3000)\n" domain else "Grafana disabled. Use `--grafana` to activate it.\n" let markdown_content ~agents = [ configuration ~agents; - grafana; - prometheus; + grafana ~agents; + prometheus ~agents; monitoring ~agents; debugging ~agents; ] @@ -156,7 +170,10 @@ let write t ~agents = ] let run () = - let dir = Temp.dir "website" in + let* () = + Process.run "mkdir" ["-p"; Filename.get_temp_dir_name () // "website"] + in + let dir = Filename.get_temp_dir_name () // "website" in let index = index dir in let port = Env.website_port in let prometheus = Env.prometheus in @@ -172,10 +189,10 @@ let run () = Filename.dirname index; ] in - {process; dir; monitoring; prometheus} + Lwt.return {process; dir; monitoring; prometheus} let start ~agents = - let t = run () in + let* t = run () in let* () = write t ~agents in Lwt.return t diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 2a2d82715c3a..1c7a511926e6 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -1518,7 +1518,8 @@ let benchmark () = vms |> List.map (function | `Bootstrap -> - Configuration.make ~docker_image:Octez_latest_release () + (* Configuration.make ~docker_image:Octez_latest_release () *) + default_vm_configuration | `Baker i -> ( match configuration.stake_machine_type with | None -> default_vm_configuration @@ -1537,6 +1538,12 @@ let benchmark () = Cloud.register (* docker images are pushed before executing the test in case binaries are modified locally. This way we always use the latest ones. *) ~vms + ~proxy_files: + [ + Format.asprintf + "src/%s/parameters/mainnet-parameters.json" + (Protocol.directory configuration.protocol); + ] ~__FILE__ ~title:"DAL node benchmark" ~tags:[Tag.cloud; "dal"; "benchmark"] -- GitLab From b26c0cdb35677c44e9d892a9230144a9689cbf4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Mon, 15 Jul 2024 16:03:56 +0200 Subject: [PATCH 17/20] Tezt/Cloud: Add a new job for creating a DNS subdomain --- tezt/lib_cloud/cli.ml | 7 +++++++ tezt/lib_cloud/cli.mli | 3 +++ tezt/lib_cloud/env.ml | 2 ++ tezt/lib_cloud/env.mli | 2 ++ tezt/lib_cloud/gcloud.ml | 25 +++++++++++++++++++++++++ tezt/lib_cloud/gcloud.mli | 6 ++++++ tezt/lib_cloud/tezt_cloud.ml | 31 ++++++++++++++++++++++++++++++- 7 files changed, 75 insertions(+), 1 deletion(-) diff --git a/tezt/lib_cloud/cli.ml b/tezt/lib_cloud/cli.ml index e6dc2da97bc0..0929f246a45e 100644 --- a/tezt/lib_cloud/cli.ml +++ b/tezt/lib_cloud/cli.ml @@ -195,3 +195,10 @@ let tezt_cloud = ~long:"tezt-cloud" ~description:"Overwrite the TEZT_CLOUD variable" () + +let dns_domain = + Clap.optional_string + ~section + ~long:"dns-domain" + ~description:"DNS domain to use for the website" + () diff --git a/tezt/lib_cloud/cli.mli b/tezt/lib_cloud/cli.mli index 97124643562e..d18ad0419ac6 100644 --- a/tezt/lib_cloud/cli.mli +++ b/tezt/lib_cloud/cli.mli @@ -88,3 +88,6 @@ val no_max_run_duration : bool val proxy : bool val tezt_cloud : string option + +(** DNS domain to use for the website. *) +val dns_domain : string option diff --git a/tezt/lib_cloud/env.ml b/tezt/lib_cloud/env.ml index 49bdff650f9a..09f834e5f180 100644 --- a/tezt/lib_cloud/env.ml +++ b/tezt/lib_cloud/env.ml @@ -64,6 +64,8 @@ let max_run_duration = Cli.max_run_duration let no_max_run_duration = Cli.no_max_run_duration +let dns_domain = Cli.dns_domain + let docker_image = (* In localhost mode, we don't want to interact with GCP. The image is taken locally. *) diff --git a/tezt/lib_cloud/env.mli b/tezt/lib_cloud/env.mli index 8f259aa416a7..27e531c90b95 100644 --- a/tezt/lib_cloud/env.mli +++ b/tezt/lib_cloud/env.mli @@ -73,6 +73,8 @@ val max_run_duration : int val no_max_run_duration : bool +val dns_domain : string option + val init : unit -> unit Lwt.t val project_id : unit -> string Lwt.t diff --git a/tezt/lib_cloud/gcloud.ml b/tezt/lib_cloud/gcloud.ml index e05a9ef9dff2..6935d99bb9cb 100644 --- a/tezt/lib_cloud/gcloud.ml +++ b/tezt/lib_cloud/gcloud.ml @@ -79,3 +79,28 @@ let list_vms ~prefix = ["compute"; "instances"; "list"; "--filter"; filter] in Lwt.return (String.trim output) + +module DNS = struct + let create_zone ~domain ~zone () = + let dns_name = Format.asprintf "%s.%s" zone domain in + let description = + Format.asprintf "Managed zone for tezt-cloud for %s project" zone + in + Process.run + "gcloud" + [ + "dns"; + "managed-zones"; + "create"; + zone; + "--dns-name"; + dns_name; + "--description"; + description; + ] + + let describe ~zone () = + Process.run_and_read_stdout + "gcloud" + ["dns"; "managed-zones"; "describe"; zone] +end diff --git a/tezt/lib_cloud/gcloud.mli b/tezt/lib_cloud/gcloud.mli index 1b7b7e5b042f..4d99995fa8a4 100644 --- a/tezt/lib_cloud/gcloud.mli +++ b/tezt/lib_cloud/gcloud.mli @@ -43,3 +43,9 @@ val cmd_wrapper : val get_ip_address_from_name : zone:string -> string -> string Lwt.t val list_vms : prefix:string -> string Lwt.t + +module DNS : sig + val create_zone : domain:string -> zone:string -> unit -> unit Lwt.t + + val describe : zone:string -> unit -> string Lwt.t +end diff --git a/tezt/lib_cloud/tezt_cloud.ml b/tezt/lib_cloud/tezt_cloud.ml index a3435e140039..75aca5e39fe1 100644 --- a/tezt/lib_cloud/tezt_cloud.ml +++ b/tezt/lib_cloud/tezt_cloud.ml @@ -80,6 +80,33 @@ let register_list_vms ~tags = let* _ = Gcloud.list_vms ~prefix:Env.tezt_cloud in Lwt.return_unit +let register_create_dns_zone ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Create a new DNS zone" + ~tags:("create" :: "dns" :: "zone" :: tags) + @@ fun _cloud -> + let domain = + match Env.dns_domain with + | None -> + Test.fail "You must specify the domain to use via --dns-domain option." + | Some domain -> domain + in + let* () = Gcloud.DNS.create_zone ~domain ~zone:"tezt-cloud" () in + let* _ = Gcloud.DNS.describe ~zone:"tezt-cloud" () in + unit + +let register_describe_dns_zone ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Describe a new DNS zone" + ~tags:("describe" :: "dns" :: "zone" :: tags) + @@ fun _cloud -> + let* _ = Gcloud.DNS.describe ~zone:"tezt-cloud" () in + unit + let register ~tags = register_docker_push ~tags ; register_docker_build ~tags ; @@ -87,4 +114,6 @@ let register ~tags = register_destroy_vms ~tags ; register_prometheus_import ~tags ; register_clean_up_vms ~tags ; - register_list_vms ~tags + register_list_vms ~tags ; + register_create_dns_zone ~tags ; + register_describe_dns_zone ~tags -- GitLab From 9f83a73c8a0a4978981348a0ad5a9f06aa00e9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Mon, 15 Jul 2024 22:45:29 +0200 Subject: [PATCH 18/20] Tezt/Cloud: Add jobs to add/remove a DNS record --- tezt/lib_cloud/gcloud.ml | 61 ++++++++++++++++++++++++++++++++++++ tezt/lib_cloud/gcloud.mli | 6 ++++ tezt/lib_cloud/tezt_cloud.ml | 34 +++++++++++++++++++- 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/tezt/lib_cloud/gcloud.ml b/tezt/lib_cloud/gcloud.ml index 6935d99bb9cb..ba8e7d0611e3 100644 --- a/tezt/lib_cloud/gcloud.ml +++ b/tezt/lib_cloud/gcloud.ml @@ -103,4 +103,65 @@ module DNS = struct Process.run_and_read_stdout "gcloud" ["dns"; "managed-zones"; "describe"; zone] + + module Transaction = struct + let start ~zone () = + Process.run + "gcloud" + ["dns"; "record-sets"; "transaction"; "start"; "--zone"; zone] + + let add ?(ttl = 300) ?(typ = "A") ~domain ~zone ~ip () = + let ttl = ["--ttl"; Format.asprintf "%d" ttl] in + let typ = ["--type"; Format.asprintf "%s" typ] in + let domain = ["--name"; domain] in + let zone = ["--zone"; zone] in + Process.run + "gcloud" + (["dns"; "record-sets"; "transaction"; "add"] + @ zone @ ttl @ typ @ domain @ [ip]) + + let remove ?(ttl = 300) ?(typ = "A") ~domain ~zone ~ip () = + let ttl = ["--ttl"; Format.asprintf "%d" ttl] in + let typ = ["--type"; Format.asprintf "%s" typ] in + let domain = ["--name"; domain] in + let zone = ["--zone"; zone] in + Process.run + "gcloud" + (["dns"; "record-sets"; "transaction"; "remove"] + @ zone @ ttl @ typ @ domain @ [ip]) + + let execute ~zone () = + Process.run + "gcloud" + ["dns"; "record-sets"; "transaction"; "execute"; "--zone"; zone] + end + + let get_domain ~tezt_cloud ~zone = + let* output = describe ~zone () in + let line = + String.trim output |> String.split_on_char '\n' + |> List.find_opt (String.starts_with ~prefix:"dnsName:") + in + match line with + | None -> Test.fail "Unable to find a managed zone. Have you create it?" + | Some line -> + Lwt.return + (Format.asprintf + "%s.%s" + tezt_cloud + (String.split_on_char ' ' line |> Fun.flip List.nth 1)) + + let add ~tezt_cloud ~zone ~ip = + let* domain = get_domain ~tezt_cloud ~zone in + let* () = Transaction.start ~zone () in + let* () = Transaction.add ~domain ~zone ~ip () in + let* () = Transaction.execute ~zone () in + unit + + let remove ~tezt_cloud ~zone ~ip = + let* domain = get_domain ~tezt_cloud ~zone in + let* () = Transaction.start ~zone () in + let* () = Transaction.remove ~domain ~zone ~ip () in + let* () = Transaction.execute ~zone () in + unit end diff --git a/tezt/lib_cloud/gcloud.mli b/tezt/lib_cloud/gcloud.mli index 4d99995fa8a4..7b04b2a91f51 100644 --- a/tezt/lib_cloud/gcloud.mli +++ b/tezt/lib_cloud/gcloud.mli @@ -48,4 +48,10 @@ module DNS : sig val create_zone : domain:string -> zone:string -> unit -> unit Lwt.t val describe : zone:string -> unit -> string Lwt.t + + val get_domain : tezt_cloud:string -> zone:string -> string Lwt.t + + val add : tezt_cloud:string -> zone:string -> ip:string -> unit Lwt.t + + val remove : tezt_cloud:string -> zone:string -> ip:string -> unit Lwt.t end diff --git a/tezt/lib_cloud/tezt_cloud.ml b/tezt/lib_cloud/tezt_cloud.ml index 75aca5e39fe1..1e0ae5ebd4f8 100644 --- a/tezt/lib_cloud/tezt_cloud.ml +++ b/tezt/lib_cloud/tezt_cloud.ml @@ -107,6 +107,36 @@ let register_describe_dns_zone ~tags = let* _ = Gcloud.DNS.describe ~zone:"tezt-cloud" () in unit +let register_dns_add ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Register a new DNS entry" + ~tags:("dns" :: "add" :: tags) + @@ fun _cloud -> + let tezt_cloud = Env.tezt_cloud in + let ip = Cli.get_string_opt "ip" in + match ip with + | None -> Test.fail "You must provide an IP address via -a ip=" + | Some ip -> + let* () = Gcloud.DNS.add ~tezt_cloud ~zone:"tezt-cloud" ~ip in + unit + +let register_dns_remove ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Remove a DNS entry" + ~tags:("dns" :: "remove" :: tags) + @@ fun _cloud -> + let tezt_cloud = Env.tezt_cloud in + let ip = Cli.get_string_opt "ip" in + match ip with + | None -> Test.fail "You must provide an IP address via -a ip=" + | Some ip -> + let* () = Gcloud.DNS.remove ~tezt_cloud ~zone:"tezt-cloud" ~ip in + unit + let register ~tags = register_docker_push ~tags ; register_docker_build ~tags ; @@ -116,4 +146,6 @@ let register ~tags = register_clean_up_vms ~tags ; register_list_vms ~tags ; register_create_dns_zone ~tags ; - register_describe_dns_zone ~tags + register_describe_dns_zone ~tags ; + register_dns_add ~tags ; + register_dns_remove ~tags -- GitLab From 87a5a789e4d52acf746d682264301e72907e80cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Mon, 15 Jul 2024 22:55:24 +0200 Subject: [PATCH 19/20] Tezt/Cloud: A domain can be associated with the website --- tezt/lib_cloud/cli.ml | 8 ++++++++ tezt/lib_cloud/cli.mli | 3 +++ tezt/lib_cloud/cloud.ml | 24 +++++++++++++++++++----- tezt/lib_cloud/deployement.ml | 19 +++++++++++++++---- tezt/lib_cloud/env.ml | 2 ++ tezt/lib_cloud/env.mli | 2 ++ tezt/lib_cloud/gcloud.ml | 27 +++++++++++++++++++-------- tezt/lib_cloud/prometheus.ml | 9 ++++++++- tezt/tests/cloud/dal.ml | 1 + 9 files changed, 77 insertions(+), 18 deletions(-) diff --git a/tezt/lib_cloud/cli.ml b/tezt/lib_cloud/cli.ml index 0929f246a45e..508c806d4fb6 100644 --- a/tezt/lib_cloud/cli.ml +++ b/tezt/lib_cloud/cli.ml @@ -202,3 +202,11 @@ let dns_domain = ~long:"dns-domain" ~description:"DNS domain to use for the website" () + +let dns = + Clap.flag + ~section + ~set_long:"dns" + ~unset_long:"no-dns" + ~description:"Associate a domain name with the website" + (proxy && website) diff --git a/tezt/lib_cloud/cli.mli b/tezt/lib_cloud/cli.mli index d18ad0419ac6..25cf21760eb0 100644 --- a/tezt/lib_cloud/cli.mli +++ b/tezt/lib_cloud/cli.mli @@ -91,3 +91,6 @@ val tezt_cloud : string option (** DNS domain to use for the website. *) val dns_domain : string option + +(** Associate a DNS domain to the website when the proxy mode is used. *) +val dns : bool diff --git a/tezt/lib_cloud/cloud.ml b/tezt/lib_cloud/cloud.ml index 6e533c7fd507..3b08bea6b76a 100644 --- a/tezt/lib_cloud/cloud.ml +++ b/tezt/lib_cloud/cloud.ml @@ -92,6 +92,14 @@ let shutdown ?exn t = let* () = Option.fold ~none:Lwt.return_unit ~some:Grafana.shutdown t.grafana in + let* () = + if Env.dns then + Gcloud.DNS.remove + ~tezt_cloud:Env.tezt_cloud + ~zone:"tezt-cloud" + ~ip:(Proxy.get_agent t.agents |> Agent.point |> fst) + else Lwt.return_unit + in let* () = Option.fold ~none:Lwt.return_unit @@ -223,11 +231,17 @@ let attach agent = let* () = eof in Log.info "Detach from the proxy process." ; if !has_sigint then on_sigint - else ( - Log.info - "Deployement website can be accessed here: http://%s:8080" - (Agent.point agent |> fst) ; - Lwt.return_unit) + else + let* uri = + if Env.dns then + let* domain = + Gcloud.DNS.get_domain ~tezt_cloud:Env.tezt_cloud ~zone:"tezt-cloud" + in + Lwt.return (Format.asprintf "http://%s" domain) + else Lwt.return (Format.asprintf "http://%s" (Agent.point agent |> fst)) + in + Log.info "Deployement website can be accessed here: %s" uri ; + Lwt.return_unit in Log.Style.set_prefix Log.Style.Hidden ; Log.Style.set_timestamp Log.Style.Hidden ; diff --git a/tezt/lib_cloud/deployement.ml b/tezt/lib_cloud/deployement.ml index 0f4c55acf067..d1e5e4ed1029 100644 --- a/tezt/lib_cloud/deployement.ml +++ b/tezt/lib_cloud/deployement.ml @@ -179,10 +179,21 @@ module Remote = struct configurations. *) order_agents (List.concat agents) configurations in - if proxy then - let* agent = deploy_proxy () in - Lwt.return {agents = agent :: agents} - else Lwt.return {agents} + let* t = + if proxy then + let* agent = deploy_proxy () in + let* () = + if Env.dns then + Gcloud.DNS.add + ~tezt_cloud:Env.tezt_cloud + ~zone:"tezt-cloud" + ~ip:(Agent.point agent |> fst) + else Lwt.return_unit + in + Lwt.return {agents = agent :: agents} + else Lwt.return {agents} + in + Lwt.return t let agents t = t.agents diff --git a/tezt/lib_cloud/env.ml b/tezt/lib_cloud/env.ml index 09f834e5f180..55a69be711bd 100644 --- a/tezt/lib_cloud/env.ml +++ b/tezt/lib_cloud/env.ml @@ -66,6 +66,8 @@ let no_max_run_duration = Cli.no_max_run_duration let dns_domain = Cli.dns_domain +let dns = Cli.dns + let docker_image = (* In localhost mode, we don't want to interact with GCP. The image is taken locally. *) diff --git a/tezt/lib_cloud/env.mli b/tezt/lib_cloud/env.mli index 27e531c90b95..a0acc73c3c11 100644 --- a/tezt/lib_cloud/env.mli +++ b/tezt/lib_cloud/env.mli @@ -75,6 +75,8 @@ val no_max_run_duration : bool val dns_domain : string option +val dns : bool + val init : unit -> unit Lwt.t val project_id : unit -> string Lwt.t diff --git a/tezt/lib_cloud/gcloud.ml b/tezt/lib_cloud/gcloud.ml index ba8e7d0611e3..be45ddfaae93 100644 --- a/tezt/lib_cloud/gcloud.ml +++ b/tezt/lib_cloud/gcloud.ml @@ -134,6 +134,11 @@ module DNS = struct Process.run "gcloud" ["dns"; "record-sets"; "transaction"; "execute"; "--zone"; zone] + + let abort ~zone () = + Process.run + "gcloud" + ["dns"; "record-sets"; "transaction"; "abort"; "--zone"; zone] end let get_domain ~tezt_cloud ~zone = @@ -153,15 +158,21 @@ module DNS = struct let add ~tezt_cloud ~zone ~ip = let* domain = get_domain ~tezt_cloud ~zone in - let* () = Transaction.start ~zone () in - let* () = Transaction.add ~domain ~zone ~ip () in - let* () = Transaction.execute ~zone () in - unit + Lwt.catch + (fun () -> + let* () = Transaction.start ~zone () in + let* () = Transaction.add ~domain ~zone ~ip () in + let* () = Transaction.execute ~zone () in + unit) + (fun _ -> Transaction.abort ~zone ()) let remove ~tezt_cloud ~zone ~ip = let* domain = get_domain ~tezt_cloud ~zone in - let* () = Transaction.start ~zone () in - let* () = Transaction.remove ~domain ~zone ~ip () in - let* () = Transaction.execute ~zone () in - unit + Lwt.catch + (fun () -> + let* () = Transaction.start ~zone () in + let* () = Transaction.remove ~domain ~zone ~ip () in + let* () = Transaction.execute ~zone () in + unit) + (fun _ -> Transaction.abort ~zone ()) end diff --git a/tezt/lib_cloud/prometheus.ml b/tezt/lib_cloud/prometheus.ml index 27e78423209f..50fcfb03855e 100644 --- a/tezt/lib_cloud/prometheus.ml +++ b/tezt/lib_cloud/prometheus.ml @@ -66,7 +66,14 @@ let tezt_source = { job_name = "tezt_metrics"; metric_path = "/metrics.txt"; - targets = [{address = "localhost"; port = 8080; app_name = "tezt"}]; + targets = + [ + { + address = "localhost"; + port = (if Env.mode = `Orchestrator then 80 else 8080); + app_name = "tezt"; + }; + ]; } let config ~scrape_interval sources = diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 1c7a511926e6..807a0c2a6b04 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -1543,6 +1543,7 @@ let benchmark () = Format.asprintf "src/%s/parameters/mainnet-parameters.json" (Protocol.directory configuration.protocol); + "evm_kernel.wasm"; ] ~__FILE__ ~title:"DAL node benchmark" -- GitLab From 7583993ddbf3a4c4b6980516c4c7dfc62bcc51e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Sun, 7 Jul 2024 20:32:09 +0200 Subject: [PATCH 20/20] Tezt/Cloud: Update README --- tezt/lib_cloud/README.md | 294 ++++++++++++++++++++++++++------------- 1 file changed, 195 insertions(+), 99 deletions(-) diff --git a/tezt/lib_cloud/README.md b/tezt/lib_cloud/README.md index 2dc82486dc3a..fc21a44975a4 100644 --- a/tezt/lib_cloud/README.md +++ b/tezt/lib_cloud/README.md @@ -1,11 +1,20 @@ # Tezt cloud -This library can used to write Tezt tests that can use machines deployed in the -cloud. At the moment those machines must be deployed onto GCP, and hence require -a GCP account to be used. +This library can be used to write Tezt tests that can use VMs deployed in the +cloud. At the moment those VMs must be deployed onto GCP, and hence require a +GCP account to be used. Scenarios written with tezt cloud can also be run in +localhost without using GCP. # Initialisation +There are two important configuration steps before using the library: + +- Having a GCP account +- Have your own dockerfile (or choose an existent one that suits you) + +Do note that the GCP account is not necessary if you use the library +in `localhost` mode (and so you don't need `gcloud` and `terraform`). + ## Creating a GCP account Check with your manager how you can have a GCP account and to which @@ -42,7 +51,7 @@ Run the following commands: with your login. 4. `gcloud services enable artifactregistry.googleapis.com`: Allow using the - artifact registries of Google (useful for pushing docker images). + artifact registries of Google (necessary for pushing docker images). 5. `sudo usermod -aG docker $USER` where `$USER` is your user. This is to enable `docker` commands to be run without being sudo. @@ -50,22 +59,47 @@ Run the following commands: 6. `gcloud config set project [YOUR_PROJECT_ID]`: For configuring the default project. -7. (optional) `gcloud auth configure-docker europe-west1-docker.pkg.dev`: - Authorize the artifact registry used by this project. +7. (optional) `gcloud auth configure-docker + europe-west1-docker.pkg.dev`: Authorize the artifact registry used + by this project. This can be useful if you plan to push docker + images without tezt cloud. + +# Tezt cloud variable -Please set an environment variable `TEZT_CLOUD` which will be used to identify -the resources you deploy onto the cloud. The value should be unique for the -given project. We recommend using a similar username as the one you use on -Gitlab. +Any time you use the Tezt cloud library, a namespace for your +experiment must be provided. Most of the time, if you only run one +experiment at a time, you can use your gitlab username for example. +Such a namespace can be provided via the environment variable +`TEZT_CLOUD` or via the CLI option `--tezt-cloud`. It may be +interesting to set the `TEZT_CLOUD` environment variable with your +gitlab username since this is the one you will use the most. + +This variable is also used to determine the dockerfile to be used for +your experiment. In case you want to use a different dockerfile, you +can use the CLI option `--dockerfile-alias`. # Docker -The library assumes that you have a dockerfile image in the subdirectory of the -library `tezt/lib_cloud/dockerfiles` (it can be committed so that it gives -people an easy way to copy/paste the dockerfile for they own purpose). This -image will be the one that will be installed on each running VM. The library -takes care for you to push this image on a dedicated GCP registry by running the -appropriate command (see below). +The library relies mainly on docker. Each VM that will be deployed +will run a docker image. This docker image must follow some conditions +(mainly running an SSH server), but apart from that you are free to +use your own dockerfile. + +To help you write such an image, a default one based on debian is +provided in `tezt/lib_cloud/saroupille.Dockerfile`. + +The library takes care for you to push this image on a dedicated GCP +registry. A docker image such as the one generated from +`saroupille.Dockerfile` may contain some binaries. Tezt cloud always +try to rebuild the docker image to be sure it uses the last version, +however be sure to compile the binaries beforehand. + +Finally, do note that with Tezt cloud you can specify for each VM +individually the docker image to be used. The one provided by the +dockerfile as discussed above will be the one by default. It is also +possible to use docker images published by the Octez CI. At the +moment, the docker image from the latest released of Octez is +supported. ## Write your own dockerfile @@ -73,44 +107,29 @@ Depending on your local setup, we recommend to copy/paste an image that is similar to your local setup. An example for a ubuntu/debian-like machine is given by `saroupille.Dockerfile`. At the moment, several assumptions are made: -1. An ssh server is running and you have access to it. -2. The ssh key should be in the default ssh directory and named - `$(TF_WORKSPACE)-tf`. The docker variable for specifying the ssh key should - be named `SSH_PUBLIC_KEY`. -3. For convenience, we recommend the image to contain the minimum necessary to +1. An ssh server is running on port `30000` +2. For convenience, we recommend the image to contain the minimum necessary to run Octez binaries that were compiled on your local setup. In particular, we use the docker variable `ZCASH_PARAMS_PATH` to provide the place where the zcash parameters are used. -4. Optionally, you can also put the binaries into the docker image (see +3. Optionally, you can also put the binaries into the docker image (see `saroupille.Dockerfile`). -# One-time deployment - -The library `tezt-cloud` needs to deploy resources that will be used throughout -your different experimentations and these resources should not be destroyed. - -At the moment there are two resources of this kind which are detailed -below. +To ensure you can connect to the ssh server, the library generates (if +it does not exist) and ssh key based on the `tezt-cloud` variable. -## Terraform state - -Terraform maintains a state of the deployed resources that are *owned* by -`terraform`. A standard way to store them is to use a GCP bucket. +# One-time deployment -Once your `TEZT_CLOUD` value is set and you have the appropriate -rights, you should be able to create the bucket using: - -```bash -dune exec tezt/tests/cloud/main.exe -- terraform state deploy bucket -v -``` +The library `tezt-cloud` needs to deploy resources that will be used +and reused throughout your different experimentations. Tezt cloud +should deploy them for you automatically. Hence the commands in this +section are optional. ## Docker registry -A docker registry is needed so that, during initialisation of the VMs, -the docker image can be pulled from the cloud. - -Once your `TEZT_CLOUD` value is set and you have the appropriate -rights, you should be able to create the registry using: +Tezt cloud does a best effort to create a docker registry if it does +not exists already. The name of the docker registry depends on the +`TEZT_CLOUD` value. ```bash dune exec tezt/tests/cloud/main.exe -- docker registry deploy -v @@ -118,7 +137,7 @@ dune exec tezt/tests/cloud/main.exe -- docker registry deploy -v ## Deployment -To push your image, you should run: +If you want to push your image manually on the registry you can do: ```bash dune exec tezt/tests/cloud/main.exe -- docker push -v @@ -126,12 +145,17 @@ dune exec tezt/tests/cloud/main.exe -- docker push -v ## Run your first scenario -Any scenario can be run on the local host for checking any deployment -error. Hence, when testing any scenario we recommend to provide the -`--localhost` parameter with a low number of machines to check whether your -scenario works as expected. The number of VMs can be specified on the -command-line via `--vms`. But it can also be computed by the scenario, so that -`--vms` is not mandatory. +Any Tezt cloud scenario can be run on the host machine for checking +any deployment error, this is the `localhost` mode. Hence, when +testing any scenario we recommend to provide the `--localhost` +parameter with a low number of machines to check whether your scenario +works as expected. + +When you are ready to deploy resources, you can remove the +`--localhost` option, or alternatively use the `--cloud` option. When +deploying machines on the cloud, the number of VMs can be specified on +the command-line via `--vms`. But it can also be computed by the +scenario, so that `--vms` is not mandatory. We recommend to test it twice: using localhost or not: @@ -148,7 +172,7 @@ dune exec tezt/tests/cloud/main.exe -- cloud health -v (Any scenario using this library should contain the tag `cloud`.) If these two tests succeed, you are ready to create your own scenarios with -`tezt-cloud`. +`tezt-cloud` or use existent ones. ## Destroying VMs @@ -162,15 +186,16 @@ running: dune exec tezt/tests/cloud/main.exe -- terraform destroy -v ``` -(It is actually quite difficult to provide an automatic way to destroy resources -published onto the cloud. A possible solution in the future would be to deploy a -cron job locally.) +However, all your VMs may come with a time to live parameter. For the +`--cloud` mode, it is set to 2 hours. For the proxy mode (see below), +it is not set by default. ## Monitoring Tezt-cloud offers you a way to monitor your VMs via [netdata](https://www.netdata.cloud/). To enable monitoring, you only have to provide the option `--monitoring` while running your scenario. +In which case, all the VMs will be monitored. The netdata dashboard can be accessed on port `19999` on the remote machine. Do note that if you use localhost, the monitoring will monitor your entire machine. @@ -184,45 +209,132 @@ stopped until the VM is being teared down. ## Website -A website can be run to summarize information about the experiment. This can be -activated with the option `--website`. The website runs at -`http://localhost:8080`. The port can be changed with option `--website-port`. -The website can also be used to report ad-hoc metrics (see below). +A website can be run to summarize information about the experiment. +This can be activated with the option `--website`. The website runs at +`http://localhost:8080`. The port can be changed with option +`--website-port`. The website can also be used to report ad-hoc +metrics (see in the next section). ## Prometheus -Prometheus can be used to export metrics from `netdata`. A prometheus container -can be spawned on the host machine via the option `--prometheus`. As a -side-effect, this will also export a snapshot at the end of the experiment. By -default, the snapshot will be stored in the `/tmp` directory. The directory can -be modified with the `--prometheus-snapshot-directory` option. The name of the -snapshot can be set via `--prometheus-snapshot`. +Prometheus can be used to export metrics from `netdata` or export +Octez metrics. A prometheus container can be spawned on the host +machine via the option `--prometheus`. As a side-effect, this will +also export a snapshot at the end of the experiment. By default, the +snapshot will be stored in the `/tmp` directory. The name of the +snapshot can be specified via `--prometheus-snapshot-filename`. After being exported, a snapshot can be imported by running: ```bash -dune exec tezt/tests/cloud/main.exe -- prometheus import --prometheus-snapshot +dune exec tezt/tests/cloud/main.exe -- prometheus import --prometheus-snapshot-filename ``` -## Exporting ad-hoc metrics +### Exporting ad-hoc metrics -While writing an experiment, it may be interesting to export metrics. This is -doable if the experiment is run with options `--website` and `--prometheus`. By -doing so, metrics can be exported via the function `Cloud.push_metric`. It takes -as parameters the name of the metric as well as optional labels and the value to -export. Metrics are reported on a special page `/metrics.txt` on the website and -will be scrapped by the Prometheus instance. Such metrics will be automatically -snapshotted at the end of the experiments. +While writing an experiment, it may be interesting to export metrics +to the scenario run (for example to ensure the scenario behaves as +expected). This feature requires the experiment to be run with +`--website` and `--prometheus` (which is the case by default). By +doing so, metrics can be exported via the function +`Cloud.push_metric`. It takes as parameters the name of the metric as +well as optional labels and the value to export. Metrics are provided +on the website at the following URI `/metrics.txt` and will be +scrapped by the Prometheus instance. Those metrics are also exported +at the end of the experiment in the Prometheus snapshot. -## Exporting daemons metrics +### Exporting daemons metrics -Metrics exported by daemons such as `octez-node` or `octez-dal-node` can also be -exported to the Prometheus instance using the function +When writing a new scenario, you can instruct how to export daemons +(e.g. `octez-node`) metrics to Prometheus by calling the function `Cloud.add_prometheus_source`. +## Grafana + +Grafana can be used to visualize prometheus metrics (activated by +default). This can be accessed on `http://localhost:3000`. Tezt cloud +comes with a set of dashboards so that it is ready to use. Feel free +to update/add new dashboards in +`tezt/lib_cloud/grafana/dasboards.json`. + +## Proxy mode + +**EXPERIMENTAL** + +The `--proxy` option allow you to run your experiment onto a VM +instead of your local machine. This may have several benefits, but in +particular, it can be used to set up long experiments (that can last +for weeks). + +When running a test in `--proxy` mode, you can detach it after the +deployement has been made and the scenario start to kick in. In that +case, you can detach from your experiment by pressing `ctrl+d`. By +doing so, you can do whatever you want, but the experiment is still +executed on a remote VM. + +To reattach the test, simply rerun the command. Normally, it should +reattach back the running experiment. + +Killing an experiment manually requires some care. You must do it in +two steps: + +1. Press Ctrl+C. This will trigger the shutdown of the +experiment. + +2. Once you see the scenario has ended properly, you can press Ctrl+D. + +If you make a mistake during the shutdown, the scenario may not end +properly. In that case, you can clean up the VMs by running: + +``` +dune exec tezt/tests/cloud/main.exe -- cloud clean up -v +``` + +UX-wise, the proxy mode has still some awkwards behaviour regarding +how log events are printed on `Ctrl+D` or `Ctrl+C`. Moreover, the +handling of `sigint` (Ctrl+C) may be a bit fragile. + +We recommend to use first the `Cloud` mode before using the `Proxy` +mode to ensure the scenario behaves as expected. + +With the proxy mode, we recommend to use the website if you want to +follow your experiment. The website should be provided in the logs +once you detach from the experiment. + +Otherwise, you can connect to it by looking at the external IP address +of the `proxy` machine on GCP and use the port `8080`. + +### DNS + +The website deployed with the proxy mode can be associated with a +domain name if the `--dns` argument is provided (true by default). + +The domain associated with the website will be: +`http://..`. + +To make it work, any project must register a subdomain zone +beforehand. This can be done via: + +``` +dune exec tezt/tests/cloud/main.exe -- cloud create dns zone -v --dns-domain +``` + +The name servers associated with this domain must be added manually to +the domain regstriy associated with ``. + +You can check this work as expected by running: + +``` +dig ns . +``` + +You should see the name servers you just added. + + ## Recommendation when writing experiments -0. Never forget to destroy your VMs. +0. Even though there is a time to live for the VM, please don't forget + to destroy your VMs. 1. You can spawn mutiple daemons on the same VM. @@ -233,9 +345,11 @@ exported to the Prometheus instance using the function use `Agent.next_port`. 3. The runner that will be used (this can be given to the agent). -3. Whenever it is possible, write your test without depending specifically on a - fixed number of machines. Whenver you want to increase or decrease the number - of machines, you can just run your test again and it will just work. +3. Whenever it is possible, write your test without depending + specifically on a fixed number of machines. Whenver you want to + increase or decrease the number of machines, you can just run your + test again and it will just work. This can be done by using a + dispenser (via the OCaml module `Seq` for example). 4. Use `--log-file` option from Tezt so that you can inspect errors a posteriori. @@ -305,21 +419,3 @@ exported to the Prometheus instance using the function ControlPath ~/.ssh/master-%r@%h:%p ControlPersist 120 ``` - -## Limitations and future work - -At the moment, there are two main limitations to `tezt-cloud`: - -- The host machine receives and parses all the events fromm all the daemons. It - could be interesting instead to have a `proxy` mode so that intermediate - processes are hosted on VMs themselves to catch some of those events. This - could be also used as a basis to implement `detach/attach` capabilities to - Tezt cloud so that you don't have to be connected all the time on your machine - during the experiment. - -- All the VMs must use the same kind of machine. For performance-related - experiments, it may be interesting to have different kinds of machines for - different kinds of VMs. While this could be supported in future versions of - `tezt-cloud`, the only way to by-pass this limitation is to use multiple - `workspace`s (you need to implement the logic to handle multiple workspaces - yourself). -- GitLab